diff --git "a/checkpoint-25177/trainer_state.json" "b/checkpoint-25177/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-25177/trainer_state.json" @@ -0,0 +1,181823 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "global_step": 25177, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-06, + "loss": 1.3412, + "theoretical_loss": 3.515379199078347, + "tokens_seen": 1650065408 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-06, + "loss": 1.2724, + "theoretical_loss": 3.515367210984307, + "tokens_seen": 1650130944 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809525e-06, + "loss": 1.2857, + "theoretical_loss": 3.515355223499678, + "tokens_seen": 1650196480 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 1.3914, + "theoretical_loss": 3.515343236624406, + "tokens_seen": 1650262016 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-06, + "loss": 1.2866, + "theoretical_loss": 3.5153312503584346, + "tokens_seen": 1650327552 + }, + { + "epoch": 0.0, + "learning_rate": 1.1904761904761905e-05, + "loss": 1.3488, + "theoretical_loss": 3.5153192647017093, + "tokens_seen": 1650393088 + }, + { + "epoch": 0.0, + "learning_rate": 1.3888888888888888e-05, + "loss": 1.2748, + "theoretical_loss": 3.5153072796541744, + "tokens_seen": 1650458624 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 1.4147, + "theoretical_loss": 3.5152952952157754, + "tokens_seen": 1650524160 + }, + { + "epoch": 0.0, + "learning_rate": 1.7857142857142855e-05, + "loss": 1.3383, + "theoretical_loss": 3.515283311386457, + "tokens_seen": 1650589696 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-05, + "loss": 1.2727, + "theoretical_loss": 3.515271328166164, + "tokens_seen": 1650655232 + }, + { + "epoch": 0.0, + "learning_rate": 2.1825396825396824e-05, + "loss": 1.2951, + "theoretical_loss": 3.515259345554841, + "tokens_seen": 1650720768 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 1.2859, + "theoretical_loss": 3.515247363552433, + "tokens_seen": 1650786304 + }, + { + "epoch": 0.0, + "learning_rate": 2.5793650793650793e-05, + "loss": 1.2871, + "theoretical_loss": 3.515235382158885, + "tokens_seen": 1650851840 + }, + { + "epoch": 0.0, + "learning_rate": 2.7777777777777776e-05, + "loss": 1.2359, + "theoretical_loss": 3.5152234013741417, + "tokens_seen": 1650917376 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904762e-05, + "loss": 1.3175, + "theoretical_loss": 3.5152114211981487, + "tokens_seen": 1650982912 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 1.2909, + "theoretical_loss": 3.51519944163085, + "tokens_seen": 1651048448 + }, + { + "epoch": 0.0, + "learning_rate": 3.373015873015873e-05, + "loss": 1.3605, + "theoretical_loss": 3.5151874626721904, + "tokens_seen": 1651113984 + }, + { + "epoch": 0.0, + "learning_rate": 3.571428571428571e-05, + "loss": 1.3939, + "theoretical_loss": 3.515175484322116, + "tokens_seen": 1651179520 + }, + { + "epoch": 0.0, + "learning_rate": 3.76984126984127e-05, + "loss": 1.3198, + "theoretical_loss": 3.5151635065805706, + "tokens_seen": 1651245056 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 1.3317, + "theoretical_loss": 3.5151515294474995, + "tokens_seen": 1651310592 + }, + { + "epoch": 0.0, + "learning_rate": 4.1666666666666665e-05, + "loss": 1.3075, + "theoretical_loss": 3.515139552922848, + "tokens_seen": 1651376128 + }, + { + "epoch": 0.0, + "learning_rate": 4.365079365079365e-05, + "loss": 1.2531, + "theoretical_loss": 3.5151275770065604, + "tokens_seen": 1651441664 + }, + { + "epoch": 0.0, + "learning_rate": 4.563492063492063e-05, + "loss": 1.2712, + "theoretical_loss": 3.5151156016985823, + "tokens_seen": 1651507200 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 1.318, + "theoretical_loss": 3.5151036269988585, + "tokens_seen": 1651572736 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-05, + "loss": 1.3133, + "theoretical_loss": 3.5150916529073335, + "tokens_seen": 1651638272 + }, + { + "epoch": 0.0, + "learning_rate": 5.1587301587301586e-05, + "loss": 1.3218, + "theoretical_loss": 3.5150796794239527, + "tokens_seen": 1651703808 + }, + { + "epoch": 0.0, + "learning_rate": 5.357142857142857e-05, + "loss": 1.332, + "theoretical_loss": 3.5150677065486615, + "tokens_seen": 1651769344 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555555e-05, + "loss": 1.3484, + "theoretical_loss": 3.515055734281404, + "tokens_seen": 1651834880 + }, + { + "epoch": 0.0, + "learning_rate": 5.753968253968254e-05, + "loss": 1.3053, + "theoretical_loss": 3.515043762622125, + "tokens_seen": 1651900416 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809524e-05, + "loss": 1.2697, + "theoretical_loss": 3.5150317915707707, + "tokens_seen": 1651965952 + }, + { + "epoch": 0.0, + "learning_rate": 6.15079365079365e-05, + "loss": 1.212, + "theoretical_loss": 3.5150198211272854, + "tokens_seen": 1652031488 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 1.3364, + "theoretical_loss": 3.5150078512916143, + "tokens_seen": 1652097024 + }, + { + "epoch": 0.0, + "learning_rate": 6.547619047619048e-05, + "loss": 1.41, + "theoretical_loss": 3.514995882063702, + "tokens_seen": 1652162560 + }, + { + "epoch": 0.0, + "learning_rate": 6.746031746031745e-05, + "loss": 1.3038, + "theoretical_loss": 3.5149839134434946, + "tokens_seen": 1652228096 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-05, + "loss": 1.3606, + "theoretical_loss": 3.5149719454309363, + "tokens_seen": 1652293632 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857142e-05, + "loss": 1.4097, + "theoretical_loss": 3.5149599780259715, + "tokens_seen": 1652359168 + }, + { + "epoch": 0.0, + "learning_rate": 7.341269841269842e-05, + "loss": 1.2951, + "theoretical_loss": 3.514948011228547, + "tokens_seen": 1652424704 + }, + { + "epoch": 0.0, + "learning_rate": 7.53968253968254e-05, + "loss": 1.3278, + "theoretical_loss": 3.5149360450386062, + "tokens_seen": 1652490240 + }, + { + "epoch": 0.0, + "learning_rate": 7.738095238095239e-05, + "loss": 1.2061, + "theoretical_loss": 3.514924079456095, + "tokens_seen": 1652555776 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 1.305, + "theoretical_loss": 3.5149121144809587, + "tokens_seen": 1652621312 + }, + { + "epoch": 0.0, + "learning_rate": 8.134920634920635e-05, + "loss": 1.2988, + "theoretical_loss": 3.5149001501131423, + "tokens_seen": 1652686848 + }, + { + "epoch": 0.0, + "learning_rate": 8.333333333333333e-05, + "loss": 1.227, + "theoretical_loss": 3.5148881863525903, + "tokens_seen": 1652752384 + }, + { + "epoch": 0.0, + "learning_rate": 8.531746031746032e-05, + "loss": 1.3487, + "theoretical_loss": 3.5148762231992476, + "tokens_seen": 1652817920 + }, + { + "epoch": 0.0, + "learning_rate": 8.73015873015873e-05, + "loss": 1.3457, + "theoretical_loss": 3.5148642606530602, + "tokens_seen": 1652883456 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571429e-05, + "loss": 1.3175, + "theoretical_loss": 3.514852298713973, + "tokens_seen": 1652948992 + }, + { + "epoch": 0.0, + "learning_rate": 9.126984126984126e-05, + "loss": 1.3399, + "theoretical_loss": 3.514840337381931, + "tokens_seen": 1653014528 + }, + { + "epoch": 0.0, + "learning_rate": 9.325396825396825e-05, + "loss": 1.2559, + "theoretical_loss": 3.5148283766568795, + "tokens_seen": 1653080064 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 1.3705, + "theoretical_loss": 3.514816416538763, + "tokens_seen": 1653145600 + }, + { + "epoch": 0.0, + "learning_rate": 9.722222222222223e-05, + "loss": 1.3992, + "theoretical_loss": 3.5148044570275276, + "tokens_seen": 1653211136 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 603740, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2631263732910156, + "objective/train/theoretical_loss": 3.5147924981231173, + "objective/train/tokens_used": 23735776, + "theoretical_loss": 3.5147924981231173, + "tokens_seen": 1653276672 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-05, + "loss": 1.261, + "theoretical_loss": 3.5147924981231173, + "tokens_seen": 1653276672 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001011904761904762, + "loss": 1.3669, + "theoretical_loss": 3.5147805398254786, + "tokens_seen": 1653342208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460317, + "loss": 1.3943, + "theoretical_loss": 3.514768582134555, + "tokens_seen": 1653407744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010515873015873016, + "loss": 1.2742, + "theoretical_loss": 3.5147566250502935, + "tokens_seen": 1653473280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010714285714285714, + "loss": 1.2705, + "theoretical_loss": 3.514744668572638, + "tokens_seen": 1653538816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010912698412698413, + "loss": 1.3269, + "theoretical_loss": 3.514732712701534, + "tokens_seen": 1653604352 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001111111111111111, + "loss": 1.3336, + "theoretical_loss": 3.514720757436927, + "tokens_seen": 1653669888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011309523809523809, + "loss": 1.2208, + "theoretical_loss": 3.5147088027787623, + "tokens_seen": 1653735424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011507936507936508, + "loss": 1.3446, + "theoretical_loss": 3.5146968487269845, + "tokens_seen": 1653800960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011706349206349207, + "loss": 1.2881, + "theoretical_loss": 3.5146848952815386, + "tokens_seen": 1653866496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 1.3218, + "theoretical_loss": 3.5146729424423713, + "tokens_seen": 1653932032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012103174603174604, + "loss": 1.2394, + "theoretical_loss": 3.514660990209426, + "tokens_seen": 1653997568 + }, + { + "epoch": 0.0, + "learning_rate": 0.000123015873015873, + "loss": 1.3168, + "theoretical_loss": 3.5146490385826494, + "tokens_seen": 1654063104 + }, + { + "epoch": 0.0, + "learning_rate": 0.000125, + "loss": 1.3394, + "theoretical_loss": 3.5146370875619857, + "tokens_seen": 1654128640 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 1.1952, + "theoretical_loss": 3.5146251371473807, + "tokens_seen": 1654194176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012896825396825398, + "loss": 1.4024, + "theoretical_loss": 3.51461318733878, + "tokens_seen": 1654259712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013095238095238096, + "loss": 1.3534, + "theoretical_loss": 3.5146012381361276, + "tokens_seen": 1654325248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013293650793650793, + "loss": 1.2637, + "theoretical_loss": 3.51458928953937, + "tokens_seen": 1654390784 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001349206349206349, + "loss": 1.3883, + "theoretical_loss": 3.514577341548452, + "tokens_seen": 1654456320 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001369047619047619, + "loss": 1.3304, + "theoretical_loss": 3.5145653941633186, + "tokens_seen": 1654521856 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001388888888888889, + "loss": 1.3348, + "theoretical_loss": 3.514553447383916, + "tokens_seen": 1654587392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014087301587301586, + "loss": 1.3753, + "theoretical_loss": 3.5145415012101884, + "tokens_seen": 1654652928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714284, + "loss": 1.2047, + "theoretical_loss": 3.514529555642082, + "tokens_seen": 1654718464 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014484126984126984, + "loss": 1.4171, + "theoretical_loss": 3.5145176106795417, + "tokens_seen": 1654784000 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014682539682539685, + "loss": 1.2871, + "theoretical_loss": 3.514505666322513, + "tokens_seen": 1654849536 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014880952380952382, + "loss": 1.2237, + "theoretical_loss": 3.5144937225709407, + "tokens_seen": 1654915072 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 1.3116, + "theoretical_loss": 3.514481779424771, + "tokens_seen": 1654980608 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001527777777777778, + "loss": 1.3522, + "theoretical_loss": 3.5144698368839484, + "tokens_seen": 1655046144 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015476190476190478, + "loss": 1.2708, + "theoretical_loss": 3.5144578949484186, + "tokens_seen": 1655111680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015674603174603175, + "loss": 1.2083, + "theoretical_loss": 3.514445953618127, + "tokens_seen": 1655177216 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 1.3442, + "theoretical_loss": 3.5144340128930196, + "tokens_seen": 1655242752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016071428571428573, + "loss": 1.3725, + "theoretical_loss": 3.51442207277304, + "tokens_seen": 1655308288 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001626984126984127, + "loss": 1.2751, + "theoretical_loss": 3.5144101332581354, + "tokens_seen": 1655373824 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016468253968253969, + "loss": 1.3079, + "theoretical_loss": 3.5143981943482503, + "tokens_seen": 1655439360 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016666666666666666, + "loss": 1.4171, + "theoretical_loss": 3.5143862560433305, + "tokens_seen": 1655504896 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016865079365079366, + "loss": 1.3876, + "theoretical_loss": 3.5143743183433207, + "tokens_seen": 1655570432 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017063492063492064, + "loss": 1.2589, + "theoretical_loss": 3.514362381248167, + "tokens_seen": 1655635968 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017261904761904762, + "loss": 1.2565, + "theoretical_loss": 3.514350444757815, + "tokens_seen": 1655701504 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001746031746031746, + "loss": 1.3118, + "theoretical_loss": 3.5143385088722088, + "tokens_seen": 1655767040 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001765873015873016, + "loss": 1.3472, + "theoretical_loss": 3.5143265735912954, + "tokens_seen": 1655832576 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017857142857142857, + "loss": 1.2835, + "theoretical_loss": 3.5143146389150193, + "tokens_seen": 1655898112 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018055555555555555, + "loss": 1.2546, + "theoretical_loss": 3.5143027048433266, + "tokens_seen": 1655963648 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968252, + "loss": 1.3388, + "theoretical_loss": 3.514290771376162, + "tokens_seen": 1656029184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018452380952380953, + "loss": 1.3113, + "theoretical_loss": 3.514278838513471, + "tokens_seen": 1656094720 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001865079365079365, + "loss": 1.3084, + "theoretical_loss": 3.5142669062552, + "tokens_seen": 1656160256 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001884920634920635, + "loss": 1.2925, + "theoretical_loss": 3.514254974601293, + "tokens_seen": 1656225792 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 1.2899, + "theoretical_loss": 3.5142430435516974, + "tokens_seen": 1656291328 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019246031746031748, + "loss": 1.2437, + "theoretical_loss": 3.514231113106357, + "tokens_seen": 1656356864 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019444444444444446, + "loss": 1.2868, + "theoretical_loss": 3.514219183265218, + "tokens_seen": 1656422400 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019642857142857144, + "loss": 1.3238, + "theoretical_loss": 3.514207254028226, + "tokens_seen": 1656487936 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 604805, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4689288139343262, + "objective/train/theoretical_loss": 3.5141953253953258, + "objective/train/tokens_used": 27012576, + "theoretical_loss": 3.5141953253953258, + "tokens_seen": 1656553472 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001984126984126984, + "loss": 1.3694, + "theoretical_loss": 3.5141953253953258, + "tokens_seen": 1656553472 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020039682539682542, + "loss": 1.3317, + "theoretical_loss": 3.5141833973664633, + "tokens_seen": 1656619008 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002023809523809524, + "loss": 1.2069, + "theoretical_loss": 3.5141714699415845, + "tokens_seen": 1656684544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020436507936507937, + "loss": 1.3143, + "theoretical_loss": 3.5141595431206345, + "tokens_seen": 1656750080 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920634, + "loss": 1.287, + "theoretical_loss": 3.514147616903559, + "tokens_seen": 1656815616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020833333333333335, + "loss": 1.3504, + "theoretical_loss": 3.5141356912903032, + "tokens_seen": 1656881152 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021031746031746032, + "loss": 1.2455, + "theoretical_loss": 3.5141237662808127, + "tokens_seen": 1656946688 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002123015873015873, + "loss": 1.5026, + "theoretical_loss": 3.5141118418750334, + "tokens_seen": 1657012224 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021428571428571427, + "loss": 1.2715, + "theoretical_loss": 3.514099918072911, + "tokens_seen": 1657077760 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021626984126984128, + "loss": 1.2512, + "theoretical_loss": 3.514087994874391, + "tokens_seen": 1657143296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021825396825396825, + "loss": 1.2572, + "theoretical_loss": 3.5140760722794178, + "tokens_seen": 1657208832 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022023809523809523, + "loss": 1.2935, + "theoretical_loss": 3.5140641502879384, + "tokens_seen": 1657274368 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002222222222222222, + "loss": 1.2324, + "theoretical_loss": 3.5140522288998977, + "tokens_seen": 1657339904 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002242063492063492, + "loss": 1.2474, + "theoretical_loss": 3.514040308115242, + "tokens_seen": 1657405440 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022619047619047618, + "loss": 1.3313, + "theoretical_loss": 3.514028387933916, + "tokens_seen": 1657470976 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022817460317460316, + "loss": 1.3137, + "theoretical_loss": 3.514016468355866, + "tokens_seen": 1657536512 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 1.1736, + "theoretical_loss": 3.5140045493810366, + "tokens_seen": 1657602048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023214285714285717, + "loss": 1.3316, + "theoretical_loss": 3.5139926310093745, + "tokens_seen": 1657667584 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023412698412698414, + "loss": 1.3608, + "theoretical_loss": 3.5139807132408256, + "tokens_seen": 1657733120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023611111111111112, + "loss": 1.2163, + "theoretical_loss": 3.513968796075334, + "tokens_seen": 1657798656 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 1.3069, + "theoretical_loss": 3.513956879512847, + "tokens_seen": 1657864192 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002400793650793651, + "loss": 1.2888, + "theoretical_loss": 3.513944963553309, + "tokens_seen": 1657929728 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024206349206349207, + "loss": 1.3412, + "theoretical_loss": 3.513933048196667, + "tokens_seen": 1657995264 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024404761904761905, + "loss": 1.4273, + "theoretical_loss": 3.5139211334428646, + "tokens_seen": 1658060800 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246031746031746, + "loss": 1.2545, + "theoretical_loss": 3.5139092192918495, + "tokens_seen": 1658126336 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248015873015873, + "loss": 1.2022, + "theoretical_loss": 3.5138973057435665, + "tokens_seen": 1658191872 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 1.2708, + "theoretical_loss": 3.5138853927979614, + "tokens_seen": 1658257408 + }, + { + "epoch": 0.01, + "learning_rate": 0.000251984126984127, + "loss": 1.2957, + "theoretical_loss": 3.51387348045498, + "tokens_seen": 1658322944 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025396825396825396, + "loss": 1.3885, + "theoretical_loss": 3.513861568714568, + "tokens_seen": 1658388480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025595238095238096, + "loss": 1.4945, + "theoretical_loss": 3.513849657576671, + "tokens_seen": 1658454016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025793650793650796, + "loss": 1.254, + "theoretical_loss": 3.513837747041234, + "tokens_seen": 1658519552 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025992063492063497, + "loss": 1.3749, + "theoretical_loss": 3.513825837108204, + "tokens_seen": 1658585088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002619047619047619, + "loss": 1.3178, + "theoretical_loss": 3.513813927777526, + "tokens_seen": 1658650624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002638888888888889, + "loss": 1.3493, + "theoretical_loss": 3.5138020190491464, + "tokens_seen": 1658716160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026587301587301587, + "loss": 1.304, + "theoretical_loss": 3.5137901109230096, + "tokens_seen": 1658781696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026785714285714287, + "loss": 1.3476, + "theoretical_loss": 3.513778203399063, + "tokens_seen": 1658847232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002698412698412698, + "loss": 1.2261, + "theoretical_loss": 3.513766296477251, + "tokens_seen": 1658912768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002718253968253968, + "loss": 1.3979, + "theoretical_loss": 3.51375439015752, + "tokens_seen": 1658978304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002738095238095238, + "loss": 1.3722, + "theoretical_loss": 3.5137424844398164, + "tokens_seen": 1659043840 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027579365079365083, + "loss": 1.3695, + "theoretical_loss": 3.5137305793240845, + "tokens_seen": 1659109376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002777777777777778, + "loss": 1.3552, + "theoretical_loss": 3.5137186748102716, + "tokens_seen": 1659174912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002797619047619048, + "loss": 1.32, + "theoretical_loss": 3.513706770898322, + "tokens_seen": 1659240448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028174603174603173, + "loss": 1.398, + "theoretical_loss": 3.5136948675881827, + "tokens_seen": 1659305984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028373015873015873, + "loss": 1.2293, + "theoretical_loss": 3.5136829648797985, + "tokens_seen": 1659371520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002857142857142857, + "loss": 1.3544, + "theoretical_loss": 3.5136710627731165, + "tokens_seen": 1659437056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002876984126984127, + "loss": 1.2462, + "theoretical_loss": 3.5136591612680816, + "tokens_seen": 1659502592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002896825396825397, + "loss": 1.2311, + "theoretical_loss": 3.51364726036464, + "tokens_seen": 1659568128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002916666666666667, + "loss": 1.4142, + "theoretical_loss": 3.513635360062737, + "tokens_seen": 1659633664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002936507936507937, + "loss": 1.4822, + "theoretical_loss": 3.513623460362319, + "tokens_seen": 1659699200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029563492063492064, + "loss": 1.3401, + "theoretical_loss": 3.5136115612633314, + "tokens_seen": 1659764736 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 606029, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3237011432647705, + "objective/train/theoretical_loss": 3.5135996627657207, + "objective/train/tokens_used": 30289376, + "theoretical_loss": 3.5135996627657207, + "tokens_seen": 1659830272 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029761904761904765, + "loss": 1.3534, + "theoretical_loss": 3.5135996627657207, + "tokens_seen": 1659830272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002996031746031746, + "loss": 1.3512, + "theoretical_loss": 3.513587764869432, + "tokens_seen": 1659895808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003015873015873016, + "loss": 1.3128, + "theoretical_loss": 3.5135758675744118, + "tokens_seen": 1659961344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030357142857142855, + "loss": 1.1756, + "theoretical_loss": 3.5135639708806057, + "tokens_seen": 1660026880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003055555555555556, + "loss": 1.3124, + "theoretical_loss": 3.5135520747879596, + "tokens_seen": 1660092416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030753968253968255, + "loss": 1.3834, + "theoretical_loss": 3.51354017929642, + "tokens_seen": 1660157952 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030952380952380956, + "loss": 1.4062, + "theoretical_loss": 3.5135282844059317, + "tokens_seen": 1660223488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003115079365079365, + "loss": 1.2078, + "theoretical_loss": 3.5135163901164406, + "tokens_seen": 1660289024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003134920634920635, + "loss": 1.4016, + "theoretical_loss": 3.5135044964278936, + "tokens_seen": 1660354560 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031547619047619046, + "loss": 1.3976, + "theoretical_loss": 3.5134926033402367, + "tokens_seen": 1660420096 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031746031746031746, + "loss": 1.213, + "theoretical_loss": 3.5134807108534147, + "tokens_seen": 1660485632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003194444444444444, + "loss": 1.2538, + "theoretical_loss": 3.513468818967374, + "tokens_seen": 1660551168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032142857142857147, + "loss": 1.2863, + "theoretical_loss": 3.5134569276820606, + "tokens_seen": 1660616704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003234126984126984, + "loss": 1.2696, + "theoretical_loss": 3.513445036997421, + "tokens_seen": 1660682240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003253968253968254, + "loss": 1.2223, + "theoretical_loss": 3.5134331469134006, + "tokens_seen": 1660747776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032738095238095237, + "loss": 1.301, + "theoretical_loss": 3.513421257429945, + "tokens_seen": 1660813312 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032936507936507937, + "loss": 1.2914, + "theoretical_loss": 3.5134093685470007, + "tokens_seen": 1660878848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003313492063492063, + "loss": 1.3476, + "theoretical_loss": 3.513397480264514, + "tokens_seen": 1660944384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003333333333333333, + "loss": 1.3696, + "theoretical_loss": 3.5133855925824298, + "tokens_seen": 1661009920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003353174603174603, + "loss": 1.4239, + "theoretical_loss": 3.513373705500695, + "tokens_seen": 1661075456 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033730158730158733, + "loss": 1.3598, + "theoretical_loss": 3.5133618190192553, + "tokens_seen": 1661140992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033928571428571433, + "loss": 1.3815, + "theoretical_loss": 3.513349933138057, + "tokens_seen": 1661206528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003412698412698413, + "loss": 1.3253, + "theoretical_loss": 3.5133380478570455, + "tokens_seen": 1661272064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003432539682539683, + "loss": 1.3166, + "theoretical_loss": 3.513326163176167, + "tokens_seen": 1661337600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034523809523809523, + "loss": 1.2531, + "theoretical_loss": 3.5133142790953684, + "tokens_seen": 1661403136 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034722222222222224, + "loss": 1.2607, + "theoretical_loss": 3.513302395614595, + "tokens_seen": 1661468672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003492063492063492, + "loss": 1.4398, + "theoretical_loss": 3.513290512733792, + "tokens_seen": 1661534208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003511904761904762, + "loss": 1.3071, + "theoretical_loss": 3.513278630452907, + "tokens_seen": 1661599744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003531746031746032, + "loss": 1.3396, + "theoretical_loss": 3.513266748771885, + "tokens_seen": 1661665280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003551587301587302, + "loss": 1.2607, + "theoretical_loss": 3.5132548676906725, + "tokens_seen": 1661730816 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035714285714285714, + "loss": 1.3685, + "theoretical_loss": 3.5132429872092157, + "tokens_seen": 1661796352 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035912698412698415, + "loss": 1.2958, + "theoretical_loss": 3.51323110732746, + "tokens_seen": 1661861888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003611111111111111, + "loss": 1.2959, + "theoretical_loss": 3.5132192280453527, + "tokens_seen": 1661927424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003630952380952381, + "loss": 1.3178, + "theoretical_loss": 3.5132073493628386, + "tokens_seen": 1661992960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036507936507936505, + "loss": 1.398, + "theoretical_loss": 3.5131954712798645, + "tokens_seen": 1662058496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036706349206349205, + "loss": 1.3103, + "theoretical_loss": 3.513183593796376, + "tokens_seen": 1662124032 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036904761904761905, + "loss": 1.3852, + "theoretical_loss": 3.5131717169123196, + "tokens_seen": 1662189568 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037103174603174606, + "loss": 1.3375, + "theoretical_loss": 3.5131598406276416, + "tokens_seen": 1662255104 + }, + { + "epoch": 0.01, + "learning_rate": 0.000373015873015873, + "loss": 1.2439, + "theoretical_loss": 3.513147964942288, + "tokens_seen": 1662320640 + }, + { + "epoch": 0.01, + "learning_rate": 0.000375, + "loss": 1.3022, + "theoretical_loss": 3.5131360898562045, + "tokens_seen": 1662386176 + }, + { + "epoch": 0.01, + "learning_rate": 0.000376984126984127, + "loss": 1.2795, + "theoretical_loss": 3.513124215369338, + "tokens_seen": 1662451712 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037896825396825396, + "loss": 1.3061, + "theoretical_loss": 3.5131123414816336, + "tokens_seen": 1662517248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038095238095238096, + "loss": 1.4298, + "theoretical_loss": 3.513100468193038, + "tokens_seen": 1662582784 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038293650793650797, + "loss": 1.2707, + "theoretical_loss": 3.513088595503498, + "tokens_seen": 1662648320 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038492063492063497, + "loss": 1.3499, + "theoretical_loss": 3.5130767234129587, + "tokens_seen": 1662713856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003869047619047619, + "loss": 1.3215, + "theoretical_loss": 3.5130648519213663, + "tokens_seen": 1662779392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003888888888888889, + "loss": 1.3652, + "theoretical_loss": 3.513052981028668, + "tokens_seen": 1662844928 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039087301587301587, + "loss": 1.3625, + "theoretical_loss": 3.513041110734809, + "tokens_seen": 1662910464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003928571428571429, + "loss": 1.4092, + "theoretical_loss": 3.513029241039737, + "tokens_seen": 1662976000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003948412698412698, + "loss": 1.2667, + "theoretical_loss": 3.5130173719433957, + "tokens_seen": 1663041536 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 607267, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.356825590133667, + "objective/train/theoretical_loss": 3.5130055034457337, + "objective/train/tokens_used": 33566176, + "theoretical_loss": 3.5130055034457337, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003968253968253968, + "loss": 1.3268, + "theoretical_loss": 3.5130055034457337, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039880952380952383, + "loss": 1.3616, + "theoretical_loss": 3.5129936355466955, + "tokens_seen": 1663172608 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040079365079365083, + "loss": 1.3558, + "theoretical_loss": 3.5129817682462283, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004027777777777778, + "loss": 1.3607, + "theoretical_loss": 3.512969901544278, + "tokens_seen": 1663303680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 1.3915, + "theoretical_loss": 3.512958035440791, + "tokens_seen": 1663369216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040674603174603173, + "loss": 1.3687, + "theoretical_loss": 3.512946169935713, + "tokens_seen": 1663434752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040873015873015874, + "loss": 1.2725, + "theoretical_loss": 3.512934305028991, + "tokens_seen": 1663500288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004107142857142857, + "loss": 1.3532, + "theoretical_loss": 3.512922440720571, + "tokens_seen": 1663565824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004126984126984127, + "loss": 1.2948, + "theoretical_loss": 3.512910577010399, + "tokens_seen": 1663631360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004146825396825397, + "loss": 1.3592, + "theoretical_loss": 3.5128987138984216, + "tokens_seen": 1663696896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004166666666666667, + "loss": 1.3072, + "theoretical_loss": 3.512886851384585, + "tokens_seen": 1663762432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004186507936507937, + "loss": 1.3437, + "theoretical_loss": 3.512874989468835, + "tokens_seen": 1663827968 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 1.3196, + "theoretical_loss": 3.5128631281511185, + "tokens_seen": 1663893504 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042261904761904765, + "loss": 1.3875, + "theoretical_loss": 3.512851267431382, + "tokens_seen": 1663959040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004246031746031746, + "loss": 1.3927, + "theoretical_loss": 3.5128394073095706, + "tokens_seen": 1664024576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004265873015873016, + "loss": 1.2842, + "theoretical_loss": 3.512827547785632, + "tokens_seen": 1664090112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042857142857142855, + "loss": 1.3583, + "theoretical_loss": 3.512815688859512, + "tokens_seen": 1664155648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004305555555555556, + "loss": 1.2375, + "theoretical_loss": 3.5128038305311566, + "tokens_seen": 1664221184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043253968253968256, + "loss": 1.4035, + "theoretical_loss": 3.512791972800512, + "tokens_seen": 1664286720 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043452380952380956, + "loss": 1.2929, + "theoretical_loss": 3.5127801156675256, + "tokens_seen": 1664352256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 1.3587, + "theoretical_loss": 3.5127682591321427, + "tokens_seen": 1664417792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004384920634920635, + "loss": 1.3087, + "theoretical_loss": 3.5127564031943104, + "tokens_seen": 1664483328 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044047619047619046, + "loss": 1.269, + "theoretical_loss": 3.5127445478539743, + "tokens_seen": 1664548864 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044246031746031746, + "loss": 1.2273, + "theoretical_loss": 3.512732693111081, + "tokens_seen": 1664614400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004444444444444444, + "loss": 1.3967, + "theoretical_loss": 3.5127208389655777, + "tokens_seen": 1664679936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044642857142857147, + "loss": 1.3524, + "theoretical_loss": 3.51270898541741, + "tokens_seen": 1664745472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004484126984126984, + "loss": 1.3109, + "theoretical_loss": 3.5126971324665237, + "tokens_seen": 1664811008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004503968253968254, + "loss": 1.3021, + "theoretical_loss": 3.512685280112866, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 1.3615, + "theoretical_loss": 3.5126734283563836, + "tokens_seen": 1664942080 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045436507936507937, + "loss": 1.264, + "theoretical_loss": 3.512661577197022, + "tokens_seen": 1665007616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004563492063492063, + "loss": 1.401, + "theoretical_loss": 3.5126497266347285, + "tokens_seen": 1665073152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004583333333333333, + "loss": 1.3086, + "theoretical_loss": 3.512637876669449, + "tokens_seen": 1665138688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 1.2858, + "theoretical_loss": 3.5126260273011303, + "tokens_seen": 1665204224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046230158730158733, + "loss": 1.3467, + "theoretical_loss": 3.5126141785297182, + "tokens_seen": 1665269760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046428571428571433, + "loss": 1.4068, + "theoretical_loss": 3.5126023303551595, + "tokens_seen": 1665335296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004662698412698413, + "loss": 1.3382, + "theoretical_loss": 3.5125904827774006, + "tokens_seen": 1665400832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 1.3011, + "theoretical_loss": 3.5125786357963884, + "tokens_seen": 1665466368 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047023809523809523, + "loss": 1.3789, + "theoretical_loss": 3.5125667894120687, + "tokens_seen": 1665531904 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047222222222222224, + "loss": 1.472, + "theoretical_loss": 3.5125549436243886, + "tokens_seen": 1665597440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004742063492063492, + "loss": 1.3007, + "theoretical_loss": 3.512543098433294, + "tokens_seen": 1665662976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 1.3371, + "theoretical_loss": 3.5125312538387314, + "tokens_seen": 1665728512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004781746031746032, + "loss": 1.3715, + "theoretical_loss": 3.512519409840648, + "tokens_seen": 1665794048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004801587301587302, + "loss": 1.4025, + "theoretical_loss": 3.5125075664389893, + "tokens_seen": 1665859584 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048214285714285715, + "loss": 1.3333, + "theoretical_loss": 3.5124957236337027, + "tokens_seen": 1665925120 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048412698412698415, + "loss": 1.4573, + "theoretical_loss": 3.512483881424734, + "tokens_seen": 1665990656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004861111111111111, + "loss": 1.3031, + "theoretical_loss": 3.5124720398120304, + "tokens_seen": 1666056192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004880952380952381, + "loss": 1.317, + "theoretical_loss": 3.5124601987955377, + "tokens_seen": 1666121728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004900793650793651, + "loss": 1.3426, + "theoretical_loss": 3.5124483583752033, + "tokens_seen": 1666187264 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492063492063492, + "loss": 1.4375, + "theoretical_loss": 3.5124365185509725, + "tokens_seen": 1666252800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004940476190476191, + "loss": 1.2994, + "theoretical_loss": 3.512424679322793, + "tokens_seen": 1666318336 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 608508, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4184010028839111, + "objective/train/theoretical_loss": 3.5124128406906108, + "objective/train/tokens_used": 36842976, + "theoretical_loss": 3.5124128406906108, + "tokens_seen": 1666383872 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496031746031746, + "loss": 1.3533, + "theoretical_loss": 3.5124128406906108, + "tokens_seen": 1666383872 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498015873015873, + "loss": 1.4404, + "theoretical_loss": 3.512401002654373, + "tokens_seen": 1666449408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 1.4442, + "theoretical_loss": 3.5123891652140253, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999799398194584, + "loss": 1.3187, + "theoretical_loss": 3.5123773283695146, + "tokens_seen": 1666580480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999598796389167, + "loss": 1.4164, + "theoretical_loss": 3.512365492120788, + "tokens_seen": 1666646016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999398194583751, + "loss": 1.4281, + "theoretical_loss": 3.5123536564677913, + "tokens_seen": 1666711552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999197592778335, + "loss": 1.3973, + "theoretical_loss": 3.5123418214104722, + "tokens_seen": 1666777088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998996990972919, + "loss": 1.2053, + "theoretical_loss": 3.5123299869487763, + "tokens_seen": 1666842624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998796389167503, + "loss": 1.2739, + "theoretical_loss": 3.51231815308265, + "tokens_seen": 1666908160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998595787362087, + "loss": 1.3377, + "theoretical_loss": 3.5123063198120406, + "tokens_seen": 1666973696 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499839518555667, + "loss": 1.3845, + "theoretical_loss": 3.5122944871368946, + "tokens_seen": 1667039232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998194583751254, + "loss": 1.2625, + "theoretical_loss": 3.5122826550571586, + "tokens_seen": 1667104768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997993981945837, + "loss": 1.2543, + "theoretical_loss": 3.5122708235727798, + "tokens_seen": 1667170304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997793380140421, + "loss": 1.3257, + "theoretical_loss": 3.5122589926837033, + "tokens_seen": 1667235840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997592778335005, + "loss": 1.3879, + "theoretical_loss": 3.512247162389877, + "tokens_seen": 1667301376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997392176529588, + "loss": 1.3611, + "theoretical_loss": 3.5122353326912474, + "tokens_seen": 1667366912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997191574724173, + "loss": 1.3385, + "theoretical_loss": 3.512223503587761, + "tokens_seen": 1667432448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996990972918757, + "loss": 1.2879, + "theoretical_loss": 3.512211675079364, + "tokens_seen": 1667497984 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499679037111334, + "loss": 1.2707, + "theoretical_loss": 3.512199847166004, + "tokens_seen": 1667563520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996589769307924, + "loss": 1.3643, + "theoretical_loss": 3.512188019847627, + "tokens_seen": 1667629056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996389167502507, + "loss": 1.3053, + "theoretical_loss": 3.5121761931241804, + "tokens_seen": 1667694592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996188565697091, + "loss": 1.2215, + "theoretical_loss": 3.5121643669956097, + "tokens_seen": 1667760128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995987963891675, + "loss": 1.4085, + "theoretical_loss": 3.512152541461863, + "tokens_seen": 1667825664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995787362086258, + "loss": 1.5177, + "theoretical_loss": 3.512140716522886, + "tokens_seen": 1667891200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995586760280842, + "loss": 1.2917, + "theoretical_loss": 3.512128892178626, + "tokens_seen": 1667956736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995386158475427, + "loss": 1.261, + "theoretical_loss": 3.5121170684290286, + "tokens_seen": 1668022272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499518555667001, + "loss": 1.3779, + "theoretical_loss": 3.5121052452740424, + "tokens_seen": 1668087808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994984954864594, + "loss": 1.2618, + "theoretical_loss": 3.5120934227136127, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994784353059178, + "loss": 1.3569, + "theoretical_loss": 3.512081600747687, + "tokens_seen": 1668218880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994583751253761, + "loss": 1.2593, + "theoretical_loss": 3.5120697793762115, + "tokens_seen": 1668284416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994383149448345, + "loss": 1.3794, + "theoretical_loss": 3.512057958599133, + "tokens_seen": 1668349952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994182547642928, + "loss": 1.3364, + "theoretical_loss": 3.512046138416399, + "tokens_seen": 1668415488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993981945837512, + "loss": 1.5521, + "theoretical_loss": 3.512034318827955, + "tokens_seen": 1668481024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993781344032096, + "loss": 1.2601, + "theoretical_loss": 3.512022499833749, + "tokens_seen": 1668546560 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499358074222668, + "loss": 1.3203, + "theoretical_loss": 3.5120106814337273, + "tokens_seen": 1668612096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993380140421264, + "loss": 1.3877, + "theoretical_loss": 3.511998863627837, + "tokens_seen": 1668677632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993179538615848, + "loss": 1.4574, + "theoretical_loss": 3.511987046416024, + "tokens_seen": 1668743168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992978936810431, + "loss": 1.3442, + "theoretical_loss": 3.511975229798236, + "tokens_seen": 1668808704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992778335005015, + "loss": 1.3632, + "theoretical_loss": 3.5119634137744193, + "tokens_seen": 1668874240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992577733199598, + "loss": 1.3682, + "theoretical_loss": 3.5119515983445213, + "tokens_seen": 1668939776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992377131394183, + "loss": 1.2747, + "theoretical_loss": 3.5119397835084882, + "tokens_seen": 1669005312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992176529588767, + "loss": 1.4697, + "theoretical_loss": 3.5119279692662673, + "tokens_seen": 1669070848 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499197592778335, + "loss": 1.2718, + "theoretical_loss": 3.5119161556178047, + "tokens_seen": 1669136384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991775325977934, + "loss": 1.4316, + "theoretical_loss": 3.511904342563048, + "tokens_seen": 1669201920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991574724172518, + "loss": 1.3667, + "theoretical_loss": 3.511892530101944, + "tokens_seen": 1669267456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991374122367101, + "loss": 1.3733, + "theoretical_loss": 3.5118807182344396, + "tokens_seen": 1669332992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991173520561685, + "loss": 1.4507, + "theoretical_loss": 3.511868906960481, + "tokens_seen": 1669398528 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499097291875627, + "loss": 1.4666, + "theoretical_loss": 3.511857096280016, + "tokens_seen": 1669464064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990772316950853, + "loss": 1.3904, + "theoretical_loss": 3.5118452861929907, + "tokens_seen": 1669529600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990571715145437, + "loss": 1.3522, + "theoretical_loss": 3.5118334766993526, + "tokens_seen": 1669595136 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 609762, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4398548603057861, + "objective/train/theoretical_loss": 3.5118216677990484, + "objective/train/tokens_used": 40119776, + "theoretical_loss": 3.5118216677990484, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499037111334002, + "loss": 1.4915, + "theoretical_loss": 3.5118216677990484, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990170511534604, + "loss": 1.3718, + "theoretical_loss": 3.5118098594920246, + "tokens_seen": 1669726208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989969909729188, + "loss": 1.4447, + "theoretical_loss": 3.5117980517782286, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989769307923771, + "loss": 1.2739, + "theoretical_loss": 3.511786244657607, + "tokens_seen": 1669857280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989568706118355, + "loss": 1.347, + "theoretical_loss": 3.5117744381301064, + "tokens_seen": 1669922816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989368104312939, + "loss": 1.3521, + "theoretical_loss": 3.5117626321956745, + "tokens_seen": 1669988352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989167502507523, + "loss": 1.3728, + "theoretical_loss": 3.5117508268542585, + "tokens_seen": 1670053888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988966900702107, + "loss": 1.4433, + "theoretical_loss": 3.5117390221058042, + "tokens_seen": 1670119424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988766298896691, + "loss": 1.4599, + "theoretical_loss": 3.511727217950259, + "tokens_seen": 1670184960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988565697091274, + "loss": 1.4139, + "theoretical_loss": 3.5117154143875706, + "tokens_seen": 1670250496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988365095285858, + "loss": 1.3556, + "theoretical_loss": 3.511703611417685, + "tokens_seen": 1670316032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988164493480441, + "loss": 1.4031, + "theoretical_loss": 3.5116918090405496, + "tokens_seen": 1670381568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987963891675025, + "loss": 1.3096, + "theoretical_loss": 3.5116800072561114, + "tokens_seen": 1670447104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987763289869609, + "loss": 1.2903, + "theoretical_loss": 3.511668206064317, + "tokens_seen": 1670512640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987562688064192, + "loss": 1.3447, + "theoretical_loss": 3.5116564054651134, + "tokens_seen": 1670578176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987362086258777, + "loss": 1.2898, + "theoretical_loss": 3.5116446054584483, + "tokens_seen": 1670643712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987161484453361, + "loss": 1.5065, + "theoretical_loss": 3.5116328060442683, + "tokens_seen": 1670709248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986960882647944, + "loss": 1.3288, + "theoretical_loss": 3.5116210072225202, + "tokens_seen": 1670774784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986760280842528, + "loss": 1.3256, + "theoretical_loss": 3.511609208993151, + "tokens_seen": 1670840320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986559679037111, + "loss": 1.3815, + "theoretical_loss": 3.5115974113561084, + "tokens_seen": 1670905856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986359077231695, + "loss": 1.3216, + "theoretical_loss": 3.511585614311339, + "tokens_seen": 1670971392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986158475426279, + "loss": 1.3473, + "theoretical_loss": 3.5115738178587894, + "tokens_seen": 1671036928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985957873620862, + "loss": 1.3758, + "theoretical_loss": 3.511562021998407, + "tokens_seen": 1671102464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985757271815446, + "loss": 1.3575, + "theoretical_loss": 3.5115502267301393, + "tokens_seen": 1671168000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985556670010031, + "loss": 1.3945, + "theoretical_loss": 3.5115384320539325, + "tokens_seen": 1671233536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985356068204614, + "loss": 1.5017, + "theoretical_loss": 3.5115266379697343, + "tokens_seen": 1671299072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985155466399198, + "loss": 1.2951, + "theoretical_loss": 3.5115148444774915, + "tokens_seen": 1671364608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984954864593782, + "loss": 1.4262, + "theoretical_loss": 3.5115030515771517, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984754262788365, + "loss": 1.3975, + "theoretical_loss": 3.511491259268661, + "tokens_seen": 1671495680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984553660982949, + "loss": 1.3532, + "theoretical_loss": 3.5114794675519674, + "tokens_seen": 1671561216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984353059177532, + "loss": 1.4, + "theoretical_loss": 3.5114676764270176, + "tokens_seen": 1671626752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984152457372116, + "loss": 1.3306, + "theoretical_loss": 3.5114558858937586, + "tokens_seen": 1671692288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049839518555667, + "loss": 1.4179, + "theoretical_loss": 3.511444095952138, + "tokens_seen": 1671757824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983751253761284, + "loss": 1.4374, + "theoretical_loss": 3.511432306602102, + "tokens_seen": 1671823360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983550651955868, + "loss": 1.4393, + "theoretical_loss": 3.5114205178435984, + "tokens_seen": 1671888896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983350050150452, + "loss": 1.456, + "theoretical_loss": 3.511408729676575, + "tokens_seen": 1671954432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983149448345035, + "loss": 1.2084, + "theoretical_loss": 3.5113969421009776, + "tokens_seen": 1672019968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982948846539619, + "loss": 1.3846, + "theoretical_loss": 3.511385155116754, + "tokens_seen": 1672085504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982748244734202, + "loss": 1.4001, + "theoretical_loss": 3.5113733687238513, + "tokens_seen": 1672151040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982547642928786, + "loss": 1.4125, + "theoretical_loss": 3.511361582922217, + "tokens_seen": 1672216576 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498234704112337, + "loss": 1.407, + "theoretical_loss": 3.5113497977117976, + "tokens_seen": 1672282112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982146439317953, + "loss": 1.3244, + "theoretical_loss": 3.5113380130925407, + "tokens_seen": 1672347648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981945837512538, + "loss": 1.3708, + "theoretical_loss": 3.5113262290643936, + "tokens_seen": 1672413184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981745235707122, + "loss": 1.3375, + "theoretical_loss": 3.5113144456273027, + "tokens_seen": 1672478720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981544633901705, + "loss": 1.4217, + "theoretical_loss": 3.5113026627812163, + "tokens_seen": 1672544256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981344032096289, + "loss": 1.2906, + "theoretical_loss": 3.5112908805260803, + "tokens_seen": 1672609792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981143430290873, + "loss": 1.4327, + "theoretical_loss": 3.5112790988618436, + "tokens_seen": 1672675328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980942828485456, + "loss": 1.3056, + "theoretical_loss": 3.511267317788452, + "tokens_seen": 1672740864 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498074222668004, + "loss": 1.3627, + "theoretical_loss": 3.5112555373058534, + "tokens_seen": 1672806400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980541624874623, + "loss": 1.4094, + "theoretical_loss": 3.511243757413995, + "tokens_seen": 1672871936 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 611052, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.390806794166565, + "objective/train/theoretical_loss": 3.511231978112823, + "objective/train/tokens_used": 43396576, + "theoretical_loss": 3.511231978112823, + "tokens_seen": 1672937472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980341023069207, + "loss": 1.3867, + "theoretical_loss": 3.511231978112823, + "tokens_seen": 1672937472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980140421263792, + "loss": 1.4314, + "theoretical_loss": 3.5112201994022865, + "tokens_seen": 1673003008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979939819458375, + "loss": 1.3568, + "theoretical_loss": 3.5112084212823316, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979739217652959, + "loss": 1.4667, + "theoretical_loss": 3.5111966437529056, + "tokens_seen": 1673134080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979538615847543, + "loss": 1.297, + "theoretical_loss": 3.511184866813956, + "tokens_seen": 1673199616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979338014042126, + "loss": 1.5068, + "theoretical_loss": 3.5111730904654297, + "tokens_seen": 1673265152 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497913741223671, + "loss": 1.3578, + "theoretical_loss": 3.5111613147072744, + "tokens_seen": 1673330688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978936810431293, + "loss": 1.4304, + "theoretical_loss": 3.511149539539437, + "tokens_seen": 1673396224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978736208625877, + "loss": 1.3073, + "theoretical_loss": 3.5111377649618656, + "tokens_seen": 1673461760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978535606820461, + "loss": 1.4159, + "theoretical_loss": 3.5111259909745067, + "tokens_seen": 1673527296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978335005015045, + "loss": 1.3471, + "theoretical_loss": 3.5111142175773074, + "tokens_seen": 1673592832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978134403209629, + "loss": 1.3852, + "theoretical_loss": 3.5111024447702155, + "tokens_seen": 1673658368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977933801404213, + "loss": 1.2641, + "theoretical_loss": 3.5110906725531787, + "tokens_seen": 1673723904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977733199598796, + "loss": 1.2829, + "theoretical_loss": 3.5110789009261434, + "tokens_seen": 1673789440 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497753259779338, + "loss": 1.4219, + "theoretical_loss": 3.5110671298890574, + "tokens_seen": 1673854976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977331995987965, + "loss": 1.4023, + "theoretical_loss": 3.511055359441868, + "tokens_seen": 1673920512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977131394182548, + "loss": 1.3657, + "theoretical_loss": 3.5110435895845233, + "tokens_seen": 1673986048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976930792377132, + "loss": 1.413, + "theoretical_loss": 3.511031820316969, + "tokens_seen": 1674051584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976730190571715, + "loss": 1.3866, + "theoretical_loss": 3.511020051639154, + "tokens_seen": 1674117120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976529588766299, + "loss": 1.3012, + "theoretical_loss": 3.511008283551025, + "tokens_seen": 1674182656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976328986960883, + "loss": 1.246, + "theoretical_loss": 3.510996516052529, + "tokens_seen": 1674248192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976128385155466, + "loss": 1.4133, + "theoretical_loss": 3.5109847491436144, + "tokens_seen": 1674313728 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497592778335005, + "loss": 1.4379, + "theoretical_loss": 3.5109729828242275, + "tokens_seen": 1674379264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975727181544635, + "loss": 1.4259, + "theoretical_loss": 3.5109612170943163, + "tokens_seen": 1674444800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975526579739218, + "loss": 1.2859, + "theoretical_loss": 3.5109494519538282, + "tokens_seen": 1674510336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975325977933802, + "loss": 1.475, + "theoretical_loss": 3.5109376874027105, + "tokens_seen": 1674575872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975125376128386, + "loss": 1.4248, + "theoretical_loss": 3.5109259234409107, + "tokens_seen": 1674641408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974924774322969, + "loss": 1.497, + "theoretical_loss": 3.510914160068376, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974724172517553, + "loss": 1.307, + "theoretical_loss": 3.5109023972850535, + "tokens_seen": 1674772480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974523570712136, + "loss": 1.3168, + "theoretical_loss": 3.5108906350908917, + "tokens_seen": 1674838016 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497432296890672, + "loss": 1.355, + "theoretical_loss": 3.510878873485837, + "tokens_seen": 1674903552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974122367101304, + "loss": 1.3286, + "theoretical_loss": 3.510867112469837, + "tokens_seen": 1674969088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973921765295888, + "loss": 1.3835, + "theoretical_loss": 3.51085535204284, + "tokens_seen": 1675034624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973721163490472, + "loss": 1.3501, + "theoretical_loss": 3.5108435922047927, + "tokens_seen": 1675100160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973520561685056, + "loss": 1.4653, + "theoretical_loss": 3.5108318329556427, + "tokens_seen": 1675165696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973319959879639, + "loss": 1.3224, + "theoretical_loss": 3.5108200742953377, + "tokens_seen": 1675231232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973119358074223, + "loss": 1.3738, + "theoretical_loss": 3.510808316223825, + "tokens_seen": 1675296768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972918756268806, + "loss": 1.3234, + "theoretical_loss": 3.5107965587410517, + "tokens_seen": 1675362304 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497271815446339, + "loss": 1.1811, + "theoretical_loss": 3.510784801846966, + "tokens_seen": 1675427840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972517552657974, + "loss": 1.2313, + "theoretical_loss": 3.5107730455415145, + "tokens_seen": 1675493376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972316950852557, + "loss": 1.3912, + "theoretical_loss": 3.5107612898246456, + "tokens_seen": 1675558912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972116349047142, + "loss": 1.2141, + "theoretical_loss": 3.5107495346963065, + "tokens_seen": 1675624448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971915747241726, + "loss": 1.3425, + "theoretical_loss": 3.510737780156445, + "tokens_seen": 1675689984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971715145436309, + "loss": 1.2733, + "theoretical_loss": 3.510726026205008, + "tokens_seen": 1675755520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971514543630893, + "loss": 1.3946, + "theoretical_loss": 3.5107142728419434, + "tokens_seen": 1675821056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313941825477, + "loss": 1.4869, + "theoretical_loss": 3.5107025200671984, + "tokens_seen": 1675886592 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497111334002006, + "loss": 1.3577, + "theoretical_loss": 3.510690767880721, + "tokens_seen": 1675952128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970912738214644, + "loss": 1.4416, + "theoretical_loss": 3.510679016282459, + "tokens_seen": 1676017664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970712136409227, + "loss": 1.2491, + "theoretical_loss": 3.5106672652723594, + "tokens_seen": 1676083200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970511534603811, + "loss": 1.2802, + "theoretical_loss": 3.51065551485037, + "tokens_seen": 1676148736 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 612232, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.205704689025879, + "objective/train/theoretical_loss": 3.5106437650164377, + "objective/train/tokens_used": 46673376, + "theoretical_loss": 3.5106437650164377, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970310932798396, + "loss": 1.3453, + "theoretical_loss": 3.5106437650164377, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970110330992979, + "loss": 1.2999, + "theoretical_loss": 3.5106320157705113, + "tokens_seen": 1676279808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969909729187563, + "loss": 1.3165, + "theoretical_loss": 3.5106202671125377, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969709127382147, + "loss": 1.3722, + "theoretical_loss": 3.5106085190424645, + "tokens_seen": 1676410880 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496950852557673, + "loss": 1.3046, + "theoretical_loss": 3.5105967715602393, + "tokens_seen": 1676476416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969307923771314, + "loss": 1.3223, + "theoretical_loss": 3.5105850246658097, + "tokens_seen": 1676541952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969107321965897, + "loss": 1.288, + "theoretical_loss": 3.5105732783591237, + "tokens_seen": 1676607488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968906720160481, + "loss": 1.3162, + "theoretical_loss": 3.5105615326401285, + "tokens_seen": 1676673024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968706118355065, + "loss": 1.3171, + "theoretical_loss": 3.5105497875087717, + "tokens_seen": 1676738560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968505516549649, + "loss": 1.3306, + "theoretical_loss": 3.510538042965001, + "tokens_seen": 1676804096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968304914744233, + "loss": 1.3371, + "theoretical_loss": 3.510526299008764, + "tokens_seen": 1676869632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968104312938817, + "loss": 1.4059, + "theoretical_loss": 3.5105145556400092, + "tokens_seen": 1676935168 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049679037111334, + "loss": 1.2618, + "theoretical_loss": 3.5105028128586824, + "tokens_seen": 1677000704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967703109327984, + "loss": 1.327, + "theoretical_loss": 3.5104910706647336, + "tokens_seen": 1677066240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967502507522568, + "loss": 1.3904, + "theoretical_loss": 3.5104793290581084, + "tokens_seen": 1677131776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967301905717151, + "loss": 1.2958, + "theoretical_loss": 3.5104675880387552, + "tokens_seen": 1677197312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967101303911735, + "loss": 1.3325, + "theoretical_loss": 3.510455847606622, + "tokens_seen": 1677262848 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496690070210632, + "loss": 1.4188, + "theoretical_loss": 3.5104441077616566, + "tokens_seen": 1677328384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966700100300903, + "loss": 1.4538, + "theoretical_loss": 3.510432368503806, + "tokens_seen": 1677393920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966499498495487, + "loss": 1.2618, + "theoretical_loss": 3.5104206298330185, + "tokens_seen": 1677459456 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496629889669007, + "loss": 1.247, + "theoretical_loss": 3.5104088917492415, + "tokens_seen": 1677524992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966098294884654, + "loss": 1.2767, + "theoretical_loss": 3.510397154252423, + "tokens_seen": 1677590528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965897693079238, + "loss": 1.274, + "theoretical_loss": 3.51038541734251, + "tokens_seen": 1677656064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965697091273821, + "loss": 1.3477, + "theoretical_loss": 3.5103736810194515, + "tokens_seen": 1677721600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965496489468405, + "loss": 1.261, + "theoretical_loss": 3.5103619452831936, + "tokens_seen": 1677787136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965295887662988, + "loss": 1.4634, + "theoretical_loss": 3.5103502101336854, + "tokens_seen": 1677852672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965095285857573, + "loss": 1.3715, + "theoretical_loss": 3.510338475570874, + "tokens_seen": 1677918208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964894684052157, + "loss": 1.3701, + "theoretical_loss": 3.510326741594708, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496469408224674, + "loss": 1.3344, + "theoretical_loss": 3.5103150082051338, + "tokens_seen": 1678049280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964493480441324, + "loss": 1.4372, + "theoretical_loss": 3.5103032754021, + "tokens_seen": 1678114816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964292878635908, + "loss": 1.3899, + "theoretical_loss": 3.5102915431855544, + "tokens_seen": 1678180352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964092276830491, + "loss": 1.2802, + "theoretical_loss": 3.5102798115554448, + "tokens_seen": 1678245888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963891675025075, + "loss": 1.4888, + "theoretical_loss": 3.510268080511718, + "tokens_seen": 1678311424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963691073219659, + "loss": 1.4316, + "theoretical_loss": 3.5102563500543233, + "tokens_seen": 1678376960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963490471414242, + "loss": 1.3811, + "theoretical_loss": 3.510244620183207, + "tokens_seen": 1678442496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963289869608827, + "loss": 1.3533, + "theoretical_loss": 3.510232890898319, + "tokens_seen": 1678508032 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496308926780341, + "loss": 1.402, + "theoretical_loss": 3.510221162199605, + "tokens_seen": 1678573568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962888665997994, + "loss": 1.3002, + "theoretical_loss": 3.5102094340870136, + "tokens_seen": 1678639104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962688064192578, + "loss": 1.4567, + "theoretical_loss": 3.5101977065604926, + "tokens_seen": 1678704640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962487462387161, + "loss": 1.3611, + "theoretical_loss": 3.5101859796199903, + "tokens_seen": 1678770176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962286860581746, + "loss": 1.3742, + "theoretical_loss": 3.510174253265454, + "tokens_seen": 1678835712 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496208625877633, + "loss": 1.3704, + "theoretical_loss": 3.5101625274968318, + "tokens_seen": 1678901248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961885656970913, + "loss": 1.3149, + "theoretical_loss": 3.510150802314071, + "tokens_seen": 1678966784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961685055165497, + "loss": 1.3109, + "theoretical_loss": 3.5101390777171204, + "tokens_seen": 1679032320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961484453360081, + "loss": 1.4759, + "theoretical_loss": 3.5101273537059274, + "tokens_seen": 1679097856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961283851554664, + "loss": 1.4503, + "theoretical_loss": 3.5101156302804393, + "tokens_seen": 1679163392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961083249749248, + "loss": 1.3239, + "theoretical_loss": 3.5101039074406053, + "tokens_seen": 1679228928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960882647943831, + "loss": 1.4195, + "theoretical_loss": 3.5100921851863722, + "tokens_seen": 1679294464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960682046138415, + "loss": 1.3329, + "theoretical_loss": 3.5100804635176877, + "tokens_seen": 1679360000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960481444333, + "loss": 1.2157, + "theoretical_loss": 3.510068742434501, + "tokens_seen": 1679425536 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 613519, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4023518562316895, + "objective/train/theoretical_loss": 3.510057021936759, + "objective/train/tokens_used": 49950176, + "theoretical_loss": 3.510057021936759, + "tokens_seen": 1679491072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960280842527583, + "loss": 1.3508, + "theoretical_loss": 3.510057021936759, + "tokens_seen": 1679491072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960080240722167, + "loss": 1.3704, + "theoretical_loss": 3.5100453020244093, + "tokens_seen": 1679556608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959879638916751, + "loss": 1.2502, + "theoretical_loss": 3.510033582697401, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959679037111334, + "loss": 1.3835, + "theoretical_loss": 3.5100218639556813, + "tokens_seen": 1679687680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959478435305918, + "loss": 1.3863, + "theoretical_loss": 3.510010145799198, + "tokens_seen": 1679753216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959277833500501, + "loss": 1.2468, + "theoretical_loss": 3.5099984282278993, + "tokens_seen": 1679818752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959077231695085, + "loss": 1.3369, + "theoretical_loss": 3.5099867112417336, + "tokens_seen": 1679884288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958876629889669, + "loss": 1.3993, + "theoretical_loss": 3.509974994840648, + "tokens_seen": 1679949824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958676028084253, + "loss": 1.2679, + "theoretical_loss": 3.5099632790245905, + "tokens_seen": 1680015360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958475426278837, + "loss": 1.386, + "theoretical_loss": 3.5099515637935097, + "tokens_seen": 1680080896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958274824473421, + "loss": 1.3126, + "theoretical_loss": 3.5099398491473535, + "tokens_seen": 1680146432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958074222668004, + "loss": 1.342, + "theoretical_loss": 3.5099281350860694, + "tokens_seen": 1680211968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957873620862588, + "loss": 1.3853, + "theoretical_loss": 3.5099164216096055, + "tokens_seen": 1680277504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957673019057172, + "loss": 1.2512, + "theoretical_loss": 3.5099047087179107, + "tokens_seen": 1680343040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957472417251755, + "loss": 1.2624, + "theoretical_loss": 3.5098929964109313, + "tokens_seen": 1680408576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957271815446339, + "loss": 1.3223, + "theoretical_loss": 3.5098812846886167, + "tokens_seen": 1680474112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957071213640923, + "loss": 1.2682, + "theoretical_loss": 3.5098695735509144, + "tokens_seen": 1680539648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956870611835507, + "loss": 1.2981, + "theoretical_loss": 3.509857862997773, + "tokens_seen": 1680605184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956670010030091, + "loss": 1.2197, + "theoretical_loss": 3.509846153029139, + "tokens_seen": 1680670720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956469408224674, + "loss": 1.3112, + "theoretical_loss": 3.509834443644962, + "tokens_seen": 1680736256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956268806419258, + "loss": 1.4174, + "theoretical_loss": 3.50982273484519, + "tokens_seen": 1680801792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956068204613842, + "loss": 1.443, + "theoretical_loss": 3.50981102662977, + "tokens_seen": 1680867328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955867602808425, + "loss": 1.4172, + "theoretical_loss": 3.5097993189986507, + "tokens_seen": 1680932864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955667001003009, + "loss": 1.399, + "theoretical_loss": 3.50978761195178, + "tokens_seen": 1680998400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955466399197592, + "loss": 1.4391, + "theoretical_loss": 3.5097759054891062, + "tokens_seen": 1681063936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955265797392177, + "loss": 1.3421, + "theoretical_loss": 3.5097641996105766, + "tokens_seen": 1681129472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955065195586761, + "loss": 1.3453, + "theoretical_loss": 3.5097524943161407, + "tokens_seen": 1681195008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954864593781344, + "loss": 1.4281, + "theoretical_loss": 3.5097407896057455, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954663991975928, + "loss": 1.2651, + "theoretical_loss": 3.5097290854793393, + "tokens_seen": 1681326080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954463390170512, + "loss": 1.4019, + "theoretical_loss": 3.50971738193687, + "tokens_seen": 1681391616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954262788365095, + "loss": 1.2531, + "theoretical_loss": 3.5097056789782863, + "tokens_seen": 1681457152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954062186559679, + "loss": 1.3387, + "theoretical_loss": 3.509693976603536, + "tokens_seen": 1681522688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953861584754263, + "loss": 1.4915, + "theoretical_loss": 3.509682274812567, + "tokens_seen": 1681588224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953660982948846, + "loss": 1.426, + "theoretical_loss": 3.509670573605328, + "tokens_seen": 1681653760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953460381143431, + "loss": 1.2807, + "theoretical_loss": 3.509658872981767, + "tokens_seen": 1681719296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953259779338014, + "loss": 1.3989, + "theoretical_loss": 3.509647172941831, + "tokens_seen": 1681784832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953059177532598, + "loss": 1.2947, + "theoretical_loss": 3.50963547348547, + "tokens_seen": 1681850368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952858575727182, + "loss": 1.3678, + "theoretical_loss": 3.5096237746126304, + "tokens_seen": 1681915904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952657973921765, + "loss": 1.4324, + "theoretical_loss": 3.509612076323261, + "tokens_seen": 1681981440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952457372116349, + "loss": 1.3791, + "theoretical_loss": 3.5096003786173116, + "tokens_seen": 1682046976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952256770310933, + "loss": 1.3376, + "theoretical_loss": 3.5095886814947277, + "tokens_seen": 1682112512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952056168505516, + "loss": 1.3442, + "theoretical_loss": 3.509576984955459, + "tokens_seen": 1682178048 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049518555667001, + "loss": 1.376, + "theoretical_loss": 3.509565288999453, + "tokens_seen": 1682243584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951654964894685, + "loss": 1.4491, + "theoretical_loss": 3.5095535936266593, + "tokens_seen": 1682309120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951454363089268, + "loss": 1.3967, + "theoretical_loss": 3.509541898837024, + "tokens_seen": 1682374656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951253761283852, + "loss": 1.3527, + "theoretical_loss": 3.5095302046304973, + "tokens_seen": 1682440192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951053159478435, + "loss": 1.3168, + "theoretical_loss": 3.509518511007026, + "tokens_seen": 1682505728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950852557673019, + "loss": 1.3596, + "theoretical_loss": 3.5095068179665585, + "tokens_seen": 1682571264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950651955867603, + "loss": 1.3057, + "theoretical_loss": 3.5094951255090443, + "tokens_seen": 1682636800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950451354062186, + "loss": 1.3464, + "theoretical_loss": 3.50948343363443, + "tokens_seen": 1682702336 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 614786, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2073283195495605, + "objective/train/theoretical_loss": 3.5094717423426642, + "objective/train/tokens_used": 53226976, + "theoretical_loss": 3.5094717423426642, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495025075225677, + "loss": 1.3024, + "theoretical_loss": 3.5094717423426642, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950050150451354, + "loss": 1.3789, + "theoretical_loss": 3.5094600516336962, + "tokens_seen": 1682833408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949849548645938, + "loss": 1.2626, + "theoretical_loss": 3.5094483615074736, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949648946840522, + "loss": 1.3576, + "theoretical_loss": 3.509436671963944, + "tokens_seen": 1682964480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949448345035105, + "loss": 1.415, + "theoretical_loss": 3.5094249830030564, + "tokens_seen": 1683030016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949247743229689, + "loss": 1.3767, + "theoretical_loss": 3.509413294624759, + "tokens_seen": 1683095552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949047141424273, + "loss": 1.3648, + "theoretical_loss": 3.509401606829, + "tokens_seen": 1683161088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948846539618856, + "loss": 1.3451, + "theoretical_loss": 3.509389919615728, + "tokens_seen": 1683226624 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494864593781344, + "loss": 1.4432, + "theoretical_loss": 3.509378232984891, + "tokens_seen": 1683292160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948445336008024, + "loss": 1.3289, + "theoretical_loss": 3.5093665469364366, + "tokens_seen": 1683357696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948244734202607, + "loss": 1.3216, + "theoretical_loss": 3.509354861470314, + "tokens_seen": 1683423232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948044132397192, + "loss": 1.3884, + "theoretical_loss": 3.509343176586472, + "tokens_seen": 1683488768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947843530591776, + "loss": 1.337, + "theoretical_loss": 3.5093314922848577, + "tokens_seen": 1683554304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947642928786359, + "loss": 1.3516, + "theoretical_loss": 3.5093198085654196, + "tokens_seen": 1683619840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947442326980943, + "loss": 1.4022, + "theoretical_loss": 3.509308125428107, + "tokens_seen": 1683685376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947241725175527, + "loss": 1.3991, + "theoretical_loss": 3.5092964428728672, + "tokens_seen": 1683750912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947041123370111, + "loss": 1.4298, + "theoretical_loss": 3.509284760899649, + "tokens_seen": 1683816448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946840521564695, + "loss": 1.5053, + "theoretical_loss": 3.509273079508401, + "tokens_seen": 1683881984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946639919759278, + "loss": 1.2488, + "theoretical_loss": 3.5092613986990715, + "tokens_seen": 1683947520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946439317953862, + "loss": 1.5306, + "theoretical_loss": 3.509249718471608, + "tokens_seen": 1684013056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946238716148446, + "loss": 1.3912, + "theoretical_loss": 3.50923803882596, + "tokens_seen": 1684078592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946038114343029, + "loss": 1.3082, + "theoretical_loss": 3.509226359762075, + "tokens_seen": 1684144128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945837512537613, + "loss": 1.4419, + "theoretical_loss": 3.5092146812799023, + "tokens_seen": 1684209664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945636910732196, + "loss": 1.4211, + "theoretical_loss": 3.5092030033793895, + "tokens_seen": 1684275200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945436308926781, + "loss": 1.399, + "theoretical_loss": 3.5091913260604857, + "tokens_seen": 1684340736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945235707121365, + "loss": 1.2682, + "theoretical_loss": 3.509179649323138, + "tokens_seen": 1684406272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945035105315948, + "loss": 1.3093, + "theoretical_loss": 3.5091679731672967, + "tokens_seen": 1684471808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944834503510532, + "loss": 1.4958, + "theoretical_loss": 3.5091562975929085, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944633901705116, + "loss": 1.3395, + "theoretical_loss": 3.5091446225999228, + "tokens_seen": 1684602880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944433299899699, + "loss": 1.3117, + "theoretical_loss": 3.509132948188287, + "tokens_seen": 1684668416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944232698094283, + "loss": 1.4741, + "theoretical_loss": 3.5091212743579514, + "tokens_seen": 1684733952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944032096288867, + "loss": 1.423, + "theoretical_loss": 3.5091096011088627, + "tokens_seen": 1684799488 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494383149448345, + "loss": 1.4385, + "theoretical_loss": 3.5090979284409705, + "tokens_seen": 1684865024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943630892678035, + "loss": 1.554, + "theoretical_loss": 3.5090862563542227, + "tokens_seen": 1684930560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943430290872618, + "loss": 1.5009, + "theoretical_loss": 3.509074584848568, + "tokens_seen": 1684996096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943229689067202, + "loss": 1.2959, + "theoretical_loss": 3.5090629139239535, + "tokens_seen": 1685061632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943029087261786, + "loss": 1.3549, + "theoretical_loss": 3.50905124358033, + "tokens_seen": 1685127168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828485456369, + "loss": 1.3463, + "theoretical_loss": 3.5090395738176445, + "tokens_seen": 1685192704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942627883650953, + "loss": 1.337, + "theoretical_loss": 3.509027904635846, + "tokens_seen": 1685258240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942427281845537, + "loss": 1.2665, + "theoretical_loss": 3.509016236034883, + "tokens_seen": 1685323776 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494222668004012, + "loss": 1.2958, + "theoretical_loss": 3.5090045680147037, + "tokens_seen": 1685389312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942026078234704, + "loss": 1.2319, + "theoretical_loss": 3.5089929005752563, + "tokens_seen": 1685454848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941825476429289, + "loss": 1.556, + "theoretical_loss": 3.50898123371649, + "tokens_seen": 1685520384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941624874623872, + "loss": 1.4128, + "theoretical_loss": 3.5089695674383536, + "tokens_seen": 1685585920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941424272818456, + "loss": 1.4533, + "theoretical_loss": 3.5089579017407946, + "tokens_seen": 1685651456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941223671013039, + "loss": 1.3248, + "theoretical_loss": 3.5089462366237623, + "tokens_seen": 1685716992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941023069207623, + "loss": 1.356, + "theoretical_loss": 3.508934572087205, + "tokens_seen": 1685782528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940822467402207, + "loss": 1.3832, + "theoretical_loss": 3.5089229081310713, + "tokens_seen": 1685848064 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494062186559679, + "loss": 1.3853, + "theoretical_loss": 3.5089112447553097, + "tokens_seen": 1685913600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940421263791374, + "loss": 1.4201, + "theoretical_loss": 3.508899581959869, + "tokens_seen": 1685979136 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 615881, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4075050354003906, + "objective/train/theoretical_loss": 3.5088879197446974, + "objective/train/tokens_used": 56503776, + "theoretical_loss": 3.5088879197446974, + "tokens_seen": 1686044672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940220661985958, + "loss": 1.4282, + "theoretical_loss": 3.5088879197446974, + "tokens_seen": 1686044672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940020060180542, + "loss": 1.3533, + "theoretical_loss": 3.5088762581097432, + "tokens_seen": 1686110208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939819458375126, + "loss": 1.4034, + "theoretical_loss": 3.5088645970549557, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493961885656971, + "loss": 1.4432, + "theoretical_loss": 3.5088529365802836, + "tokens_seen": 1686241280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939418254764293, + "loss": 1.3309, + "theoretical_loss": 3.5088412766856747, + "tokens_seen": 1686306816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939217652958877, + "loss": 1.3937, + "theoretical_loss": 3.5088296173710782, + "tokens_seen": 1686372352 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493901705115346, + "loss": 1.366, + "theoretical_loss": 3.508817958636442, + "tokens_seen": 1686437888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938816449348044, + "loss": 1.3288, + "theoretical_loss": 3.508806300481716, + "tokens_seen": 1686503424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938615847542628, + "loss": 1.4774, + "theoretical_loss": 3.5087946429068477, + "tokens_seen": 1686568960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938415245737211, + "loss": 1.4116, + "theoretical_loss": 3.5087829859117856, + "tokens_seen": 1686634496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938214643931796, + "loss": 1.4398, + "theoretical_loss": 3.5087713294964793, + "tokens_seen": 1686700032 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493801404212638, + "loss": 1.3074, + "theoretical_loss": 3.508759673660877, + "tokens_seen": 1686765568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937813440320963, + "loss": 1.2896, + "theoretical_loss": 3.508748018404927, + "tokens_seen": 1686831104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937612838515547, + "loss": 1.5198, + "theoretical_loss": 3.5087363637285787, + "tokens_seen": 1686896640 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493741223671013, + "loss": 1.3359, + "theoretical_loss": 3.50872470963178, + "tokens_seen": 1686962176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937211634904714, + "loss": 1.3666, + "theoretical_loss": 3.5087130561144795, + "tokens_seen": 1687027712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937011033099298, + "loss": 1.2792, + "theoretical_loss": 3.5087014031766266, + "tokens_seen": 1687093248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936810431293881, + "loss": 1.2595, + "theoretical_loss": 3.50868975081817, + "tokens_seen": 1687158784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936609829488465, + "loss": 1.3133, + "theoretical_loss": 3.5086780990390576, + "tokens_seen": 1687224320 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493640922768305, + "loss": 1.3364, + "theoretical_loss": 3.5086664478392384, + "tokens_seen": 1687289856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936208625877633, + "loss": 1.2262, + "theoretical_loss": 3.5086547972186617, + "tokens_seen": 1687355392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936008024072217, + "loss": 1.3995, + "theoretical_loss": 3.5086431471772754, + "tokens_seen": 1687420928 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049358074222668, + "loss": 1.4076, + "theoretical_loss": 3.5086314977150286, + "tokens_seen": 1687486464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935606820461384, + "loss": 1.3261, + "theoretical_loss": 3.50861984883187, + "tokens_seen": 1687552000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935406218655968, + "loss": 1.3928, + "theoretical_loss": 3.508608200527749, + "tokens_seen": 1687617536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935205616850551, + "loss": 1.3427, + "theoretical_loss": 3.5085965528026124, + "tokens_seen": 1687683072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935005015045135, + "loss": 1.4534, + "theoretical_loss": 3.5085849056564107, + "tokens_seen": 1687748608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934804413239719, + "loss": 1.3425, + "theoretical_loss": 3.508573259089092, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934603811434303, + "loss": 1.3352, + "theoretical_loss": 3.508561613100605, + "tokens_seen": 1687879680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934403209628887, + "loss": 1.4193, + "theoretical_loss": 3.5085499676908993, + "tokens_seen": 1687945216 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493420260782347, + "loss": 1.4887, + "theoretical_loss": 3.5085383228599225, + "tokens_seen": 1688010752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934002006018054, + "loss": 1.3698, + "theoretical_loss": 3.5085266786076237, + "tokens_seen": 1688076288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933801404212638, + "loss": 1.551, + "theoretical_loss": 3.5085150349339522, + "tokens_seen": 1688141824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933600802407221, + "loss": 1.3699, + "theoretical_loss": 3.508503391838856, + "tokens_seen": 1688207360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933400200601805, + "loss": 1.4732, + "theoretical_loss": 3.5084917493222845, + "tokens_seen": 1688272896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933199598796389, + "loss": 1.4211, + "theoretical_loss": 3.5084801073841865, + "tokens_seen": 1688338432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932998996990972, + "loss": 1.3232, + "theoretical_loss": 3.5084684660245107, + "tokens_seen": 1688403968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932798395185557, + "loss": 1.2814, + "theoretical_loss": 3.5084568252432056, + "tokens_seen": 1688469504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932597793380141, + "loss": 1.355, + "theoretical_loss": 3.5084451850402205, + "tokens_seen": 1688535040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932397191574724, + "loss": 1.3167, + "theoretical_loss": 3.5084335454155036, + "tokens_seen": 1688600576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932196589769308, + "loss": 1.2502, + "theoretical_loss": 3.508421906369004, + "tokens_seen": 1688666112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931995987963893, + "loss": 1.2834, + "theoretical_loss": 3.508410267900671, + "tokens_seen": 1688731648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931795386158476, + "loss": 1.3767, + "theoretical_loss": 3.5083986300104533, + "tokens_seen": 1688797184 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493159478435306, + "loss": 1.3759, + "theoretical_loss": 3.508386992698299, + "tokens_seen": 1688862720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931394182547643, + "loss": 1.297, + "theoretical_loss": 3.5083753559641577, + "tokens_seen": 1688928256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931193580742227, + "loss": 1.46, + "theoretical_loss": 3.508363719807978, + "tokens_seen": 1688993792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930992978936811, + "loss": 1.2731, + "theoretical_loss": 3.508352084229709, + "tokens_seen": 1689059328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930792377131394, + "loss": 1.3691, + "theoretical_loss": 3.5083404492292996, + "tokens_seen": 1689124864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930591775325978, + "loss": 1.4393, + "theoretical_loss": 3.508328814806698, + "tokens_seen": 1689190400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930391173520562, + "loss": 1.3628, + "theoretical_loss": 3.5083171809618534, + "tokens_seen": 1689255936 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 617011, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4949119091033936, + "objective/train/theoretical_loss": 3.5083055476947154, + "objective/train/tokens_used": 59780576, + "theoretical_loss": 3.5083055476947154, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930190571715146, + "loss": 1.507, + "theoretical_loss": 3.5083055476947154, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492998996990973, + "loss": 1.3042, + "theoretical_loss": 3.508293915005232, + "tokens_seen": 1689387008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929789368104313, + "loss": 1.2017, + "theoretical_loss": 3.5082822828933526, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929588766298897, + "loss": 1.2654, + "theoretical_loss": 3.5082706513590263, + "tokens_seen": 1689518080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929388164493481, + "loss": 1.3117, + "theoretical_loss": 3.5082590204022015, + "tokens_seen": 1689583616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929187562688064, + "loss": 1.3386, + "theoretical_loss": 3.5082473900228273, + "tokens_seen": 1689649152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928986960882648, + "loss": 1.4195, + "theoretical_loss": 3.5082357602208525, + "tokens_seen": 1689714688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928786359077232, + "loss": 1.4252, + "theoretical_loss": 3.5082241309962265, + "tokens_seen": 1689780224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928585757271815, + "loss": 1.2852, + "theoretical_loss": 3.508212502348898, + "tokens_seen": 1689845760 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049283851554664, + "loss": 1.3667, + "theoretical_loss": 3.508200874278816, + "tokens_seen": 1689911296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928184553660984, + "loss": 1.3233, + "theoretical_loss": 3.5081892467859292, + "tokens_seen": 1689976832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927983951855567, + "loss": 1.3363, + "theoretical_loss": 3.5081776198701866, + "tokens_seen": 1690042368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927783350050151, + "loss": 1.3989, + "theoretical_loss": 3.5081659935315375, + "tokens_seen": 1690107904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927582748244734, + "loss": 1.3096, + "theoretical_loss": 3.5081543677699307, + "tokens_seen": 1690173440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927382146439318, + "loss": 1.2772, + "theoretical_loss": 3.5081427425853153, + "tokens_seen": 1690238976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927181544633902, + "loss": 1.3291, + "theoretical_loss": 3.5081311179776398, + "tokens_seen": 1690304512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926980942828485, + "loss": 1.3775, + "theoretical_loss": 3.508119493946854, + "tokens_seen": 1690370048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926780341023069, + "loss": 1.4747, + "theoretical_loss": 3.5081078704929065, + "tokens_seen": 1690435584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926579739217654, + "loss": 1.3508, + "theoretical_loss": 3.5080962476157462, + "tokens_seen": 1690501120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926379137412237, + "loss": 1.4896, + "theoretical_loss": 3.5080846253153224, + "tokens_seen": 1690566656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926178535606821, + "loss": 1.5249, + "theoretical_loss": 3.508073003591584, + "tokens_seen": 1690632192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925977933801404, + "loss": 1.3723, + "theoretical_loss": 3.5080613824444793, + "tokens_seen": 1690697728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925777331995988, + "loss": 1.3005, + "theoretical_loss": 3.508049761873959, + "tokens_seen": 1690763264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925576730190572, + "loss": 1.463, + "theoretical_loss": 3.5080381418799704, + "tokens_seen": 1690828800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925376128385155, + "loss": 1.3577, + "theoretical_loss": 3.5080265224624636, + "tokens_seen": 1690894336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925175526579739, + "loss": 1.3403, + "theoretical_loss": 3.5080149036213877, + "tokens_seen": 1690959872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924974924774323, + "loss": 1.3324, + "theoretical_loss": 3.508003285356691, + "tokens_seen": 1691025408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924774322968907, + "loss": 1.4333, + "theoretical_loss": 3.5079916676683234, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924573721163491, + "loss": 1.3066, + "theoretical_loss": 3.507980050556233, + "tokens_seen": 1691156480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924373119358075, + "loss": 1.3331, + "theoretical_loss": 3.50796843402037, + "tokens_seen": 1691222016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924172517552658, + "loss": 1.3279, + "theoretical_loss": 3.507956818060683, + "tokens_seen": 1691287552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923971915747242, + "loss": 1.4721, + "theoretical_loss": 3.5079452026771207, + "tokens_seen": 1691353088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923771313941825, + "loss": 1.3156, + "theoretical_loss": 3.507933587869633, + "tokens_seen": 1691418624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923570712136409, + "loss": 1.5105, + "theoretical_loss": 3.5079219736381684, + "tokens_seen": 1691484160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923370110330993, + "loss": 1.4068, + "theoretical_loss": 3.507910359982676, + "tokens_seen": 1691549696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923169508525576, + "loss": 1.2907, + "theoretical_loss": 3.5078987469031055, + "tokens_seen": 1691615232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922968906720161, + "loss": 1.3368, + "theoretical_loss": 3.507887134399405, + "tokens_seen": 1691680768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922768304914745, + "loss": 1.4479, + "theoretical_loss": 3.507875522471525, + "tokens_seen": 1691746304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922567703109328, + "loss": 1.3563, + "theoretical_loss": 3.5078639111194136, + "tokens_seen": 1691811840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922367101303912, + "loss": 1.4157, + "theoretical_loss": 3.5078523003430204, + "tokens_seen": 1691877376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922166499498495, + "loss": 1.2107, + "theoretical_loss": 3.5078406901422943, + "tokens_seen": 1691942912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921965897693079, + "loss": 1.2928, + "theoretical_loss": 3.5078290805171846, + "tokens_seen": 1692008448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921765295887663, + "loss": 1.4624, + "theoretical_loss": 3.5078174714676407, + "tokens_seen": 1692073984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921564694082246, + "loss": 1.3345, + "theoretical_loss": 3.507805862993611, + "tokens_seen": 1692139520 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492136409227683, + "loss": 1.3296, + "theoretical_loss": 3.507794255095046, + "tokens_seen": 1692205056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921163490471415, + "loss": 1.3384, + "theoretical_loss": 3.5077826477718936, + "tokens_seen": 1692270592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920962888665998, + "loss": 1.4161, + "theoretical_loss": 3.5077710410241036, + "tokens_seen": 1692336128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920762286860582, + "loss": 1.3567, + "theoretical_loss": 3.507759434851625, + "tokens_seen": 1692401664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920561685055166, + "loss": 1.4252, + "theoretical_loss": 3.5077478292544075, + "tokens_seen": 1692467200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920361083249749, + "loss": 1.4003, + "theoretical_loss": 3.5077362242323993, + "tokens_seen": 1692532736 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 618329, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.301423192024231, + "objective/train/theoretical_loss": 3.5077246197855505, + "objective/train/tokens_used": 63057376, + "theoretical_loss": 3.5077246197855505, + "tokens_seen": 1692598272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920160481444333, + "loss": 1.2787, + "theoretical_loss": 3.5077246197855505, + "tokens_seen": 1692598272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919959879638916, + "loss": 1.4332, + "theoretical_loss": 3.5077130159138106, + "tokens_seen": 1692663808 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049197592778335, + "loss": 1.3741, + "theoretical_loss": 3.507701412617128, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919558676028084, + "loss": 1.4026, + "theoretical_loss": 3.507689809895452, + "tokens_seen": 1692794880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919358074222668, + "loss": 1.4147, + "theoretical_loss": 3.5076782077487323, + "tokens_seen": 1692860416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919157472417252, + "loss": 1.3272, + "theoretical_loss": 3.507666606176918, + "tokens_seen": 1692925952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918956870611836, + "loss": 1.41, + "theoretical_loss": 3.5076550051799575, + "tokens_seen": 1692991488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918756268806419, + "loss": 1.3977, + "theoretical_loss": 3.507643404757802, + "tokens_seen": 1693057024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918555667001003, + "loss": 1.3732, + "theoretical_loss": 3.5076318049103987, + "tokens_seen": 1693122560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918355065195586, + "loss": 1.471, + "theoretical_loss": 3.507620205637698, + "tokens_seen": 1693188096 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491815446339017, + "loss": 1.2343, + "theoretical_loss": 3.50760860693965, + "tokens_seen": 1693253632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917953861584754, + "loss": 1.4344, + "theoretical_loss": 3.507597008816202, + "tokens_seen": 1693319168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917753259779337, + "loss": 1.3792, + "theoretical_loss": 3.5075854112673044, + "tokens_seen": 1693384704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917552657973922, + "loss": 1.2945, + "theoretical_loss": 3.5075738142929067, + "tokens_seen": 1693450240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917352056168506, + "loss": 1.3872, + "theoretical_loss": 3.507562217892958, + "tokens_seen": 1693515776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917151454363089, + "loss": 1.2753, + "theoretical_loss": 3.5075506220674066, + "tokens_seen": 1693581312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916950852557673, + "loss": 1.3919, + "theoretical_loss": 3.507539026816204, + "tokens_seen": 1693646848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916750250752258, + "loss": 1.4268, + "theoretical_loss": 3.5075274321392973, + "tokens_seen": 1693712384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916549648946841, + "loss": 1.3616, + "theoretical_loss": 3.507515838036637, + "tokens_seen": 1693777920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916349047141425, + "loss": 1.468, + "theoretical_loss": 3.5075042445081728, + "tokens_seen": 1693843456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916148445336008, + "loss": 1.4037, + "theoretical_loss": 3.5074926515538527, + "tokens_seen": 1693908992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915947843530592, + "loss": 1.3366, + "theoretical_loss": 3.507481059173627, + "tokens_seen": 1693974528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915747241725176, + "loss": 1.3876, + "theoretical_loss": 3.5074694673674456, + "tokens_seen": 1694040064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915546639919759, + "loss": 1.4898, + "theoretical_loss": 3.507457876135257, + "tokens_seen": 1694105600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915346038114343, + "loss": 1.4334, + "theoretical_loss": 3.5074462854770103, + "tokens_seen": 1694171136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915145436308927, + "loss": 1.3677, + "theoretical_loss": 3.5074346953926554, + "tokens_seen": 1694236672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914944834503511, + "loss": 1.2385, + "theoretical_loss": 3.507423105882142, + "tokens_seen": 1694302208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914744232698095, + "loss": 1.4356, + "theoretical_loss": 3.5074115169454188, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914543630892679, + "loss": 1.4225, + "theoretical_loss": 3.5073999285824353, + "tokens_seen": 1694433280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343029087262, + "loss": 1.3465, + "theoretical_loss": 3.507388340793142, + "tokens_seen": 1694498816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914142427281846, + "loss": 1.3653, + "theoretical_loss": 3.5073767535774865, + "tokens_seen": 1694564352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913941825476429, + "loss": 1.4899, + "theoretical_loss": 3.5073651669354193, + "tokens_seen": 1694629888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913741223671013, + "loss": 1.3602, + "theoretical_loss": 3.50735358086689, + "tokens_seen": 1694695424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913540621865597, + "loss": 1.3722, + "theoretical_loss": 3.5073419953718474, + "tokens_seen": 1694760960 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491334002006018, + "loss": 1.3544, + "theoretical_loss": 3.507330410450242, + "tokens_seen": 1694826496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913139418254765, + "loss": 1.4186, + "theoretical_loss": 3.507318826102022, + "tokens_seen": 1694892032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912938816449349, + "loss": 1.4275, + "theoretical_loss": 3.507307242327137, + "tokens_seen": 1694957568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912738214643932, + "loss": 1.4761, + "theoretical_loss": 3.5072956591255373, + "tokens_seen": 1695023104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912537612838516, + "loss": 1.2628, + "theoretical_loss": 3.5072840764971716, + "tokens_seen": 1695088640 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049123370110331, + "loss": 1.3808, + "theoretical_loss": 3.50727249444199, + "tokens_seen": 1695154176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912136409227683, + "loss": 1.344, + "theoretical_loss": 3.507260912959941, + "tokens_seen": 1695219712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911935807422267, + "loss": 1.3068, + "theoretical_loss": 3.507249332050975, + "tokens_seen": 1695285248 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491173520561685, + "loss": 1.3323, + "theoretical_loss": 3.5072377517150413, + "tokens_seen": 1695350784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911534603811434, + "loss": 1.2759, + "theoretical_loss": 3.507226171952089, + "tokens_seen": 1695416320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911334002006019, + "loss": 1.4506, + "theoretical_loss": 3.507214592762068, + "tokens_seen": 1695481856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911133400200602, + "loss": 1.3608, + "theoretical_loss": 3.5072030141449284, + "tokens_seen": 1695547392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910932798395186, + "loss": 1.319, + "theoretical_loss": 3.507191436100618, + "tokens_seen": 1695612928 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491073219658977, + "loss": 1.3914, + "theoretical_loss": 3.5071798586290877, + "tokens_seen": 1695678464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910531594784353, + "loss": 1.3683, + "theoretical_loss": 3.507168281730287, + "tokens_seen": 1695744000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910330992978937, + "loss": 1.3267, + "theoretical_loss": 3.5071567054041646, + "tokens_seen": 1695809536 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 619583, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.226139783859253, + "objective/train/theoretical_loss": 3.507145129650671, + "objective/train/tokens_used": 66334176, + "theoretical_loss": 3.507145129650671, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491013039117352, + "loss": 1.3059, + "theoretical_loss": 3.507145129650671, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909929789368104, + "loss": 1.4155, + "theoretical_loss": 3.507133554469755, + "tokens_seen": 1695940608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909729187562688, + "loss": 1.4839, + "theoretical_loss": 3.507121979861367, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909528585757272, + "loss": 1.336, + "theoretical_loss": 3.507110405825455, + "tokens_seen": 1696071680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909327983951856, + "loss": 1.3128, + "theoretical_loss": 3.50709883236197, + "tokens_seen": 1696137216 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490912738214644, + "loss": 1.3394, + "theoretical_loss": 3.507087259470861, + "tokens_seen": 1696202752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908926780341023, + "loss": 1.4315, + "theoretical_loss": 3.5070756871520783, + "tokens_seen": 1696268288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908726178535607, + "loss": 1.4136, + "theoretical_loss": 3.50706411540557, + "tokens_seen": 1696333824 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490852557673019, + "loss": 1.4009, + "theoretical_loss": 3.5070525442312874, + "tokens_seen": 1696399360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908324974924774, + "loss": 1.3483, + "theoretical_loss": 3.507040973629179, + "tokens_seen": 1696464896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908124373119358, + "loss": 1.5329, + "theoretical_loss": 3.507029403599195, + "tokens_seen": 1696530432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907923771313941, + "loss": 1.3032, + "theoretical_loss": 3.5070178341412843, + "tokens_seen": 1696595968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907723169508526, + "loss": 1.393, + "theoretical_loss": 3.5070062652553977, + "tokens_seen": 1696661504 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490752256770311, + "loss": 1.2946, + "theoretical_loss": 3.506994696941483, + "tokens_seen": 1696727040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907321965897693, + "loss": 1.2365, + "theoretical_loss": 3.5069831291994915, + "tokens_seen": 1696792576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907121364092277, + "loss": 1.3889, + "theoretical_loss": 3.506971562029372, + "tokens_seen": 1696858112 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490692076228686, + "loss": 1.3606, + "theoretical_loss": 3.5069599954310746, + "tokens_seen": 1696923648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906720160481444, + "loss": 1.2779, + "theoretical_loss": 3.5069484294045483, + "tokens_seen": 1696989184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906519558676028, + "loss": 1.2498, + "theoretical_loss": 3.5069368639497434, + "tokens_seen": 1697054720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906318956870611, + "loss": 1.3094, + "theoretical_loss": 3.5069252990666095, + "tokens_seen": 1697120256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906118355065195, + "loss": 1.5626, + "theoretical_loss": 3.506913734755096, + "tokens_seen": 1697185792 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490591775325978, + "loss": 1.3086, + "theoretical_loss": 3.5069021710151524, + "tokens_seen": 1697251328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905717151454363, + "loss": 1.4392, + "theoretical_loss": 3.506890607846729, + "tokens_seen": 1697316864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905516549648947, + "loss": 1.4032, + "theoretical_loss": 3.506879045249775, + "tokens_seen": 1697382400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905315947843531, + "loss": 1.0952, + "theoretical_loss": 3.5068674832242404, + "tokens_seen": 1697447936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905115346038114, + "loss": 1.3024, + "theoretical_loss": 3.5068559217700748, + "tokens_seen": 1697513472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904914744232698, + "loss": 1.4544, + "theoretical_loss": 3.5068443608872277, + "tokens_seen": 1697579008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904714142427281, + "loss": 1.3557, + "theoretical_loss": 3.5068328005756486, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904513540621865, + "loss": 1.3338, + "theoretical_loss": 3.506821240835288, + "tokens_seen": 1697710080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904312938816449, + "loss": 1.4531, + "theoretical_loss": 3.5068096816660956, + "tokens_seen": 1697775616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904112337011033, + "loss": 1.351, + "theoretical_loss": 3.5067981230680196, + "tokens_seen": 1697841152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903911735205617, + "loss": 1.3425, + "theoretical_loss": 3.506786565041012, + "tokens_seen": 1697906688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903711133400201, + "loss": 1.3297, + "theoretical_loss": 3.506775007585021, + "tokens_seen": 1697972224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903510531594784, + "loss": 1.3589, + "theoretical_loss": 3.5067634506999967, + "tokens_seen": 1698037760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903309929789368, + "loss": 1.2881, + "theoretical_loss": 3.5067518943858893, + "tokens_seen": 1698103296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903109327983952, + "loss": 1.4676, + "theoretical_loss": 3.506740338642648, + "tokens_seen": 1698168832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902908726178535, + "loss": 1.3875, + "theoretical_loss": 3.5067287834702223, + "tokens_seen": 1698234368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902708124373119, + "loss": 1.4076, + "theoretical_loss": 3.506717228868563, + "tokens_seen": 1698299904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902507522567703, + "loss": 1.367, + "theoretical_loss": 3.506705674837619, + "tokens_seen": 1698365440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902306920762287, + "loss": 1.5286, + "theoretical_loss": 3.5066941213773406, + "tokens_seen": 1698430976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902106318956871, + "loss": 1.4007, + "theoretical_loss": 3.5066825684876775, + "tokens_seen": 1698496512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901905717151454, + "loss": 1.2717, + "theoretical_loss": 3.5066710161685792, + "tokens_seen": 1698562048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901705115346038, + "loss": 1.4175, + "theoretical_loss": 3.5066594644199958, + "tokens_seen": 1698627584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901504513540623, + "loss": 1.4019, + "theoretical_loss": 3.506647913241877, + "tokens_seen": 1698693120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901303911735206, + "loss": 1.4839, + "theoretical_loss": 3.5066363626341728, + "tokens_seen": 1698758656 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490110330992979, + "loss": 1.3258, + "theoretical_loss": 3.506624812596833, + "tokens_seen": 1698824192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900902708124374, + "loss": 1.2434, + "theoretical_loss": 3.5066132631298075, + "tokens_seen": 1698889728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900702106318957, + "loss": 1.3587, + "theoretical_loss": 3.506601714233045, + "tokens_seen": 1698955264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900501504513541, + "loss": 1.3124, + "theoretical_loss": 3.5065901659064975, + "tokens_seen": 1699020800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900300902708124, + "loss": 1.423, + "theoretical_loss": 3.5065786181501126, + "tokens_seen": 1699086336 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 620634, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3232345581054688, + "objective/train/theoretical_loss": 3.5065670709638423, + "objective/train/tokens_used": 69610976, + "theoretical_loss": 3.5065670709638423, + "tokens_seen": 1699151872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900100300902708, + "loss": 1.3566, + "theoretical_loss": 3.5065670709638423, + "tokens_seen": 1699151872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899899699097292, + "loss": 1.2686, + "theoretical_loss": 3.5065555243476347, + "tokens_seen": 1699217408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899699097291876, + "loss": 1.2973, + "theoretical_loss": 3.5065439783014405, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489949849548646, + "loss": 1.2781, + "theoretical_loss": 3.5065324328252094, + "tokens_seen": 1699348480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899297893681044, + "loss": 1.3311, + "theoretical_loss": 3.5065208879188914, + "tokens_seen": 1699414016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899097291875627, + "loss": 1.3235, + "theoretical_loss": 3.506509343582436, + "tokens_seen": 1699479552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898896690070211, + "loss": 1.3341, + "theoretical_loss": 3.506497799815794, + "tokens_seen": 1699545088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898696088264794, + "loss": 1.2423, + "theoretical_loss": 3.5064862566189143, + "tokens_seen": 1699610624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898495486459378, + "loss": 1.3397, + "theoretical_loss": 3.5064747139917474, + "tokens_seen": 1699676160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898294884653962, + "loss": 1.2578, + "theoretical_loss": 3.506463171934243, + "tokens_seen": 1699741696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898094282848545, + "loss": 1.3833, + "theoretical_loss": 3.5064516304463513, + "tokens_seen": 1699807232 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489789368104313, + "loss": 1.2578, + "theoretical_loss": 3.506440089528022, + "tokens_seen": 1699872768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897693079237714, + "loss": 1.4603, + "theoretical_loss": 3.5064285491792044, + "tokens_seen": 1699938304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897492477432297, + "loss": 1.2667, + "theoretical_loss": 3.5064170093998497, + "tokens_seen": 1700003840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897291875626881, + "loss": 1.2995, + "theoretical_loss": 3.506405470189907, + "tokens_seen": 1700069376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897091273821465, + "loss": 1.2886, + "theoretical_loss": 3.5063939315493267, + "tokens_seen": 1700134912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896890672016048, + "loss": 1.3382, + "theoretical_loss": 3.5063823934780585, + "tokens_seen": 1700200448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896690070210632, + "loss": 1.3299, + "theoretical_loss": 3.506370855976052, + "tokens_seen": 1700265984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896489468405215, + "loss": 1.2978, + "theoretical_loss": 3.5063593190432574, + "tokens_seen": 1700331520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896288866599799, + "loss": 1.2663, + "theoretical_loss": 3.506347782679626, + "tokens_seen": 1700397056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896088264794384, + "loss": 1.2953, + "theoretical_loss": 3.5063362468851054, + "tokens_seen": 1700462592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895887662988967, + "loss": 1.3638, + "theoretical_loss": 3.5063247116596474, + "tokens_seen": 1700528128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895687061183551, + "loss": 1.2951, + "theoretical_loss": 3.506313177003202, + "tokens_seen": 1700593664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895486459378135, + "loss": 1.4755, + "theoretical_loss": 3.5063016429157177, + "tokens_seen": 1700659200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895285857572718, + "loss": 1.4268, + "theoretical_loss": 3.506290109397146, + "tokens_seen": 1700724736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895085255767302, + "loss": 1.296, + "theoretical_loss": 3.5062785764474365, + "tokens_seen": 1700790272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894884653961885, + "loss": 1.3229, + "theoretical_loss": 3.506267044066538, + "tokens_seen": 1700855808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894684052156469, + "loss": 1.2576, + "theoretical_loss": 3.506255512254403, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894483450351053, + "loss": 1.3534, + "theoretical_loss": 3.5062439810109796, + "tokens_seen": 1700986880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894282848545637, + "loss": 1.4438, + "theoretical_loss": 3.5062324503362183, + "tokens_seen": 1701052416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894082246740221, + "loss": 1.5134, + "theoretical_loss": 3.5062209202300694, + "tokens_seen": 1701117952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893881644934805, + "loss": 1.4263, + "theoretical_loss": 3.506209390692483, + "tokens_seen": 1701183488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893681043129388, + "loss": 1.3592, + "theoretical_loss": 3.5061978617234084, + "tokens_seen": 1701249024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893480441323972, + "loss": 1.418, + "theoretical_loss": 3.5061863333227965, + "tokens_seen": 1701314560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893279839518556, + "loss": 1.3557, + "theoretical_loss": 3.5061748054905975, + "tokens_seen": 1701380096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893079237713139, + "loss": 1.2816, + "theoretical_loss": 3.5061632782267607, + "tokens_seen": 1701445632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892878635907723, + "loss": 1.3639, + "theoretical_loss": 3.506151751531237, + "tokens_seen": 1701511168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892678034102307, + "loss": 1.4028, + "theoretical_loss": 3.5061402254039757, + "tokens_seen": 1701576704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892477432296891, + "loss": 1.3536, + "theoretical_loss": 3.5061286998449273, + "tokens_seen": 1701642240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892276830491475, + "loss": 1.3034, + "theoretical_loss": 3.506117174854042, + "tokens_seen": 1701707776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892076228686058, + "loss": 1.2685, + "theoretical_loss": 3.50610565043127, + "tokens_seen": 1701773312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891875626880642, + "loss": 1.3486, + "theoretical_loss": 3.506094126576561, + "tokens_seen": 1701838848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891675025075226, + "loss": 1.3492, + "theoretical_loss": 3.5060826032898653, + "tokens_seen": 1701904384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891474423269809, + "loss": 1.5114, + "theoretical_loss": 3.5060710805711333, + "tokens_seen": 1701969920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891273821464393, + "loss": 1.3318, + "theoretical_loss": 3.506059558420314, + "tokens_seen": 1702035456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891073219658976, + "loss": 1.3173, + "theoretical_loss": 3.5060480368373597, + "tokens_seen": 1702100992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890872617853561, + "loss": 1.4413, + "theoretical_loss": 3.5060365158222186, + "tokens_seen": 1702166528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890672016048145, + "loss": 1.3575, + "theoretical_loss": 3.5060249953748417, + "tokens_seen": 1702232064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890471414242728, + "loss": 1.2915, + "theoretical_loss": 3.506013475495179, + "tokens_seen": 1702297600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890270812437312, + "loss": 1.345, + "theoretical_loss": 3.506001956183181, + "tokens_seen": 1702363136 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 621885, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3620576858520508, + "objective/train/theoretical_loss": 3.5059904374387973, + "objective/train/tokens_used": 72887776, + "theoretical_loss": 3.5059904374387973, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890070210631896, + "loss": 1.3207, + "theoretical_loss": 3.5059904374387973, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889869608826479, + "loss": 1.2116, + "theoretical_loss": 3.5059789192619784, + "tokens_seen": 1702494208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889669007021063, + "loss": 1.3397, + "theoretical_loss": 3.5059674016526747, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889468405215647, + "loss": 1.2122, + "theoretical_loss": 3.5059558846108354, + "tokens_seen": 1702625280 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488926780341023, + "loss": 1.4189, + "theoretical_loss": 3.5059443681364124, + "tokens_seen": 1702690816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889067201604815, + "loss": 1.3155, + "theoretical_loss": 3.5059328522293542, + "tokens_seen": 1702756352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888866599799398, + "loss": 1.4403, + "theoretical_loss": 3.505921336889612, + "tokens_seen": 1702821888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888665997993982, + "loss": 1.4239, + "theoretical_loss": 3.505909822117136, + "tokens_seen": 1702887424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888465396188566, + "loss": 1.3553, + "theoretical_loss": 3.5058983079118757, + "tokens_seen": 1702952960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888264794383149, + "loss": 1.3793, + "theoretical_loss": 3.505886794273782, + "tokens_seen": 1703018496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888064192577733, + "loss": 1.375, + "theoretical_loss": 3.505875281202805, + "tokens_seen": 1703084032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887863590772317, + "loss": 1.3817, + "theoretical_loss": 3.5058637686988954, + "tokens_seen": 1703149568 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048876629889669, + "loss": 1.3402, + "theoretical_loss": 3.5058522567620023, + "tokens_seen": 1703215104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887462387161484, + "loss": 1.3355, + "theoretical_loss": 3.505840745392077, + "tokens_seen": 1703280640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887261785356069, + "loss": 1.3791, + "theoretical_loss": 3.5058292345890694, + "tokens_seen": 1703346176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887061183550652, + "loss": 1.326, + "theoretical_loss": 3.505817724352929, + "tokens_seen": 1703411712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886860581745236, + "loss": 1.292, + "theoretical_loss": 3.505806214683608, + "tokens_seen": 1703477248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886659979939819, + "loss": 1.4999, + "theoretical_loss": 3.5057947055810548, + "tokens_seen": 1703542784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886459378134403, + "loss": 1.4128, + "theoretical_loss": 3.5057831970452207, + "tokens_seen": 1703608320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886258776328988, + "loss": 1.3775, + "theoretical_loss": 3.5057716890760555, + "tokens_seen": 1703673856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886058174523571, + "loss": 1.4581, + "theoretical_loss": 3.5057601816735096, + "tokens_seen": 1703739392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885857572718155, + "loss": 1.2917, + "theoretical_loss": 3.505748674837534, + "tokens_seen": 1703804928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656970912739, + "loss": 1.3024, + "theoretical_loss": 3.5057371685680776, + "tokens_seen": 1703870464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885456369107322, + "loss": 1.2958, + "theoretical_loss": 3.505725662865092, + "tokens_seen": 1703936000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885255767301906, + "loss": 1.3506, + "theoretical_loss": 3.5057141577285273, + "tokens_seen": 1704001536 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488505516549649, + "loss": 1.2579, + "theoretical_loss": 3.505702653158333, + "tokens_seen": 1704067072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884854563691073, + "loss": 1.3549, + "theoretical_loss": 3.5056911491544605, + "tokens_seen": 1704132608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884653961885657, + "loss": 1.3368, + "theoretical_loss": 3.5056796457168593, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884453360080241, + "loss": 1.4262, + "theoretical_loss": 3.5056681428454803, + "tokens_seen": 1704263680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884252758274825, + "loss": 1.4651, + "theoretical_loss": 3.5056566405402743, + "tokens_seen": 1704329216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884052156469409, + "loss": 1.4421, + "theoretical_loss": 3.5056451388011904, + "tokens_seen": 1704394752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883851554663992, + "loss": 1.2227, + "theoretical_loss": 3.5056336376281796, + "tokens_seen": 1704460288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883650952858576, + "loss": 1.4228, + "theoretical_loss": 3.5056221370211924, + "tokens_seen": 1704525824 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488345035105316, + "loss": 1.2801, + "theoretical_loss": 3.505610636980179, + "tokens_seen": 1704591360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883249749247743, + "loss": 1.4061, + "theoretical_loss": 3.50559913750509, + "tokens_seen": 1704656896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883049147442327, + "loss": 1.4225, + "theoretical_loss": 3.505587638595876, + "tokens_seen": 1704722432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882848545636911, + "loss": 1.4614, + "theoretical_loss": 3.5055761402524865, + "tokens_seen": 1704787968 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048826479438314946, + "loss": 1.3331, + "theoretical_loss": 3.5055646424748725, + "tokens_seen": 1704853504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882447342026078, + "loss": 1.3728, + "theoretical_loss": 3.5055531452629847, + "tokens_seen": 1704919040 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822467402206624, + "loss": 1.3744, + "theoretical_loss": 3.505541648616773, + "tokens_seen": 1704984576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882046138415246, + "loss": 1.4004, + "theoretical_loss": 3.5055301525361884, + "tokens_seen": 1705050112 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818455366098296, + "loss": 1.3302, + "theoretical_loss": 3.5055186570211805, + "tokens_seen": 1705115648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881644934804413, + "loss": 1.3399, + "theoretical_loss": 3.5055071620717007, + "tokens_seen": 1705181184 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048814443329989974, + "loss": 1.439, + "theoretical_loss": 3.5054956676876987, + "tokens_seen": 1705246720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881243731193581, + "loss": 1.4232, + "theoretical_loss": 3.505484173869125, + "tokens_seen": 1705312256 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048810431293881646, + "loss": 1.3036, + "theoretical_loss": 3.5054726806159304, + "tokens_seen": 1705377792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880842527582748, + "loss": 1.4074, + "theoretical_loss": 3.505461187928065, + "tokens_seen": 1705443328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880641925777332, + "loss": 1.3335, + "theoretical_loss": 3.50544969580548, + "tokens_seen": 1705508864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880441323971916, + "loss": 1.3578, + "theoretical_loss": 3.505438204248125, + "tokens_seen": 1705574400 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802407221664997, + "loss": 1.2749, + "theoretical_loss": 3.505426713255951, + "tokens_seen": 1705639936 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 623014, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2867580652236938, + "objective/train/theoretical_loss": 3.5054152228289084, + "objective/train/tokens_used": 76164576, + "theoretical_loss": 3.5054152228289084, + "tokens_seen": 1705705472 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048800401203610833, + "loss": 1.408, + "theoretical_loss": 3.5054152228289084, + "tokens_seen": 1705705472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879839518555667, + "loss": 1.3039, + "theoretical_loss": 3.5054037329669474, + "tokens_seen": 1705771008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879638916750251, + "loss": 1.4341, + "theoretical_loss": 3.505392243670019, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048794383149448347, + "loss": 1.4318, + "theoretical_loss": 3.5053807549380736, + "tokens_seen": 1705902080 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048792377131394183, + "loss": 1.3985, + "theoretical_loss": 3.5053692667710608, + "tokens_seen": 1705967616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879037111334002, + "loss": 1.4275, + "theoretical_loss": 3.5053577791689325, + "tokens_seen": 1706033152 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048788365095285856, + "loss": 1.2929, + "theoretical_loss": 3.5053462921316383, + "tokens_seen": 1706098688 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048786359077231697, + "loss": 1.2899, + "theoretical_loss": 3.505334805659129, + "tokens_seen": 1706164224 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048784353059177533, + "loss": 1.3528, + "theoretical_loss": 3.505323319751356, + "tokens_seen": 1706229760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878234704112337, + "loss": 1.3581, + "theoretical_loss": 3.505311834408268, + "tokens_seen": 1706295296 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048780341023069206, + "loss": 1.3595, + "theoretical_loss": 3.505300349629817, + "tokens_seen": 1706360832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877833500501505, + "loss": 1.1994, + "theoretical_loss": 3.5052888654159533, + "tokens_seen": 1706426368 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048776328986960884, + "loss": 1.3109, + "theoretical_loss": 3.5052773817666267, + "tokens_seen": 1706491904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877432296890672, + "loss": 1.379, + "theoretical_loss": 3.505265898681789, + "tokens_seen": 1706557440 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048772316950852556, + "loss": 1.3817, + "theoretical_loss": 3.5052544161613897, + "tokens_seen": 1706622976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877031093279839, + "loss": 1.4001, + "theoretical_loss": 3.50524293420538, + "tokens_seen": 1706688512 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048768304914744234, + "loss": 1.4308, + "theoretical_loss": 3.5052314528137103, + "tokens_seen": 1706754048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876629889669007, + "loss": 1.4139, + "theoretical_loss": 3.5052199719863317, + "tokens_seen": 1706819584 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048764292878635907, + "loss": 1.4029, + "theoretical_loss": 3.5052084917231934, + "tokens_seen": 1706885120 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048762286860581743, + "loss": 1.3839, + "theoretical_loss": 3.5051970120242473, + "tokens_seen": 1706950656 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048760280842527584, + "loss": 1.3622, + "theoretical_loss": 3.5051855328894437, + "tokens_seen": 1707016192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875827482447342, + "loss": 1.1993, + "theoretical_loss": 3.5051740543187333, + "tokens_seen": 1707081728 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048756268806419257, + "loss": 1.3528, + "theoretical_loss": 3.5051625763120664, + "tokens_seen": 1707147264 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048754262788365093, + "loss": 1.2254, + "theoretical_loss": 3.5051510988693937, + "tokens_seen": 1707212800 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048752256770310935, + "loss": 1.3726, + "theoretical_loss": 3.505139621990666, + "tokens_seen": 1707278336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875025075225677, + "loss": 1.2674, + "theoretical_loss": 3.505128145675834, + "tokens_seen": 1707343872 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048748244734202607, + "loss": 1.4298, + "theoretical_loss": 3.505116669924848, + "tokens_seen": 1707409408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048746238716148443, + "loss": 1.3453, + "theoretical_loss": 3.5051051947376592, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874423269809428, + "loss": 1.3674, + "theoretical_loss": 3.5050937201142176, + "tokens_seen": 1707540480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874222668004012, + "loss": 1.4249, + "theoretical_loss": 3.5050822460544744, + "tokens_seen": 1707606016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874022066198596, + "loss": 1.3695, + "theoretical_loss": 3.5050707725583803, + "tokens_seen": 1707671552 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048738214643931794, + "loss": 1.2728, + "theoretical_loss": 3.5050592996258856, + "tokens_seen": 1707737088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873620862587763, + "loss": 1.2962, + "theoretical_loss": 3.5050478272569414, + "tokens_seen": 1707802624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873420260782347, + "loss": 1.3047, + "theoretical_loss": 3.505036355451498, + "tokens_seen": 1707868160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873219658976931, + "loss": 1.4268, + "theoretical_loss": 3.505024884209506, + "tokens_seen": 1707933696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048730190571715144, + "loss": 1.2318, + "theoretical_loss": 3.5050134135309166, + "tokens_seen": 1707999232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872818455366098, + "loss": 1.332, + "theoretical_loss": 3.50500194341568, + "tokens_seen": 1708064768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048726178535606816, + "loss": 1.3295, + "theoretical_loss": 3.5049904738637476, + "tokens_seen": 1708130304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872417251755266, + "loss": 1.2741, + "theoretical_loss": 3.5049790048750697, + "tokens_seen": 1708195840 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487221664994985, + "loss": 1.4285, + "theoretical_loss": 3.504967536449597, + "tokens_seen": 1708261376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048720160481444336, + "loss": 1.3439, + "theoretical_loss": 3.50495606858728, + "tokens_seen": 1708326912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871815446339017, + "loss": 1.3742, + "theoretical_loss": 3.50494460128807, + "tokens_seen": 1708392448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048716148445336014, + "loss": 1.4036, + "theoretical_loss": 3.5049331345519175, + "tokens_seen": 1708457984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871414242728185, + "loss": 1.2504, + "theoretical_loss": 3.504921668378773, + "tokens_seen": 1708523520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048712136409227686, + "loss": 1.3234, + "theoretical_loss": 3.504910202768588, + "tokens_seen": 1708589056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871013039117352, + "loss": 1.2954, + "theoretical_loss": 3.5048987377213123, + "tokens_seen": 1708654592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870812437311936, + "loss": 1.3709, + "theoretical_loss": 3.5048872732368976, + "tokens_seen": 1708720128 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487061183550652, + "loss": 1.3592, + "theoretical_loss": 3.504875809315294, + "tokens_seen": 1708785664 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048704112337011037, + "loss": 1.3201, + "theoretical_loss": 3.5048643459564524, + "tokens_seen": 1708851200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048702106318956873, + "loss": 1.2678, + "theoretical_loss": 3.5048528831603236, + "tokens_seen": 1708916736 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 624223, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4896975755691528, + "objective/train/theoretical_loss": 3.5048414209268586, + "objective/train/tokens_used": 79441376, + "theoretical_loss": 3.5048414209268586, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870010030090271, + "loss": 1.3843, + "theoretical_loss": 3.5048414209268586, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869809428284855, + "loss": 1.331, + "theoretical_loss": 3.504829959256009, + "tokens_seen": 1709047808 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048696088264794387, + "loss": 1.227, + "theoretical_loss": 3.5048184981477233, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048694082246740223, + "loss": 1.3152, + "theoretical_loss": 3.504807037601955, + "tokens_seen": 1709178880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869207622868606, + "loss": 1.3221, + "theoretical_loss": 3.504795577618653, + "tokens_seen": 1709244416 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048690070210631896, + "loss": 1.2703, + "theoretical_loss": 3.504784118197769, + "tokens_seen": 1709309952 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688064192577737, + "loss": 1.3755, + "theoretical_loss": 3.5047726593392534, + "tokens_seen": 1709375488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686058174523573, + "loss": 1.3504, + "theoretical_loss": 3.504761201043057, + "tokens_seen": 1709441024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868405215646941, + "loss": 1.3398, + "theoretical_loss": 3.5047497433091315, + "tokens_seen": 1709506560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682046138415246, + "loss": 1.2513, + "theoretical_loss": 3.504738286137427, + "tokens_seen": 1709572096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868004012036109, + "loss": 1.3342, + "theoretical_loss": 3.5047268295278946, + "tokens_seen": 1709637632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678034102306924, + "loss": 1.444, + "theoretical_loss": 3.504715373480485, + "tokens_seen": 1709703168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867602808425276, + "loss": 1.3006, + "theoretical_loss": 3.5047039179951494, + "tokens_seen": 1709768704 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048674022066198596, + "loss": 1.4039, + "theoretical_loss": 3.504692463071838, + "tokens_seen": 1709834240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867201604814443, + "loss": 1.3323, + "theoretical_loss": 3.5046810087105027, + "tokens_seen": 1709899776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048670010030090274, + "loss": 1.3463, + "theoretical_loss": 3.5046695549110938, + "tokens_seen": 1709965312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866800401203611, + "loss": 1.3981, + "theoretical_loss": 3.504658101673562, + "tokens_seen": 1710030848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048665997993981947, + "loss": 1.4365, + "theoretical_loss": 3.5046466489978583, + "tokens_seen": 1710096384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048663991975927783, + "loss": 1.4214, + "theoretical_loss": 3.504635196883934, + "tokens_seen": 1710161920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048661985957873624, + "loss": 1.3643, + "theoretical_loss": 3.50462374533174, + "tokens_seen": 1710227456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865997993981946, + "loss": 1.3654, + "theoretical_loss": 3.504612294341227, + "tokens_seen": 1710292992 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048657973921765297, + "loss": 1.3913, + "theoretical_loss": 3.5046008439123453, + "tokens_seen": 1710358528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048655967903711133, + "loss": 1.408, + "theoretical_loss": 3.5045893940450465, + "tokens_seen": 1710424064 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048653961885656975, + "loss": 1.383, + "theoretical_loss": 3.504577944739282, + "tokens_seen": 1710489600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865195586760281, + "loss": 1.3865, + "theoretical_loss": 3.504566495995002, + "tokens_seen": 1710555136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048649949849548647, + "loss": 1.3072, + "theoretical_loss": 3.504555047812158, + "tokens_seen": 1710620672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048647943831494483, + "loss": 1.4701, + "theoretical_loss": 3.5045436001907007, + "tokens_seen": 1710686208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864593781344032, + "loss": 1.4357, + "theoretical_loss": 3.5045321531305804, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864393179538616, + "loss": 1.3332, + "theoretical_loss": 3.5045207066317494, + "tokens_seen": 1710817280 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048641925777332, + "loss": 1.3726, + "theoretical_loss": 3.5045092606941575, + "tokens_seen": 1710882816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048639919759277834, + "loss": 1.2636, + "theoretical_loss": 3.5044978153177566, + "tokens_seen": 1710948352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863791374122367, + "loss": 1.2637, + "theoretical_loss": 3.504486370502497, + "tokens_seen": 1711013888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863590772316951, + "loss": 1.5042, + "theoretical_loss": 3.5044749262483297, + "tokens_seen": 1711079424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863390170511535, + "loss": 1.3395, + "theoretical_loss": 3.504463482555207, + "tokens_seen": 1711144960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048631895687061184, + "loss": 1.2796, + "theoretical_loss": 3.504452039423078, + "tokens_seen": 1711210496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862988966900702, + "loss": 1.185, + "theoretical_loss": 3.5044405968518944, + "tokens_seen": 1711276032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048627883650952857, + "loss": 1.3957, + "theoretical_loss": 3.5044291548416076, + "tokens_seen": 1711341568 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486258776328987, + "loss": 1.3358, + "theoretical_loss": 3.5044177133921686, + "tokens_seen": 1711407104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048623871614844534, + "loss": 1.3734, + "theoretical_loss": 3.504406272503528, + "tokens_seen": 1711472640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862186559679037, + "loss": 1.3995, + "theoretical_loss": 3.5043948321756373, + "tokens_seen": 1711538176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048619859578736207, + "loss": 1.2497, + "theoretical_loss": 3.5043833924084473, + "tokens_seen": 1711603712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861785356068205, + "loss": 1.4278, + "theoretical_loss": 3.5043719532019093, + "tokens_seen": 1711669248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048615847542627885, + "loss": 1.3903, + "theoretical_loss": 3.504360514555974, + "tokens_seen": 1711734784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861384152457372, + "loss": 1.3357, + "theoretical_loss": 3.504349076470593, + "tokens_seen": 1711800320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048611835506519557, + "loss": 1.4167, + "theoretical_loss": 3.5043376389457164, + "tokens_seen": 1711865856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048609829488465393, + "loss": 1.4046, + "theoretical_loss": 3.504326201981296, + "tokens_seen": 1711931392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048607823470411235, + "loss": 1.4093, + "theoretical_loss": 3.504314765577283, + "tokens_seen": 1711996928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860581745235707, + "loss": 1.405, + "theoretical_loss": 3.504303329733628, + "tokens_seen": 1712062464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860381143430291, + "loss": 1.3021, + "theoretical_loss": 3.5042918944502826, + "tokens_seen": 1712128000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048601805416248744, + "loss": 1.3389, + "theoretical_loss": 3.5042804597271973, + "tokens_seen": 1712193536 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 625444, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4292044639587402, + "objective/train/theoretical_loss": 3.504269025564324, + "objective/train/tokens_used": 82718176, + "theoretical_loss": 3.504269025564324, + "tokens_seen": 1712259072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048599799398194585, + "loss": 1.3556, + "theoretical_loss": 3.504269025564324, + "tokens_seen": 1712259072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859779338014042, + "loss": 1.3043, + "theoretical_loss": 3.504257591961613, + "tokens_seen": 1712324608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859578736208626, + "loss": 1.3576, + "theoretical_loss": 3.504246158919016, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048593781344032094, + "loss": 1.2208, + "theoretical_loss": 3.5042347264364837, + "tokens_seen": 1712455680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859177532597793, + "loss": 1.4373, + "theoretical_loss": 3.5042232945139675, + "tokens_seen": 1712521216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858976930792377, + "loss": 1.217, + "theoretical_loss": 3.5042118631514185, + "tokens_seen": 1712586752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858776328986961, + "loss": 1.3577, + "theoretical_loss": 3.504200432348788, + "tokens_seen": 1712652288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048585757271815444, + "loss": 1.3441, + "theoretical_loss": 3.5041890021060267, + "tokens_seen": 1712717824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858375125376128, + "loss": 1.3392, + "theoretical_loss": 3.504177572423086, + "tokens_seen": 1712783360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858174523570712, + "loss": 1.2145, + "theoretical_loss": 3.5041661432999174, + "tokens_seen": 1712848896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857973921765296, + "loss": 1.4326, + "theoretical_loss": 3.504154714736472, + "tokens_seen": 1712914432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577733199598795, + "loss": 1.2853, + "theoretical_loss": 3.5041432867327007, + "tokens_seen": 1712979968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857572718154463, + "loss": 1.3071, + "theoretical_loss": 3.504131859288554, + "tokens_seen": 1713045504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573721163490467, + "loss": 1.3519, + "theoretical_loss": 3.504120432403984, + "tokens_seen": 1713111040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048571715145436314, + "loss": 1.3422, + "theoretical_loss": 3.5041090060789424, + "tokens_seen": 1713176576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856970912738215, + "loss": 1.3945, + "theoretical_loss": 3.504097580313379, + "tokens_seen": 1713242112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048567703109327987, + "loss": 1.4079, + "theoretical_loss": 3.504086155107246, + "tokens_seen": 1713307648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048565697091273823, + "loss": 1.3968, + "theoretical_loss": 3.504074730460494, + "tokens_seen": 1713373184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563691073219664, + "loss": 1.3522, + "theoretical_loss": 3.504063306373075, + "tokens_seen": 1713438720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485616850551655, + "loss": 1.3877, + "theoretical_loss": 3.50405188284494, + "tokens_seen": 1713504256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559679037111337, + "loss": 1.222, + "theoretical_loss": 3.5040404598760393, + "tokens_seen": 1713569792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048557673019057173, + "loss": 1.3387, + "theoretical_loss": 3.504029037466325, + "tokens_seen": 1713635328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048555667001003015, + "loss": 1.3555, + "theoretical_loss": 3.504017615615748, + "tokens_seen": 1713700864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855366098294885, + "loss": 1.3985, + "theoretical_loss": 3.50400619432426, + "tokens_seen": 1713766400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048551654964894687, + "loss": 1.357, + "theoretical_loss": 3.5039947735918124, + "tokens_seen": 1713831936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048549648946840523, + "loss": 1.3677, + "theoretical_loss": 3.5039833534183553, + "tokens_seen": 1713897472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854764292878636, + "loss": 1.4683, + "theoretical_loss": 3.5039719338038413, + "tokens_seen": 1713963008 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485456369107322, + "loss": 1.3917, + "theoretical_loss": 3.5039605147482202, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854363089267804, + "loss": 1.4392, + "theoretical_loss": 3.503949096251445, + "tokens_seen": 1714094080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048541624874623874, + "loss": 1.2508, + "theoretical_loss": 3.5039376783134655, + "tokens_seen": 1714159616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853961885656971, + "loss": 1.3869, + "theoretical_loss": 3.503926260934234, + "tokens_seen": 1714225152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853761283851555, + "loss": 1.3878, + "theoretical_loss": 3.5039148441137016, + "tokens_seen": 1714290688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853560682046139, + "loss": 1.3385, + "theoretical_loss": 3.503903427851819, + "tokens_seen": 1714356224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533600802407224, + "loss": 1.3191, + "theoretical_loss": 3.5038920121485386, + "tokens_seen": 1714421760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853159478435306, + "loss": 1.3496, + "theoretical_loss": 3.5038805970038105, + "tokens_seen": 1714487296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048529588766298897, + "loss": 1.3658, + "theoretical_loss": 3.5038691824175867, + "tokens_seen": 1714552832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852758274824474, + "loss": 1.3793, + "theoretical_loss": 3.5038577683898184, + "tokens_seen": 1714618368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525576730190574, + "loss": 1.4162, + "theoretical_loss": 3.503846354920457, + "tokens_seen": 1714683904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852357071213641, + "loss": 1.3681, + "theoretical_loss": 3.503834942009454, + "tokens_seen": 1714749440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521564694082247, + "loss": 1.5114, + "theoretical_loss": 3.50382352965676, + "tokens_seen": 1714814976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851955867602809, + "loss": 1.3666, + "theoretical_loss": 3.5038121178623274, + "tokens_seen": 1714880512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048517552657973925, + "loss": 1.2265, + "theoretical_loss": 3.503800706626107, + "tokens_seen": 1714946048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851554663991976, + "loss": 1.356, + "theoretical_loss": 3.5037892959480494, + "tokens_seen": 1715011584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513540621865597, + "loss": 1.3333, + "theoretical_loss": 3.5037778858281077, + "tokens_seen": 1715077120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048511534603811433, + "loss": 1.3985, + "theoretical_loss": 3.503766476266232, + "tokens_seen": 1715142656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048509528585757275, + "loss": 1.4035, + "theoretical_loss": 3.503755067262374, + "tokens_seen": 1715208192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850752256770311, + "loss": 1.3749, + "theoretical_loss": 3.503743658816485, + "tokens_seen": 1715273728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850551654964895, + "loss": 1.4813, + "theoretical_loss": 3.503732250928517, + "tokens_seen": 1715339264 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503510531594784, + "loss": 1.3899, + "theoretical_loss": 3.5037208435984204, + "tokens_seen": 1715404800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048501504513540625, + "loss": 1.3947, + "theoretical_loss": 3.5037094368261474, + "tokens_seen": 1715470336 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 626732, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3518413305282593, + "objective/train/theoretical_loss": 3.503698030611649, + "objective/train/tokens_used": 85994976, + "theoretical_loss": 3.503698030611649, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849949849548646, + "loss": 1.2875, + "theoretical_loss": 3.503698030611649, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484974924774323, + "loss": 1.3052, + "theoretical_loss": 3.503686624954877, + "tokens_seen": 1715601408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048495486459378134, + "loss": 1.3779, + "theoretical_loss": 3.503675219855782, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849348044132397, + "loss": 1.3558, + "theoretical_loss": 3.503663815314317, + "tokens_seen": 1715732480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849147442326981, + "loss": 1.4397, + "theoretical_loss": 3.5036524113304317, + "tokens_seen": 1715798016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848946840521565, + "loss": 1.3833, + "theoretical_loss": 3.5036410079040783, + "tokens_seen": 1715863552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048487462387161484, + "loss": 1.4652, + "theoretical_loss": 3.5036296050352087, + "tokens_seen": 1715929088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848545636910732, + "loss": 1.4268, + "theoretical_loss": 3.5036182027237732, + "tokens_seen": 1715994624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848345035105316, + "loss": 1.3413, + "theoretical_loss": 3.5036068009697248, + "tokens_seen": 1716060160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048481444332999, + "loss": 1.518, + "theoretical_loss": 3.5035953997730136, + "tokens_seen": 1716125696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048479438314944835, + "loss": 1.4168, + "theoretical_loss": 3.5035839991335918, + "tokens_seen": 1716191232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847743229689067, + "loss": 1.4196, + "theoretical_loss": 3.5035725990514104, + "tokens_seen": 1716256768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048475426278836507, + "loss": 1.2809, + "theoretical_loss": 3.5035611995264215, + "tokens_seen": 1716322304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847342026078235, + "loss": 1.3607, + "theoretical_loss": 3.503549800558576, + "tokens_seen": 1716387840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048471414242728185, + "loss": 1.3481, + "theoretical_loss": 3.503538402147826, + "tokens_seen": 1716453376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846940822467402, + "loss": 1.3558, + "theoretical_loss": 3.5035270042941224, + "tokens_seen": 1716518912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846740220661986, + "loss": 1.3838, + "theoretical_loss": 3.503515606997417, + "tokens_seen": 1716584448 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484653961885657, + "loss": 1.4044, + "theoretical_loss": 3.5035042102576615, + "tokens_seen": 1716649984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048463390170511535, + "loss": 1.4107, + "theoretical_loss": 3.5034928140748067, + "tokens_seen": 1716715520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846138415245737, + "loss": 1.3627, + "theoretical_loss": 3.5034814184488052, + "tokens_seen": 1716781056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845937813440321, + "loss": 1.3421, + "theoretical_loss": 3.5034700233796077, + "tokens_seen": 1716846592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048457372116349044, + "loss": 1.4453, + "theoretical_loss": 3.5034586288671656, + "tokens_seen": 1716912128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048455366098294886, + "loss": 1.4054, + "theoretical_loss": 3.5034472349114316, + "tokens_seen": 1716977664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845336008024072, + "loss": 1.34, + "theoretical_loss": 3.5034358415123563, + "tokens_seen": 1717043200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845135406218656, + "loss": 1.3819, + "theoretical_loss": 3.5034244486698913, + "tokens_seen": 1717108736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048449348044132394, + "loss": 1.405, + "theoretical_loss": 3.5034130563839887, + "tokens_seen": 1717174272 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048447342026078236, + "loss": 1.3497, + "theoretical_loss": 3.5034016646545996, + "tokens_seen": 1717239808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844533600802407, + "loss": 1.3935, + "theoretical_loss": 3.503390273481676, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844332998996991, + "loss": 1.4651, + "theoretical_loss": 3.503378882865168, + "tokens_seen": 1717370880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048441323971915745, + "loss": 1.2457, + "theoretical_loss": 3.5033674928050296, + "tokens_seen": 1717436416 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048439317953861586, + "loss": 1.3766, + "theoretical_loss": 3.5033561033012104, + "tokens_seen": 1717501952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843731193580742, + "loss": 1.3019, + "theoretical_loss": 3.5033447143536636, + "tokens_seen": 1717567488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843530591775326, + "loss": 1.4416, + "theoretical_loss": 3.5033333259623394, + "tokens_seen": 1717633024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048433299899699095, + "loss": 1.3507, + "theoretical_loss": 3.50332193812719, + "tokens_seen": 1717698560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843129388164493, + "loss": 1.3759, + "theoretical_loss": 3.503310550848167, + "tokens_seen": 1717764096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048429287863590773, + "loss": 1.4532, + "theoretical_loss": 3.503299164125222, + "tokens_seen": 1717829632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842728184553661, + "loss": 1.401, + "theoretical_loss": 3.503287777958307, + "tokens_seen": 1717895168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048425275827482445, + "loss": 1.3517, + "theoretical_loss": 3.503276392347373, + "tokens_seen": 1717960704 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048423269809428287, + "loss": 1.3692, + "theoretical_loss": 3.503265007292372, + "tokens_seen": 1718026240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842126379137413, + "loss": 1.2823, + "theoretical_loss": 3.5032536227932556, + "tokens_seen": 1718091776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048419257773319965, + "loss": 1.3413, + "theoretical_loss": 3.5032422388499755, + "tokens_seen": 1718157312 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484172517552658, + "loss": 1.3555, + "theoretical_loss": 3.503230855462483, + "tokens_seen": 1718222848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048415245737211637, + "loss": 1.4239, + "theoretical_loss": 3.5032194726307306, + "tokens_seen": 1718288384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048413239719157473, + "loss": 1.2772, + "theoretical_loss": 3.503208090354669, + "tokens_seen": 1718353920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048411233701103315, + "loss": 1.3275, + "theoretical_loss": 3.5031967086342504, + "tokens_seen": 1718419456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840922768304915, + "loss": 1.397, + "theoretical_loss": 3.5031853274694265, + "tokens_seen": 1718484992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840722166499499, + "loss": 1.3658, + "theoretical_loss": 3.503173946860149, + "tokens_seen": 1718550528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048405215646940824, + "loss": 1.2916, + "theoretical_loss": 3.5031625668063695, + "tokens_seen": 1718616064 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048403209628886665, + "loss": 1.3655, + "theoretical_loss": 3.5031511873080396, + "tokens_seen": 1718681600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484012036108325, + "loss": 1.3443, + "theoretical_loss": 3.5031398083651113, + "tokens_seen": 1718747136 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 627826, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4458614587783813, + "objective/train/theoretical_loss": 3.503128429977536, + "objective/train/tokens_used": 89271776, + "theoretical_loss": 3.503128429977536, + "tokens_seen": 1718812672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839919759277834, + "loss": 1.4504, + "theoretical_loss": 3.503128429977536, + "tokens_seen": 1718812672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397191574724174, + "loss": 1.4615, + "theoretical_loss": 3.5031170521452655, + "tokens_seen": 1718878208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839518555667001, + "loss": 1.3933, + "theoretical_loss": 3.5031056748682516, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839317953861585, + "loss": 1.3026, + "theoretical_loss": 3.503094298146446, + "tokens_seen": 1719009280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839117352056169, + "loss": 1.3633, + "theoretical_loss": 3.503082921979801, + "tokens_seen": 1719074816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048389167502507524, + "loss": 1.2215, + "theoretical_loss": 3.5030715463682673, + "tokens_seen": 1719140352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838716148445336, + "loss": 1.3471, + "theoretical_loss": 3.5030601713117973, + "tokens_seen": 1719205888 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483851554663992, + "loss": 1.3934, + "theoretical_loss": 3.5030487968103428, + "tokens_seen": 1719271424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838314944834504, + "loss": 1.3297, + "theoretical_loss": 3.503037422863855, + "tokens_seen": 1719336960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381143430290875, + "loss": 1.4072, + "theoretical_loss": 3.5030260494722865, + "tokens_seen": 1719402496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837913741223671, + "loss": 1.4298, + "theoretical_loss": 3.5030146766355887, + "tokens_seen": 1719468032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377131394182547, + "loss": 1.437, + "theoretical_loss": 3.5030033043537125, + "tokens_seen": 1719533568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837512537612839, + "loss": 1.4109, + "theoretical_loss": 3.5029919326266112, + "tokens_seen": 1719599104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373119358074225, + "loss": 1.333, + "theoretical_loss": 3.502980561454236, + "tokens_seen": 1719664640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837111334002006, + "loss": 1.3641, + "theoretical_loss": 3.502969190836538, + "tokens_seen": 1719730176 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483691073219659, + "loss": 1.4565, + "theoretical_loss": 3.50295782077347, + "tokens_seen": 1719795712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836710130391174, + "loss": 1.428, + "theoretical_loss": 3.5029464512649833, + "tokens_seen": 1719861248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048365095285857575, + "loss": 1.3461, + "theoretical_loss": 3.50293508231103, + "tokens_seen": 1719926784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836308926780341, + "loss": 1.3766, + "theoretical_loss": 3.502923713911562, + "tokens_seen": 1719992320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836108324974925, + "loss": 1.3903, + "theoretical_loss": 3.5029123460665303, + "tokens_seen": 1720057856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359077231695084, + "loss": 1.4031, + "theoretical_loss": 3.502900978775888, + "tokens_seen": 1720123392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357071213640926, + "loss": 1.3642, + "theoretical_loss": 3.502889612039586, + "tokens_seen": 1720188928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835506519558676, + "loss": 1.3222, + "theoretical_loss": 3.502878245857576, + "tokens_seen": 1720254464 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483530591775326, + "loss": 1.3895, + "theoretical_loss": 3.502866880229811, + "tokens_seen": 1720320000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048351053159478434, + "loss": 1.3602, + "theoretical_loss": 3.5028555151562415, + "tokens_seen": 1720385536 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349047141424276, + "loss": 1.3234, + "theoretical_loss": 3.5028441506368204, + "tokens_seen": 1720451072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834704112337011, + "loss": 1.3973, + "theoretical_loss": 3.502832786671499, + "tokens_seen": 1720516608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834503510531595, + "loss": 1.3449, + "theoretical_loss": 3.5028214232602295, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343029087261785, + "loss": 1.3186, + "theoretical_loss": 3.5028100604029637, + "tokens_seen": 1720647680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048341023069207626, + "loss": 1.4168, + "theoretical_loss": 3.5027986980996535, + "tokens_seen": 1720713216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833901705115346, + "loss": 1.4152, + "theoretical_loss": 3.5027873363502504, + "tokens_seen": 1720778752 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483370110330993, + "loss": 1.4194, + "theoretical_loss": 3.502775975154707, + "tokens_seen": 1720844288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048335005015045135, + "loss": 1.3233, + "theoretical_loss": 3.5027646145129747, + "tokens_seen": 1720909824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833299899699097, + "loss": 1.4071, + "theoretical_loss": 3.5027532544250057, + "tokens_seen": 1720975360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048330992978936813, + "loss": 1.5229, + "theoretical_loss": 3.502741894890751, + "tokens_seen": 1721040896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832898696088265, + "loss": 1.463, + "theoretical_loss": 3.5027305359101644, + "tokens_seen": 1721106432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048326980942828485, + "loss": 1.4471, + "theoretical_loss": 3.502719177483196, + "tokens_seen": 1721171968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832497492477432, + "loss": 1.4777, + "theoretical_loss": 3.502707819609799, + "tokens_seen": 1721237504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048322968906720163, + "loss": 1.5016, + "theoretical_loss": 3.502696462289925, + "tokens_seen": 1721303040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048320962888666, + "loss": 1.3798, + "theoretical_loss": 3.502685105523525, + "tokens_seen": 1721368576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048318956870611836, + "loss": 1.3582, + "theoretical_loss": 3.5026737493105524, + "tokens_seen": 1721434112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831695085255767, + "loss": 1.3654, + "theoretical_loss": 3.502662393650958, + "tokens_seen": 1721499648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831494483450351, + "loss": 1.231, + "theoretical_loss": 3.5026510385446947, + "tokens_seen": 1721565184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831293881644935, + "loss": 1.3885, + "theoretical_loss": 3.502639683991714, + "tokens_seen": 1721630720 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048310932798395186, + "loss": 1.4827, + "theoretical_loss": 3.5026283299919676, + "tokens_seen": 1721696256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830892678034102, + "loss": 1.3953, + "theoretical_loss": 3.5026169765454083, + "tokens_seen": 1721761792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830692076228686, + "loss": 1.2148, + "theoretical_loss": 3.5026056236519874, + "tokens_seen": 1721827328 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483049147442327, + "loss": 1.404, + "theoretical_loss": 3.502594271311657, + "tokens_seen": 1721892864 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048302908726178536, + "loss": 1.3541, + "theoretical_loss": 3.502582919524369, + "tokens_seen": 1721958400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830090270812437, + "loss": 1.3703, + "theoretical_loss": 3.502571568290076, + "tokens_seen": 1722023936 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 629016, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3245419263839722, + "objective/train/theoretical_loss": 3.50256021760873, + "objective/train/tokens_used": 92548576, + "theoretical_loss": 3.50256021760873, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829889669007021, + "loss": 1.3607, + "theoretical_loss": 3.50256021760873, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048296890672016045, + "loss": 1.2648, + "theoretical_loss": 3.502548867480282, + "tokens_seen": 1722155008 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294884653961887, + "loss": 1.3391, + "theoretical_loss": 3.5025375179046847, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292878635907723, + "loss": 1.3666, + "theoretical_loss": 3.5025261688818903, + "tokens_seen": 1722286080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829087261785356, + "loss": 1.4693, + "theoretical_loss": 3.502514820411851, + "tokens_seen": 1722351616 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048288866599799395, + "loss": 1.358, + "theoretical_loss": 3.502503472494518, + "tokens_seen": 1722417152 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048286860581745237, + "loss": 1.3005, + "theoretical_loss": 3.5024921251298444, + "tokens_seen": 1722482688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048284854563691073, + "loss": 1.5898, + "theoretical_loss": 3.502480778317781, + "tokens_seen": 1722548224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828284854563691, + "loss": 1.4709, + "theoretical_loss": 3.502469432058281, + "tokens_seen": 1722613760 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048280842527582746, + "loss": 1.3743, + "theoretical_loss": 3.5024580863512966, + "tokens_seen": 1722679296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827883650952858, + "loss": 1.3715, + "theoretical_loss": 3.5024467411967786, + "tokens_seen": 1722744832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048276830491474423, + "loss": 1.5312, + "theoretical_loss": 3.50243539659468, + "tokens_seen": 1722810368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827482447342026, + "loss": 1.4025, + "theoretical_loss": 3.502424052544953, + "tokens_seen": 1722875904 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482728184553661, + "loss": 1.3609, + "theoretical_loss": 3.502412709047549, + "tokens_seen": 1722941440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827081243731194, + "loss": 1.3602, + "theoretical_loss": 3.502401366102421, + "tokens_seen": 1723006976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826880641925778, + "loss": 1.3103, + "theoretical_loss": 3.5023900237095202, + "tokens_seen": 1723072512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266800401203615, + "loss": 1.4195, + "theoretical_loss": 3.502378681868799, + "tokens_seen": 1723138048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826479438314945, + "loss": 1.4357, + "theoretical_loss": 3.50236734058021, + "tokens_seen": 1723203584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826278836509529, + "loss": 1.3346, + "theoretical_loss": 3.5023559998437053, + "tokens_seen": 1723269120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048260782347041124, + "loss": 1.4596, + "theoretical_loss": 3.5023446596592365, + "tokens_seen": 1723334656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258776328986966, + "loss": 1.2937, + "theoretical_loss": 3.5023333200267555, + "tokens_seen": 1723400192 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482567703109328, + "loss": 1.2587, + "theoretical_loss": 3.502321980946215, + "tokens_seen": 1723465728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825476429287864, + "loss": 1.4569, + "theoretical_loss": 3.502310642417567, + "tokens_seen": 1723531264 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252758274824474, + "loss": 1.4138, + "theoretical_loss": 3.502299304440764, + "tokens_seen": 1723596800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048250752256770316, + "loss": 1.354, + "theoretical_loss": 3.5022879670157576, + "tokens_seen": 1723662336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824874623871615, + "loss": 1.4462, + "theoretical_loss": 3.5022766301425, + "tokens_seen": 1723727872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824674022066199, + "loss": 1.3097, + "theoretical_loss": 3.502265293820944, + "tokens_seen": 1723793408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244734202607825, + "loss": 1.3613, + "theoretical_loss": 3.5022539580510417, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048242728184553666, + "loss": 1.4121, + "theoretical_loss": 3.5022426228327443, + "tokens_seen": 1723924480 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482407221664995, + "loss": 1.2609, + "theoretical_loss": 3.502231288166005, + "tokens_seen": 1723990016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823871614844534, + "loss": 1.3218, + "theoretical_loss": 3.5022199540507755, + "tokens_seen": 1724055552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048236710130391175, + "loss": 1.3748, + "theoretical_loss": 3.502208620487008, + "tokens_seen": 1724121088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823470411233701, + "loss": 1.2851, + "theoretical_loss": 3.5021972874746554, + "tokens_seen": 1724186624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048232698094282853, + "loss": 1.4481, + "theoretical_loss": 3.502185955013669, + "tokens_seen": 1724252160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823069207622869, + "loss": 1.4375, + "theoretical_loss": 3.5021746231040014, + "tokens_seen": 1724317696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048228686058174525, + "loss": 1.3691, + "theoretical_loss": 3.5021632917456045, + "tokens_seen": 1724383232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822668004012036, + "loss": 1.4047, + "theoretical_loss": 3.502151960938431, + "tokens_seen": 1724448768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224674022066203, + "loss": 1.3064, + "theoretical_loss": 3.502140630682433, + "tokens_seen": 1724514304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822266800401204, + "loss": 1.3829, + "theoretical_loss": 3.502129300977563, + "tokens_seen": 1724579840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048220661985957876, + "loss": 1.4479, + "theoretical_loss": 3.5021179718237727, + "tokens_seen": 1724645376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821865596790371, + "loss": 1.3896, + "theoretical_loss": 3.5021066432210146, + "tokens_seen": 1724710912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821664994984955, + "loss": 1.4652, + "theoretical_loss": 3.5020953151692416, + "tokens_seen": 1724776448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821464393179539, + "loss": 1.3009, + "theoretical_loss": 3.5020839876684047, + "tokens_seen": 1724841984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048212637913741226, + "loss": 1.4205, + "theoretical_loss": 3.502072660718457, + "tokens_seen": 1724907520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821063189568706, + "loss": 1.4712, + "theoretical_loss": 3.5020613343193507, + "tokens_seen": 1724973056 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482086258776329, + "loss": 1.4408, + "theoretical_loss": 3.5020500084710378, + "tokens_seen": 1725038592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820661985957874, + "loss": 1.3621, + "theoretical_loss": 3.502038683173471, + "tokens_seen": 1725104128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204613841524576, + "loss": 1.4467, + "theoretical_loss": 3.502027358426602, + "tokens_seen": 1725169664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820260782347041, + "loss": 1.35, + "theoretical_loss": 3.502016034230384, + "tokens_seen": 1725235200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820060180541625, + "loss": 1.4556, + "theoretical_loss": 3.5020047105847687, + "tokens_seen": 1725300736 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 630228, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3771824836730957, + "objective/train/theoretical_loss": 3.5019933874897085, + "objective/train/tokens_used": 95825376, + "theoretical_loss": 3.5019933874897085, + "tokens_seen": 1725366272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048198595787362085, + "loss": 1.3527, + "theoretical_loss": 3.5019933874897085, + "tokens_seen": 1725366272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048196589769307927, + "loss": 1.4271, + "theoretical_loss": 3.5019820649451554, + "tokens_seen": 1725431808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048194583751253763, + "loss": 1.3421, + "theoretical_loss": 3.5019707429510625, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481925777331996, + "loss": 1.378, + "theoretical_loss": 3.5019594215073813, + "tokens_seen": 1725562880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048190571715145435, + "loss": 1.3448, + "theoretical_loss": 3.501948100614065, + "tokens_seen": 1725628416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048188565697091277, + "loss": 1.3443, + "theoretical_loss": 3.5019367802710653, + "tokens_seen": 1725693952 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048186559679037113, + "loss": 1.3398, + "theoretical_loss": 3.5019254604783345, + "tokens_seen": 1725759488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818455366098295, + "loss": 1.4324, + "theoretical_loss": 3.5019141412358255, + "tokens_seen": 1725825024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048182547642928786, + "loss": 1.4844, + "theoretical_loss": 3.5019028225434905, + "tokens_seen": 1725890560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818054162487462, + "loss": 1.4636, + "theoretical_loss": 3.5018915044012813, + "tokens_seen": 1725956096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048178535606820463, + "loss": 1.4421, + "theoretical_loss": 3.5018801868091507, + "tokens_seen": 1726021632 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481765295887663, + "loss": 1.4172, + "theoretical_loss": 3.5018688697670513, + "tokens_seen": 1726087168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048174523570712136, + "loss": 1.4705, + "theoretical_loss": 3.501857553274935, + "tokens_seen": 1726152704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817251755265797, + "loss": 1.3969, + "theoretical_loss": 3.501846237332755, + "tokens_seen": 1726218240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048170511534603814, + "loss": 1.3441, + "theoretical_loss": 3.5018349219404628, + "tokens_seen": 1726283776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816850551654965, + "loss": 1.4434, + "theoretical_loss": 3.501823607098011, + "tokens_seen": 1726349312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048166499498495486, + "loss": 1.342, + "theoretical_loss": 3.5018122928053526, + "tokens_seen": 1726414848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816449348044132, + "loss": 1.2928, + "theoretical_loss": 3.5018009790624394, + "tokens_seen": 1726480384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816248746238716, + "loss": 1.3533, + "theoretical_loss": 3.5017896658692234, + "tokens_seen": 1726545920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048160481444333, + "loss": 1.3742, + "theoretical_loss": 3.5017783532256583, + "tokens_seen": 1726611456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048158475426278837, + "loss": 1.5971, + "theoretical_loss": 3.501767041131696, + "tokens_seen": 1726676992 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048156469408224673, + "loss": 1.38, + "theoretical_loss": 3.5017557295872885, + "tokens_seen": 1726742528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815446339017051, + "loss": 1.321, + "theoretical_loss": 3.5017444185923887, + "tokens_seen": 1726808064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815245737211635, + "loss": 1.2945, + "theoretical_loss": 3.501733108146949, + "tokens_seen": 1726873600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048150451354062187, + "loss": 1.3117, + "theoretical_loss": 3.5017217982509212, + "tokens_seen": 1726939136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048148445336008023, + "loss": 1.4456, + "theoretical_loss": 3.5017104889042585, + "tokens_seen": 1727004672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814643931795386, + "loss": 1.3466, + "theoretical_loss": 3.5016991801069137, + "tokens_seen": 1727070208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048144433299899696, + "loss": 1.3321, + "theoretical_loss": 3.501687871858839, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048142427281845537, + "loss": 1.2479, + "theoretical_loss": 3.501676564159986, + "tokens_seen": 1727201280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048140421263791373, + "loss": 1.3769, + "theoretical_loss": 3.501665257010308, + "tokens_seen": 1727266816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813841524573721, + "loss": 1.4701, + "theoretical_loss": 3.5016539504097572, + "tokens_seen": 1727332352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048136409227683046, + "loss": 1.4552, + "theoretical_loss": 3.501642644358286, + "tokens_seen": 1727397888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813440320962889, + "loss": 1.3864, + "theoretical_loss": 3.501631338855848, + "tokens_seen": 1727463424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048132397191574724, + "loss": 1.37, + "theoretical_loss": 3.5016200339023946, + "tokens_seen": 1727528960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813039117352056, + "loss": 1.336, + "theoretical_loss": 3.501608729497878, + "tokens_seen": 1727594496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048128385155466396, + "loss": 1.3311, + "theoretical_loss": 3.5015974256422515, + "tokens_seen": 1727660032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812637913741223, + "loss": 1.411, + "theoretical_loss": 3.501586122335468, + "tokens_seen": 1727725568 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048124373119358074, + "loss": 1.3059, + "theoretical_loss": 3.501574819577479, + "tokens_seen": 1727791104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048122367101303916, + "loss": 1.4278, + "theoretical_loss": 3.5015635173682376, + "tokens_seen": 1727856640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812036108324975, + "loss": 1.4128, + "theoretical_loss": 3.5015522157076964, + "tokens_seen": 1727922176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811835506519559, + "loss": 1.4261, + "theoretical_loss": 3.5015409145958074, + "tokens_seen": 1727987712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811634904714143, + "loss": 1.4068, + "theoretical_loss": 3.5015296140325236, + "tokens_seen": 1728053248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048114343029087266, + "loss": 1.3934, + "theoretical_loss": 3.5015183140177975, + "tokens_seen": 1728118784 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481123370110331, + "loss": 1.3494, + "theoretical_loss": 3.5015070145515823, + "tokens_seen": 1728184320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811033099297894, + "loss": 1.366, + "theoretical_loss": 3.5014957156338293, + "tokens_seen": 1728249856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810832497492478, + "loss": 1.3325, + "theoretical_loss": 3.501484417264492, + "tokens_seen": 1728315392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048106318956870616, + "loss": 1.3124, + "theoretical_loss": 3.5014731194435225, + "tokens_seen": 1728380928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810431293881645, + "loss": 1.3541, + "theoretical_loss": 3.501461822170874, + "tokens_seen": 1728446464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810230692076229, + "loss": 1.347, + "theoretical_loss": 3.5014505254464985, + "tokens_seen": 1728512000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048100300902708125, + "loss": 1.3788, + "theoretical_loss": 3.501439229270349, + "tokens_seen": 1728577536 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 631392, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.398906946182251, + "objective/train/theoretical_loss": 3.5014279336423777, + "objective/train/tokens_used": 99102176, + "theoretical_loss": 3.5014279336423777, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048098294884653967, + "loss": 1.3433, + "theoretical_loss": 3.5014279336423777, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048096288866599803, + "loss": 1.3671, + "theoretical_loss": 3.501416638562538, + "tokens_seen": 1728708608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809428284854564, + "loss": 1.4467, + "theoretical_loss": 3.501405344030781, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048092276830491475, + "loss": 1.3963, + "theoretical_loss": 3.501394050047061, + "tokens_seen": 1728839680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048090270812437317, + "loss": 1.3759, + "theoretical_loss": 3.5013827566113296, + "tokens_seen": 1728905216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048088264794383153, + "loss": 1.261, + "theoretical_loss": 3.5013714637235402, + "tokens_seen": 1728970752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808625877632899, + "loss": 1.3582, + "theoretical_loss": 3.501360171383645, + "tokens_seen": 1729036288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084252758274826, + "loss": 1.3878, + "theoretical_loss": 3.5013488795915966, + "tokens_seen": 1729101824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808224674022066, + "loss": 1.2432, + "theoretical_loss": 3.5013375883473477, + "tokens_seen": 1729167360 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048080240722166503, + "loss": 1.359, + "theoretical_loss": 3.501326297650851, + "tokens_seen": 1729232896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807823470411234, + "loss": 1.4311, + "theoretical_loss": 3.5013150075020594, + "tokens_seen": 1729298432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048076228686058176, + "loss": 1.403, + "theoretical_loss": 3.5013037179009245, + "tokens_seen": 1729363968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807422266800401, + "loss": 1.4158, + "theoretical_loss": 3.5012924288474006, + "tokens_seen": 1729429504 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072216649949854, + "loss": 1.3937, + "theoretical_loss": 3.5012811403414394, + "tokens_seen": 1729495040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807021063189569, + "loss": 1.2947, + "theoretical_loss": 3.501269852382994, + "tokens_seen": 1729560576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068204613841526, + "loss": 1.4609, + "theoretical_loss": 3.501258564972017, + "tokens_seen": 1729626112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806619859578736, + "loss": 1.413, + "theoretical_loss": 3.501247278108461, + "tokens_seen": 1729691648 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480641925777332, + "loss": 1.4556, + "theoretical_loss": 3.501235991792279, + "tokens_seen": 1729757184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806218655967904, + "loss": 1.3312, + "theoretical_loss": 3.5012247060234225, + "tokens_seen": 1729822720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048060180541624877, + "loss": 1.3286, + "theoretical_loss": 3.5012134208018457, + "tokens_seen": 1729888256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058174523570713, + "loss": 1.3236, + "theoretical_loss": 3.501202136127501, + "tokens_seen": 1729953792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805616850551655, + "loss": 1.3335, + "theoretical_loss": 3.5011908520003407, + "tokens_seen": 1730019328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805416248746239, + "loss": 1.3722, + "theoretical_loss": 3.501179568420318, + "tokens_seen": 1730084864 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052156469408227, + "loss": 1.5094, + "theoretical_loss": 3.5011682853873856, + "tokens_seen": 1730150400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048050150451354063, + "loss": 1.4556, + "theoretical_loss": 3.501157002901495, + "tokens_seen": 1730215936 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480481444332999, + "loss": 1.3884, + "theoretical_loss": 3.501145720962601, + "tokens_seen": 1730281472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048046138415245736, + "loss": 1.278, + "theoretical_loss": 3.5011344395706554, + "tokens_seen": 1730347008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048044132397191577, + "loss": 1.352, + "theoretical_loss": 3.501123158725611, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042126379137413, + "loss": 1.3292, + "theoretical_loss": 3.5011118784274204, + "tokens_seen": 1730478080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804012036108325, + "loss": 1.37, + "theoretical_loss": 3.501100598676036, + "tokens_seen": 1730543616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038114343029086, + "loss": 1.3221, + "theoretical_loss": 3.501089319471412, + "tokens_seen": 1730609152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803610832497493, + "loss": 1.4695, + "theoretical_loss": 3.5010780408135, + "tokens_seen": 1730674688 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034102306920764, + "loss": 1.3348, + "theoretical_loss": 3.501066762702253, + "tokens_seen": 1730740224 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480320962888666, + "loss": 1.3532, + "theoretical_loss": 3.501055485137624, + "tokens_seen": 1730805760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048030090270812436, + "loss": 1.2408, + "theoretical_loss": 3.5010442081195654, + "tokens_seen": 1730871296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802808425275827, + "loss": 1.2848, + "theoretical_loss": 3.5010329316480306, + "tokens_seen": 1730936832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048026078234704114, + "loss": 1.3686, + "theoretical_loss": 3.501021655722972, + "tokens_seen": 1731002368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802407221664995, + "loss": 1.3355, + "theoretical_loss": 3.501010380344343, + "tokens_seen": 1731067904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048022066198595786, + "loss": 1.4343, + "theoretical_loss": 3.5009991055120957, + "tokens_seen": 1731133440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020060180541623, + "loss": 1.4387, + "theoretical_loss": 3.5009878312261833, + "tokens_seen": 1731198976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048018054162487464, + "loss": 1.4586, + "theoretical_loss": 3.500976557486559, + "tokens_seen": 1731264512 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480160481444333, + "loss": 1.3992, + "theoretical_loss": 3.5009652842931747, + "tokens_seen": 1731330048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014042126379137, + "loss": 1.4603, + "theoretical_loss": 3.5009540116459843, + "tokens_seen": 1731395584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048012036108324973, + "loss": 1.3138, + "theoretical_loss": 3.5009427395449397, + "tokens_seen": 1731461120 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048010030090270815, + "loss": 1.2857, + "theoretical_loss": 3.5009314679899948, + "tokens_seen": 1731526656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800802407221665, + "loss": 1.2783, + "theoretical_loss": 3.5009201969811015, + "tokens_seen": 1731592192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048006018054162487, + "loss": 1.3111, + "theoretical_loss": 3.500908926518213, + "tokens_seen": 1731657728 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004012036108323, + "loss": 1.3926, + "theoretical_loss": 3.500897656601283, + "tokens_seen": 1731723264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800200601805416, + "loss": 1.3168, + "theoretical_loss": 3.5008863872302634, + "tokens_seen": 1731788800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 1.4196, + "theoretical_loss": 3.5008751184051072, + "tokens_seen": 1731854336 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 632543, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4212391376495361, + "objective/train/theoretical_loss": 3.500863850125768, + "objective/train/tokens_used": 102378976, + "theoretical_loss": 3.500863850125768, + "tokens_seen": 1731919872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799799398194584, + "loss": 1.3597, + "theoretical_loss": 3.500863850125768, + "tokens_seen": 1731919872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047995987963891674, + "loss": 1.4753, + "theoretical_loss": 3.5008525823921977, + "tokens_seen": 1731985408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799398194583751, + "loss": 1.3734, + "theoretical_loss": 3.5008413152043496, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799197592778335, + "loss": 1.3823, + "theoretical_loss": 3.500830048562177, + "tokens_seen": 1732116480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798996990972919, + "loss": 1.3213, + "theoretical_loss": 3.5008187824656325, + "tokens_seen": 1732182016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047987963891675024, + "loss": 1.356, + "theoretical_loss": 3.5008075169146693, + "tokens_seen": 1732247552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798595787362086, + "loss": 1.4404, + "theoretical_loss": 3.50079625190924, + "tokens_seen": 1732313088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983951855566696, + "loss": 1.4043, + "theoretical_loss": 3.500784987449298, + "tokens_seen": 1732378624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798194583751254, + "loss": 1.4762, + "theoretical_loss": 3.5007737235347953, + "tokens_seen": 1732444160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047979939819458374, + "loss": 1.4518, + "theoretical_loss": 3.500762460165686, + "tokens_seen": 1732509696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797793380140421, + "loss": 1.1211, + "theoretical_loss": 3.5007511973419225, + "tokens_seen": 1732575232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975927783350047, + "loss": 1.3428, + "theoretical_loss": 3.5007399350634576, + "tokens_seen": 1732640768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797392176529589, + "loss": 1.3472, + "theoretical_loss": 3.5007286733302445, + "tokens_seen": 1732706304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797191574724173, + "loss": 1.2619, + "theoretical_loss": 3.5007174121422366, + "tokens_seen": 1732771840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047969909729187566, + "loss": 1.5019, + "theoretical_loss": 3.5007061514993865, + "tokens_seen": 1732837376 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479679037111334, + "loss": 1.3659, + "theoretical_loss": 3.500694891401647, + "tokens_seen": 1732902912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796589769307924, + "loss": 1.4453, + "theoretical_loss": 3.500683631848971, + "tokens_seen": 1732968448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796389167502508, + "loss": 1.3936, + "theoretical_loss": 3.5006723728413123, + "tokens_seen": 1733033984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047961885656970917, + "loss": 1.4513, + "theoretical_loss": 3.500661114378623, + "tokens_seen": 1733099520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047959879638916753, + "loss": 1.4493, + "theoretical_loss": 3.500649856460856, + "tokens_seen": 1733165056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795787362086259, + "loss": 1.3189, + "theoretical_loss": 3.5006385990879663, + "tokens_seen": 1733230592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795586760280843, + "loss": 1.3582, + "theoretical_loss": 3.500627342259904, + "tokens_seen": 1733296128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953861584754267, + "loss": 1.4295, + "theoretical_loss": 3.5006160859766244, + "tokens_seen": 1733361664 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047951855566700103, + "loss": 1.3399, + "theoretical_loss": 3.5006048302380792, + "tokens_seen": 1733427200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794984954864594, + "loss": 1.4912, + "theoretical_loss": 3.5005935750442223, + "tokens_seen": 1733492736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047947843530591776, + "loss": 1.3689, + "theoretical_loss": 3.5005823203950066, + "tokens_seen": 1733558272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047945837512537617, + "loss": 1.4018, + "theoretical_loss": 3.5005710662903846, + "tokens_seen": 1733623808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943831494483453, + "loss": 1.4469, + "theoretical_loss": 3.50055981273031, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794182547642929, + "loss": 1.4087, + "theoretical_loss": 3.500548559714735, + "tokens_seen": 1733754880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939819458375126, + "loss": 1.3906, + "theoretical_loss": 3.500537307243614, + "tokens_seen": 1733820416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793781344032097, + "loss": 1.3194, + "theoretical_loss": 3.5005260553168993, + "tokens_seen": 1733885952 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047935807422266804, + "loss": 1.3877, + "theoretical_loss": 3.5005148039345437, + "tokens_seen": 1733951488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793380140421264, + "loss": 1.4208, + "theoretical_loss": 3.500503553096501, + "tokens_seen": 1734017024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047931795386158476, + "loss": 1.4336, + "theoretical_loss": 3.5004923028027237, + "tokens_seen": 1734082560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792978936810431, + "loss": 1.3924, + "theoretical_loss": 3.500481053053165, + "tokens_seen": 1734148096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047927783350050154, + "loss": 1.3703, + "theoretical_loss": 3.5004698038477784, + "tokens_seen": 1734213632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792577733199599, + "loss": 1.3303, + "theoretical_loss": 3.500458555186517, + "tokens_seen": 1734279168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923771313941827, + "loss": 1.3545, + "theoretical_loss": 3.500447307069333, + "tokens_seen": 1734344704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047921765295887663, + "loss": 1.3707, + "theoretical_loss": 3.500436059496181, + "tokens_seen": 1734410240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919759277833504, + "loss": 1.3634, + "theoretical_loss": 3.5004248124670125, + "tokens_seen": 1734475776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791775325977934, + "loss": 1.3367, + "theoretical_loss": 3.5004135659817823, + "tokens_seen": 1734541312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047915747241725177, + "loss": 1.3838, + "theoretical_loss": 3.500402320040442, + "tokens_seen": 1734606848 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913741223671013, + "loss": 1.4155, + "theoretical_loss": 3.5003910746429456, + "tokens_seen": 1734672384 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047911735205616855, + "loss": 1.5309, + "theoretical_loss": 3.5003798297892468, + "tokens_seen": 1734737920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790972918756269, + "loss": 1.5158, + "theoretical_loss": 3.500368585479297, + "tokens_seen": 1734803456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047907723169508527, + "loss": 1.4727, + "theoretical_loss": 3.500357341713051, + "tokens_seen": 1734868992 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905717151454363, + "loss": 1.3598, + "theoretical_loss": 3.5003460984904615, + "tokens_seen": 1734934528 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479037111334002, + "loss": 1.3277, + "theoretical_loss": 3.5003348558114817, + "tokens_seen": 1735000064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790170511534604, + "loss": 1.4234, + "theoretical_loss": 3.5003236136760645, + "tokens_seen": 1735065600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789969909729188, + "loss": 1.404, + "theoretical_loss": 3.5003123720841636, + "tokens_seen": 1735131136 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 633815, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3423707485198975, + "objective/train/theoretical_loss": 3.500301131035731, + "objective/train/tokens_used": 105655776, + "theoretical_loss": 3.500301131035731, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047897693079237714, + "loss": 1.4197, + "theoretical_loss": 3.500301131035731, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789568706118355, + "loss": 1.3932, + "theoretical_loss": 3.5002898905307216, + "tokens_seen": 1735262208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789368104312939, + "loss": 1.4013, + "theoretical_loss": 3.5002786505690873, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789167502507523, + "loss": 1.3346, + "theoretical_loss": 3.5002674111507823, + "tokens_seen": 1735393280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047889669007021064, + "loss": 1.3498, + "theoretical_loss": 3.5002561722757592, + "tokens_seen": 1735458816 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478876629889669, + "loss": 1.4119, + "theoretical_loss": 3.500244933943971, + "tokens_seen": 1735524352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047885656970912736, + "loss": 1.282, + "theoretical_loss": 3.5002336961553713, + "tokens_seen": 1735589888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788365095285858, + "loss": 1.4133, + "theoretical_loss": 3.5002224589099136, + "tokens_seen": 1735655424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047881644934804414, + "loss": 1.4711, + "theoretical_loss": 3.5002112222075508, + "tokens_seen": 1735720960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787963891675025, + "loss": 1.2595, + "theoretical_loss": 3.5001999860482362, + "tokens_seen": 1735786496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047877632898696087, + "loss": 1.3564, + "theoretical_loss": 3.500188750431923, + "tokens_seen": 1735852032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787562688064193, + "loss": 1.4286, + "theoretical_loss": 3.5001775153585646, + "tokens_seen": 1735917568 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047873620862587765, + "loss": 1.3318, + "theoretical_loss": 3.5001662808281138, + "tokens_seen": 1735983104 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478716148445336, + "loss": 1.3822, + "theoretical_loss": 3.5001550468405247, + "tokens_seen": 1736048640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047869608826479437, + "loss": 1.4199, + "theoretical_loss": 3.5001438133957494, + "tokens_seen": 1736114176 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047867602808425273, + "loss": 1.3039, + "theoretical_loss": 3.5001325804937427, + "tokens_seen": 1736179712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047865596790371115, + "loss": 1.381, + "theoretical_loss": 3.500121348134457, + "tokens_seen": 1736245248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786359077231695, + "loss": 1.4469, + "theoretical_loss": 3.5001101163178454, + "tokens_seen": 1736310784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786158475426279, + "loss": 1.508, + "theoretical_loss": 3.5000988850438617, + "tokens_seen": 1736376320 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047859578736208624, + "loss": 1.338, + "theoretical_loss": 3.500087654312459, + "tokens_seen": 1736441856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047857572718154465, + "loss": 1.444, + "theoretical_loss": 3.5000764241235904, + "tokens_seen": 1736507392 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478555667001003, + "loss": 1.3398, + "theoretical_loss": 3.500065194477209, + "tokens_seen": 1736572928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785356068204614, + "loss": 1.471, + "theoretical_loss": 3.5000539653732696, + "tokens_seen": 1736638464 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047851554663991974, + "loss": 1.4408, + "theoretical_loss": 3.5000427368117233, + "tokens_seen": 1736704000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784954864593781, + "loss": 1.4286, + "theoretical_loss": 3.5000315087925253, + "tokens_seen": 1736769536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784754262788365, + "loss": 1.3556, + "theoretical_loss": 3.5000202813156287, + "tokens_seen": 1736835072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784553660982949, + "loss": 1.2849, + "theoretical_loss": 3.5000090543809854, + "tokens_seen": 1736900608 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047843530591775324, + "loss": 1.3791, + "theoretical_loss": 3.4999978279885506, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784152457372116, + "loss": 1.3739, + "theoretical_loss": 3.4999866021382764, + "tokens_seen": 1737031680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047839518555667, + "loss": 1.4587, + "theoretical_loss": 3.4999753768301165, + "tokens_seen": 1737097216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783751253761284, + "loss": 1.4282, + "theoretical_loss": 3.499964152064025, + "tokens_seen": 1737162752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047835506519558675, + "loss": 1.3448, + "theoretical_loss": 3.4999529278399537, + "tokens_seen": 1737228288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783350050150451, + "loss": 1.4438, + "theoretical_loss": 3.499941704157857, + "tokens_seen": 1737293824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047831494483450347, + "loss": 1.5265, + "theoretical_loss": 3.4999304810176888, + "tokens_seen": 1737359360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782948846539619, + "loss": 1.32, + "theoretical_loss": 3.499919258419401, + "tokens_seen": 1737424896 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047827482447342025, + "loss": 1.3087, + "theoretical_loss": 3.4999080363629487, + "tokens_seen": 1737490432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782547642928786, + "loss": 1.3742, + "theoretical_loss": 3.4998968148482845, + "tokens_seen": 1737555968 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478234704112337, + "loss": 1.373, + "theoretical_loss": 3.4998855938753612, + "tokens_seen": 1737621504 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047821464393179544, + "loss": 1.4785, + "theoretical_loss": 3.499874373444133, + "tokens_seen": 1737687040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781945837512538, + "loss": 1.382, + "theoretical_loss": 3.4998631535545535, + "tokens_seen": 1737752576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047817452357071217, + "loss": 1.2707, + "theoretical_loss": 3.4998519342065753, + "tokens_seen": 1737818112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047815446339017053, + "loss": 1.3979, + "theoretical_loss": 3.4998407154001523, + "tokens_seen": 1737883648 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047813440320962895, + "loss": 1.4073, + "theoretical_loss": 3.499829497135238, + "tokens_seen": 1737949184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781143430290873, + "loss": 1.3476, + "theoretical_loss": 3.4998182794117856, + "tokens_seen": 1738014720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047809428284854567, + "loss": 1.2186, + "theoretical_loss": 3.499807062229749, + "tokens_seen": 1738080256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047807422266800403, + "loss": 1.5325, + "theoretical_loss": 3.4997958455890816, + "tokens_seen": 1738145792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780541624874624, + "loss": 1.5235, + "theoretical_loss": 3.4997846294897363, + "tokens_seen": 1738211328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780341023069208, + "loss": 1.342, + "theoretical_loss": 3.499773413931667, + "tokens_seen": 1738276864 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780140421263792, + "loss": 1.378, + "theoretical_loss": 3.4997621989148273, + "tokens_seen": 1738342400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799398194583754, + "loss": 1.3128, + "theoretical_loss": 3.49975098443917, + "tokens_seen": 1738407936 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 635089, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5819060802459717, + "objective/train/theoretical_loss": 3.4997397705046493, + "objective/train/tokens_used": 108932576, + "theoretical_loss": 3.4997397705046493, + "tokens_seen": 1738473472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779739217652959, + "loss": 1.421, + "theoretical_loss": 3.4997397705046493, + "tokens_seen": 1738473472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779538615847543, + "loss": 1.3741, + "theoretical_loss": 3.4997285571112178, + "tokens_seen": 1738539008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779338014042127, + "loss": 1.4196, + "theoretical_loss": 3.4997173442588307, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047791374122367104, + "loss": 1.3153, + "theoretical_loss": 3.4997061319474394, + "tokens_seen": 1738670080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778936810431294, + "loss": 1.3222, + "theoretical_loss": 3.4996949201769993, + "tokens_seen": 1738735616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047787362086258776, + "loss": 1.3881, + "theoretical_loss": 3.4996837089474626, + "tokens_seen": 1738801152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778535606820462, + "loss": 1.3621, + "theoretical_loss": 3.4996724982587835, + "tokens_seen": 1738866688 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783350050150454, + "loss": 1.3975, + "theoretical_loss": 3.4996612881109153, + "tokens_seen": 1738932224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778134403209629, + "loss": 1.3734, + "theoretical_loss": 3.4996500785038114, + "tokens_seen": 1738997760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047779338014042127, + "loss": 1.4063, + "theoretical_loss": 3.499638869437425, + "tokens_seen": 1739063296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777733199598797, + "loss": 1.4812, + "theoretical_loss": 3.4996276609117105, + "tokens_seen": 1739128832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775325977933805, + "loss": 1.422, + "theoretical_loss": 3.499616452926621, + "tokens_seen": 1739194368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777331995987964, + "loss": 1.2819, + "theoretical_loss": 3.4996052454821105, + "tokens_seen": 1739259904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771313941825477, + "loss": 1.4095, + "theoretical_loss": 3.499594038578132, + "tokens_seen": 1739325440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047769307923771313, + "loss": 1.3387, + "theoretical_loss": 3.4995828322146387, + "tokens_seen": 1739390976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047767301905717155, + "loss": 1.4609, + "theoretical_loss": 3.4995716263915853, + "tokens_seen": 1739456512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776529588766299, + "loss": 1.3585, + "theoretical_loss": 3.4995604211089244, + "tokens_seen": 1739522048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776328986960883, + "loss": 1.2814, + "theoretical_loss": 3.49954921636661, + "tokens_seen": 1739587584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761283851554664, + "loss": 1.3707, + "theoretical_loss": 3.499538012164596, + "tokens_seen": 1739653120 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047759277833500505, + "loss": 1.3301, + "theoretical_loss": 3.499526808502835, + "tokens_seen": 1739718656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775727181544634, + "loss": 1.5041, + "theoretical_loss": 3.4995156053812817, + "tokens_seen": 1739784192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775526579739218, + "loss": 1.302, + "theoretical_loss": 3.4995044027998894, + "tokens_seen": 1739849728 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753259779338014, + "loss": 1.4329, + "theoretical_loss": 3.499493200758611, + "tokens_seen": 1739915264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775125376128385, + "loss": 1.44, + "theoretical_loss": 3.499481999257401, + "tokens_seen": 1739980800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774924774322969, + "loss": 1.3856, + "theoretical_loss": 3.4994707982962128, + "tokens_seen": 1740046336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774724172517553, + "loss": 1.4457, + "theoretical_loss": 3.499459597875, + "tokens_seen": 1740111872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745235707121364, + "loss": 1.4626, + "theoretical_loss": 3.4994483979937154, + "tokens_seen": 1740177408 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477432296890672, + "loss": 1.3459, + "theoretical_loss": 3.4994371986523145, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774122367101304, + "loss": 1.2833, + "theoretical_loss": 3.499425999850749, + "tokens_seen": 1740308480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773921765295888, + "loss": 1.3325, + "theoretical_loss": 3.4994148015889737, + "tokens_seen": 1740374016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047737211634904715, + "loss": 1.4385, + "theoretical_loss": 3.499403603866942, + "tokens_seen": 1740439552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773520561685055, + "loss": 1.3918, + "theoretical_loss": 3.499392406684607, + "tokens_seen": 1740505088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733199598796387, + "loss": 1.359, + "theoretical_loss": 3.4993812100419235, + "tokens_seen": 1740570624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773119358074223, + "loss": 1.2948, + "theoretical_loss": 3.4993700139388446, + "tokens_seen": 1740636160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729187562688065, + "loss": 1.4623, + "theoretical_loss": 3.4993588183753235, + "tokens_seen": 1740701696 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477271815446339, + "loss": 1.4959, + "theoretical_loss": 3.4993476233513148, + "tokens_seen": 1740767232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772517552657974, + "loss": 1.4357, + "theoretical_loss": 3.499336428866771, + "tokens_seen": 1740832768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772316950852558, + "loss": 1.367, + "theoretical_loss": 3.4993252349216473, + "tokens_seen": 1740898304 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047721163490471415, + "loss": 1.4615, + "theoretical_loss": 3.4993140415158956, + "tokens_seen": 1740963840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771915747241725, + "loss": 1.4295, + "theoretical_loss": 3.4993028486494717, + "tokens_seen": 1741029376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771715145436309, + "loss": 1.3367, + "theoretical_loss": 3.4992916563223275, + "tokens_seen": 1741094912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047715145436308924, + "loss": 1.3049, + "theoretical_loss": 3.4992804645344178, + "tokens_seen": 1741160448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713139418254766, + "loss": 1.4341, + "theoretical_loss": 3.499269273285696, + "tokens_seen": 1741225984 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477111334002006, + "loss": 1.3332, + "theoretical_loss": 3.499258082576116, + "tokens_seen": 1741291520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770912738214644, + "loss": 1.3339, + "theoretical_loss": 3.4992468924056306, + "tokens_seen": 1741357056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047707121364092274, + "loss": 1.3419, + "theoretical_loss": 3.499235702774195, + "tokens_seen": 1741422592 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047705115346038116, + "loss": 1.3719, + "theoretical_loss": 3.499224513681762, + "tokens_seen": 1741488128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770310932798395, + "loss": 1.3671, + "theoretical_loss": 3.4992133251282853, + "tokens_seen": 1741553664 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770110330992979, + "loss": 1.3919, + "theoretical_loss": 3.499202137113719, + "tokens_seen": 1741619200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047699097291875625, + "loss": 1.4125, + "theoretical_loss": 3.499190949638017, + "tokens_seen": 1741684736 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 636395, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3723151683807373, + "objective/train/theoretical_loss": 3.499179762701133, + "objective/train/tokens_used": 112209376, + "theoretical_loss": 3.499179762701133, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047697091273821466, + "loss": 1.3412, + "theoretical_loss": 3.499179762701133, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476950852557673, + "loss": 1.3023, + "theoretical_loss": 3.4991685763030205, + "tokens_seen": 1741815808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769307923771314, + "loss": 1.347, + "theoretical_loss": 3.4991573904436333, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047691073219658975, + "loss": 1.4426, + "theoretical_loss": 3.499146205122926, + "tokens_seen": 1741946880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768906720160481, + "loss": 1.3363, + "theoretical_loss": 3.4991350203408507, + "tokens_seen": 1742012416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047687061183550653, + "loss": 1.3294, + "theoretical_loss": 3.499123836097363, + "tokens_seen": 1742077952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768505516549649, + "loss": 1.4609, + "theoretical_loss": 3.499112652392416, + "tokens_seen": 1742143488 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047683049147442325, + "loss": 1.3622, + "theoretical_loss": 3.499101469225963, + "tokens_seen": 1742209024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768104312938816, + "loss": 1.2921, + "theoretical_loss": 3.4990902865979585, + "tokens_seen": 1742274560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047679037111334003, + "loss": 1.3533, + "theoretical_loss": 3.499079104508356, + "tokens_seen": 1742340096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767703109327984, + "loss": 1.3406, + "theoretical_loss": 3.499067922957109, + "tokens_seen": 1742405632 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047675025075225676, + "loss": 1.4049, + "theoretical_loss": 3.4990567419441723, + "tokens_seen": 1742471168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767301905717151, + "loss": 1.4386, + "theoretical_loss": 3.499045561469499, + "tokens_seen": 1742536704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047671013039117353, + "loss": 1.3344, + "theoretical_loss": 3.4990343815330434, + "tokens_seen": 1742602240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047669007021063195, + "loss": 1.3613, + "theoretical_loss": 3.4990232021347585, + "tokens_seen": 1742667776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766700100300903, + "loss": 1.427, + "theoretical_loss": 3.4990120232745987, + "tokens_seen": 1742733312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766499498495487, + "loss": 1.4005, + "theoretical_loss": 3.4990008449525183, + "tokens_seen": 1742798848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662988966900704, + "loss": 1.2999, + "theoretical_loss": 3.4989896671684706, + "tokens_seen": 1742864384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047660982948846545, + "loss": 1.4541, + "theoretical_loss": 3.49897848992241, + "tokens_seen": 1742929920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765897693079238, + "loss": 1.4126, + "theoretical_loss": 3.4989673132142896, + "tokens_seen": 1742995456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765697091273822, + "loss": 1.4026, + "theoretical_loss": 3.498956137044064, + "tokens_seen": 1743060992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654964894684054, + "loss": 1.4616, + "theoretical_loss": 3.498944961411687, + "tokens_seen": 1743126528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765295887662989, + "loss": 1.4183, + "theoretical_loss": 3.4989337863171115, + "tokens_seen": 1743192064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765095285857573, + "loss": 1.3282, + "theoretical_loss": 3.4989226117602925, + "tokens_seen": 1743257600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764894684052157, + "loss": 1.5221, + "theoretical_loss": 3.498911437741184, + "tokens_seen": 1743323136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047646940822467404, + "loss": 1.4154, + "theoretical_loss": 3.4989002642597393, + "tokens_seen": 1743388672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764493480441324, + "loss": 1.4993, + "theoretical_loss": 3.4988890913159123, + "tokens_seen": 1743454208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764292878635908, + "loss": 1.3779, + "theoretical_loss": 3.498877918909657, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764092276830492, + "loss": 1.432, + "theoretical_loss": 3.4988667470409283, + "tokens_seen": 1743585280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047638916750250755, + "loss": 1.5108, + "theoretical_loss": 3.498855575709679, + "tokens_seen": 1743650816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763691073219659, + "loss": 1.3134, + "theoretical_loss": 3.4988444049158627, + "tokens_seen": 1743716352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047634904714142427, + "loss": 1.3464, + "theoretical_loss": 3.4988332346594344, + "tokens_seen": 1743781888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763289869608827, + "loss": 1.3202, + "theoretical_loss": 3.498822064940348, + "tokens_seen": 1743847424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047630892678034105, + "loss": 1.4056, + "theoretical_loss": 3.498810895758557, + "tokens_seen": 1743912960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762888665997994, + "loss": 1.417, + "theoretical_loss": 3.498799727114015, + "tokens_seen": 1743978496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762688064192578, + "loss": 1.4262, + "theoretical_loss": 3.4987885590066767, + "tokens_seen": 1744044032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762487462387162, + "loss": 1.3868, + "theoretical_loss": 3.498777391436496, + "tokens_seen": 1744109568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047622868605817455, + "loss": 1.3644, + "theoretical_loss": 3.4987662244034268, + "tokens_seen": 1744175104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762086258776329, + "loss": 1.4063, + "theoretical_loss": 3.498755057907423, + "tokens_seen": 1744240640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761885656970913, + "loss": 1.426, + "theoretical_loss": 3.4987438919484384, + "tokens_seen": 1744306176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047616850551654964, + "loss": 1.3395, + "theoretical_loss": 3.498732726526427, + "tokens_seen": 1744371712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614844533600806, + "loss": 1.3326, + "theoretical_loss": 3.4987215616413434, + "tokens_seen": 1744437248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761283851554664, + "loss": 1.3448, + "theoretical_loss": 3.498710397293141, + "tokens_seen": 1744502784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761083249749248, + "loss": 1.3836, + "theoretical_loss": 3.498699233481774, + "tokens_seen": 1744568320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047608826479438314, + "loss": 1.4019, + "theoretical_loss": 3.498688070207197, + "tokens_seen": 1744633856 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047606820461384156, + "loss": 1.4755, + "theoretical_loss": 3.4986769074693624, + "tokens_seen": 1744699392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760481444332999, + "loss": 1.4124, + "theoretical_loss": 3.498665745268226, + "tokens_seen": 1744764928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760280842527583, + "loss": 1.4381, + "theoretical_loss": 3.498654583603741, + "tokens_seen": 1744830464 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047600802407221665, + "loss": 1.395, + "theoretical_loss": 3.498643422475862, + "tokens_seen": 1744896000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047598796389167506, + "loss": 1.3212, + "theoretical_loss": 3.4986322618845422, + "tokens_seen": 1744961536 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 637583, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.238525152206421, + "objective/train/theoretical_loss": 3.4986211018297357, + "objective/train/tokens_used": 115486176, + "theoretical_loss": 3.4986211018297357, + "tokens_seen": 1745027072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759679037111334, + "loss": 1.2066, + "theoretical_loss": 3.4986211018297357, + "tokens_seen": 1745027072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759478435305918, + "loss": 1.4097, + "theoretical_loss": 3.498609942311398, + "tokens_seen": 1745092608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047592778335005015, + "loss": 1.2245, + "theoretical_loss": 3.4985987833294816, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759077231695085, + "loss": 1.3761, + "theoretical_loss": 3.4985876248839407, + "tokens_seen": 1745223680 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047588766298896693, + "loss": 1.4018, + "theoretical_loss": 3.49857646697473, + "tokens_seen": 1745289216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758676028084253, + "loss": 1.4224, + "theoretical_loss": 3.498565309601804, + "tokens_seen": 1745354752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047584754262788365, + "loss": 1.3306, + "theoretical_loss": 3.498554152765115, + "tokens_seen": 1745420288 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475827482447342, + "loss": 1.4405, + "theoretical_loss": 3.498542996464619, + "tokens_seen": 1745485824 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047580742226680043, + "loss": 1.3593, + "theoretical_loss": 3.4985318407002692, + "tokens_seen": 1745551360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757873620862588, + "loss": 1.367, + "theoretical_loss": 3.4985206854720197, + "tokens_seen": 1745616896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047576730190571716, + "loss": 1.4049, + "theoretical_loss": 3.498509530779825, + "tokens_seen": 1745682432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757472417251755, + "loss": 1.4652, + "theoretical_loss": 3.4984983766236386, + "tokens_seen": 1745747968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757271815446339, + "loss": 1.3683, + "theoretical_loss": 3.498487223003415, + "tokens_seen": 1745813504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757071213640923, + "loss": 1.3818, + "theoretical_loss": 3.498476069919109, + "tokens_seen": 1745879040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047568706118355066, + "loss": 1.3599, + "theoretical_loss": 3.498464917370673, + "tokens_seen": 1745944576 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475667001003009, + "loss": 1.4126, + "theoretical_loss": 3.498453765358063, + "tokens_seen": 1746010112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756469408224674, + "loss": 1.5066, + "theoretical_loss": 3.498442613881232, + "tokens_seen": 1746075648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756268806419258, + "loss": 1.4062, + "theoretical_loss": 3.4984314629401347, + "tokens_seen": 1746141184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047560682046138416, + "loss": 1.4216, + "theoretical_loss": 3.498420312534725, + "tokens_seen": 1746206720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755867602808425, + "loss": 1.3739, + "theoretical_loss": 3.498409162664957, + "tokens_seen": 1746272256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755667001003009, + "loss": 1.3176, + "theoretical_loss": 3.4983980133307853, + "tokens_seen": 1746337792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047554663991975925, + "loss": 1.3238, + "theoretical_loss": 3.498386864532163, + "tokens_seen": 1746403328 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047552657973921766, + "loss": 1.348, + "theoretical_loss": 3.498375716269046, + "tokens_seen": 1746468864 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047550651955867603, + "loss": 1.3714, + "theoretical_loss": 3.498364568541387, + "tokens_seen": 1746534400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754864593781344, + "loss": 1.3557, + "theoretical_loss": 3.4983534213491407, + "tokens_seen": 1746599936 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047546639919759275, + "loss": 1.3873, + "theoretical_loss": 3.4983422746922614, + "tokens_seen": 1746665472 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047544633901705117, + "loss": 1.442, + "theoretical_loss": 3.4983311285707033, + "tokens_seen": 1746731008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047542627883650953, + "loss": 1.3813, + "theoretical_loss": 3.4983199829844205, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754062186559679, + "loss": 1.26, + "theoretical_loss": 3.498308837933367, + "tokens_seen": 1746862080 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047538615847542625, + "loss": 1.3855, + "theoretical_loss": 3.4982976934174976, + "tokens_seen": 1746927616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753660982948846, + "loss": 1.3722, + "theoretical_loss": 3.4982865494367656, + "tokens_seen": 1746993152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047534603811434303, + "loss": 1.4065, + "theoretical_loss": 3.4982754059911265, + "tokens_seen": 1747058688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753259779338014, + "loss": 1.309, + "theoretical_loss": 3.498264263080533, + "tokens_seen": 1747124224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047530591775325976, + "loss": 1.5057, + "theoretical_loss": 3.498253120704941, + "tokens_seen": 1747189760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752858575727181, + "loss": 1.4015, + "theoretical_loss": 3.4982419788643035, + "tokens_seen": 1747255296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047526579739217654, + "loss": 1.5058, + "theoretical_loss": 3.498230837558576, + "tokens_seen": 1747320832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752457372116349, + "loss": 1.433, + "theoretical_loss": 3.498219696787711, + "tokens_seen": 1747386368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047522567703109326, + "loss": 1.2959, + "theoretical_loss": 3.498208556551664, + "tokens_seen": 1747451904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752056168505517, + "loss": 1.4287, + "theoretical_loss": 3.4981974168503887, + "tokens_seen": 1747517440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047518555667001004, + "loss": 1.3092, + "theoretical_loss": 3.49818627768384, + "tokens_seen": 1747582976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047516549648946846, + "loss": 1.4277, + "theoretical_loss": 3.4981751390519715, + "tokens_seen": 1747648512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751454363089268, + "loss": 1.4031, + "theoretical_loss": 3.498164000954738, + "tokens_seen": 1747714048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751253761283852, + "loss": 1.3672, + "theoretical_loss": 3.498152863392094, + "tokens_seen": 1747779584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047510531594784354, + "loss": 1.3672, + "theoretical_loss": 3.498141726363993, + "tokens_seen": 1747845120 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047508525576730196, + "loss": 1.3441, + "theoretical_loss": 3.4981305898703896, + "tokens_seen": 1747910656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750651955867603, + "loss": 1.4591, + "theoretical_loss": 3.4981194539112384, + "tokens_seen": 1747976192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750451354062187, + "loss": 1.4302, + "theoretical_loss": 3.4981083184864934, + "tokens_seen": 1748041728 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047502507522567705, + "loss": 1.3723, + "theoretical_loss": 3.4980971835961094, + "tokens_seen": 1748107264 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047500501504513546, + "loss": 1.3964, + "theoretical_loss": 3.49808604924004, + "tokens_seen": 1748172800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749849548645938, + "loss": 1.3725, + "theoretical_loss": 3.49807491541824, + "tokens_seen": 1748238336 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 638722, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1972347497940063, + "objective/train/theoretical_loss": 3.4980637821306635, + "objective/train/tokens_used": 118762976, + "theoretical_loss": 3.4980637821306635, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749648946840522, + "loss": 1.2796, + "theoretical_loss": 3.4980637821306635, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494483450351055, + "loss": 1.3572, + "theoretical_loss": 3.4980526493772652, + "tokens_seen": 1748369408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749247743229689, + "loss": 1.3555, + "theoretical_loss": 3.498041517157999, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047490471414242733, + "loss": 1.3755, + "theoretical_loss": 3.49803038547282, + "tokens_seen": 1748500480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748846539618857, + "loss": 1.3053, + "theoretical_loss": 3.4980192543216813, + "tokens_seen": 1748566016 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047486459378134405, + "loss": 1.3668, + "theoretical_loss": 3.4980081237045386, + "tokens_seen": 1748631552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748445336008024, + "loss": 1.2702, + "theoretical_loss": 3.4979969936213458, + "tokens_seen": 1748697088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047482447342026083, + "loss": 1.4395, + "theoretical_loss": 3.497985864072057, + "tokens_seen": 1748762624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748044132397192, + "loss": 1.341, + "theoretical_loss": 3.4979747350566264, + "tokens_seen": 1748828160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478435305917756, + "loss": 1.3135, + "theoretical_loss": 3.4979636065750093, + "tokens_seen": 1748893696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747642928786359, + "loss": 1.4415, + "theoretical_loss": 3.497952478627159, + "tokens_seen": 1748959232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747442326980943, + "loss": 1.3595, + "theoretical_loss": 3.4979413512130306, + "tokens_seen": 1749024768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747241725175527, + "loss": 1.4819, + "theoretical_loss": 3.4979302243325785, + "tokens_seen": 1749090304 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470411233701106, + "loss": 1.4374, + "theoretical_loss": 3.497919097985757, + "tokens_seen": 1749155840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746840521564694, + "loss": 1.3396, + "theoretical_loss": 3.49790797217252, + "tokens_seen": 1749221376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746639919759278, + "loss": 1.3657, + "theoretical_loss": 3.4978968468928224, + "tokens_seen": 1749286912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746439317953862, + "loss": 1.5177, + "theoretical_loss": 3.497885722146619, + "tokens_seen": 1749352448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047462387161484456, + "loss": 1.3515, + "theoretical_loss": 3.4978745979338637, + "tokens_seen": 1749417984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746038114343029, + "loss": 1.3026, + "theoretical_loss": 3.497863474254511, + "tokens_seen": 1749483520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745837512537613, + "loss": 1.4378, + "theoretical_loss": 3.4978523511085156, + "tokens_seen": 1749549056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047456369107321965, + "loss": 1.3463, + "theoretical_loss": 3.4978412284958313, + "tokens_seen": 1749614592 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454363089267807, + "loss": 1.3778, + "theoretical_loss": 3.497830106416414, + "tokens_seen": 1749680128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047452357071213643, + "loss": 1.4166, + "theoretical_loss": 3.497818984870216, + "tokens_seen": 1749745664 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745035105315948, + "loss": 1.4328, + "theoretical_loss": 3.4978078638571937, + "tokens_seen": 1749811200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047448345035105315, + "loss": 1.4418, + "theoretical_loss": 3.4977967433773003, + "tokens_seen": 1749876736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047446339017051157, + "loss": 1.406, + "theoretical_loss": 3.497785623430491, + "tokens_seen": 1749942272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444332998996993, + "loss": 1.4632, + "theoretical_loss": 3.4977745040167205, + "tokens_seen": 1750007808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744232698094283, + "loss": 1.3397, + "theoretical_loss": 3.4977633851359426, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047440320962888666, + "loss": 1.3761, + "theoretical_loss": 3.497752266788112, + "tokens_seen": 1750138880 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474383149448345, + "loss": 1.4273, + "theoretical_loss": 3.497741148973183, + "tokens_seen": 1750204416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436308926780343, + "loss": 1.3322, + "theoretical_loss": 3.497730031691111, + "tokens_seen": 1750269952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743430290872618, + "loss": 1.3073, + "theoretical_loss": 3.4977189149418493, + "tokens_seen": 1750335488 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432296890672016, + "loss": 1.4261, + "theoretical_loss": 3.497707798725353, + "tokens_seen": 1750401024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743029087261785, + "loss": 1.3149, + "theoretical_loss": 3.497696683041577, + "tokens_seen": 1750466560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047428284854563694, + "loss": 1.3635, + "theoretical_loss": 3.4976855678904752, + "tokens_seen": 1750532096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742627883650953, + "loss": 1.4252, + "theoretical_loss": 3.497674453272002, + "tokens_seen": 1750597632 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047424272818455366, + "loss": 1.4385, + "theoretical_loss": 3.497663339186113, + "tokens_seen": 1750663168 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474222668004012, + "loss": 1.3353, + "theoretical_loss": 3.497652225632762, + "tokens_seen": 1750728704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742026078234704, + "loss": 1.4279, + "theoretical_loss": 3.4976411126119036, + "tokens_seen": 1750794240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741825476429288, + "loss": 1.3647, + "theoretical_loss": 3.497630000123492, + "tokens_seen": 1750859776 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416248746238716, + "loss": 1.4138, + "theoretical_loss": 3.497618888167482, + "tokens_seen": 1750925312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741424272818455, + "loss": 1.3681, + "theoretical_loss": 3.4976077767438287, + "tokens_seen": 1750990848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741223671013039, + "loss": 1.3681, + "theoretical_loss": 3.497596665852486, + "tokens_seen": 1751056384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741023069207623, + "loss": 1.4527, + "theoretical_loss": 3.497585555493409, + "tokens_seen": 1751121920 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047408224674022067, + "loss": 1.4242, + "theoretical_loss": 3.497574445666552, + "tokens_seen": 1751187456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047406218655967903, + "loss": 1.4123, + "theoretical_loss": 3.4975633363718694, + "tokens_seen": 1751252992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740421263791374, + "loss": 1.4234, + "theoretical_loss": 3.497552227609316, + "tokens_seen": 1751318528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047402206619859575, + "loss": 1.4451, + "theoretical_loss": 3.4975411193788464, + "tokens_seen": 1751384064 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047400200601805417, + "loss": 1.3812, + "theoretical_loss": 3.4975300116804156, + "tokens_seen": 1751449600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047398194583751253, + "loss": 1.3308, + "theoretical_loss": 3.497518904513978, + "tokens_seen": 1751515136 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 639884, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3613932132720947, + "objective/train/theoretical_loss": 3.4975077978794875, + "objective/train/tokens_used": 122039776, + "theoretical_loss": 3.4975077978794875, + "tokens_seen": 1751580672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739618856569709, + "loss": 1.3875, + "theoretical_loss": 3.4975077978794875, + "tokens_seen": 1751580672 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047394182547642926, + "loss": 1.3947, + "theoretical_loss": 3.497496691776899, + "tokens_seen": 1751646208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739217652958877, + "loss": 1.3981, + "theoretical_loss": 3.497485586206168, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047390170511534604, + "loss": 1.4743, + "theoretical_loss": 3.497474481167248, + "tokens_seen": 1751777280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738816449348044, + "loss": 1.4963, + "theoretical_loss": 3.4974633766600944, + "tokens_seen": 1751842816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047386158475426276, + "loss": 1.3637, + "theoretical_loss": 3.4974522726846615, + "tokens_seen": 1751908352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738415245737211, + "loss": 1.3598, + "theoretical_loss": 3.497441169240904, + "tokens_seen": 1751973888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047382146439317954, + "loss": 1.3887, + "theoretical_loss": 3.497430066328777, + "tokens_seen": 1752039424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738014042126379, + "loss": 1.4186, + "theoretical_loss": 3.4974189639482347, + "tokens_seen": 1752104960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047378134403209626, + "loss": 1.4263, + "theoretical_loss": 3.497407862099232, + "tokens_seen": 1752170496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737612838515546, + "loss": 1.3414, + "theoretical_loss": 3.497396760781723, + "tokens_seen": 1752236032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047374122367101304, + "loss": 1.3561, + "theoretical_loss": 3.497385659995663, + "tokens_seen": 1752301568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047372116349047146, + "loss": 1.3877, + "theoretical_loss": 3.497374559741006, + "tokens_seen": 1752367104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737011033099298, + "loss": 1.3865, + "theoretical_loss": 3.4973634600177075, + "tokens_seen": 1752432640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736810431293882, + "loss": 1.5212, + "theoretical_loss": 3.497352360825722, + "tokens_seen": 1752498176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736609829488466, + "loss": 1.4743, + "theoretical_loss": 3.497341262165004, + "tokens_seen": 1752563712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047364092276830496, + "loss": 1.4386, + "theoretical_loss": 3.4973301640355086, + "tokens_seen": 1752629248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736208625877633, + "loss": 1.3088, + "theoretical_loss": 3.497319066437189, + "tokens_seen": 1752694784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736008024072217, + "loss": 1.378, + "theoretical_loss": 3.497307969370002, + "tokens_seen": 1752760320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047358074222668005, + "loss": 1.3621, + "theoretical_loss": 3.4972968728339016, + "tokens_seen": 1752825856 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047356068204613847, + "loss": 1.344, + "theoretical_loss": 3.497285776828842, + "tokens_seen": 1752891392 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047354062186559683, + "loss": 1.4438, + "theoretical_loss": 3.4972746813547784, + "tokens_seen": 1752956928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735205616850552, + "loss": 1.5445, + "theoretical_loss": 3.4972635864116652, + "tokens_seen": 1753022464 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047350050150451355, + "loss": 1.3919, + "theoretical_loss": 3.4972524919994576, + "tokens_seen": 1753088000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047348044132397197, + "loss": 1.29, + "theoretical_loss": 3.49724139811811, + "tokens_seen": 1753153536 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047346038114343033, + "loss": 1.3707, + "theoretical_loss": 3.4972303047675775, + "tokens_seen": 1753219072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734403209628887, + "loss": 1.4446, + "theoretical_loss": 3.4972192119478143, + "tokens_seen": 1753284608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047342026078234706, + "loss": 1.3172, + "theoretical_loss": 3.497208119658775, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734002006018054, + "loss": 1.3996, + "theoretical_loss": 3.4971970279004156, + "tokens_seen": 1753415680 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047338014042126383, + "loss": 1.3655, + "theoretical_loss": 3.4971859366726905, + "tokens_seen": 1753481216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733600802407222, + "loss": 1.4249, + "theoretical_loss": 3.497174845975553, + "tokens_seen": 1753546752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047334002006018056, + "loss": 1.487, + "theoretical_loss": 3.49716375580896, + "tokens_seen": 1753612288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733199598796389, + "loss": 1.4138, + "theoretical_loss": 3.4971526661728647, + "tokens_seen": 1753677824 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047329989969909734, + "loss": 1.3696, + "theoretical_loss": 3.4971415770672225, + "tokens_seen": 1753743360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732798395185557, + "loss": 1.4004, + "theoretical_loss": 3.4971304884919885, + "tokens_seen": 1753808896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325977933801406, + "loss": 1.3973, + "theoretical_loss": 3.4971194004471173, + "tokens_seen": 1753874432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732397191574724, + "loss": 1.4326, + "theoretical_loss": 3.4971083129325633, + "tokens_seen": 1753939968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732196589769308, + "loss": 1.3029, + "theoretical_loss": 3.4970972259482824, + "tokens_seen": 1754005504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731995987963892, + "loss": 1.4806, + "theoretical_loss": 3.497086139494228, + "tokens_seen": 1754071040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047317953861584757, + "loss": 1.4118, + "theoretical_loss": 3.497075053570356, + "tokens_seen": 1754136576 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047315947843530593, + "loss": 1.4521, + "theoretical_loss": 3.4970639681766205, + "tokens_seen": 1754202112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731394182547643, + "loss": 1.3827, + "theoretical_loss": 3.4970528833129766, + "tokens_seen": 1754267648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731193580742227, + "loss": 1.4467, + "theoretical_loss": 3.49704179897938, + "tokens_seen": 1754333184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047309929789368107, + "loss": 1.437, + "theoretical_loss": 3.497030715175784, + "tokens_seen": 1754398720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047307923771313943, + "loss": 1.4981, + "theoretical_loss": 3.4970196319021447, + "tokens_seen": 1754464256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730591775325978, + "loss": 1.4951, + "theoretical_loss": 3.4970085491584166, + "tokens_seen": 1754529792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303911735205615, + "loss": 1.4327, + "theoretical_loss": 3.496997466944554, + "tokens_seen": 1754595328 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047301905717151457, + "loss": 1.3146, + "theoretical_loss": 3.496986385260513, + "tokens_seen": 1754660864 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047299899699097293, + "loss": 1.3945, + "theoretical_loss": 3.496975304106247, + "tokens_seen": 1754726400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729789368104313, + "loss": 1.4097, + "theoretical_loss": 3.4969642234817124, + "tokens_seen": 1754791936 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 641116, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3436782360076904, + "objective/train/theoretical_loss": 3.496953143386863, + "objective/train/tokens_used": 125316576, + "theoretical_loss": 3.496953143386863, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047295887662988966, + "loss": 1.373, + "theoretical_loss": 3.496953143386863, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729388164493481, + "loss": 1.3959, + "theoretical_loss": 3.4969420638216535, + "tokens_seen": 1754923008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047291875626880644, + "loss": 1.3779, + "theoretical_loss": 3.49693098478604, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728986960882648, + "loss": 1.3608, + "theoretical_loss": 3.496919906279977, + "tokens_seen": 1755054080 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047287863590772316, + "loss": 1.4135, + "theoretical_loss": 3.496908828303419, + "tokens_seen": 1755119616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728585757271815, + "loss": 1.3838, + "theoretical_loss": 3.496897750856321, + "tokens_seen": 1755185152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047283851554663994, + "loss": 1.3669, + "theoretical_loss": 3.496886673938638, + "tokens_seen": 1755250688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728184553660983, + "loss": 1.4353, + "theoretical_loss": 3.496875597550325, + "tokens_seen": 1755316224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047279839518555666, + "loss": 1.3751, + "theoretical_loss": 3.4968645216913368, + "tokens_seen": 1755381760 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472778335005015, + "loss": 1.323, + "theoretical_loss": 3.4968534463616283, + "tokens_seen": 1755447296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047275827482447344, + "loss": 1.4803, + "theoretical_loss": 3.496842371561155, + "tokens_seen": 1755512832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727382146439318, + "loss": 1.3426, + "theoretical_loss": 3.496831297289871, + "tokens_seen": 1755578368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047271815446339017, + "loss": 1.3724, + "theoretical_loss": 3.4968202235477324, + "tokens_seen": 1755643904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047269809428284853, + "loss": 1.3883, + "theoretical_loss": 3.496809150334693, + "tokens_seen": 1755709440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047267803410230695, + "loss": 1.3686, + "theoretical_loss": 3.4967980776507086, + "tokens_seen": 1755774976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726579739217653, + "loss": 1.3677, + "theoretical_loss": 3.4967870054957335, + "tokens_seen": 1755840512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047263791374122367, + "loss": 1.3445, + "theoretical_loss": 3.4967759338697233, + "tokens_seen": 1755906048 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047261785356068203, + "loss": 1.3832, + "theoretical_loss": 3.496764862772632, + "tokens_seen": 1755971584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725977933801404, + "loss": 1.3187, + "theoretical_loss": 3.496753792204416, + "tokens_seen": 1756037120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725777331995988, + "loss": 1.253, + "theoretical_loss": 3.4967427221650293, + "tokens_seen": 1756102656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725576730190572, + "loss": 1.3705, + "theoretical_loss": 3.4967316526544274, + "tokens_seen": 1756168192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047253761283851554, + "loss": 1.3044, + "theoretical_loss": 3.496720583672565, + "tokens_seen": 1756233728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725175526579739, + "loss": 1.4567, + "theoretical_loss": 3.496709515219397, + "tokens_seen": 1756299264 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724974924774323, + "loss": 1.4206, + "theoretical_loss": 3.4966984472948788, + "tokens_seen": 1756364800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724774322968907, + "loss": 1.4351, + "theoretical_loss": 3.4966873798989653, + "tokens_seen": 1756430336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047245737211634904, + "loss": 1.4001, + "theoretical_loss": 3.4966763130316116, + "tokens_seen": 1756495872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724373119358074, + "loss": 1.4328, + "theoretical_loss": 3.4966652466927726, + "tokens_seen": 1756561408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047241725175526576, + "loss": 1.4283, + "theoretical_loss": 3.496654180882403, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723971915747242, + "loss": 1.3985, + "theoretical_loss": 3.4966431156004587, + "tokens_seen": 1756692480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047237713139418254, + "loss": 1.2532, + "theoretical_loss": 3.496632050846894, + "tokens_seen": 1756758016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723570712136409, + "loss": 1.4727, + "theoretical_loss": 3.4966209866216644, + "tokens_seen": 1756823552 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047233701103309927, + "loss": 1.4377, + "theoretical_loss": 3.496609922924725, + "tokens_seen": 1756889088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723169508525577, + "loss": 1.4425, + "theoretical_loss": 3.4965988597560305, + "tokens_seen": 1756954624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047229689067201605, + "loss": 1.4734, + "theoretical_loss": 3.496587797115536, + "tokens_seen": 1757020160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722768304914744, + "loss": 1.4572, + "theoretical_loss": 3.496576735003197, + "tokens_seen": 1757085696 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047225677031093277, + "loss": 1.4508, + "theoretical_loss": 3.496565673418968, + "tokens_seen": 1757151232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047223671013039113, + "loss": 1.4301, + "theoretical_loss": 3.4965546123628046, + "tokens_seen": 1757216768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722166499498496, + "loss": 1.2344, + "theoretical_loss": 3.4965435518346615, + "tokens_seen": 1757282304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047219658976930797, + "loss": 1.3642, + "theoretical_loss": 3.496532491834494, + "tokens_seen": 1757347840 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047217652958876633, + "loss": 1.3074, + "theoretical_loss": 3.4965214323622575, + "tokens_seen": 1757413376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721564694082247, + "loss": 1.4659, + "theoretical_loss": 3.496510373417907, + "tokens_seen": 1757478912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721364092276831, + "loss": 1.3674, + "theoretical_loss": 3.4964993150013974, + "tokens_seen": 1757544448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047211634904714147, + "loss": 1.4215, + "theoretical_loss": 3.4964882571126834, + "tokens_seen": 1757609984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047209628886659983, + "loss": 1.4338, + "theoretical_loss": 3.496477199751721, + "tokens_seen": 1757675520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720762286860582, + "loss": 1.304, + "theoretical_loss": 3.496466142918465, + "tokens_seen": 1757741056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047205616850551656, + "loss": 1.4645, + "theoretical_loss": 3.4964550866128703, + "tokens_seen": 1757806592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047203610832497497, + "loss": 1.4003, + "theoretical_loss": 3.4964440308348923, + "tokens_seen": 1757872128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047201604814443333, + "loss": 1.4411, + "theoretical_loss": 3.4964329755844856, + "tokens_seen": 1757937664 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719959879638917, + "loss": 1.3551, + "theoretical_loss": 3.4964219208616067, + "tokens_seen": 1758003200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047197592778335006, + "loss": 1.3761, + "theoretical_loss": 3.49641086666621, + "tokens_seen": 1758068736 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 642262, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.330387830734253, + "objective/train/theoretical_loss": 3.4963998129982494, + "objective/train/tokens_used": 128593376, + "theoretical_loss": 3.4963998129982494, + "tokens_seen": 1758134272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719558676028085, + "loss": 1.3445, + "theoretical_loss": 3.4963998129982494, + "tokens_seen": 1758134272 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193580742226684, + "loss": 1.3591, + "theoretical_loss": 3.496388759857682, + "tokens_seen": 1758199808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719157472417252, + "loss": 1.3132, + "theoretical_loss": 3.4963777072444624, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047189568706118356, + "loss": 1.2909, + "theoretical_loss": 3.4963666551585453, + "tokens_seen": 1758330880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718756268806419, + "loss": 1.5081, + "theoretical_loss": 3.496355603599887, + "tokens_seen": 1758396416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047185556670010034, + "loss": 1.288, + "theoretical_loss": 3.496344552568441, + "tokens_seen": 1758461952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718355065195587, + "loss": 1.3416, + "theoretical_loss": 3.4963335020641635, + "tokens_seen": 1758527488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047181544633901706, + "loss": 1.4727, + "theoretical_loss": 3.4963224520870098, + "tokens_seen": 1758593024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717953861584754, + "loss": 1.4002, + "theoretical_loss": 3.496311402636935, + "tokens_seen": 1758658560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177532597793384, + "loss": 1.4526, + "theoretical_loss": 3.4963003537138944, + "tokens_seen": 1758724096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717552657973922, + "loss": 1.473, + "theoretical_loss": 3.4962893053178425, + "tokens_seen": 1758789632 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173520561685057, + "loss": 1.4001, + "theoretical_loss": 3.4962782574487354, + "tokens_seen": 1758855168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047171514543630893, + "loss": 1.5418, + "theoretical_loss": 3.496267210106528, + "tokens_seen": 1758920704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169508525576735, + "loss": 1.4355, + "theoretical_loss": 3.496256163291176, + "tokens_seen": 1758986240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716750250752257, + "loss": 1.4155, + "theoretical_loss": 3.496245117002634, + "tokens_seen": 1759051776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047165496489468407, + "loss": 1.3922, + "theoretical_loss": 3.496234071240857, + "tokens_seen": 1759117312 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163490471414243, + "loss": 1.3249, + "theoretical_loss": 3.4962230260058016, + "tokens_seen": 1759182848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716148445336008, + "loss": 1.3263, + "theoretical_loss": 3.4962119812974217, + "tokens_seen": 1759248384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715947843530592, + "loss": 1.3491, + "theoretical_loss": 3.496200937115673, + "tokens_seen": 1759313920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715747241725176, + "loss": 1.384, + "theoretical_loss": 3.496189893460511, + "tokens_seen": 1759379456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047155466399197594, + "loss": 1.3865, + "theoretical_loss": 3.4961788503318907, + "tokens_seen": 1759444992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715346038114343, + "loss": 1.3548, + "theoretical_loss": 3.4961678077297673, + "tokens_seen": 1759510528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715145436308927, + "loss": 1.4088, + "theoretical_loss": 3.4961567656540966, + "tokens_seen": 1759576064 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714944834503511, + "loss": 1.2849, + "theoretical_loss": 3.4961457241048337, + "tokens_seen": 1759641600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047147442326980944, + "loss": 1.385, + "theoretical_loss": 3.4961346830819338, + "tokens_seen": 1759707136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714543630892678, + "loss": 1.34, + "theoretical_loss": 3.496123642585352, + "tokens_seen": 1759772672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047143430290872616, + "loss": 1.3061, + "theoretical_loss": 3.4961126026150433, + "tokens_seen": 1759838208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714142427281846, + "loss": 1.4748, + "theoretical_loss": 3.4961015631709644, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139418254764294, + "loss": 1.4464, + "theoretical_loss": 3.496090524253069, + "tokens_seen": 1759969280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713741223671013, + "loss": 1.5164, + "theoretical_loss": 3.496079485861314, + "tokens_seen": 1760034816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135406218655967, + "loss": 1.4403, + "theoretical_loss": 3.496068447995653, + "tokens_seen": 1760100352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713340020060181, + "loss": 1.4814, + "theoretical_loss": 3.496057410656043, + "tokens_seen": 1760165888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047131394182547645, + "loss": 1.363, + "theoretical_loss": 3.4960463738424385, + "tokens_seen": 1760231424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712938816449348, + "loss": 1.2892, + "theoretical_loss": 3.496035337554795, + "tokens_seen": 1760296960 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047127382146439317, + "loss": 1.352, + "theoretical_loss": 3.4960243017930672, + "tokens_seen": 1760362496 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125376128385153, + "loss": 1.4501, + "theoretical_loss": 3.4960132665572115, + "tokens_seen": 1760428032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047123370110330995, + "loss": 1.4287, + "theoretical_loss": 3.496002231847182, + "tokens_seen": 1760493568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712136409227683, + "loss": 1.3261, + "theoretical_loss": 3.495991197662936, + "tokens_seen": 1760559104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711935807422267, + "loss": 1.398, + "theoretical_loss": 3.4959801640044272, + "tokens_seen": 1760624640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047117352056168504, + "loss": 1.421, + "theoretical_loss": 3.495969130871612, + "tokens_seen": 1760690176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115346038114345, + "loss": 1.2906, + "theoretical_loss": 3.4959580982644445, + "tokens_seen": 1760755712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711334002006018, + "loss": 1.3436, + "theoretical_loss": 3.4959470661828815, + "tokens_seen": 1760821248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711133400200602, + "loss": 1.4988, + "theoretical_loss": 3.495936034626878, + "tokens_seen": 1760886784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047109327983951854, + "loss": 1.4439, + "theoretical_loss": 3.4959250035963887, + "tokens_seen": 1760952320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710732196589769, + "loss": 1.3507, + "theoretical_loss": 3.49591397309137, + "tokens_seen": 1761017856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710531594784353, + "loss": 1.3673, + "theoretical_loss": 3.495902943111777, + "tokens_seen": 1761083392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710330992978937, + "loss": 1.3384, + "theoretical_loss": 3.4958919136575646, + "tokens_seen": 1761148928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047101303911735204, + "loss": 1.3313, + "theoretical_loss": 3.4958808847286886, + "tokens_seen": 1761214464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709929789368104, + "loss": 1.4544, + "theoretical_loss": 3.495869856325105, + "tokens_seen": 1761280000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709729187562688, + "loss": 1.3167, + "theoretical_loss": 3.4958588284467678, + "tokens_seen": 1761345536 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 643454, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3653559684753418, + "objective/train/theoretical_loss": 3.4958478010936336, + "objective/train/tokens_used": 131870176, + "theoretical_loss": 3.4958478010936336, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709528585757272, + "loss": 1.3219, + "theoretical_loss": 3.4958478010936336, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047093279839518555, + "loss": 1.4581, + "theoretical_loss": 3.4958367742656575, + "tokens_seen": 1761476608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709127382146439, + "loss": 1.4421, + "theoretical_loss": 3.4958257479627957, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047089267803410227, + "loss": 1.3858, + "theoretical_loss": 3.495814722185002, + "tokens_seen": 1761607680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708726178535607, + "loss": 1.4404, + "theoretical_loss": 3.4958036969322337, + "tokens_seen": 1761673216 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047085255767301905, + "loss": 1.3771, + "theoretical_loss": 3.495792672204445, + "tokens_seen": 1761738752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708324974924774, + "loss": 1.4397, + "theoretical_loss": 3.495781648001592, + "tokens_seen": 1761804288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708124373119358, + "loss": 1.3939, + "theoretical_loss": 3.4957706243236295, + "tokens_seen": 1761869824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707923771313942, + "loss": 1.2862, + "theoretical_loss": 3.495759601170514, + "tokens_seen": 1761935360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047077231695085255, + "loss": 1.501, + "theoretical_loss": 3.4957485785422, + "tokens_seen": 1762000896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707522567703109, + "loss": 1.5213, + "theoretical_loss": 3.4957375564386437, + "tokens_seen": 1762066432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707321965897693, + "loss": 1.3949, + "theoretical_loss": 3.4957265348598, + "tokens_seen": 1762131968 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047071213640922775, + "loss": 1.3351, + "theoretical_loss": 3.495715513805625, + "tokens_seen": 1762197504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706920762286861, + "loss": 1.3716, + "theoretical_loss": 3.495704493276074, + "tokens_seen": 1762263040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047067201604814447, + "loss": 1.4119, + "theoretical_loss": 3.4956934732711025, + "tokens_seen": 1762328576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047065195586760283, + "loss": 1.4479, + "theoretical_loss": 3.4956824537906654, + "tokens_seen": 1762394112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706318956870612, + "loss": 1.5315, + "theoretical_loss": 3.4956714348347195, + "tokens_seen": 1762459648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706118355065196, + "loss": 1.345, + "theoretical_loss": 3.4956604164032195, + "tokens_seen": 1762525184 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470591775325978, + "loss": 1.4292, + "theoretical_loss": 3.4956493984961208, + "tokens_seen": 1762590720 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047057171514543634, + "loss": 1.4123, + "theoretical_loss": 3.4956383811133795, + "tokens_seen": 1762656256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705516549648947, + "loss": 1.3416, + "theoretical_loss": 3.4956273642549514, + "tokens_seen": 1762721792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705315947843531, + "loss": 1.2949, + "theoretical_loss": 3.495616347920791, + "tokens_seen": 1762787328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705115346038115, + "loss": 1.6214, + "theoretical_loss": 3.4956053321108547, + "tokens_seen": 1762852864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047049147442326984, + "loss": 1.334, + "theoretical_loss": 3.4955943168250974, + "tokens_seen": 1762918400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704714142427282, + "loss": 1.4307, + "theoretical_loss": 3.4955833020634755, + "tokens_seen": 1762983936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047045135406218656, + "loss": 1.4839, + "theoretical_loss": 3.4955722878259436, + "tokens_seen": 1763049472 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470431293881645, + "loss": 1.4356, + "theoretical_loss": 3.495561274112458, + "tokens_seen": 1763115008 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047041123370110334, + "loss": 1.3556, + "theoretical_loss": 3.4955502609229745, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703911735205617, + "loss": 1.3959, + "theoretical_loss": 3.4955392482574474, + "tokens_seen": 1763246080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047037111334002007, + "loss": 1.3551, + "theoretical_loss": 3.4955282361158337, + "tokens_seen": 1763311616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703510531594785, + "loss": 1.4848, + "theoretical_loss": 3.495517224498089, + "tokens_seen": 1763377152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047033099297893685, + "loss": 1.488, + "theoretical_loss": 3.495506213404168, + "tokens_seen": 1763442688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703109327983952, + "loss": 1.3503, + "theoretical_loss": 3.495495202834027, + "tokens_seen": 1763508224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047029087261785357, + "loss": 1.5253, + "theoretical_loss": 3.495484192787621, + "tokens_seen": 1763573760 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047027081243731193, + "loss": 1.6191, + "theoretical_loss": 3.495473183264906, + "tokens_seen": 1763639296 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047025075225677035, + "loss": 1.4056, + "theoretical_loss": 3.495462174265838, + "tokens_seen": 1763704832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702306920762287, + "loss": 1.3762, + "theoretical_loss": 3.4954511657903717, + "tokens_seen": 1763770368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702106318956871, + "loss": 1.3131, + "theoretical_loss": 3.4954401578384635, + "tokens_seen": 1763835904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047019057171514544, + "loss": 1.274, + "theoretical_loss": 3.495429150410069, + "tokens_seen": 1763901440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047017051153460385, + "loss": 1.4417, + "theoretical_loss": 3.4954181435051437, + "tokens_seen": 1763966976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701504513540622, + "loss": 1.4518, + "theoretical_loss": 3.495407137123643, + "tokens_seen": 1764032512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701303911735206, + "loss": 1.3252, + "theoretical_loss": 3.495396131265523, + "tokens_seen": 1764098048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047011033099297894, + "loss": 1.3624, + "theoretical_loss": 3.495385125930739, + "tokens_seen": 1764163584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700902708124373, + "loss": 1.4579, + "theoretical_loss": 3.495374121119247, + "tokens_seen": 1764229120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700702106318957, + "loss": 1.3647, + "theoretical_loss": 3.4953631168310024, + "tokens_seen": 1764294656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700501504513541, + "loss": 1.4022, + "theoretical_loss": 3.4953521130659615, + "tokens_seen": 1764360192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047003009027081244, + "loss": 1.3839, + "theoretical_loss": 3.4953411098240785, + "tokens_seen": 1764425728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700100300902708, + "loss": 1.4308, + "theoretical_loss": 3.4953301071053113, + "tokens_seen": 1764491264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699899699097292, + "loss": 1.3421, + "theoretical_loss": 3.4953191049096137, + "tokens_seen": 1764556800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699699097291876, + "loss": 1.356, + "theoretical_loss": 3.495308103236942, + "tokens_seen": 1764622336 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 644554, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3437608480453491, + "objective/train/theoretical_loss": 3.4952971020872527, + "objective/train/tokens_used": 135146976, + "theoretical_loss": 3.4952971020872527, + "tokens_seen": 1764687872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046994984954864595, + "loss": 1.408, + "theoretical_loss": 3.4952971020872527, + "tokens_seen": 1764687872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699297893681043, + "loss": 1.4481, + "theoretical_loss": 3.4952861014605006, + "tokens_seen": 1764753408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046990972918756267, + "loss": 1.4172, + "theoretical_loss": 3.4952751013566417, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698896690070211, + "loss": 1.4879, + "theoretical_loss": 3.4952641017756316, + "tokens_seen": 1764884480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046986960882647945, + "loss": 1.3973, + "theoretical_loss": 3.495253102717426, + "tokens_seen": 1764950016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698495486459378, + "loss": 1.4776, + "theoretical_loss": 3.495242104181981, + "tokens_seen": 1765015552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698294884653962, + "loss": 1.4463, + "theoretical_loss": 3.495231106169252, + "tokens_seen": 1765081088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698094282848546, + "loss": 1.3859, + "theoretical_loss": 3.495220108679195, + "tokens_seen": 1765146624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046978936810431295, + "loss": 1.3913, + "theoretical_loss": 3.4952091117117656, + "tokens_seen": 1765212160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697693079237713, + "loss": 1.3257, + "theoretical_loss": 3.49519811526692, + "tokens_seen": 1765277696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697492477432297, + "loss": 1.3308, + "theoretical_loss": 3.495187119344613, + "tokens_seen": 1765343232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046972918756268804, + "loss": 1.4673, + "theoretical_loss": 3.495176123944801, + "tokens_seen": 1765408768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046970912738214646, + "loss": 1.4629, + "theoretical_loss": 3.4951651290674395, + "tokens_seen": 1765474304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696890672016048, + "loss": 1.4301, + "theoretical_loss": 3.495154134712485, + "tokens_seen": 1765539840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696690070210632, + "loss": 1.3324, + "theoretical_loss": 3.4951431408798928, + "tokens_seen": 1765605376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046964894684052154, + "loss": 1.2896, + "theoretical_loss": 3.4951321475696187, + "tokens_seen": 1765670912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046962888665997996, + "loss": 1.448, + "theoretical_loss": 3.495121154781618, + "tokens_seen": 1765736448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696088264794383, + "loss": 1.3542, + "theoretical_loss": 3.4951101625158474, + "tokens_seen": 1765801984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695887662988967, + "loss": 1.3436, + "theoretical_loss": 3.495099170772262, + "tokens_seen": 1765867520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046956870611835505, + "loss": 1.3527, + "theoretical_loss": 3.4950881795508177, + "tokens_seen": 1765933056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046954864593781346, + "loss": 1.5647, + "theoretical_loss": 3.4950771888514716, + "tokens_seen": 1765998592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695285857572718, + "loss": 1.2566, + "theoretical_loss": 3.495066198674177, + "tokens_seen": 1766064128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695085255767302, + "loss": 1.4403, + "theoretical_loss": 3.495055209018892, + "tokens_seen": 1766129664 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046948846539618855, + "loss": 1.3159, + "theoretical_loss": 3.495044219885572, + "tokens_seen": 1766195200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694684052156469, + "loss": 1.355, + "theoretical_loss": 3.4950332312741716, + "tokens_seen": 1766260736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694483450351053, + "loss": 1.4411, + "theoretical_loss": 3.495022243184648, + "tokens_seen": 1766326272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694282848545637, + "loss": 1.3657, + "theoretical_loss": 3.495011255616957, + "tokens_seen": 1766391808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046940822467402205, + "loss": 1.3799, + "theoretical_loss": 3.4950002685710535, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693881644934804, + "loss": 1.3737, + "theoretical_loss": 3.4949892820468937, + "tokens_seen": 1766522880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046936810431293883, + "loss": 1.3805, + "theoretical_loss": 3.494978296044434, + "tokens_seen": 1766588416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693480441323972, + "loss": 1.3766, + "theoretical_loss": 3.49496731056363, + "tokens_seen": 1766653952 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046932798395185555, + "loss": 1.6006, + "theoretical_loss": 3.494956325604437, + "tokens_seen": 1766719488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693079237713139, + "loss": 1.3969, + "theoretical_loss": 3.4949453411668117, + "tokens_seen": 1766785024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692878635907723, + "loss": 1.2856, + "theoretical_loss": 3.4949343572507097, + "tokens_seen": 1766850560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692678034102307, + "loss": 1.3832, + "theoretical_loss": 3.4949233738560874, + "tokens_seen": 1766916096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046924774322968906, + "loss": 1.3627, + "theoretical_loss": 3.4949123909828996, + "tokens_seen": 1766981632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692276830491474, + "loss": 1.4127, + "theoretical_loss": 3.4949014086311028, + "tokens_seen": 1767047168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046920762286860584, + "loss": 1.4154, + "theoretical_loss": 3.4948904268006533, + "tokens_seen": 1767112704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046918756268806425, + "loss": 1.3805, + "theoretical_loss": 3.494879445491506, + "tokens_seen": 1767178240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691675025075226, + "loss": 1.4497, + "theoretical_loss": 3.494868464703618, + "tokens_seen": 1767243776 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469147442326981, + "loss": 1.3187, + "theoretical_loss": 3.4948574844369444, + "tokens_seen": 1767309312 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046912738214643934, + "loss": 1.3534, + "theoretical_loss": 3.4948465046914414, + "tokens_seen": 1767374848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691073219658977, + "loss": 1.3963, + "theoretical_loss": 3.494835525467065, + "tokens_seen": 1767440384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690872617853561, + "loss": 1.3614, + "theoretical_loss": 3.494824546763771, + "tokens_seen": 1767505920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690672016048145, + "loss": 1.3758, + "theoretical_loss": 3.494813568581516, + "tokens_seen": 1767571456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046904714142427284, + "loss": 1.3334, + "theoretical_loss": 3.494802590920255, + "tokens_seen": 1767636992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690270812437312, + "loss": 1.3513, + "theoretical_loss": 3.494791613779944, + "tokens_seen": 1767702528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690070210631896, + "loss": 1.2967, + "theoretical_loss": 3.49478063716054, + "tokens_seen": 1767768064 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468986960882648, + "loss": 1.6072, + "theoretical_loss": 3.494769661061998, + "tokens_seen": 1767833600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046896690070210635, + "loss": 1.3412, + "theoretical_loss": 3.494758685484274, + "tokens_seen": 1767899136 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 645687, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4466071128845215, + "objective/train/theoretical_loss": 3.4947477104273243, + "objective/train/tokens_used": 138423776, + "theoretical_loss": 3.4947477104273243, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689468405215647, + "loss": 1.3827, + "theoretical_loss": 3.4947477104273243, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892678034102307, + "loss": 1.4484, + "theoretical_loss": 3.494736735891105, + "tokens_seen": 1768030208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689067201604815, + "loss": 1.3213, + "theoretical_loss": 3.494725761875572, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046888665997993985, + "loss": 1.3091, + "theoretical_loss": 3.494714788380681, + "tokens_seen": 1768161280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688665997993982, + "loss": 1.3501, + "theoretical_loss": 3.494703815406388, + "tokens_seen": 1768226816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688465396188566, + "loss": 1.3446, + "theoretical_loss": 3.4946928429526496, + "tokens_seen": 1768292352 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468826479438315, + "loss": 1.3405, + "theoretical_loss": 3.494681871019422, + "tokens_seen": 1768357888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046880641925777335, + "loss": 1.4153, + "theoretical_loss": 3.4946708996066596, + "tokens_seen": 1768423424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687863590772317, + "loss": 1.4501, + "theoretical_loss": 3.49465992871432, + "tokens_seen": 1768488960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687662988966901, + "loss": 1.4606, + "theoretical_loss": 3.494648958342359, + "tokens_seen": 1768554496 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046874623871614844, + "loss": 1.3804, + "theoretical_loss": 3.494637988490732, + "tokens_seen": 1768620032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872617853560686, + "loss": 1.474, + "theoretical_loss": 3.4946270191593958, + "tokens_seen": 1768685568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687061183550652, + "loss": 1.4744, + "theoretical_loss": 3.4946160503483057, + "tokens_seen": 1768751104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686860581745236, + "loss": 1.3864, + "theoretical_loss": 3.494605082057418, + "tokens_seen": 1768816640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046866599799398194, + "loss": 1.3426, + "theoretical_loss": 3.4945941142866888, + "tokens_seen": 1768882176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864593781344036, + "loss": 1.3869, + "theoretical_loss": 3.494583147036075, + "tokens_seen": 1768947712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686258776328987, + "loss": 1.3047, + "theoretical_loss": 3.494572180305531, + "tokens_seen": 1769013248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686058174523571, + "loss": 1.3638, + "theoretical_loss": 3.494561214095014, + "tokens_seen": 1769078784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046858575727181545, + "loss": 1.2741, + "theoretical_loss": 3.4945502484044804, + "tokens_seen": 1769144320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656970912738, + "loss": 1.3958, + "theoretical_loss": 3.494539283233885, + "tokens_seen": 1769209856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685456369107322, + "loss": 1.38, + "theoretical_loss": 3.4945283185831846, + "tokens_seen": 1769275392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685255767301906, + "loss": 1.4113, + "theoretical_loss": 3.4945173544523356, + "tokens_seen": 1769340928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046850551654964895, + "loss": 1.3416, + "theoretical_loss": 3.494506390841294, + "tokens_seen": 1769406464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684854563691073, + "loss": 1.4602, + "theoretical_loss": 3.4944954277500155, + "tokens_seen": 1769472000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046846539618856573, + "loss": 1.4125, + "theoretical_loss": 3.4944844651784566, + "tokens_seen": 1769537536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684453360080241, + "loss": 1.426, + "theoretical_loss": 3.4944735031265726, + "tokens_seen": 1769603072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046842527582748245, + "loss": 1.3728, + "theoretical_loss": 3.494462541594321, + "tokens_seen": 1769668608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684052156469408, + "loss": 1.3692, + "theoretical_loss": 3.4944515805816563, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838515546639923, + "loss": 1.3873, + "theoretical_loss": 3.494440620088536, + "tokens_seen": 1769799680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683650952858576, + "loss": 1.3983, + "theoretical_loss": 3.494429660114916, + "tokens_seen": 1769865216 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834503510531595, + "loss": 1.2393, + "theoretical_loss": 3.494418700660752, + "tokens_seen": 1769930752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683249749247743, + "loss": 1.403, + "theoretical_loss": 3.4944077417260004, + "tokens_seen": 1769996288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683049147442327, + "loss": 1.3028, + "theoretical_loss": 3.4943967833106173, + "tokens_seen": 1770061824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682848545636911, + "loss": 1.3154, + "theoretical_loss": 3.4943858254145583, + "tokens_seen": 1770127360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046826479438314946, + "loss": 1.4234, + "theoretical_loss": 3.494374868037781, + "tokens_seen": 1770192896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682447342026078, + "loss": 1.2604, + "theoretical_loss": 3.49436391118024, + "tokens_seen": 1770258432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682246740220662, + "loss": 1.3015, + "theoretical_loss": 3.4943529548418923, + "tokens_seen": 1770323968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682046138415246, + "loss": 1.4155, + "theoretical_loss": 3.4943419990226943, + "tokens_seen": 1770389504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046818455366098296, + "loss": 1.3454, + "theoretical_loss": 3.4943310437226014, + "tokens_seen": 1770455040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681644934804413, + "loss": 1.4116, + "theoretical_loss": 3.49432008894157, + "tokens_seen": 1770520576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681444332998997, + "loss": 1.3459, + "theoretical_loss": 3.4943091346795567, + "tokens_seen": 1770586112 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046812437311935805, + "loss": 1.393, + "theoretical_loss": 3.494298180936518, + "tokens_seen": 1770651648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810431293881646, + "loss": 1.4032, + "theoretical_loss": 3.4942872277124093, + "tokens_seen": 1770717184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680842527582748, + "loss": 1.4136, + "theoretical_loss": 3.494276275007187, + "tokens_seen": 1770782720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680641925777332, + "loss": 1.3419, + "theoretical_loss": 3.494265322820808, + "tokens_seen": 1770848256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046804413239719155, + "loss": 1.3964, + "theoretical_loss": 3.494254371153227, + "tokens_seen": 1770913792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802407221664997, + "loss": 1.3462, + "theoretical_loss": 3.4942434200044015, + "tokens_seen": 1770979328 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800401203610833, + "loss": 1.3377, + "theoretical_loss": 3.494232469374287, + "tokens_seen": 1771044864 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679839518555667, + "loss": 1.3578, + "theoretical_loss": 3.494221519262841, + "tokens_seen": 1771110400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046796389167502505, + "loss": 1.3526, + "theoretical_loss": 3.4942105696700185, + "tokens_seen": 1771175936 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 646815, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4491500854492188, + "objective/train/theoretical_loss": 3.4941996205957766, + "objective/train/tokens_used": 141700576, + "theoretical_loss": 3.4941996205957766, + "tokens_seen": 1771241472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679438314944834, + "loss": 1.4018, + "theoretical_loss": 3.4941996205957766, + "tokens_seen": 1771241472 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046792377131394183, + "loss": 1.3682, + "theoretical_loss": 3.4941886720400706, + "tokens_seen": 1771307008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679037111334002, + "loss": 1.3883, + "theoretical_loss": 3.4941777240028573, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046788365095285856, + "loss": 1.3136, + "theoretical_loss": 3.494166776484093, + "tokens_seen": 1771438080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678635907723169, + "loss": 1.3225, + "theoretical_loss": 3.494155829483734, + "tokens_seen": 1771503616 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046784353059177534, + "loss": 1.4145, + "theoretical_loss": 3.494144883001736, + "tokens_seen": 1771569152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678234704112337, + "loss": 1.3247, + "theoretical_loss": 3.4941339370380566, + "tokens_seen": 1771634688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046780341023069206, + "loss": 1.307, + "theoretical_loss": 3.4941229915926506, + "tokens_seen": 1771700224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677833500501504, + "loss": 1.4132, + "theoretical_loss": 3.4941120466654745, + "tokens_seen": 1771765760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677632898696088, + "loss": 1.4054, + "theoretical_loss": 3.4941011022564856, + "tokens_seen": 1771831296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677432296890672, + "loss": 1.3943, + "theoretical_loss": 3.4940901583656396, + "tokens_seen": 1771896832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046772316950852556, + "loss": 1.4279, + "theoretical_loss": 3.494079214992893, + "tokens_seen": 1771962368 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467703109327984, + "loss": 1.2983, + "theoretical_loss": 3.4940682721382017, + "tokens_seen": 1772027904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046768304914744234, + "loss": 1.2707, + "theoretical_loss": 3.4940573298015223, + "tokens_seen": 1772093440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046766298896690076, + "loss": 1.3194, + "theoretical_loss": 3.494046387982811, + "tokens_seen": 1772158976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676429287863591, + "loss": 1.4656, + "theoretical_loss": 3.494035446682024, + "tokens_seen": 1772224512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676228686058175, + "loss": 1.493, + "theoretical_loss": 3.494024505899118, + "tokens_seen": 1772290048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046760280842527585, + "loss": 1.3873, + "theoretical_loss": 3.4940135656340496, + "tokens_seen": 1772355584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675827482447342, + "loss": 1.3803, + "theoretical_loss": 3.4940026258867745, + "tokens_seen": 1772421120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675626880641926, + "loss": 1.4669, + "theoretical_loss": 3.4939916866572487, + "tokens_seen": 1772486656 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467542627883651, + "loss": 1.3403, + "theoretical_loss": 3.4939807479454297, + "tokens_seen": 1772552192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046752256770310935, + "loss": 1.353, + "theoretical_loss": 3.493969809751273, + "tokens_seen": 1772617728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675025075225677, + "loss": 1.3699, + "theoretical_loss": 3.4939588720747357, + "tokens_seen": 1772683264 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046748244734202613, + "loss": 1.5135, + "theoretical_loss": 3.493947934915773, + "tokens_seen": 1772748800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674623871614845, + "loss": 1.2624, + "theoretical_loss": 3.493936998274343, + "tokens_seen": 1772814336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046744232698094285, + "loss": 1.3377, + "theoretical_loss": 3.4939260621504005, + "tokens_seen": 1772879872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674222668004012, + "loss": 1.3574, + "theoretical_loss": 3.493915126543902, + "tokens_seen": 1772945408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046740220661985963, + "loss": 1.3531, + "theoretical_loss": 3.493904191454805, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467382146439318, + "loss": 1.3937, + "theoretical_loss": 3.493893256883065, + "tokens_seen": 1773076480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046736208625877636, + "loss": 1.4041, + "theoretical_loss": 3.4938823228286386, + "tokens_seen": 1773142016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673420260782347, + "loss": 1.3208, + "theoretical_loss": 3.493871389291482, + "tokens_seen": 1773207552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673219658976931, + "loss": 1.3034, + "theoretical_loss": 3.4938604562715527, + "tokens_seen": 1773273088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673019057171515, + "loss": 1.4102, + "theoretical_loss": 3.4938495237688056, + "tokens_seen": 1773338624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046728184553660986, + "loss": 1.4702, + "theoretical_loss": 3.4938385917831978, + "tokens_seen": 1773404160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672617853560682, + "loss": 1.3546, + "theoretical_loss": 3.493827660314686, + "tokens_seen": 1773469696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672417251755266, + "loss": 1.4124, + "theoretical_loss": 3.4938167293632265, + "tokens_seen": 1773535232 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467221664994985, + "loss": 1.345, + "theoretical_loss": 3.493805798928775, + "tokens_seen": 1773600768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046720160481444336, + "loss": 1.3367, + "theoretical_loss": 3.493794869011289, + "tokens_seen": 1773666304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671815446339017, + "loss": 1.4365, + "theoretical_loss": 3.493783939610725, + "tokens_seen": 1773731840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671614844533601, + "loss": 1.518, + "theoretical_loss": 3.493773010727038, + "tokens_seen": 1773797376 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046714142427281845, + "loss": 1.5115, + "theoretical_loss": 3.493762082360186, + "tokens_seen": 1773862912 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046712136409227686, + "loss": 1.3766, + "theoretical_loss": 3.4937511545101243, + "tokens_seen": 1773928448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671013039117352, + "loss": 1.532, + "theoretical_loss": 3.4937402271768105, + "tokens_seen": 1773993984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670812437311936, + "loss": 1.3404, + "theoretical_loss": 3.493729300360201, + "tokens_seen": 1774059520 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046706118355065195, + "loss": 1.324, + "theoretical_loss": 3.4937183740602507, + "tokens_seen": 1774125056 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046704112337011037, + "loss": 1.4054, + "theoretical_loss": 3.4937074482769175, + "tokens_seen": 1774190592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046702106318956873, + "loss": 1.3717, + "theoretical_loss": 3.4936965230101578, + "tokens_seen": 1774256128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670010030090271, + "loss": 1.3153, + "theoretical_loss": 3.493685598259928, + "tokens_seen": 1774321664 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046698094282848545, + "loss": 1.3947, + "theoretical_loss": 3.4936746740261846, + "tokens_seen": 1774387200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669608826479438, + "loss": 1.4154, + "theoretical_loss": 3.4936637503088837, + "tokens_seen": 1774452736 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 647978, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3683605194091797, + "objective/train/theoretical_loss": 3.493652827107982, + "objective/train/tokens_used": 144977376, + "theoretical_loss": 3.493652827107982, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694082246740223, + "loss": 1.393, + "theoretical_loss": 3.493652827107982, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669207622868606, + "loss": 1.3967, + "theoretical_loss": 3.493641904423437, + "tokens_seen": 1774583808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046690070210631896, + "loss": 1.3406, + "theoretical_loss": 3.493630982255203, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668806419257773, + "loss": 1.2683, + "theoretical_loss": 3.493620060603239, + "tokens_seen": 1774714880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046686058174523574, + "loss": 1.2875, + "theoretical_loss": 3.4936091394674995, + "tokens_seen": 1774780416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668405215646941, + "loss": 1.4032, + "theoretical_loss": 3.4935982188479424, + "tokens_seen": 1774845952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682046138415246, + "loss": 1.4913, + "theoretical_loss": 3.493587298744524, + "tokens_seen": 1774911488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668004012036108, + "loss": 1.3395, + "theoretical_loss": 3.4935763791572003, + "tokens_seen": 1774977024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667803410230692, + "loss": 1.3878, + "theoretical_loss": 3.4935654600859283, + "tokens_seen": 1775042560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667602808425276, + "loss": 1.3406, + "theoretical_loss": 3.4935545415306644, + "tokens_seen": 1775108096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046674022066198596, + "loss": 1.402, + "theoretical_loss": 3.4935436234913655, + "tokens_seen": 1775173632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667201604814443, + "loss": 1.3797, + "theoretical_loss": 3.4935327059679873, + "tokens_seen": 1775239168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667001003009027, + "loss": 1.2969, + "theoretical_loss": 3.4935217889604875, + "tokens_seen": 1775304704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666800401203611, + "loss": 1.3547, + "theoretical_loss": 3.493510872468822, + "tokens_seen": 1775370240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046665997993981947, + "loss": 1.3587, + "theoretical_loss": 3.493499956492948, + "tokens_seen": 1775435776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046663991975927783, + "loss": 1.3586, + "theoretical_loss": 3.493489041032821, + "tokens_seen": 1775501312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666198595787362, + "loss": 1.409, + "theoretical_loss": 3.493478126088398, + "tokens_seen": 1775566848 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046659979939819455, + "loss": 1.4863, + "theoretical_loss": 3.4934672116596364, + "tokens_seen": 1775632384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046657973921765297, + "loss": 1.3135, + "theoretical_loss": 3.4934562977464925, + "tokens_seen": 1775697920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046655967903711133, + "loss": 1.3717, + "theoretical_loss": 3.493445384348922, + "tokens_seen": 1775763456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665396188565697, + "loss": 1.4024, + "theoretical_loss": 3.493434471466882, + "tokens_seen": 1775828992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046651955867602806, + "loss": 1.3746, + "theoretical_loss": 3.49342355910033, + "tokens_seen": 1775894528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664994984954865, + "loss": 1.5006, + "theoretical_loss": 3.4934126472492215, + "tokens_seen": 1775960064 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046647943831494484, + "loss": 1.4287, + "theoretical_loss": 3.4934017359135137, + "tokens_seen": 1776025600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664593781344032, + "loss": 1.3496, + "theoretical_loss": 3.4933908250931633, + "tokens_seen": 1776091136 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046643931795386156, + "loss": 1.3728, + "theoretical_loss": 3.493379914788126, + "tokens_seen": 1776156672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664192577733199, + "loss": 1.4248, + "theoretical_loss": 3.49336900499836, + "tokens_seen": 1776222208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046639919759277834, + "loss": 1.4654, + "theoretical_loss": 3.4933580957238206, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663791374122367, + "loss": 1.2819, + "theoretical_loss": 3.4933471869644652, + "tokens_seen": 1776353280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046635907723169506, + "loss": 1.3475, + "theoretical_loss": 3.4933362787202507, + "tokens_seen": 1776418816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663390170511534, + "loss": 1.3553, + "theoretical_loss": 3.4933253709911325, + "tokens_seen": 1776484352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046631895687061184, + "loss": 1.4399, + "theoretical_loss": 3.493314463777069, + "tokens_seen": 1776549888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662988966900702, + "loss": 1.2961, + "theoretical_loss": 3.493303557078015, + "tokens_seen": 1776615424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046627883650952857, + "loss": 1.4466, + "theoretical_loss": 3.493292650893929, + "tokens_seen": 1776680960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046625877632898693, + "loss": 1.4076, + "theoretical_loss": 3.4932817452247664, + "tokens_seen": 1776746496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046623871614844535, + "loss": 1.3671, + "theoretical_loss": 3.493270840070484, + "tokens_seen": 1776812032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662186559679037, + "loss": 1.2753, + "theoretical_loss": 3.4932599354310403, + "tokens_seen": 1776877568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661985957873621, + "loss": 1.3254, + "theoretical_loss": 3.493249031306389, + "tokens_seen": 1776943104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661785356068205, + "loss": 1.336, + "theoretical_loss": 3.4932381276964897, + "tokens_seen": 1777008640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046615847542627885, + "loss": 1.3294, + "theoretical_loss": 3.4932272246012968, + "tokens_seen": 1777074176 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046613841524573727, + "loss": 1.4117, + "theoretical_loss": 3.4932163220207686, + "tokens_seen": 1777139712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046611835506519563, + "loss": 1.3507, + "theoretical_loss": 3.493205419954861, + "tokens_seen": 1777205248 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466098294884654, + "loss": 1.3653, + "theoretical_loss": 3.4931945184035307, + "tokens_seen": 1777270784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046607823470411235, + "loss": 1.317, + "theoretical_loss": 3.493183617366735, + "tokens_seen": 1777336320 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046605817452357077, + "loss": 1.4495, + "theoretical_loss": 3.4931727168444304, + "tokens_seen": 1777401856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046603811434302913, + "loss": 1.398, + "theoretical_loss": 3.4931618168365737, + "tokens_seen": 1777467392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660180541624875, + "loss": 1.387, + "theoretical_loss": 3.4931509173431214, + "tokens_seen": 1777532928 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046599799398194586, + "loss": 1.4403, + "theoretical_loss": 3.4931400183640307, + "tokens_seen": 1777598464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659779338014042, + "loss": 1.4114, + "theoretical_loss": 3.4931291198992582, + "tokens_seen": 1777664000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595787362086263, + "loss": 1.4083, + "theoretical_loss": 3.49311822194876, + "tokens_seen": 1777729536 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 649180, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1798685789108276, + "objective/train/theoretical_loss": 3.493107324512494, + "objective/train/tokens_used": 148254176, + "theoretical_loss": 3.493107324512494, + "tokens_seen": 1777795072 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465937813440321, + "loss": 1.2022, + "theoretical_loss": 3.493107324512494, + "tokens_seen": 1777795072 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591775325977936, + "loss": 1.3655, + "theoretical_loss": 3.493096427590416, + "tokens_seen": 1777860608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658976930792377, + "loss": 1.3358, + "theoretical_loss": 3.4930855311824835, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046587763289869614, + "loss": 1.3148, + "theoretical_loss": 3.493074635288653, + "tokens_seen": 1777991680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658575727181545, + "loss": 1.407, + "theoretical_loss": 3.493063739908881, + "tokens_seen": 1778057216 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046583751253761286, + "loss": 1.3564, + "theoretical_loss": 3.493052845043125, + "tokens_seen": 1778122752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658174523570712, + "loss": 1.3304, + "theoretical_loss": 3.493041950691341, + "tokens_seen": 1778188288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657973921765296, + "loss": 1.3414, + "theoretical_loss": 3.4930310568534866, + "tokens_seen": 1778253824 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465777331995988, + "loss": 1.3462, + "theoretical_loss": 3.4930201635295175, + "tokens_seen": 1778319360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575727181544636, + "loss": 1.3212, + "theoretical_loss": 3.493009270719392, + "tokens_seen": 1778384896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657372116349047, + "loss": 1.4643, + "theoretical_loss": 3.4929983784230663, + "tokens_seen": 1778450432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171514543631, + "loss": 1.2515, + "theoretical_loss": 3.492987486640496, + "tokens_seen": 1778515968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656970912738215, + "loss": 1.3084, + "theoretical_loss": 3.4929765953716405, + "tokens_seen": 1778581504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046567703109327987, + "loss": 1.3553, + "theoretical_loss": 3.4929657046164544, + "tokens_seen": 1778647040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565697091273823, + "loss": 1.3112, + "theoretical_loss": 3.4929548143748956, + "tokens_seen": 1778712576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656369107321966, + "loss": 1.2687, + "theoretical_loss": 3.4929439246469203, + "tokens_seen": 1778778112 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046561685055165495, + "loss": 1.325, + "theoretical_loss": 3.492933035432486, + "tokens_seen": 1778843648 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046559679037111337, + "loss": 1.3414, + "theoretical_loss": 3.492922146731549, + "tokens_seen": 1778909184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046557673019057173, + "loss": 1.3544, + "theoretical_loss": 3.4929112585440674, + "tokens_seen": 1778974720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655566700100301, + "loss": 1.2733, + "theoretical_loss": 3.492900370869996, + "tokens_seen": 1779040256 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553660982948846, + "loss": 1.3441, + "theoretical_loss": 3.492889483709294, + "tokens_seen": 1779105792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655165496489469, + "loss": 1.3427, + "theoretical_loss": 3.4928785970619165, + "tokens_seen": 1779171328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046549648946840524, + "loss": 1.3447, + "theoretical_loss": 3.492867710927821, + "tokens_seen": 1779236864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654764292878636, + "loss": 1.1811, + "theoretical_loss": 3.4928568253069643, + "tokens_seen": 1779302400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046545636910732196, + "loss": 1.3992, + "theoretical_loss": 3.4928459401993033, + "tokens_seen": 1779367936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654363089267803, + "loss": 1.3751, + "theoretical_loss": 3.492835055604796, + "tokens_seen": 1779433472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046541624874623874, + "loss": 1.3041, + "theoretical_loss": 3.492824171523397, + "tokens_seen": 1779499008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653961885656971, + "loss": 1.4208, + "theoretical_loss": 3.492813287955065, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046537612838515546, + "loss": 1.2894, + "theoretical_loss": 3.4928024048997566, + "tokens_seen": 1779630080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653560682046138, + "loss": 1.4283, + "theoretical_loss": 3.492791522357429, + "tokens_seen": 1779695616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533600802407224, + "loss": 1.254, + "theoretical_loss": 3.4927806403280384, + "tokens_seen": 1779761152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653159478435306, + "loss": 1.2984, + "theoretical_loss": 3.4927697588115416, + "tokens_seen": 1779826688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046529588766298897, + "loss": 1.3202, + "theoretical_loss": 3.492758877807897, + "tokens_seen": 1779892224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046527582748244733, + "loss": 1.3428, + "theoretical_loss": 3.4927479973170596, + "tokens_seen": 1779957760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525576730190575, + "loss": 1.5392, + "theoretical_loss": 3.4927371173389874, + "tokens_seen": 1780023296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652357071213641, + "loss": 1.4185, + "theoretical_loss": 3.4927262378736375, + "tokens_seen": 1780088832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046521564694082247, + "loss": 1.3486, + "theoretical_loss": 3.4927153589209667, + "tokens_seen": 1780154368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519558676028083, + "loss": 1.4041, + "theoretical_loss": 3.492704480480932, + "tokens_seen": 1780219904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651755265797392, + "loss": 1.3262, + "theoretical_loss": 3.49269360255349, + "tokens_seen": 1780285440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651554663991976, + "loss": 1.3214, + "theoretical_loss": 3.492682725138598, + "tokens_seen": 1780350976 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465135406218656, + "loss": 1.3432, + "theoretical_loss": 3.492671848236213, + "tokens_seen": 1780416512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046511534603811434, + "loss": 1.2627, + "theoretical_loss": 3.4926609718462918, + "tokens_seen": 1780482048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650952858575727, + "loss": 1.3192, + "theoretical_loss": 3.4926500959687914, + "tokens_seen": 1780547584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650752256770311, + "loss": 1.4248, + "theoretical_loss": 3.492639220603669, + "tokens_seen": 1780613120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650551654964895, + "loss": 1.4194, + "theoretical_loss": 3.4926283457508815, + "tokens_seen": 1780678656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046503510531594784, + "loss": 1.4209, + "theoretical_loss": 3.4926174714103864, + "tokens_seen": 1780744192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650150451354062, + "loss": 1.4346, + "theoretical_loss": 3.4926065975821396, + "tokens_seen": 1780809728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046499498495486456, + "loss": 1.242, + "theoretical_loss": 3.4925957242660988, + "tokens_seen": 1780875264 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464974924774323, + "loss": 1.3773, + "theoretical_loss": 3.4925848514622215, + "tokens_seen": 1780940800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046495486459378134, + "loss": 1.3099, + "theoretical_loss": 3.4925739791704635, + "tokens_seen": 1781006336 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 650314, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2257611751556396, + "objective/train/theoretical_loss": 3.492563107390783, + "objective/train/tokens_used": 151530976, + "theoretical_loss": 3.492563107390783, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649348044132397, + "loss": 1.4112, + "theoretical_loss": 3.492563107390783, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046491474423269807, + "loss": 1.4127, + "theoretical_loss": 3.4925522361231365, + "tokens_seen": 1781137408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648946840521565, + "loss": 1.328, + "theoretical_loss": 3.4925413653674813, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046487462387161485, + "loss": 1.3604, + "theoretical_loss": 3.4925304951237734, + "tokens_seen": 1781268480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648545636910732, + "loss": 1.4498, + "theoretical_loss": 3.4925196253919717, + "tokens_seen": 1781334016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046483450351053157, + "loss": 1.563, + "theoretical_loss": 3.4925087561720325, + "tokens_seen": 1781399552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046481444332998993, + "loss": 1.3441, + "theoretical_loss": 3.4924978874639114, + "tokens_seen": 1781465088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046479438314944835, + "loss": 1.3977, + "theoretical_loss": 3.4924870192675677, + "tokens_seen": 1781530624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647743229689067, + "loss": 1.3104, + "theoretical_loss": 3.4924761515829577, + "tokens_seen": 1781596160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647542627883651, + "loss": 1.3729, + "theoretical_loss": 3.492465284410038, + "tokens_seen": 1781661696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046473420260782344, + "loss": 1.4054, + "theoretical_loss": 3.4924544177487657, + "tokens_seen": 1781727232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647141424272819, + "loss": 1.2698, + "theoretical_loss": 3.4924435515990986, + "tokens_seen": 1781792768 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046469408224674027, + "loss": 1.3376, + "theoretical_loss": 3.492432685960993, + "tokens_seen": 1781858304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046467402206619863, + "loss": 1.4646, + "theoretical_loss": 3.492421820834406, + "tokens_seen": 1781923840 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464653961885657, + "loss": 1.4831, + "theoretical_loss": 3.492410956219296, + "tokens_seen": 1781989376 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046463390170511535, + "loss": 1.4281, + "theoretical_loss": 3.492400092115619, + "tokens_seen": 1782054912 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046461384152457377, + "loss": 1.3922, + "theoretical_loss": 3.492389228523332, + "tokens_seen": 1782120448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046459378134403213, + "loss": 1.3606, + "theoretical_loss": 3.4923783654423923, + "tokens_seen": 1782185984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645737211634905, + "loss": 1.367, + "theoretical_loss": 3.4923675028727574, + "tokens_seen": 1782251520 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046455366098294886, + "loss": 1.3624, + "theoretical_loss": 3.4923566408143842, + "tokens_seen": 1782317056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645336008024073, + "loss": 1.3743, + "theoretical_loss": 3.49234577926723, + "tokens_seen": 1782382592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046451354062186564, + "loss": 1.3936, + "theoretical_loss": 3.4923349182312515, + "tokens_seen": 1782448128 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464493480441324, + "loss": 1.4711, + "theoretical_loss": 3.4923240577064067, + "tokens_seen": 1782513664 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046447342026078236, + "loss": 1.3881, + "theoretical_loss": 3.4923131976926514, + "tokens_seen": 1782579200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644533600802407, + "loss": 1.347, + "theoretical_loss": 3.492302338189944, + "tokens_seen": 1782644736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046443329989969914, + "loss": 1.4536, + "theoretical_loss": 3.492291479198241, + "tokens_seen": 1782710272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644132397191575, + "loss": 1.2682, + "theoretical_loss": 3.4922806207174997, + "tokens_seen": 1782775808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046439317953861586, + "loss": 1.2763, + "theoretical_loss": 3.4922697627476778, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643731193580742, + "loss": 1.3557, + "theoretical_loss": 3.4922589052887316, + "tokens_seen": 1782906880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046435305917753264, + "loss": 1.2785, + "theoretical_loss": 3.492248048340619, + "tokens_seen": 1782972416 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464332998996991, + "loss": 1.4066, + "theoretical_loss": 3.492237191903297, + "tokens_seen": 1783037952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046431293881644937, + "loss": 1.2322, + "theoretical_loss": 3.4922263359767225, + "tokens_seen": 1783103488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046429287863590773, + "loss": 1.3669, + "theoretical_loss": 3.492215480560853, + "tokens_seen": 1783169024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046427281845536615, + "loss": 1.3803, + "theoretical_loss": 3.4922046256556456, + "tokens_seen": 1783234560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642527582748245, + "loss": 1.3788, + "theoretical_loss": 3.4921937712610576, + "tokens_seen": 1783300096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046423269809428287, + "loss": 1.3734, + "theoretical_loss": 3.492182917377046, + "tokens_seen": 1783365632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046421263791374123, + "loss": 1.3093, + "theoretical_loss": 3.492172064003568, + "tokens_seen": 1783431168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641925777331996, + "loss": 1.4635, + "theoretical_loss": 3.492161211140581, + "tokens_seen": 1783496704 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464172517552658, + "loss": 1.2471, + "theoretical_loss": 3.4921503587880425, + "tokens_seen": 1783562240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641524573721164, + "loss": 1.3886, + "theoretical_loss": 3.4921395069459096, + "tokens_seen": 1783627776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046413239719157474, + "loss": 1.3439, + "theoretical_loss": 3.492128655614139, + "tokens_seen": 1783693312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641123370110331, + "loss": 1.507, + "theoretical_loss": 3.492117804792689, + "tokens_seen": 1783758848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640922768304915, + "loss": 1.3927, + "theoretical_loss": 3.4921069544815153, + "tokens_seen": 1783824384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640722166499499, + "loss": 1.3649, + "theoretical_loss": 3.4920961046805763, + "tokens_seen": 1783889920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046405215646940824, + "loss": 1.3722, + "theoretical_loss": 3.4920852553898296, + "tokens_seen": 1783955456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640320962888666, + "loss": 1.4852, + "theoretical_loss": 3.4920744066092313, + "tokens_seen": 1784020992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046401203610832496, + "loss": 1.4986, + "theoretical_loss": 3.4920635583387396, + "tokens_seen": 1784086528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639919759277834, + "loss": 1.348, + "theoretical_loss": 3.492052710578311, + "tokens_seen": 1784152064 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046397191574724174, + "loss": 1.3796, + "theoretical_loss": 3.492041863327904, + "tokens_seen": 1784217600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639518555667001, + "loss": 1.3677, + "theoretical_loss": 3.4920310165874744, + "tokens_seen": 1784283136 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 651558, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.542694091796875, + "objective/train/theoretical_loss": 3.49202017035698, + "objective/train/tokens_used": 154807776, + "theoretical_loss": 3.49202017035698, + "tokens_seen": 1784348672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393179538615847, + "loss": 1.3991, + "theoretical_loss": 3.49202017035698, + "tokens_seen": 1784348672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639117352056169, + "loss": 1.393, + "theoretical_loss": 3.492009324636379, + "tokens_seen": 1784414208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389167502507525, + "loss": 1.4695, + "theoretical_loss": 3.491998479425628, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638716148445336, + "loss": 1.4816, + "theoretical_loss": 3.491987634724684, + "tokens_seen": 1784545280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385155466399197, + "loss": 1.4259, + "theoretical_loss": 3.4919767905335046, + "tokens_seen": 1784610816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046383149448345033, + "loss": 1.3959, + "theoretical_loss": 3.491965946852047, + "tokens_seen": 1784676352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381143430290875, + "loss": 1.4587, + "theoretical_loss": 3.491955103680269, + "tokens_seen": 1784741888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637913741223671, + "loss": 1.3978, + "theoretical_loss": 3.4919442610181277, + "tokens_seen": 1784807424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637713139418255, + "loss": 1.5788, + "theoretical_loss": 3.49193341886558, + "tokens_seen": 1784872960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375125376128384, + "loss": 1.3937, + "theoretical_loss": 3.491922577222584, + "tokens_seen": 1784938496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046373119358074225, + "loss": 1.3733, + "theoretical_loss": 3.4919117360890963, + "tokens_seen": 1785004032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637111334002006, + "loss": 1.4229, + "theoretical_loss": 3.4919008954650748, + "tokens_seen": 1785069568 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463691073219659, + "loss": 1.5605, + "theoretical_loss": 3.491890055350477, + "tokens_seen": 1785135104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046367101303911734, + "loss": 1.4093, + "theoretical_loss": 3.491879215745259, + "tokens_seen": 1785200640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636509528585757, + "loss": 1.4728, + "theoretical_loss": 3.4918683766493794, + "tokens_seen": 1785266176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636308926780341, + "loss": 1.3199, + "theoretical_loss": 3.491857538062795, + "tokens_seen": 1785331712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636108324974925, + "loss": 1.3595, + "theoretical_loss": 3.491846699985464, + "tokens_seen": 1785397248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359077231695084, + "loss": 1.4362, + "theoretical_loss": 3.491835862417343, + "tokens_seen": 1785462784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635707121364092, + "loss": 1.4483, + "theoretical_loss": 3.491825025358389, + "tokens_seen": 1785528320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635506519558676, + "loss": 1.3289, + "theoretical_loss": 3.4918141888085605, + "tokens_seen": 1785593856 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463530591775326, + "loss": 1.5213, + "theoretical_loss": 3.4918033527678145, + "tokens_seen": 1785659392 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046351053159478434, + "loss": 1.2615, + "theoretical_loss": 3.491792517236108, + "tokens_seen": 1785724928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634904714142427, + "loss": 1.3449, + "theoretical_loss": 3.4917816822133982, + "tokens_seen": 1785790464 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046347041123370107, + "loss": 1.3547, + "theoretical_loss": 3.4917708476996436, + "tokens_seen": 1785856000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634503510531595, + "loss": 1.3417, + "theoretical_loss": 3.491760013694801, + "tokens_seen": 1785921536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343029087261785, + "loss": 1.4067, + "theoretical_loss": 3.491749180198828, + "tokens_seen": 1785987072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634102306920762, + "loss": 1.347, + "theoretical_loss": 3.4917383472116814, + "tokens_seen": 1786052608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046339017051153457, + "loss": 1.3094, + "theoretical_loss": 3.491727514733319, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463370110330993, + "loss": 1.2779, + "theoretical_loss": 3.4917166827636987, + "tokens_seen": 1786183680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046335005015045135, + "loss": 1.409, + "theoretical_loss": 3.491705851302777, + "tokens_seen": 1786249216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633299899699097, + "loss": 1.3644, + "theoretical_loss": 3.4916950203505124, + "tokens_seen": 1786314752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633099297893681, + "loss": 1.3594, + "theoretical_loss": 3.491684189906862, + "tokens_seen": 1786380288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046328986960882644, + "loss": 1.3757, + "theoretical_loss": 3.4916733599717826, + "tokens_seen": 1786445824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046326980942828485, + "loss": 1.3782, + "theoretical_loss": 3.4916625305452325, + "tokens_seen": 1786511360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632497492477432, + "loss": 1.2859, + "theoretical_loss": 3.491651701627169, + "tokens_seen": 1786576896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632296890672016, + "loss": 1.5382, + "theoretical_loss": 3.4916408732175492, + "tokens_seen": 1786642432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046320962888666, + "loss": 1.3755, + "theoretical_loss": 3.491630045316331, + "tokens_seen": 1786707968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631895687061184, + "loss": 1.4508, + "theoretical_loss": 3.4916192179234713, + "tokens_seen": 1786773504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631695085255768, + "loss": 1.5339, + "theoretical_loss": 3.491608391038928, + "tokens_seen": 1786839040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046314944834503514, + "loss": 1.31, + "theoretical_loss": 3.4915975646626594, + "tokens_seen": 1786904576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631293881644935, + "loss": 1.3931, + "theoretical_loss": 3.491586738794621, + "tokens_seen": 1786970112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631093279839519, + "loss": 1.3781, + "theoretical_loss": 3.491575913434772, + "tokens_seen": 1787035648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630892678034103, + "loss": 1.2669, + "theoretical_loss": 3.4915650885830694, + "tokens_seen": 1787101184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046306920762286864, + "loss": 1.3142, + "theoretical_loss": 3.4915542642394706, + "tokens_seen": 1787166720 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463049147442327, + "loss": 1.4238, + "theoretical_loss": 3.4915434404039334, + "tokens_seen": 1787232256 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046302908726178536, + "loss": 1.2684, + "theoretical_loss": 3.491532617076415, + "tokens_seen": 1787297792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630090270812438, + "loss": 1.4976, + "theoretical_loss": 3.491521794256873, + "tokens_seen": 1787363328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298896690070214, + "loss": 1.3864, + "theoretical_loss": 3.4915109719452646, + "tokens_seen": 1787428864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629689067201605, + "loss": 1.3209, + "theoretical_loss": 3.4915001501415484, + "tokens_seen": 1787494400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294884653961887, + "loss": 1.4777, + "theoretical_loss": 3.491489328845681, + "tokens_seen": 1787559936 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 652772, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3689403533935547, + "objective/train/theoretical_loss": 3.4914785080576207, + "objective/train/tokens_used": 158084576, + "theoretical_loss": 3.4914785080576207, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629287863590773, + "loss": 1.4206, + "theoretical_loss": 3.4914785080576207, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046290872617853565, + "loss": 1.3574, + "theoretical_loss": 3.491467687777324, + "tokens_seen": 1787691008 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462888665997994, + "loss": 1.3886, + "theoretical_loss": 3.4914568680047493, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046286860581745237, + "loss": 1.4557, + "theoretical_loss": 3.4914460487398538, + "tokens_seen": 1787822080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284854563691073, + "loss": 1.3635, + "theoretical_loss": 3.491435229982595, + "tokens_seen": 1787887616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046282848545636915, + "loss": 1.4842, + "theoretical_loss": 3.4914244117329307, + "tokens_seen": 1787953152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628084252758275, + "loss": 1.392, + "theoretical_loss": 3.491413593990819, + "tokens_seen": 1788018688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627883650952859, + "loss": 1.3914, + "theoretical_loss": 3.4914027767562166, + "tokens_seen": 1788084224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046276830491474424, + "loss": 1.3906, + "theoretical_loss": 3.4913919600290813, + "tokens_seen": 1788149760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274824473420265, + "loss": 1.2698, + "theoretical_loss": 3.491381143809371, + "tokens_seen": 1788215296 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462728184553661, + "loss": 1.3993, + "theoretical_loss": 3.4913703280970427, + "tokens_seen": 1788280832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627081243731194, + "loss": 1.4036, + "theoretical_loss": 3.4913595128920547, + "tokens_seen": 1788346368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046268806419257774, + "loss": 1.2978, + "theoretical_loss": 3.4913486981943644, + "tokens_seen": 1788411904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626680040120361, + "loss": 1.2322, + "theoretical_loss": 3.4913378840039293, + "tokens_seen": 1788477440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626479438314945, + "loss": 1.3615, + "theoretical_loss": 3.491327070320707, + "tokens_seen": 1788542976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626278836509529, + "loss": 1.3352, + "theoretical_loss": 3.491316257144655, + "tokens_seen": 1788608512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260782347041124, + "loss": 1.4185, + "theoretical_loss": 3.4913054444757314, + "tokens_seen": 1788674048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625877632898696, + "loss": 1.3946, + "theoretical_loss": 3.4912946323138936, + "tokens_seen": 1788739584 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462567703109328, + "loss": 1.2568, + "theoretical_loss": 3.491283820659099, + "tokens_seen": 1788805120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625476429287864, + "loss": 1.4322, + "theoretical_loss": 3.491273009511306, + "tokens_seen": 1788870656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252758274824475, + "loss": 1.4246, + "theoretical_loss": 3.491262198870471, + "tokens_seen": 1788936192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625075225677031, + "loss": 1.351, + "theoretical_loss": 3.491251388736553, + "tokens_seen": 1789001728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046248746238716147, + "loss": 1.357, + "theoretical_loss": 3.4912405791095082, + "tokens_seen": 1789067264 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624674022066199, + "loss": 1.3887, + "theoretical_loss": 3.491229769989296, + "tokens_seen": 1789132800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244734202607825, + "loss": 1.3465, + "theoretical_loss": 3.4912189613758726, + "tokens_seen": 1789198336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624272818455366, + "loss": 1.2618, + "theoretical_loss": 3.4912081532691963, + "tokens_seen": 1789263872 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462407221664995, + "loss": 1.3755, + "theoretical_loss": 3.491197345669225, + "tokens_seen": 1789329408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623871614844534, + "loss": 1.3202, + "theoretical_loss": 3.4911865385759158, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236710130391175, + "loss": 1.3203, + "theoretical_loss": 3.4911757319892267, + "tokens_seen": 1789460480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623470411233701, + "loss": 1.4261, + "theoretical_loss": 3.4911649259091155, + "tokens_seen": 1789526016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623269809428285, + "loss": 1.2647, + "theoretical_loss": 3.4911541203355396, + "tokens_seen": 1789591552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230692076228684, + "loss": 1.2963, + "theoretical_loss": 3.4911433152684572, + "tokens_seen": 1789657088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046228686058174525, + "loss": 1.3134, + "theoretical_loss": 3.4911325107078257, + "tokens_seen": 1789722624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622668004012036, + "loss": 1.4385, + "theoretical_loss": 3.491121706653603, + "tokens_seen": 1789788160 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462246740220662, + "loss": 1.3566, + "theoretical_loss": 3.4911109031057457, + "tokens_seen": 1789853696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046222668004012034, + "loss": 1.2555, + "theoretical_loss": 3.4911001000642132, + "tokens_seen": 1789919232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046220661985957876, + "loss": 1.4008, + "theoretical_loss": 3.491089297528963, + "tokens_seen": 1789984768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621865596790371, + "loss": 1.2429, + "theoretical_loss": 3.4910784954999516, + "tokens_seen": 1790050304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621664994984955, + "loss": 1.319, + "theoretical_loss": 3.491067693977138, + "tokens_seen": 1790115840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214643931795384, + "loss": 1.3774, + "theoretical_loss": 3.4910568929604793, + "tokens_seen": 1790181376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621263791374122, + "loss": 1.4883, + "theoretical_loss": 3.491046092449933, + "tokens_seen": 1790246912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621063189568706, + "loss": 1.5182, + "theoretical_loss": 3.4910352924454577, + "tokens_seen": 1790312448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462086258776329, + "loss": 1.4293, + "theoretical_loss": 3.4910244929470107, + "tokens_seen": 1790377984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046206619859578735, + "loss": 1.3163, + "theoretical_loss": 3.4910136939545495, + "tokens_seen": 1790443520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620461384152457, + "loss": 1.2555, + "theoretical_loss": 3.4910028954680326, + "tokens_seen": 1790509056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620260782347041, + "loss": 1.3977, + "theoretical_loss": 3.490992097487417, + "tokens_seen": 1790574592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620060180541625, + "loss": 1.4721, + "theoretical_loss": 3.490981300012661, + "tokens_seen": 1790640128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046198595787362085, + "loss": 1.2129, + "theoretical_loss": 3.490970503043722, + "tokens_seen": 1790705664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619658976930792, + "loss": 1.4094, + "theoretical_loss": 3.4909597065805587, + "tokens_seen": 1790771200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046194583751253763, + "loss": 1.4175, + "theoretical_loss": 3.4909489106231275, + "tokens_seen": 1790836736 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 653832, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5036251544952393, + "objective/train/theoretical_loss": 3.490938115171387, + "objective/train/tokens_used": 161361376, + "theoretical_loss": 3.490938115171387, + "tokens_seen": 1790902272 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461925777331996, + "loss": 1.482, + "theoretical_loss": 3.490938115171387, + "tokens_seen": 1790902272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046190571715145435, + "loss": 1.3238, + "theoretical_loss": 3.4909273202252953, + "tokens_seen": 1790967808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618856569709127, + "loss": 1.3115, + "theoretical_loss": 3.49091652578481, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618655967903711, + "loss": 1.4552, + "theoretical_loss": 3.490905731849888, + "tokens_seen": 1791098880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618455366098295, + "loss": 1.3243, + "theoretical_loss": 3.4908949384204884, + "tokens_seen": 1791164416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046182547642928786, + "loss": 1.2971, + "theoretical_loss": 3.490884145496568, + "tokens_seen": 1791229952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618054162487462, + "loss": 1.421, + "theoretical_loss": 3.4908733530780856, + "tokens_seen": 1791295488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617853560682046, + "loss": 1.2402, + "theoretical_loss": 3.490862561164999, + "tokens_seen": 1791361024 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461765295887663, + "loss": 1.3192, + "theoretical_loss": 3.490851769757265, + "tokens_seen": 1791426560 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046174523570712136, + "loss": 1.2511, + "theoretical_loss": 3.490840978854842, + "tokens_seen": 1791492096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617251755265797, + "loss": 1.2764, + "theoretical_loss": 3.4908301884576884, + "tokens_seen": 1791557632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046170511534603814, + "loss": 1.4033, + "theoretical_loss": 3.4908193985657614, + "tokens_seen": 1791623168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616850551654965, + "loss": 1.3282, + "theoretical_loss": 3.490808609179019, + "tokens_seen": 1791688704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616649949849549, + "loss": 1.4277, + "theoretical_loss": 3.490797820297419, + "tokens_seen": 1791754240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616449348044133, + "loss": 1.364, + "theoretical_loss": 3.4907870319209198, + "tokens_seen": 1791819776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046162487462387164, + "loss": 1.3934, + "theoretical_loss": 3.4907762440494787, + "tokens_seen": 1791885312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046160481444333, + "loss": 1.3087, + "theoretical_loss": 3.4907654566830537, + "tokens_seen": 1791950848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615847542627884, + "loss": 1.414, + "theoretical_loss": 3.4907546698216025, + "tokens_seen": 1792016384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615646940822468, + "loss": 1.4455, + "theoretical_loss": 3.4907438834650844, + "tokens_seen": 1792081920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046154463390170515, + "loss": 1.2248, + "theoretical_loss": 3.4907330976134547, + "tokens_seen": 1792147456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615245737211635, + "loss": 1.3332, + "theoretical_loss": 3.4907223122666737, + "tokens_seen": 1792212992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046150451354062187, + "loss": 1.5327, + "theoretical_loss": 3.490711527424698, + "tokens_seen": 1792278528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614844533600803, + "loss": 1.3328, + "theoretical_loss": 3.490700743087486, + "tokens_seen": 1792344064 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046146439317953865, + "loss": 1.3008, + "theoretical_loss": 3.4906899592549956, + "tokens_seen": 1792409600 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461444332998997, + "loss": 1.4446, + "theoretical_loss": 3.4906791759271845, + "tokens_seen": 1792475136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614242728184554, + "loss": 1.3194, + "theoretical_loss": 3.490668393104011, + "tokens_seen": 1792540672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614042126379138, + "loss": 1.2333, + "theoretical_loss": 3.4906576107854326, + "tokens_seen": 1792606208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046138415245737215, + "loss": 1.3816, + "theoretical_loss": 3.490646828971407, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613640922768305, + "loss": 1.3335, + "theoretical_loss": 3.4906360476618934, + "tokens_seen": 1792737280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613440320962889, + "loss": 1.2906, + "theoretical_loss": 3.4906252668568483, + "tokens_seen": 1792802816 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046132397191574724, + "loss": 1.3803, + "theoretical_loss": 3.4906144865562307, + "tokens_seen": 1792868352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046130391173520566, + "loss": 1.4188, + "theoretical_loss": 3.490603706759998, + "tokens_seen": 1792933888 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461283851554664, + "loss": 1.2697, + "theoretical_loss": 3.490592927468108, + "tokens_seen": 1792999424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612637913741224, + "loss": 1.3661, + "theoretical_loss": 3.4905821486805197, + "tokens_seen": 1793064960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046124373119358074, + "loss": 1.3723, + "theoretical_loss": 3.49057137039719, + "tokens_seen": 1793130496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046122367101303916, + "loss": 1.2416, + "theoretical_loss": 3.490560592618077, + "tokens_seen": 1793196032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612036108324975, + "loss": 1.3244, + "theoretical_loss": 3.4905498153431394, + "tokens_seen": 1793261568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611835506519559, + "loss": 1.4122, + "theoretical_loss": 3.4905390385723347, + "tokens_seen": 1793327104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046116349047141425, + "loss": 1.298, + "theoretical_loss": 3.49052826230562, + "tokens_seen": 1793392640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611434302908726, + "loss": 1.3589, + "theoretical_loss": 3.490517486542956, + "tokens_seen": 1793458176 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461123370110331, + "loss": 1.3116, + "theoretical_loss": 3.490506711284297, + "tokens_seen": 1793523712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611033099297894, + "loss": 1.5182, + "theoretical_loss": 3.490495936529604, + "tokens_seen": 1793589248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046108324974924775, + "loss": 1.3556, + "theoretical_loss": 3.490485162278834, + "tokens_seen": 1793654784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610631895687061, + "loss": 1.3344, + "theoretical_loss": 3.490474388531945, + "tokens_seen": 1793720320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610431293881645, + "loss": 1.4541, + "theoretical_loss": 3.4904636152888946, + "tokens_seen": 1793785856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610230692076229, + "loss": 1.3576, + "theoretical_loss": 3.4904528425496415, + "tokens_seen": 1793851392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046100300902708125, + "loss": 1.32, + "theoretical_loss": 3.490442070314143, + "tokens_seen": 1793916928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609829488465396, + "loss": 1.3711, + "theoretical_loss": 3.490431298582358, + "tokens_seen": 1793982464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046096288866599803, + "loss": 1.2296, + "theoretical_loss": 3.490420527354244, + "tokens_seen": 1794048000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609428284854564, + "loss": 1.413, + "theoretical_loss": 3.49040975662976, + "tokens_seen": 1794113536 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 655001, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.234043002128601, + "objective/train/theoretical_loss": 3.4903989864088625, + "objective/train/tokens_used": 164638176, + "theoretical_loss": 3.4903989864088625, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046092276830491475, + "loss": 1.3584, + "theoretical_loss": 3.4903989864088625, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609027081243731, + "loss": 1.3314, + "theoretical_loss": 3.4903882166915103, + "tokens_seen": 1794244608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608826479438315, + "loss": 1.2828, + "theoretical_loss": 3.4903774474776617, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608625877632899, + "loss": 1.1977, + "theoretical_loss": 3.4903666787672747, + "tokens_seen": 1794375680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084252758274826, + "loss": 1.3539, + "theoretical_loss": 3.4903559105603073, + "tokens_seen": 1794441216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608224674022066, + "loss": 1.33, + "theoretical_loss": 3.490345142856717, + "tokens_seen": 1794506752 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460802407221665, + "loss": 1.3558, + "theoretical_loss": 3.4903343756564627, + "tokens_seen": 1794572288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607823470411234, + "loss": 1.4131, + "theoretical_loss": 3.4903236089595024, + "tokens_seen": 1794637824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046076228686058176, + "loss": 1.4542, + "theoretical_loss": 3.4903128427657935, + "tokens_seen": 1794703360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607422266800401, + "loss": 1.3535, + "theoretical_loss": 3.4903020770752953, + "tokens_seen": 1794768896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607221664994985, + "loss": 1.3594, + "theoretical_loss": 3.4902913118879644, + "tokens_seen": 1794834432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070210631895685, + "loss": 1.3839, + "theoretical_loss": 3.4902805472037604, + "tokens_seen": 1794899968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046068204613841526, + "loss": 1.3381, + "theoretical_loss": 3.4902697830226406, + "tokens_seen": 1794965504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606619859578736, + "loss": 1.3986, + "theoretical_loss": 3.490259019344563, + "tokens_seen": 1795031040 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460641925777332, + "loss": 1.2497, + "theoretical_loss": 3.490248256169486, + "tokens_seen": 1795096576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046062186559679035, + "loss": 1.3083, + "theoretical_loss": 3.490237493497368, + "tokens_seen": 1795162112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060180541624877, + "loss": 1.4295, + "theoretical_loss": 3.4902267313281667, + "tokens_seen": 1795227648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046058174523570713, + "loss": 1.2517, + "theoretical_loss": 3.4902159696618402, + "tokens_seen": 1795293184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605616850551655, + "loss": 1.2911, + "theoretical_loss": 3.490205208498347, + "tokens_seen": 1795358720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054162487462385, + "loss": 1.2823, + "theoretical_loss": 3.490194447837645, + "tokens_seen": 1795424256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605215646940822, + "loss": 1.3681, + "theoretical_loss": 3.490183687679693, + "tokens_seen": 1795489792 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050150451354063, + "loss": 1.2534, + "theoretical_loss": 3.490172928024448, + "tokens_seen": 1795555328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460481444332999, + "loss": 1.2252, + "theoretical_loss": 3.490162168871869, + "tokens_seen": 1795620864 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046138415245736, + "loss": 1.2917, + "theoretical_loss": 3.4901514102219138, + "tokens_seen": 1795686400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604413239719157, + "loss": 1.3955, + "theoretical_loss": 3.490140652074541, + "tokens_seen": 1795751936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042126379137414, + "loss": 1.537, + "theoretical_loss": 3.4901298944297086, + "tokens_seen": 1795817472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604012036108325, + "loss": 1.3599, + "theoretical_loss": 3.4901191372873743, + "tokens_seen": 1795883008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046038114343029086, + "loss": 1.3454, + "theoretical_loss": 3.490108380647497, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603610832497492, + "loss": 1.4816, + "theoretical_loss": 3.4900976245100344, + "tokens_seen": 1796014080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603410230692076, + "loss": 1.3968, + "theoretical_loss": 3.4900868688749442, + "tokens_seen": 1796079616 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460320962888666, + "loss": 1.3785, + "theoretical_loss": 3.4900761137421865, + "tokens_seen": 1796145152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030090270812436, + "loss": 1.3047, + "theoretical_loss": 3.4900653591117177, + "tokens_seen": 1796210688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602808425275827, + "loss": 1.2954, + "theoretical_loss": 3.490054604983497, + "tokens_seen": 1796276224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602607823470411, + "loss": 1.2261, + "theoretical_loss": 3.4900438513574814, + "tokens_seen": 1796341760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602407221664995, + "loss": 1.3646, + "theoretical_loss": 3.490033098233631, + "tokens_seen": 1796407296 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046022066198595787, + "loss": 1.4895, + "theoretical_loss": 3.4900223456119024, + "tokens_seen": 1796472832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602006018054163, + "loss": 1.2432, + "theoretical_loss": 3.4900115934922544, + "tokens_seen": 1796538368 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046018054162487465, + "loss": 1.4947, + "theoretical_loss": 3.4900008418746458, + "tokens_seen": 1796603904 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460160481444333, + "loss": 1.365, + "theoretical_loss": 3.4899900907590338, + "tokens_seen": 1796669440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601404212637914, + "loss": 1.3777, + "theoretical_loss": 3.4899793401453776, + "tokens_seen": 1796734976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601203610832498, + "loss": 1.3186, + "theoretical_loss": 3.4899685900336346, + "tokens_seen": 1796800512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046010030090270815, + "loss": 1.2987, + "theoretical_loss": 3.489957840423764, + "tokens_seen": 1796866048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600802407221665, + "loss": 1.3973, + "theoretical_loss": 3.489947091315723, + "tokens_seen": 1796931584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046006018054162493, + "loss": 1.1846, + "theoretical_loss": 3.4899363427094707, + "tokens_seen": 1796997120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600401203610833, + "loss": 1.497, + "theoretical_loss": 3.489925594604965, + "tokens_seen": 1797062656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046002006018054165, + "loss": 1.4801, + "theoretical_loss": 3.4899148470021646, + "tokens_seen": 1797128192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 1.4469, + "theoretical_loss": 3.489904099901027, + "tokens_seen": 1797193728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045997993981945843, + "loss": 1.3574, + "theoretical_loss": 3.4898933533015115, + "tokens_seen": 1797259264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599598796389168, + "loss": 1.4101, + "theoretical_loss": 3.489882607203576, + "tokens_seen": 1797324800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993981945837515, + "loss": 1.3496, + "theoretical_loss": 3.489871861607178, + "tokens_seen": 1797390336 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 656301, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.390718936920166, + "objective/train/theoretical_loss": 3.4898611165122766, + "objective/train/tokens_used": 167914976, + "theoretical_loss": 3.4898611165122766, + "tokens_seen": 1797455872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599197592778335, + "loss": 1.4125, + "theoretical_loss": 3.4898611165122766, + "tokens_seen": 1797455872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598996990972919, + "loss": 1.4021, + "theoretical_loss": 3.4898503719188305, + "tokens_seen": 1797521408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598796389167503, + "loss": 1.4134, + "theoretical_loss": 3.489839627826797, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045985957873620866, + "loss": 1.3264, + "theoretical_loss": 3.489828884236135, + "tokens_seen": 1797652480 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459839518555667, + "loss": 1.2744, + "theoretical_loss": 3.489818141146803, + "tokens_seen": 1797718016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598194583751254, + "loss": 1.4155, + "theoretical_loss": 3.489807398558759, + "tokens_seen": 1797783552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597993981945838, + "loss": 1.3576, + "theoretical_loss": 3.4897966564719614, + "tokens_seen": 1797849088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045977933801404216, + "loss": 1.5046, + "theoretical_loss": 3.489785914886369, + "tokens_seen": 1797914624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597592778335005, + "loss": 1.3326, + "theoretical_loss": 3.489775173801939, + "tokens_seen": 1797980160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597392176529589, + "loss": 1.2927, + "theoretical_loss": 3.489764433218631, + "tokens_seen": 1798045696 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045971915747241725, + "loss": 1.4271, + "theoretical_loss": 3.489753693136402, + "tokens_seen": 1798111232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969909729187566, + "loss": 1.372, + "theoretical_loss": 3.4897429535552122, + "tokens_seen": 1798176768 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459679037111334, + "loss": 1.2677, + "theoretical_loss": 3.4897322144750182, + "tokens_seen": 1798242304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596589769307924, + "loss": 1.3884, + "theoretical_loss": 3.4897214758957795, + "tokens_seen": 1798307840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045963891675025075, + "loss": 1.4863, + "theoretical_loss": 3.489710737817454, + "tokens_seen": 1798373376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045961885656970917, + "loss": 1.3901, + "theoretical_loss": 3.48970000024, + "tokens_seen": 1798438912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045959879638916753, + "loss": 1.3629, + "theoretical_loss": 3.4896892631633762, + "tokens_seen": 1798504448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595787362086259, + "loss": 1.3527, + "theoretical_loss": 3.489678526587541, + "tokens_seen": 1798569984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045955867602808425, + "loss": 1.4832, + "theoretical_loss": 3.4896677905124527, + "tokens_seen": 1798635520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595386158475426, + "loss": 1.3014, + "theoretical_loss": 3.4896570549380694, + "tokens_seen": 1798701056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045951855566700103, + "loss": 1.3594, + "theoretical_loss": 3.4896463198643497, + "tokens_seen": 1798766592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594984954864594, + "loss": 1.4096, + "theoretical_loss": 3.4896355852912526, + "tokens_seen": 1798832128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045947843530591776, + "loss": 1.3733, + "theoretical_loss": 3.4896248512187356, + "tokens_seen": 1798897664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594583751253761, + "loss": 1.3307, + "theoretical_loss": 3.4896141176467577, + "tokens_seen": 1798963200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045943831494483454, + "loss": 1.3302, + "theoretical_loss": 3.4896033845752763, + "tokens_seen": 1799028736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594182547642929, + "loss": 1.3078, + "theoretical_loss": 3.4895926520042515, + "tokens_seen": 1799094272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045939819458375126, + "loss": 1.2989, + "theoretical_loss": 3.4895819199336406, + "tokens_seen": 1799159808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593781344032096, + "loss": 1.2548, + "theoretical_loss": 3.489571188363403, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459358074222668, + "loss": 1.3335, + "theoretical_loss": 3.4895604572934955, + "tokens_seen": 1799290880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593380140421264, + "loss": 1.3241, + "theoretical_loss": 3.489549726723878, + "tokens_seen": 1799356416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931795386158476, + "loss": 1.3731, + "theoretical_loss": 3.4895389966545087, + "tokens_seen": 1799421952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592978936810431, + "loss": 1.2981, + "theoretical_loss": 3.489528267085346, + "tokens_seen": 1799487488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592778335005015, + "loss": 1.4009, + "theoretical_loss": 3.4895175380163472, + "tokens_seen": 1799553024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592577733199599, + "loss": 1.277, + "theoretical_loss": 3.4895068094474726, + "tokens_seen": 1799618560 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923771313941827, + "loss": 1.2825, + "theoretical_loss": 3.4894960813786797, + "tokens_seen": 1799684096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045921765295887663, + "loss": 1.3936, + "theoretical_loss": 3.4894853538099273, + "tokens_seen": 1799749632 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459197592778335, + "loss": 1.337, + "theoretical_loss": 3.489474626741174, + "tokens_seen": 1799815168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045917753259779335, + "loss": 1.4326, + "theoretical_loss": 3.4894639001723773, + "tokens_seen": 1799880704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045915747241725177, + "loss": 1.3389, + "theoretical_loss": 3.4894531741034966, + "tokens_seen": 1799946240 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045913741223671013, + "loss": 1.3869, + "theoretical_loss": 3.489442448534491, + "tokens_seen": 1800011776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591173520561685, + "loss": 1.3062, + "theoretical_loss": 3.489431723465317, + "tokens_seen": 1800077312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045909729187562686, + "loss": 1.2805, + "theoretical_loss": 3.489420998895935, + "tokens_seen": 1800142848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590772316950853, + "loss": 1.334, + "theoretical_loss": 3.489410274826303, + "tokens_seen": 1800208384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045905717151454364, + "loss": 1.4646, + "theoretical_loss": 3.489399551256379, + "tokens_seen": 1800273920 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459037111334002, + "loss": 1.4175, + "theoretical_loss": 3.4893888281861223, + "tokens_seen": 1800339456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045901705115346036, + "loss": 1.5074, + "theoretical_loss": 3.489378105615491, + "tokens_seen": 1800404992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589969909729187, + "loss": 1.3638, + "theoretical_loss": 3.489367383544443, + "tokens_seen": 1800470528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045897693079237714, + "loss": 1.2953, + "theoretical_loss": 3.489356661972938, + "tokens_seen": 1800536064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589568706118355, + "loss": 1.4398, + "theoretical_loss": 3.489345940900934, + "tokens_seen": 1800601600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045893681043129386, + "loss": 1.4388, + "theoretical_loss": 3.48933522032839, + "tokens_seen": 1800667136 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 657521, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3838400840759277, + "objective/train/theoretical_loss": 3.4893245002552638, + "objective/train/tokens_used": 171191776, + "theoretical_loss": 3.4893245002552638, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589167502507522, + "loss": 1.4321, + "theoretical_loss": 3.4893245002552638, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045889669007021064, + "loss": 1.2989, + "theoretical_loss": 3.489313780681514, + "tokens_seen": 1800798208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458876629889669, + "loss": 1.3539, + "theoretical_loss": 3.4893030616070995, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045885656970912737, + "loss": 1.3443, + "theoretical_loss": 3.4892923430319795, + "tokens_seen": 1800929280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045883650952858573, + "loss": 1.3797, + "theoretical_loss": 3.4892816249561114, + "tokens_seen": 1800994816 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045881644934804415, + "loss": 1.3086, + "theoretical_loss": 3.4892709073794546, + "tokens_seen": 1801060352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587963891675025, + "loss": 1.3856, + "theoretical_loss": 3.489260190301967, + "tokens_seen": 1801125888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045877632898696087, + "loss": 1.4438, + "theoretical_loss": 3.489249473723608, + "tokens_seen": 1801191424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045875626880641923, + "loss": 1.3116, + "theoretical_loss": 3.4892387576443356, + "tokens_seen": 1801256960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587362086258776, + "loss": 1.2708, + "theoretical_loss": 3.4892280420641084, + "tokens_seen": 1801322496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458716148445336, + "loss": 1.3552, + "theoretical_loss": 3.4892173269828852, + "tokens_seen": 1801388032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586960882647944, + "loss": 1.4093, + "theoretical_loss": 3.489206612400625, + "tokens_seen": 1801453568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586760280842528, + "loss": 1.3794, + "theoretical_loss": 3.489195898317286, + "tokens_seen": 1801519104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045865596790371115, + "loss": 1.4323, + "theoretical_loss": 3.4891851847328264, + "tokens_seen": 1801584640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045863590772316957, + "loss": 1.3559, + "theoretical_loss": 3.489174471647205, + "tokens_seen": 1801650176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045861584754262793, + "loss": 1.3762, + "theoretical_loss": 3.4891637590603812, + "tokens_seen": 1801715712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585957873620863, + "loss": 1.332, + "theoretical_loss": 3.489153046972313, + "tokens_seen": 1801781248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045857572718154465, + "loss": 1.4762, + "theoretical_loss": 3.4891423353829594, + "tokens_seen": 1801846784 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458555667001003, + "loss": 1.3516, + "theoretical_loss": 3.4891316242922787, + "tokens_seen": 1801912320 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045853560682046143, + "loss": 1.4266, + "theoretical_loss": 3.489120913700229, + "tokens_seen": 1801977856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585155466399198, + "loss": 1.3399, + "theoretical_loss": 3.4891102036067703, + "tokens_seen": 1802043392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045849548645937816, + "loss": 1.3181, + "theoretical_loss": 3.4890994940118603, + "tokens_seen": 1802108928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584754262788365, + "loss": 1.3428, + "theoretical_loss": 3.4890887849154577, + "tokens_seen": 1802174464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045845536609829494, + "loss": 1.2375, + "theoretical_loss": 3.4890780763175218, + "tokens_seen": 1802240000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584353059177533, + "loss": 1.3274, + "theoretical_loss": 3.4890673682180107, + "tokens_seen": 1802305536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045841524573721166, + "loss": 1.2824, + "theoretical_loss": 3.489056660616883, + "tokens_seen": 1802371072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045839518555667, + "loss": 1.2304, + "theoretical_loss": 3.489045953514098, + "tokens_seen": 1802436608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583751253761284, + "loss": 1.4738, + "theoretical_loss": 3.489035246909614, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583550651955868, + "loss": 1.4322, + "theoretical_loss": 3.4890245408033893, + "tokens_seen": 1802567680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045833500501504516, + "loss": 1.3753, + "theoretical_loss": 3.489013835195383, + "tokens_seen": 1802633216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583149448345035, + "loss": 1.3823, + "theoretical_loss": 3.489003130085554, + "tokens_seen": 1802698752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582948846539619, + "loss": 1.3556, + "theoretical_loss": 3.488992425473861, + "tokens_seen": 1802764288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582748244734203, + "loss": 1.4368, + "theoretical_loss": 3.4889817213602625, + "tokens_seen": 1802829824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045825476429287867, + "loss": 1.3289, + "theoretical_loss": 3.488971017744717, + "tokens_seen": 1802895360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045823470411233703, + "loss": 1.2775, + "theoretical_loss": 3.4889603146271835, + "tokens_seen": 1802960896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582146439317954, + "loss": 1.437, + "theoretical_loss": 3.488949612007621, + "tokens_seen": 1803026432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045819458375125375, + "loss": 1.4029, + "theoretical_loss": 3.488938909885988, + "tokens_seen": 1803091968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045817452357071217, + "loss": 1.3429, + "theoretical_loss": 3.4889282082622426, + "tokens_seen": 1803157504 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045815446339017053, + "loss": 1.4094, + "theoretical_loss": 3.4889175071363443, + "tokens_seen": 1803223040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581344032096289, + "loss": 1.4227, + "theoretical_loss": 3.4889068065082522, + "tokens_seen": 1803288576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045811434302908726, + "loss": 1.3736, + "theoretical_loss": 3.488896106377924, + "tokens_seen": 1803354112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580942828485457, + "loss": 1.2447, + "theoretical_loss": 3.488885406745319, + "tokens_seen": 1803419648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045807422266800404, + "loss": 1.4373, + "theoretical_loss": 3.4888747076103956, + "tokens_seen": 1803485184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580541624874624, + "loss": 1.2957, + "theoretical_loss": 3.4888640089731133, + "tokens_seen": 1803550720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045803410230692076, + "loss": 1.3422, + "theoretical_loss": 3.4888533108334303, + "tokens_seen": 1803616256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580140421263791, + "loss": 1.3404, + "theoretical_loss": 3.4888426131913057, + "tokens_seen": 1803681792 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045799398194583754, + "loss": 1.2702, + "theoretical_loss": 3.4888319160466983, + "tokens_seen": 1803747328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579739217652959, + "loss": 1.3288, + "theoretical_loss": 3.488821219399566, + "tokens_seen": 1803812864 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045795386158475426, + "loss": 1.4499, + "theoretical_loss": 3.488810523249869, + "tokens_seen": 1803878400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579338014042126, + "loss": 1.4317, + "theoretical_loss": 3.488799827597565, + "tokens_seen": 1803943936 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 658552, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3652687072753906, + "objective/train/theoretical_loss": 3.4887891324426135, + "objective/train/tokens_used": 174468576, + "theoretical_loss": 3.4887891324426135, + "tokens_seen": 1804009472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045791374122367104, + "loss": 1.4031, + "theoretical_loss": 3.4887891324426135, + "tokens_seen": 1804009472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578936810431294, + "loss": 1.2971, + "theoretical_loss": 3.4887784377849727, + "tokens_seen": 1804075008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045787362086258777, + "loss": 1.3763, + "theoretical_loss": 3.488767743624602, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045785356068204613, + "loss": 1.2835, + "theoretical_loss": 3.4887570499614595, + "tokens_seen": 1804206080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783350050150455, + "loss": 1.4371, + "theoretical_loss": 3.488746356795505, + "tokens_seen": 1804271616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578134403209629, + "loss": 1.2313, + "theoretical_loss": 3.488735664126697, + "tokens_seen": 1804337152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045779338014042127, + "loss": 1.3781, + "theoretical_loss": 3.4887249719549933, + "tokens_seen": 1804402688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045777331995987963, + "loss": 1.2699, + "theoretical_loss": 3.488714280280354, + "tokens_seen": 1804468224 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457753259779338, + "loss": 1.4195, + "theoretical_loss": 3.4887035891027374, + "tokens_seen": 1804533760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577331995987964, + "loss": 1.3151, + "theoretical_loss": 3.4886928984221024, + "tokens_seen": 1804599296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577131394182548, + "loss": 1.3726, + "theoretical_loss": 3.4886822082384077, + "tokens_seen": 1804664832 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045769307923771314, + "loss": 1.4224, + "theoretical_loss": 3.4886715185516124, + "tokens_seen": 1804730368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576730190571715, + "loss": 1.3777, + "theoretical_loss": 3.4886608293616757, + "tokens_seen": 1804795904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576529588766299, + "loss": 1.2689, + "theoretical_loss": 3.4886501406685557, + "tokens_seen": 1804861440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576328986960883, + "loss": 1.2477, + "theoretical_loss": 3.4886394524722117, + "tokens_seen": 1804926976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045761283851554664, + "loss": 1.392, + "theoretical_loss": 3.488628764772603, + "tokens_seen": 1804992512 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457592778335005, + "loss": 1.2839, + "theoretical_loss": 3.4886180775696873, + "tokens_seen": 1805058048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045757271815446336, + "loss": 1.3361, + "theoretical_loss": 3.4886073908634243, + "tokens_seen": 1805123584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575526579739218, + "loss": 1.3321, + "theoretical_loss": 3.488596704653773, + "tokens_seen": 1805189120 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045753259779338014, + "loss": 1.3662, + "theoretical_loss": 3.488586018940692, + "tokens_seen": 1805254656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575125376128385, + "loss": 1.3051, + "theoretical_loss": 3.4885753337241403, + "tokens_seen": 1805320192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045749247743229687, + "loss": 1.3048, + "theoretical_loss": 3.4885646490040765, + "tokens_seen": 1805385728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574724172517553, + "loss": 1.5203, + "theoretical_loss": 3.48855396478046, + "tokens_seen": 1805451264 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045745235707121364, + "loss": 1.2229, + "theoretical_loss": 3.4885432810532495, + "tokens_seen": 1805516800 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457432296890672, + "loss": 1.4225, + "theoretical_loss": 3.4885325978224038, + "tokens_seen": 1805582336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045741223671013037, + "loss": 1.3852, + "theoretical_loss": 3.488521915087882, + "tokens_seen": 1805647872 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739217652958873, + "loss": 1.3862, + "theoretical_loss": 3.4885112328496426, + "tokens_seen": 1805713408 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045737211634904715, + "loss": 1.3808, + "theoretical_loss": 3.4885005511076455, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573520561685055, + "loss": 1.4036, + "theoretical_loss": 3.4884898698618487, + "tokens_seen": 1805844480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045733199598796387, + "loss": 1.2988, + "theoretical_loss": 3.4884791891122116, + "tokens_seen": 1805910016 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045731193580742223, + "loss": 1.5018, + "theoretical_loss": 3.488468508858693, + "tokens_seen": 1805975552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045729187562688065, + "loss": 1.2976, + "theoretical_loss": 3.488457829101252, + "tokens_seen": 1806041088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457271815446339, + "loss": 1.3444, + "theoretical_loss": 3.488447149839847, + "tokens_seen": 1806106624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572517552657974, + "loss": 1.4244, + "theoretical_loss": 3.4884364710744378, + "tokens_seen": 1806172160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045723169508525574, + "loss": 1.3512, + "theoretical_loss": 3.488425792804983, + "tokens_seen": 1806237696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572116349047141, + "loss": 1.2197, + "theoretical_loss": 3.488415115031441, + "tokens_seen": 1806303232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045719157472417257, + "loss": 1.3635, + "theoretical_loss": 3.488404437753772, + "tokens_seen": 1806368768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045717151454363093, + "loss": 1.3916, + "theoretical_loss": 3.4883937609719338, + "tokens_seen": 1806434304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571514543630893, + "loss": 1.5014, + "theoretical_loss": 3.4883830846858856, + "tokens_seen": 1806499840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045713139418254766, + "loss": 1.3277, + "theoretical_loss": 3.4883724088955876, + "tokens_seen": 1806565376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571113340020061, + "loss": 1.4299, + "theoretical_loss": 3.488361733600997, + "tokens_seen": 1806630912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045709127382146444, + "loss": 1.4107, + "theoretical_loss": 3.488351058802074, + "tokens_seen": 1806696448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570712136409228, + "loss": 1.4041, + "theoretical_loss": 3.4883403844987777, + "tokens_seen": 1806761984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045705115346038116, + "loss": 1.3683, + "theoretical_loss": 3.4883297106910662, + "tokens_seen": 1806827520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570310932798395, + "loss": 1.3187, + "theoretical_loss": 3.4883190373788997, + "tokens_seen": 1806893056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045701103309929794, + "loss": 1.3595, + "theoretical_loss": 3.4883083645622355, + "tokens_seen": 1806958592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569909729187563, + "loss": 1.3336, + "theoretical_loss": 3.4882976922410345, + "tokens_seen": 1807024128 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045697091273821466, + "loss": 1.4033, + "theoretical_loss": 3.4882870204152545, + "tokens_seen": 1807089664 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456950852557673, + "loss": 1.2869, + "theoretical_loss": 3.488276349084855, + "tokens_seen": 1807155200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045693079237713144, + "loss": 1.3787, + "theoretical_loss": 3.488265678249795, + "tokens_seen": 1807220736 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 659683, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4606060981750488, + "objective/train/theoretical_loss": 3.4882550079100336, + "objective/train/tokens_used": 177745376, + "theoretical_loss": 3.4882550079100336, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569107321965898, + "loss": 1.3264, + "theoretical_loss": 3.4882550079100336, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045689067201604817, + "loss": 1.3305, + "theoretical_loss": 3.4882443380655297, + "tokens_seen": 1807351808 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045687061183550653, + "loss": 1.4957, + "theoretical_loss": 3.4882336687162425, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045685055165496495, + "loss": 1.4547, + "theoretical_loss": 3.4882229998621304, + "tokens_seen": 1807482880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568304914744233, + "loss": 1.3334, + "theoretical_loss": 3.4882123315031537, + "tokens_seen": 1807548416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045681043129388167, + "loss": 1.4867, + "theoretical_loss": 3.4882016636392708, + "tokens_seen": 1807613952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045679037111334003, + "loss": 1.317, + "theoretical_loss": 3.4881909962704407, + "tokens_seen": 1807679488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567703109327984, + "loss": 1.3876, + "theoretical_loss": 3.4881803293966227, + "tokens_seen": 1807745024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567502507522568, + "loss": 1.3145, + "theoretical_loss": 3.488169663017775, + "tokens_seen": 1807810560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567301905717152, + "loss": 1.3685, + "theoretical_loss": 3.4881589971338585, + "tokens_seen": 1807876096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045671013039117354, + "loss": 1.2868, + "theoretical_loss": 3.4881483317448305, + "tokens_seen": 1807941632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566900702106319, + "loss": 1.3468, + "theoretical_loss": 3.488137666850651, + "tokens_seen": 1808007168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566700100300903, + "loss": 1.351, + "theoretical_loss": 3.488127002451279, + "tokens_seen": 1808072704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566499498495487, + "loss": 1.391, + "theoretical_loss": 3.488116338546674, + "tokens_seen": 1808138240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045662988966900704, + "loss": 1.4638, + "theoretical_loss": 3.488105675136794, + "tokens_seen": 1808203776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566098294884654, + "loss": 1.4516, + "theoretical_loss": 3.4880950122215992, + "tokens_seen": 1808269312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045658976930792376, + "loss": 1.281, + "theoretical_loss": 3.4880843498010483, + "tokens_seen": 1808334848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565697091273822, + "loss": 1.3846, + "theoretical_loss": 3.4880736878751, + "tokens_seen": 1808400384 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045654964894684054, + "loss": 1.314, + "theoretical_loss": 3.4880630264437142, + "tokens_seen": 1808465920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565295887662989, + "loss": 1.3936, + "theoretical_loss": 3.48805236550685, + "tokens_seen": 1808531456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045650952858575727, + "loss": 1.4567, + "theoretical_loss": 3.488041705064466, + "tokens_seen": 1808596992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564894684052157, + "loss": 1.3893, + "theoretical_loss": 3.4880310451165215, + "tokens_seen": 1808662528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045646940822467405, + "loss": 1.3522, + "theoretical_loss": 3.488020385662976, + "tokens_seen": 1808728064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564493480441324, + "loss": 1.3825, + "theoretical_loss": 3.4880097267037877, + "tokens_seen": 1808793600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045642928786359077, + "loss": 1.3196, + "theoretical_loss": 3.4879990682389175, + "tokens_seen": 1808859136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045640922768304913, + "loss": 1.2819, + "theoretical_loss": 3.487988410268323, + "tokens_seen": 1808924672 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045638916750250755, + "loss": 1.4451, + "theoretical_loss": 3.4879777527919638, + "tokens_seen": 1808990208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563691073219659, + "loss": 1.3774, + "theoretical_loss": 3.4879670958097995, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045634904714142427, + "loss": 1.3554, + "theoretical_loss": 3.4879564393217892, + "tokens_seen": 1809121280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045632898696088263, + "loss": 1.4159, + "theoretical_loss": 3.4879457833278913, + "tokens_seen": 1809186816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045630892678034105, + "loss": 1.3398, + "theoretical_loss": 3.4879351278280657, + "tokens_seen": 1809252352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562888665997994, + "loss": 1.3942, + "theoretical_loss": 3.4879244728222716, + "tokens_seen": 1809317888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562688064192578, + "loss": 1.4348, + "theoretical_loss": 3.487913818310468, + "tokens_seen": 1809383424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045624874623871614, + "loss": 1.3525, + "theoretical_loss": 3.4879031642926144, + "tokens_seen": 1809448960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562286860581745, + "loss": 1.319, + "theoretical_loss": 3.4878925107686696, + "tokens_seen": 1809514496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562086258776329, + "loss": 1.4513, + "theoretical_loss": 3.487881857738593, + "tokens_seen": 1809580032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561885656970913, + "loss": 1.3961, + "theoretical_loss": 3.4878712052023433, + "tokens_seen": 1809645568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045616850551654964, + "loss": 1.327, + "theoretical_loss": 3.487860553159881, + "tokens_seen": 1809711104 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456148445336008, + "loss": 1.3921, + "theoretical_loss": 3.487849901611164, + "tokens_seen": 1809776640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561283851554664, + "loss": 1.3918, + "theoretical_loss": 3.487839250556153, + "tokens_seen": 1809842176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561083249749248, + "loss": 1.2764, + "theoretical_loss": 3.4878285999948053, + "tokens_seen": 1809907712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045608826479438314, + "loss": 1.3769, + "theoretical_loss": 3.4878179499270816, + "tokens_seen": 1809973248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560682046138415, + "loss": 1.303, + "theoretical_loss": 3.487807300352941, + "tokens_seen": 1810038784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045604814443329987, + "loss": 1.3821, + "theoretical_loss": 3.487796651272342, + "tokens_seen": 1810104320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560280842527583, + "loss": 1.3369, + "theoretical_loss": 3.4877860026852443, + "tokens_seen": 1810169856 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045600802407221665, + "loss": 1.3372, + "theoretical_loss": 3.4877753545916077, + "tokens_seen": 1810235392 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455987963891675, + "loss": 1.4035, + "theoretical_loss": 3.487764706991391, + "tokens_seen": 1810300928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045596790371113337, + "loss": 1.3171, + "theoretical_loss": 3.487754059884553, + "tokens_seen": 1810366464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559478435305918, + "loss": 1.3921, + "theoretical_loss": 3.487743413271054, + "tokens_seen": 1810432000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045592778335005015, + "loss": 1.362, + "theoretical_loss": 3.487732767150852, + "tokens_seen": 1810497536 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 660859, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3827738761901855, + "objective/train/theoretical_loss": 3.487722121523907, + "objective/train/tokens_used": 181022176, + "theoretical_loss": 3.487722121523907, + "tokens_seen": 1810563072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559077231695085, + "loss": 1.3172, + "theoretical_loss": 3.487722121523907, + "tokens_seen": 1810563072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558876629889669, + "loss": 1.3648, + "theoretical_loss": 3.487711476390179, + "tokens_seen": 1810628608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045586760280842524, + "loss": 1.4288, + "theoretical_loss": 3.4877008317496268, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045584754262788365, + "loss": 1.4121, + "theoretical_loss": 3.4876901876022086, + "tokens_seen": 1810759680 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455827482447342, + "loss": 1.3978, + "theoretical_loss": 3.487679543947885, + "tokens_seen": 1810825216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558074222668004, + "loss": 1.4279, + "theoretical_loss": 3.4876689007866153, + "tokens_seen": 1810890752 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045578736208625874, + "loss": 1.2723, + "theoretical_loss": 3.4876582581183575, + "tokens_seen": 1810956288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045576730190571716, + "loss": 1.4098, + "theoretical_loss": 3.4876476159430725, + "tokens_seen": 1811021824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557472417251755, + "loss": 1.3927, + "theoretical_loss": 3.4876369742607194, + "tokens_seen": 1811087360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557271815446339, + "loss": 1.3357, + "theoretical_loss": 3.4876263330712565, + "tokens_seen": 1811152896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045570712136409224, + "loss": 1.4501, + "theoretical_loss": 3.4876156923746437, + "tokens_seen": 1811218432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556870611835507, + "loss": 1.3736, + "theoretical_loss": 3.487605052170841, + "tokens_seen": 1811283968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556670010030091, + "loss": 1.4069, + "theoretical_loss": 3.4875944124598064, + "tokens_seen": 1811349504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045564694082246744, + "loss": 1.3743, + "theoretical_loss": 3.487583773241501, + "tokens_seen": 1811415040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556268806419258, + "loss": 1.4264, + "theoretical_loss": 3.4875731345158822, + "tokens_seen": 1811480576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045560682046138416, + "loss": 1.4066, + "theoretical_loss": 3.4875624962829104, + "tokens_seen": 1811546112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555867602808426, + "loss": 1.3675, + "theoretical_loss": 3.4875518585425453, + "tokens_seen": 1811611648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045556670010030094, + "loss": 1.3768, + "theoretical_loss": 3.487541221294746, + "tokens_seen": 1811677184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555466399197593, + "loss": 1.3791, + "theoretical_loss": 3.4875305845394715, + "tokens_seen": 1811742720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045552657973921767, + "loss": 1.2643, + "theoretical_loss": 3.487519948276681, + "tokens_seen": 1811808256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555065195586761, + "loss": 1.443, + "theoretical_loss": 3.487509312506335, + "tokens_seen": 1811873792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045548645937813445, + "loss": 1.3575, + "theoretical_loss": 3.4874986772283916, + "tokens_seen": 1811939328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554663991975928, + "loss": 1.4148, + "theoretical_loss": 3.487488042442811, + "tokens_seen": 1812004864 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045544633901705117, + "loss": 1.3839, + "theoretical_loss": 3.4874774081495525, + "tokens_seen": 1812070400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045542627883650953, + "loss": 1.3475, + "theoretical_loss": 3.487466774348575, + "tokens_seen": 1812135936 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045540621865596795, + "loss": 1.3223, + "theoretical_loss": 3.4874561410398393, + "tokens_seen": 1812201472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553861584754263, + "loss": 1.4257, + "theoretical_loss": 3.4874455082233027, + "tokens_seen": 1812267008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553660982948847, + "loss": 1.4016, + "theoretical_loss": 3.487434875898926, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045534603811434304, + "loss": 1.2334, + "theoretical_loss": 3.4874242440666685, + "tokens_seen": 1812398080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045532597793380145, + "loss": 1.3541, + "theoretical_loss": 3.487413612726489, + "tokens_seen": 1812463616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553059177532598, + "loss": 1.3471, + "theoretical_loss": 3.4874029818783487, + "tokens_seen": 1812529152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552858575727182, + "loss": 1.3846, + "theoretical_loss": 3.4873923515222045, + "tokens_seen": 1812594688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045526579739217654, + "loss": 1.4484, + "theoretical_loss": 3.487381721658018, + "tokens_seen": 1812660224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552457372116349, + "loss": 1.3441, + "theoretical_loss": 3.487371092285747, + "tokens_seen": 1812725760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552256770310933, + "loss": 1.4135, + "theoretical_loss": 3.487360463405352, + "tokens_seen": 1812791296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552056168505517, + "loss": 1.4303, + "theoretical_loss": 3.4873498350167917, + "tokens_seen": 1812856832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045518555667001004, + "loss": 1.2703, + "theoretical_loss": 3.4873392071200264, + "tokens_seen": 1812922368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551654964894684, + "loss": 1.346, + "theoretical_loss": 3.4873285797150153, + "tokens_seen": 1812987904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551454363089268, + "loss": 1.2833, + "theoretical_loss": 3.4873179528017175, + "tokens_seen": 1813053440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551253761283852, + "loss": 1.276, + "theoretical_loss": 3.487307326380093, + "tokens_seen": 1813118976 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045510531594784354, + "loss": 1.3792, + "theoretical_loss": 3.487296700450101, + "tokens_seen": 1813184512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550852557673019, + "loss": 1.409, + "theoretical_loss": 3.487286075011701, + "tokens_seen": 1813250048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045506519558676027, + "loss": 1.3866, + "theoretical_loss": 3.4872754500648524, + "tokens_seen": 1813315584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550451354062187, + "loss": 1.3271, + "theoretical_loss": 3.4872648256095147, + "tokens_seen": 1813381120 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045502507522567705, + "loss": 1.351, + "theoretical_loss": 3.487254201645648, + "tokens_seen": 1813446656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550050150451354, + "loss": 1.3199, + "theoretical_loss": 3.4872435781732105, + "tokens_seen": 1813512192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045498495486459377, + "loss": 1.3552, + "theoretical_loss": 3.487232955192163, + "tokens_seen": 1813577728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549648946840522, + "loss": 1.3602, + "theoretical_loss": 3.487222332702464, + "tokens_seen": 1813643264 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045494483450351055, + "loss": 1.4444, + "theoretical_loss": 3.487211710704074, + "tokens_seen": 1813708800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549247743229689, + "loss": 1.2832, + "theoretical_loss": 3.487201089196952, + "tokens_seen": 1813774336 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 661946, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.294579029083252, + "objective/train/theoretical_loss": 3.4871904681810575, + "objective/train/tokens_used": 184298976, + "theoretical_loss": 3.4871904681810575, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549047141424273, + "loss": 1.2835, + "theoretical_loss": 3.4871904681810575, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045488465396188564, + "loss": 1.4588, + "theoretical_loss": 3.48717984765635, + "tokens_seen": 1813905408 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045486459378134405, + "loss": 1.4644, + "theoretical_loss": 3.4871692276227897, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548445336008024, + "loss": 1.4076, + "theoretical_loss": 3.487158608080335, + "tokens_seen": 1814036480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548244734202608, + "loss": 1.3952, + "theoretical_loss": 3.487147989028946, + "tokens_seen": 1814102016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045480441323971914, + "loss": 1.3131, + "theoretical_loss": 3.4871373704685826, + "tokens_seen": 1814167552 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045478435305917756, + "loss": 1.4066, + "theoretical_loss": 3.487126752399204, + "tokens_seen": 1814233088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547642928786359, + "loss": 1.3782, + "theoretical_loss": 3.48711613482077, + "tokens_seen": 1814298624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547442326980943, + "loss": 1.2663, + "theoretical_loss": 3.48710551773324, + "tokens_seen": 1814364160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045472417251755264, + "loss": 1.4287, + "theoretical_loss": 3.4870949011365733, + "tokens_seen": 1814429696 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454704112337011, + "loss": 1.431, + "theoretical_loss": 3.48708428503073, + "tokens_seen": 1814495232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546840521564694, + "loss": 1.4615, + "theoretical_loss": 3.487073669415669, + "tokens_seen": 1814560768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546639919759278, + "loss": 1.447, + "theoretical_loss": 3.487063054291351, + "tokens_seen": 1814626304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045464393179538615, + "loss": 1.2898, + "theoretical_loss": 3.4870524396577345, + "tokens_seen": 1814691840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546238716148445, + "loss": 1.4077, + "theoretical_loss": 3.4870418255147793, + "tokens_seen": 1814757376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546038114343029, + "loss": 1.5181, + "theoretical_loss": 3.487031211862446, + "tokens_seen": 1814822912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545837512537613, + "loss": 1.3867, + "theoretical_loss": 3.4870205987006924, + "tokens_seen": 1814888448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045456369107321965, + "loss": 1.3475, + "theoretical_loss": 3.48700998602948, + "tokens_seen": 1814953984 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454543630892678, + "loss": 1.3641, + "theoretical_loss": 3.486999373848767, + "tokens_seen": 1815019520 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045452357071213643, + "loss": 1.4215, + "theoretical_loss": 3.4869887621585134, + "tokens_seen": 1815085056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545035105315948, + "loss": 1.3982, + "theoretical_loss": 3.486978150958679, + "tokens_seen": 1815150592 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045448345035105315, + "loss": 1.4663, + "theoretical_loss": 3.486967540249224, + "tokens_seen": 1815216128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544633901705115, + "loss": 1.3237, + "theoretical_loss": 3.486956930030107, + "tokens_seen": 1815281664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544433299899699, + "loss": 1.3386, + "theoretical_loss": 3.4869463203012883, + "tokens_seen": 1815347200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544232698094283, + "loss": 1.389, + "theoretical_loss": 3.486935711062727, + "tokens_seen": 1815412736 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045440320962888666, + "loss": 1.3784, + "theoretical_loss": 3.4869251023143835, + "tokens_seen": 1815478272 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454383149448345, + "loss": 1.4512, + "theoretical_loss": 3.486914494056217, + "tokens_seen": 1815543808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543630892678034, + "loss": 1.3793, + "theoretical_loss": 3.4869038862881867, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543430290872618, + "loss": 1.3004, + "theoretical_loss": 3.4868932790102534, + "tokens_seen": 1815674880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045432296890672016, + "loss": 1.3104, + "theoretical_loss": 3.4868826722223756, + "tokens_seen": 1815740416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543029087261785, + "loss": 1.4026, + "theoretical_loss": 3.486872065924514, + "tokens_seen": 1815805952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542828485456369, + "loss": 1.3596, + "theoretical_loss": 3.486861460116627, + "tokens_seen": 1815871488 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045426278836509525, + "loss": 1.3373, + "theoretical_loss": 3.4868508547986754, + "tokens_seen": 1815937024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045424272818455366, + "loss": 1.3375, + "theoretical_loss": 3.486840249970619, + "tokens_seen": 1816002560 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454222668004012, + "loss": 1.2957, + "theoretical_loss": 3.4868296456324166, + "tokens_seen": 1816068096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045420260782347044, + "loss": 1.2682, + "theoretical_loss": 3.4868190417840284, + "tokens_seen": 1816133632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541825476429288, + "loss": 1.3165, + "theoretical_loss": 3.486808438425414, + "tokens_seen": 1816199168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541624874623872, + "loss": 1.4053, + "theoretical_loss": 3.4867978355565334, + "tokens_seen": 1816264704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541424272818456, + "loss": 1.3571, + "theoretical_loss": 3.4867872331773455, + "tokens_seen": 1816330240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045412236710130395, + "loss": 1.4024, + "theoretical_loss": 3.4867766312878112, + "tokens_seen": 1816395776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541023069207623, + "loss": 1.4612, + "theoretical_loss": 3.486766029887889, + "tokens_seen": 1816461312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045408224674022067, + "loss": 1.2546, + "theoretical_loss": 3.4867554289775398, + "tokens_seen": 1816526848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540621865596791, + "loss": 1.3132, + "theoretical_loss": 3.4867448285567226, + "tokens_seen": 1816592384 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045404212637913745, + "loss": 1.4106, + "theoretical_loss": 3.486734228625397, + "tokens_seen": 1816657920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540220661985958, + "loss": 1.3342, + "theoretical_loss": 3.486723629183523, + "tokens_seen": 1816723456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045400200601805417, + "loss": 1.4007, + "theoretical_loss": 3.4867130302310607, + "tokens_seen": 1816788992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539819458375126, + "loss": 1.3379, + "theoretical_loss": 3.486702431767969, + "tokens_seen": 1816854528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045396188565697095, + "loss": 1.3287, + "theoretical_loss": 3.4866918337942088, + "tokens_seen": 1816920064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539418254764293, + "loss": 1.3228, + "theoretical_loss": 3.486681236309739, + "tokens_seen": 1816985600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539217652958877, + "loss": 1.4388, + "theoretical_loss": 3.4866706393145197, + "tokens_seen": 1817051136 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 663068, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.466980218887329, + "objective/train/theoretical_loss": 3.48666004280851, + "objective/train/tokens_used": 187575776, + "theoretical_loss": 3.48666004280851, + "tokens_seen": 1817116672 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045390170511534604, + "loss": 1.477, + "theoretical_loss": 3.48666004280851, + "tokens_seen": 1817116672 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045388164493480445, + "loss": 1.3778, + "theoretical_loss": 3.4866494467916707, + "tokens_seen": 1817182208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538615847542628, + "loss": 1.334, + "theoretical_loss": 3.486638851263961, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538415245737212, + "loss": 1.3123, + "theoretical_loss": 3.486628256225341, + "tokens_seen": 1817313280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045382146439317954, + "loss": 1.4716, + "theoretical_loss": 3.48661766167577, + "tokens_seen": 1817378816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045380140421263796, + "loss": 1.2645, + "theoretical_loss": 3.486607067615208, + "tokens_seen": 1817444352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537813440320963, + "loss": 1.427, + "theoretical_loss": 3.486596474043615, + "tokens_seen": 1817509888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537612838515547, + "loss": 1.2429, + "theoretical_loss": 3.4865858809609507, + "tokens_seen": 1817575424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045374122367101304, + "loss": 1.3738, + "theoretical_loss": 3.486575288367175, + "tokens_seen": 1817640960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537211634904714, + "loss": 1.272, + "theoretical_loss": 3.4865646962622474, + "tokens_seen": 1817706496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537011033099298, + "loss": 1.391, + "theoretical_loss": 3.486554104646128, + "tokens_seen": 1817772032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536810431293882, + "loss": 1.3263, + "theoretical_loss": 3.486543513518776, + "tokens_seen": 1817837568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045366098294884655, + "loss": 1.2763, + "theoretical_loss": 3.4865329228801523, + "tokens_seen": 1817903104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536409227683049, + "loss": 1.2687, + "theoretical_loss": 3.486522332730216, + "tokens_seen": 1817968640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536208625877633, + "loss": 1.4156, + "theoretical_loss": 3.4865117430689274, + "tokens_seen": 1818034176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536008024072217, + "loss": 1.2561, + "theoretical_loss": 3.4865011538962456, + "tokens_seen": 1818099712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045358074222668005, + "loss": 1.3223, + "theoretical_loss": 3.486490565212131, + "tokens_seen": 1818165248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535606820461384, + "loss": 1.4152, + "theoretical_loss": 3.486479977016544, + "tokens_seen": 1818230784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045354062186559683, + "loss": 1.4437, + "theoretical_loss": 3.486469389309443, + "tokens_seen": 1818296320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535205616850552, + "loss": 1.5072, + "theoretical_loss": 3.486458802090789, + "tokens_seen": 1818361856 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045350050150451355, + "loss": 1.2958, + "theoretical_loss": 3.4864482153605416, + "tokens_seen": 1818427392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534804413239719, + "loss": 1.3456, + "theoretical_loss": 3.4864376291186607, + "tokens_seen": 1818492928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534603811434303, + "loss": 1.3086, + "theoretical_loss": 3.4864270433651052, + "tokens_seen": 1818558464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534403209628887, + "loss": 1.552, + "theoretical_loss": 3.486416458099837, + "tokens_seen": 1818624000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045342026078234706, + "loss": 1.2934, + "theoretical_loss": 3.486405873322814, + "tokens_seen": 1818689536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534002006018054, + "loss": 1.3957, + "theoretical_loss": 3.4863952890339975, + "tokens_seen": 1818755072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533801404212638, + "loss": 1.3605, + "theoretical_loss": 3.486384705233346, + "tokens_seen": 1818820608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533600802407222, + "loss": 1.388, + "theoretical_loss": 3.4863741219208206, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045334002006018056, + "loss": 1.4278, + "theoretical_loss": 3.486363539096381, + "tokens_seen": 1818951680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533199598796389, + "loss": 1.4162, + "theoretical_loss": 3.486352956759987, + "tokens_seen": 1819017216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532998996990973, + "loss": 1.4359, + "theoretical_loss": 3.4863423749115983, + "tokens_seen": 1819082752 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045327983951855565, + "loss": 1.374, + "theoretical_loss": 3.486331793551175, + "tokens_seen": 1819148288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045325977933801406, + "loss": 1.3729, + "theoretical_loss": 3.4863212126786762, + "tokens_seen": 1819213824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532397191574724, + "loss": 1.3984, + "theoretical_loss": 3.4863106322940633, + "tokens_seen": 1819279360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532196589769308, + "loss": 1.2672, + "theoretical_loss": 3.4863000523972953, + "tokens_seen": 1819344896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045319959879638915, + "loss": 1.4197, + "theoretical_loss": 3.4862894729883322, + "tokens_seen": 1819410432 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045317953861584757, + "loss": 1.3061, + "theoretical_loss": 3.4862788940671345, + "tokens_seen": 1819475968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045315947843530593, + "loss": 1.5047, + "theoretical_loss": 3.4862683156336614, + "tokens_seen": 1819541504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531394182547643, + "loss": 1.4117, + "theoretical_loss": 3.486257737687873, + "tokens_seen": 1819607040 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045311935807422265, + "loss": 1.3772, + "theoretical_loss": 3.4862471602297296, + "tokens_seen": 1819672576 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453099297893681, + "loss": 1.3383, + "theoretical_loss": 3.486236583259191, + "tokens_seen": 1819738112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045307923771313943, + "loss": 1.2922, + "theoretical_loss": 3.4862260067762176, + "tokens_seen": 1819803648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530591775325978, + "loss": 1.489, + "theoretical_loss": 3.486215430780769, + "tokens_seen": 1819869184 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045303911735205616, + "loss": 1.2933, + "theoretical_loss": 3.486204855272804, + "tokens_seen": 1819934720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530190571715145, + "loss": 1.3882, + "theoretical_loss": 3.4861942802522843, + "tokens_seen": 1820000256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045299899699097294, + "loss": 1.3231, + "theoretical_loss": 3.486183705719169, + "tokens_seen": 1820065792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529789368104313, + "loss": 1.378, + "theoretical_loss": 3.486173131673419, + "tokens_seen": 1820131328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045295887662988966, + "loss": 1.3546, + "theoretical_loss": 3.4861625581149926, + "tokens_seen": 1820196864 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452938816449348, + "loss": 1.4891, + "theoretical_loss": 3.4861519850438514, + "tokens_seen": 1820262400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529187562688064, + "loss": 1.3358, + "theoretical_loss": 3.486141412459955, + "tokens_seen": 1820327936 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 664244, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.473057746887207, + "objective/train/theoretical_loss": 3.486130840363263, + "objective/train/tokens_used": 190852576, + "theoretical_loss": 3.486130840363263, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528986960882648, + "loss": 1.4186, + "theoretical_loss": 3.486130840363263, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045287863590772316, + "loss": 1.5056, + "theoretical_loss": 3.4861202687537354, + "tokens_seen": 1820459008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528585757271815, + "loss": 1.3785, + "theoretical_loss": 3.486109697631333, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528385155466399, + "loss": 1.4482, + "theoretical_loss": 3.486099126996015, + "tokens_seen": 1820590080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528184553660983, + "loss": 1.3856, + "theoretical_loss": 3.4860885568477413, + "tokens_seen": 1820655616 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045279839518555667, + "loss": 1.2543, + "theoretical_loss": 3.486077987186473, + "tokens_seen": 1820721152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045277833500501503, + "loss": 1.2726, + "theoretical_loss": 3.486067418012169, + "tokens_seen": 1820786688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527582748244734, + "loss": 1.4595, + "theoretical_loss": 3.4860568493247897, + "tokens_seen": 1820852224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045273821464393175, + "loss": 1.4927, + "theoretical_loss": 3.4860462811242954, + "tokens_seen": 1820917760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045271815446339017, + "loss": 1.3728, + "theoretical_loss": 3.4860357134106463, + "tokens_seen": 1820983296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526980942828486, + "loss": 1.4341, + "theoretical_loss": 3.4860251461838017, + "tokens_seen": 1821048832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045267803410230695, + "loss": 1.3573, + "theoretical_loss": 3.4860145794437223, + "tokens_seen": 1821114368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526579739217653, + "loss": 1.3299, + "theoretical_loss": 3.486004013190368, + "tokens_seen": 1821179904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526379137412237, + "loss": 1.4155, + "theoretical_loss": 3.4859934474236987, + "tokens_seen": 1821245440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526178535606821, + "loss": 1.4428, + "theoretical_loss": 3.4859828821436745, + "tokens_seen": 1821310976 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045259779338014045, + "loss": 1.4274, + "theoretical_loss": 3.4859723173502557, + "tokens_seen": 1821376512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525777331995988, + "loss": 1.3713, + "theoretical_loss": 3.4859617530434024, + "tokens_seen": 1821442048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045255767301905723, + "loss": 1.2971, + "theoretical_loss": 3.485951189223074, + "tokens_seen": 1821507584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525376128385156, + "loss": 1.4422, + "theoretical_loss": 3.4859406258892314, + "tokens_seen": 1821573120 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045251755265797395, + "loss": 1.3715, + "theoretical_loss": 3.4859300630418346, + "tokens_seen": 1821638656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524974924774323, + "loss": 1.3374, + "theoretical_loss": 3.4859195006808434, + "tokens_seen": 1821704192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524774322968907, + "loss": 1.415, + "theoretical_loss": 3.485908938806218, + "tokens_seen": 1821769728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524573721163491, + "loss": 1.2894, + "theoretical_loss": 3.485898377417919, + "tokens_seen": 1821835264 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045243731193580746, + "loss": 1.3543, + "theoretical_loss": 3.4858878165159055, + "tokens_seen": 1821900800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524172517552658, + "loss": 1.4854, + "theoretical_loss": 3.485877256100138, + "tokens_seen": 1821966336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523971915747242, + "loss": 1.3344, + "theoretical_loss": 3.4858666961705773, + "tokens_seen": 1822031872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523771313941826, + "loss": 1.3174, + "theoretical_loss": 3.4858561367271825, + "tokens_seen": 1822097408 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045235707121364096, + "loss": 1.3643, + "theoretical_loss": 3.485845577769915, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523370110330993, + "loss": 1.5148, + "theoretical_loss": 3.4858350192987335, + "tokens_seen": 1822228480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523169508525577, + "loss": 1.3437, + "theoretical_loss": 3.485824461313599, + "tokens_seen": 1822294016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045229689067201605, + "loss": 1.4285, + "theoretical_loss": 3.4858139038144715, + "tokens_seen": 1822359552 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045227683049147446, + "loss": 1.4553, + "theoretical_loss": 3.485803346801311, + "tokens_seen": 1822425088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522567703109328, + "loss": 1.4443, + "theoretical_loss": 3.485792790274078, + "tokens_seen": 1822490624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522367101303912, + "loss": 1.4322, + "theoretical_loss": 3.4857822342327323, + "tokens_seen": 1822556160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045221664994984955, + "loss": 1.4931, + "theoretical_loss": 3.4857716786772346, + "tokens_seen": 1822621696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045219658976930797, + "loss": 1.443, + "theoretical_loss": 3.4857611236075443, + "tokens_seen": 1822687232 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045217652958876633, + "loss": 1.454, + "theoretical_loss": 3.4857505690236223, + "tokens_seen": 1822752768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521564694082247, + "loss": 1.2928, + "theoretical_loss": 3.485740014925428, + "tokens_seen": 1822818304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045213640922768305, + "loss": 1.3843, + "theoretical_loss": 3.485729461312922, + "tokens_seen": 1822883840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521163490471414, + "loss": 1.3998, + "theoretical_loss": 3.4857189081860644, + "tokens_seen": 1822949376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045209628886659983, + "loss": 1.396, + "theoretical_loss": 3.485708355544816, + "tokens_seen": 1823014912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520762286860582, + "loss": 1.4106, + "theoretical_loss": 3.4856978033891366, + "tokens_seen": 1823080448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045205616850551656, + "loss": 1.2949, + "theoretical_loss": 3.4856872517189856, + "tokens_seen": 1823145984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520361083249749, + "loss": 1.3768, + "theoretical_loss": 3.4856767005343245, + "tokens_seen": 1823211520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045201604814443334, + "loss": 1.3806, + "theoretical_loss": 3.4856661498351125, + "tokens_seen": 1823277056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519959879638917, + "loss": 1.2996, + "theoretical_loss": 3.4856555996213103, + "tokens_seen": 1823342592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045197592778335006, + "loss": 1.3818, + "theoretical_loss": 3.485645049892878, + "tokens_seen": 1823408128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519558676028084, + "loss": 1.2801, + "theoretical_loss": 3.485634500649777, + "tokens_seen": 1823473664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519358074222668, + "loss": 1.4582, + "theoretical_loss": 3.4856239518919647, + "tokens_seen": 1823539200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519157472417252, + "loss": 1.3983, + "theoretical_loss": 3.4856134036194044, + "tokens_seen": 1823604736 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 664954, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.428945541381836, + "objective/train/theoretical_loss": 3.485602855832054, + "objective/train/tokens_used": 194129376, + "theoretical_loss": 3.485602855832054, + "tokens_seen": 1823670272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045189568706118356, + "loss": 1.4432, + "theoretical_loss": 3.485602855832054, + "tokens_seen": 1823670272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518756268806419, + "loss": 1.3504, + "theoretical_loss": 3.4855923085298754, + "tokens_seen": 1823735808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518555667001003, + "loss": 1.4151, + "theoretical_loss": 3.4855817617128277, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518355065195587, + "loss": 1.4631, + "theoretical_loss": 3.4855712153808724, + "tokens_seen": 1823866880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045181544633901707, + "loss": 1.364, + "theoretical_loss": 3.4855606695339683, + "tokens_seen": 1823932416 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045179538615847543, + "loss": 1.341, + "theoretical_loss": 3.485550124172077, + "tokens_seen": 1823997952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517753259779338, + "loss": 1.4471, + "theoretical_loss": 3.4855395792951573, + "tokens_seen": 1824063488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045175526579739215, + "loss": 1.3904, + "theoretical_loss": 3.4855290349031707, + "tokens_seen": 1824129024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173520561685057, + "loss": 1.4009, + "theoretical_loss": 3.4855184909960775, + "tokens_seen": 1824194560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045171514543630893, + "loss": 1.3067, + "theoretical_loss": 3.4855079475738373, + "tokens_seen": 1824260096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516950852557673, + "loss": 1.3469, + "theoretical_loss": 3.4854974046364102, + "tokens_seen": 1824325632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045167502507522566, + "loss": 1.3843, + "theoretical_loss": 3.485486862183757, + "tokens_seen": 1824391168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045165496489468407, + "loss": 1.41, + "theoretical_loss": 3.4854763202158385, + "tokens_seen": 1824456704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045163490471414244, + "loss": 1.3258, + "theoretical_loss": 3.485465778732614, + "tokens_seen": 1824522240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516148445336008, + "loss": 1.3487, + "theoretical_loss": 3.485455237734045, + "tokens_seen": 1824587776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045159478435305916, + "loss": 1.3223, + "theoretical_loss": 3.4854446972200903, + "tokens_seen": 1824653312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515747241725175, + "loss": 1.3633, + "theoretical_loss": 3.485434157190711, + "tokens_seen": 1824718848 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045155466399197594, + "loss": 1.2066, + "theoretical_loss": 3.4854236176458677, + "tokens_seen": 1824784384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515346038114343, + "loss": 1.3287, + "theoretical_loss": 3.4854130785855206, + "tokens_seen": 1824849920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045151454363089266, + "loss": 1.4036, + "theoretical_loss": 3.4854025400096296, + "tokens_seen": 1824915456 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451494483450351, + "loss": 1.3777, + "theoretical_loss": 3.485392001918155, + "tokens_seen": 1824980992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045147442326980944, + "loss": 1.3453, + "theoretical_loss": 3.485381464311058, + "tokens_seen": 1825046528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514543630892678, + "loss": 1.289, + "theoretical_loss": 3.485370927188298, + "tokens_seen": 1825112064 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045143430290872617, + "loss": 1.4624, + "theoretical_loss": 3.4853603905498356, + "tokens_seen": 1825177600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045141424272818453, + "loss": 1.3451, + "theoretical_loss": 3.485349854395632, + "tokens_seen": 1825243136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045139418254764294, + "loss": 1.4064, + "theoretical_loss": 3.4853393187256456, + "tokens_seen": 1825308672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513741223671013, + "loss": 1.2947, + "theoretical_loss": 3.4853287835398388, + "tokens_seen": 1825374208 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045135406218655967, + "loss": 1.396, + "theoretical_loss": 3.485318248838171, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045133400200601803, + "loss": 1.3771, + "theoretical_loss": 3.485307714620603, + "tokens_seen": 1825505280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513139418254764, + "loss": 1.4019, + "theoretical_loss": 3.4852971808870947, + "tokens_seen": 1825570816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512938816449348, + "loss": 1.3241, + "theoretical_loss": 3.4852866476376065, + "tokens_seen": 1825636352 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045127382146439317, + "loss": 1.4185, + "theoretical_loss": 3.485276114872099, + "tokens_seen": 1825701888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125376128385153, + "loss": 1.3577, + "theoretical_loss": 3.4852655825905328, + "tokens_seen": 1825767424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512337011033099, + "loss": 1.2659, + "theoretical_loss": 3.485255050792868, + "tokens_seen": 1825832960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512136409227683, + "loss": 1.3476, + "theoretical_loss": 3.485244519479065, + "tokens_seen": 1825898496 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045119358074222673, + "loss": 1.3852, + "theoretical_loss": 3.4852339886490844, + "tokens_seen": 1825964032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511735205616851, + "loss": 1.3941, + "theoretical_loss": 3.4852234583028863, + "tokens_seen": 1826029568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045115346038114345, + "loss": 1.346, + "theoretical_loss": 3.485212928440431, + "tokens_seen": 1826095104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511334002006018, + "loss": 1.5541, + "theoretical_loss": 3.48520239906168, + "tokens_seen": 1826160640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045111334002006023, + "loss": 1.4031, + "theoretical_loss": 3.4851918701665925, + "tokens_seen": 1826226176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510932798395186, + "loss": 1.3364, + "theoretical_loss": 3.485181341755129, + "tokens_seen": 1826291712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045107321965897696, + "loss": 1.3323, + "theoretical_loss": 3.4851708138272506, + "tokens_seen": 1826357248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510531594784353, + "loss": 1.3312, + "theoretical_loss": 3.4851602863829174, + "tokens_seen": 1826422784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045103309929789374, + "loss": 1.2774, + "theoretical_loss": 3.48514975942209, + "tokens_seen": 1826488320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510130391173521, + "loss": 1.3689, + "theoretical_loss": 3.4851392329447286, + "tokens_seen": 1826553856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045099297893681046, + "loss": 1.4138, + "theoretical_loss": 3.4851287069507935, + "tokens_seen": 1826619392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509729187562688, + "loss": 1.3661, + "theoretical_loss": 3.485118181440246, + "tokens_seen": 1826684928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509528585757272, + "loss": 1.3387, + "theoretical_loss": 3.4851076564130454, + "tokens_seen": 1826750464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509327983951856, + "loss": 1.3327, + "theoretical_loss": 3.485097131869153, + "tokens_seen": 1826816000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045091273821464396, + "loss": 1.2156, + "theoretical_loss": 3.4850866078085287, + "tokens_seen": 1826881536 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 666029, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2888948917388916, + "objective/train/theoretical_loss": 3.485076084231134, + "objective/train/tokens_used": 197406176, + "theoretical_loss": 3.485076084231134, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508926780341023, + "loss": 1.3517, + "theoretical_loss": 3.485076084231134, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508726178535607, + "loss": 1.4546, + "theoretical_loss": 3.485065561136928, + "tokens_seen": 1827012608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508525576730191, + "loss": 1.2362, + "theoretical_loss": 3.4850550385258723, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045083249749247747, + "loss": 1.4626, + "theoretical_loss": 3.4850445163979265, + "tokens_seen": 1827143680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045081243731193583, + "loss": 1.4563, + "theoretical_loss": 3.485033994753052, + "tokens_seen": 1827209216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507923771313942, + "loss": 1.3422, + "theoretical_loss": 3.4850234735912085, + "tokens_seen": 1827274752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045077231695085255, + "loss": 1.3542, + "theoretical_loss": 3.4850129529123564, + "tokens_seen": 1827340288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045075225677031097, + "loss": 1.4255, + "theoretical_loss": 3.4850024327164575, + "tokens_seen": 1827405824 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045073219658976933, + "loss": 1.3217, + "theoretical_loss": 3.484991913003471, + "tokens_seen": 1827471360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507121364092277, + "loss": 1.4181, + "theoretical_loss": 3.484981393773358, + "tokens_seen": 1827536896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045069207622868606, + "loss": 1.3229, + "theoretical_loss": 3.484970875026079, + "tokens_seen": 1827602432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506720160481445, + "loss": 1.4248, + "theoretical_loss": 3.4849603567615937, + "tokens_seen": 1827667968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045065195586760284, + "loss": 1.368, + "theoretical_loss": 3.4849498389798637, + "tokens_seen": 1827733504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506318956870612, + "loss": 1.4122, + "theoretical_loss": 3.4849393216808497, + "tokens_seen": 1827799040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045061183550651956, + "loss": 1.307, + "theoretical_loss": 3.484928804864511, + "tokens_seen": 1827864576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505917753259779, + "loss": 1.2867, + "theoretical_loss": 3.4849182885308094, + "tokens_seen": 1827930112 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045057171514543634, + "loss": 1.4847, + "theoretical_loss": 3.484907772679705, + "tokens_seen": 1827995648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505516549648947, + "loss": 1.2788, + "theoretical_loss": 3.4848972573111574, + "tokens_seen": 1828061184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045053159478435306, + "loss": 1.3949, + "theoretical_loss": 3.484886742425129, + "tokens_seen": 1828126720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505115346038114, + "loss": 1.3631, + "theoretical_loss": 3.4848762280215784, + "tokens_seen": 1828192256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045049147442326984, + "loss": 1.4437, + "theoretical_loss": 3.4848657141004677, + "tokens_seen": 1828257792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504714142427282, + "loss": 1.4086, + "theoretical_loss": 3.484855200661757, + "tokens_seen": 1828323328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045045135406218657, + "loss": 1.3689, + "theoretical_loss": 3.4848446877054062, + "tokens_seen": 1828388864 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045043129388164493, + "loss": 1.3715, + "theoretical_loss": 3.484834175231377, + "tokens_seen": 1828454400 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045041123370110334, + "loss": 1.3624, + "theoretical_loss": 3.4848236632396294, + "tokens_seen": 1828519936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503911735205617, + "loss": 1.4685, + "theoretical_loss": 3.484813151730124, + "tokens_seen": 1828585472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045037111334002007, + "loss": 1.3944, + "theoretical_loss": 3.484802640702821, + "tokens_seen": 1828651008 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045035105315947843, + "loss": 1.3633, + "theoretical_loss": 3.484792130157682, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503309929789368, + "loss": 1.3572, + "theoretical_loss": 3.484781620094667, + "tokens_seen": 1828782080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503109327983952, + "loss": 1.294, + "theoretical_loss": 3.4847711105137367, + "tokens_seen": 1828847616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045029087261785357, + "loss": 1.3522, + "theoretical_loss": 3.4847606014148518, + "tokens_seen": 1828913152 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045027081243731193, + "loss": 1.3408, + "theoretical_loss": 3.484750092797972, + "tokens_seen": 1828978688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502507522567703, + "loss": 1.2818, + "theoretical_loss": 3.4847395846630596, + "tokens_seen": 1829044224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502306920762287, + "loss": 1.2774, + "theoretical_loss": 3.4847290770100736, + "tokens_seen": 1829109760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502106318956871, + "loss": 1.2751, + "theoretical_loss": 3.484718569838976, + "tokens_seen": 1829175296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045019057171514544, + "loss": 1.2768, + "theoretical_loss": 3.484708063149726, + "tokens_seen": 1829240832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501705115346038, + "loss": 1.3734, + "theoretical_loss": 3.484697556942286, + "tokens_seen": 1829306368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045015045135406216, + "loss": 1.3498, + "theoretical_loss": 3.484687051216615, + "tokens_seen": 1829371904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501303911735206, + "loss": 1.332, + "theoretical_loss": 3.4846765459726745, + "tokens_seen": 1829437440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045011033099297894, + "loss": 1.3605, + "theoretical_loss": 3.484666041210425, + "tokens_seen": 1829502976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500902708124373, + "loss": 1.34, + "theoretical_loss": 3.484655536929827, + "tokens_seen": 1829568512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045007021063189567, + "loss": 1.2935, + "theoretical_loss": 3.4846450331308416, + "tokens_seen": 1829634048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500501504513541, + "loss": 1.398, + "theoretical_loss": 3.484634529813429, + "tokens_seen": 1829699584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045003009027081244, + "loss": 1.3657, + "theoretical_loss": 3.48462402697755, + "tokens_seen": 1829765120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500100300902708, + "loss": 1.3742, + "theoretical_loss": 3.4846135246231658, + "tokens_seen": 1829830656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044998996990972917, + "loss": 1.361, + "theoretical_loss": 3.4846030227502363, + "tokens_seen": 1829896192 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044996990972918753, + "loss": 1.4564, + "theoretical_loss": 3.484592521358722, + "tokens_seen": 1829961728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044994984954864595, + "loss": 1.3517, + "theoretical_loss": 3.4845820204485847, + "tokens_seen": 1830027264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499297893681043, + "loss": 1.3877, + "theoretical_loss": 3.4845715200197844, + "tokens_seen": 1830092800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044990972918756267, + "loss": 1.3119, + "theoretical_loss": 3.484561020072282, + "tokens_seen": 1830158336 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 667256, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3890573978424072, + "objective/train/theoretical_loss": 3.484550520606038, + "objective/train/tokens_used": 200682976, + "theoretical_loss": 3.484550520606038, + "tokens_seen": 1830223872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044988966900702103, + "loss": 1.3316, + "theoretical_loss": 3.484550520606038, + "tokens_seen": 1830223872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044986960882647945, + "loss": 1.4165, + "theoretical_loss": 3.484540021621013, + "tokens_seen": 1830289408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498495486459378, + "loss": 1.3301, + "theoretical_loss": 3.484529523117168, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498294884653962, + "loss": 1.3621, + "theoretical_loss": 3.4845190250944635, + "tokens_seen": 1830420480 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044980942828485454, + "loss": 1.23, + "theoretical_loss": 3.484508527552861, + "tokens_seen": 1830486016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497893681043129, + "loss": 1.2959, + "theoretical_loss": 3.48449803049232, + "tokens_seen": 1830551552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497693079237713, + "loss": 1.2651, + "theoretical_loss": 3.4844875339128016, + "tokens_seen": 1830617088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497492477432297, + "loss": 1.3512, + "theoretical_loss": 3.4844770378142673, + "tokens_seen": 1830682624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044972918756268804, + "loss": 1.4223, + "theoretical_loss": 3.484466542196677, + "tokens_seen": 1830748160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497091273821464, + "loss": 1.346, + "theoretical_loss": 3.484456047059992, + "tokens_seen": 1830813696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496890672016049, + "loss": 1.2978, + "theoretical_loss": 3.484445552404173, + "tokens_seen": 1830879232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044966900702106324, + "loss": 1.3466, + "theoretical_loss": 3.48443505822918, + "tokens_seen": 1830944768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496489468405216, + "loss": 1.3783, + "theoretical_loss": 3.484424564534975, + "tokens_seen": 1831010304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044962888665997996, + "loss": 1.4081, + "theoretical_loss": 3.4844140713215177, + "tokens_seen": 1831075840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496088264794383, + "loss": 1.4309, + "theoretical_loss": 3.4844035785887693, + "tokens_seen": 1831141376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044958876629889674, + "loss": 1.3216, + "theoretical_loss": 3.4843930863366905, + "tokens_seen": 1831206912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495687061183551, + "loss": 1.3568, + "theoretical_loss": 3.4843825945652425, + "tokens_seen": 1831272448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044954864593781346, + "loss": 1.3302, + "theoretical_loss": 3.484372103274385, + "tokens_seen": 1831337984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495285857572718, + "loss": 1.4184, + "theoretical_loss": 3.4843616124640797, + "tokens_seen": 1831403520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044950852557673024, + "loss": 1.2208, + "theoretical_loss": 3.4843511221342873, + "tokens_seen": 1831469056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494884653961886, + "loss": 1.3212, + "theoretical_loss": 3.484340632284969, + "tokens_seen": 1831534592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044946840521564697, + "loss": 1.3291, + "theoretical_loss": 3.4843301429160842, + "tokens_seen": 1831600128 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044944834503510533, + "loss": 1.3375, + "theoretical_loss": 3.484319654027595, + "tokens_seen": 1831665664 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044942828485456375, + "loss": 1.3828, + "theoretical_loss": 3.484309165619462, + "tokens_seen": 1831731200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494082246740221, + "loss": 1.3023, + "theoretical_loss": 3.484298677691645, + "tokens_seen": 1831796736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044938816449348047, + "loss": 1.3586, + "theoretical_loss": 3.484288190244106, + "tokens_seen": 1831862272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044936810431293883, + "loss": 1.3609, + "theoretical_loss": 3.484277703276806, + "tokens_seen": 1831927808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493480441323972, + "loss": 1.3817, + "theoretical_loss": 3.484267216789705, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493279839518556, + "loss": 1.388, + "theoretical_loss": 3.4842567307827643, + "tokens_seen": 1832058880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044930792377131397, + "loss": 1.2866, + "theoretical_loss": 3.484246245255944, + "tokens_seen": 1832124416 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044928786359077234, + "loss": 1.4305, + "theoretical_loss": 3.484235760209206, + "tokens_seen": 1832189952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492678034102307, + "loss": 1.336, + "theoretical_loss": 3.4842252756425105, + "tokens_seen": 1832255488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492477432296891, + "loss": 1.5091, + "theoretical_loss": 3.4842147915558184, + "tokens_seen": 1832321024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492276830491475, + "loss": 1.289, + "theoretical_loss": 3.4842043079490908, + "tokens_seen": 1832386560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044920762286860584, + "loss": 1.4503, + "theoretical_loss": 3.4841938248222886, + "tokens_seen": 1832452096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491875626880642, + "loss": 1.3954, + "theoretical_loss": 3.4841833421753723, + "tokens_seen": 1832517632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044916750250752256, + "loss": 1.4462, + "theoretical_loss": 3.484172860008303, + "tokens_seen": 1832583168 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449147442326981, + "loss": 1.4106, + "theoretical_loss": 3.4841623783210416, + "tokens_seen": 1832648704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044912738214643934, + "loss": 1.3435, + "theoretical_loss": 3.4841518971135486, + "tokens_seen": 1832714240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491073219658977, + "loss": 1.1785, + "theoretical_loss": 3.4841414163857856, + "tokens_seen": 1832779776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044908726178535607, + "loss": 1.4378, + "theoretical_loss": 3.484130936137713, + "tokens_seen": 1832845312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490672016048145, + "loss": 1.4084, + "theoretical_loss": 3.4841204563692916, + "tokens_seen": 1832910848 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044904714142427284, + "loss": 1.3024, + "theoretical_loss": 3.484109977080483, + "tokens_seen": 1832976384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490270812437312, + "loss": 1.2725, + "theoretical_loss": 3.484099498271247, + "tokens_seen": 1833041920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044900702106318957, + "loss": 1.3649, + "theoretical_loss": 3.4840890199415453, + "tokens_seen": 1833107456 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044898696088264793, + "loss": 1.4148, + "theoretical_loss": 3.4840785420913387, + "tokens_seen": 1833172992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044896690070210635, + "loss": 1.3436, + "theoretical_loss": 3.484068064720588, + "tokens_seen": 1833238528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489468405215647, + "loss": 1.4146, + "theoretical_loss": 3.4840575878292546, + "tokens_seen": 1833304064 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044892678034102307, + "loss": 1.4709, + "theoretical_loss": 3.484047111417298, + "tokens_seen": 1833369600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044890672016048143, + "loss": 1.4155, + "theoretical_loss": 3.484036635484681, + "tokens_seen": 1833435136 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 668449, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3851547241210938, + "objective/train/theoretical_loss": 3.4840261600313633, + "objective/train/tokens_used": 203959776, + "theoretical_loss": 3.4840261600313633, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044888665997993985, + "loss": 1.3764, + "theoretical_loss": 3.4840261600313633, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488665997993982, + "loss": 1.3619, + "theoretical_loss": 3.4840156850573063, + "tokens_seen": 1833566208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488465396188566, + "loss": 1.4809, + "theoretical_loss": 3.4840052105624704, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044882647943831494, + "loss": 1.3097, + "theoretical_loss": 3.4839947365468174, + "tokens_seen": 1833697280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488064192577733, + "loss": 1.4207, + "theoretical_loss": 3.483984263010308, + "tokens_seen": 1833762816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487863590772317, + "loss": 1.3273, + "theoretical_loss": 3.483973789952903, + "tokens_seen": 1833828352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487662988966901, + "loss": 1.3316, + "theoretical_loss": 3.4839633173745628, + "tokens_seen": 1833893888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044874623871614844, + "loss": 1.3361, + "theoretical_loss": 3.4839528452752493, + "tokens_seen": 1833959424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487261785356068, + "loss": 1.3571, + "theoretical_loss": 3.483942373654923, + "tokens_seen": 1834024960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487061183550652, + "loss": 1.3918, + "theoretical_loss": 3.483931902513545, + "tokens_seen": 1834090496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486860581745236, + "loss": 1.3251, + "theoretical_loss": 3.4839214318510763, + "tokens_seen": 1834156032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044866599799398194, + "loss": 1.4931, + "theoretical_loss": 3.483910961667478, + "tokens_seen": 1834221568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486459378134403, + "loss": 1.3209, + "theoretical_loss": 3.483900491962711, + "tokens_seen": 1834287104 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044862587763289867, + "loss": 1.3772, + "theoretical_loss": 3.4838900227367358, + "tokens_seen": 1834352640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486058174523571, + "loss": 1.3448, + "theoretical_loss": 3.4838795539895138, + "tokens_seen": 1834418176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044858575727181545, + "loss": 1.4248, + "theoretical_loss": 3.483869085721006, + "tokens_seen": 1834483712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485656970912738, + "loss": 1.451, + "theoretical_loss": 3.483858617931174, + "tokens_seen": 1834549248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044854563691073217, + "loss": 1.4277, + "theoretical_loss": 3.483848150619978, + "tokens_seen": 1834614784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485255767301906, + "loss": 1.3668, + "theoretical_loss": 3.483837683787379, + "tokens_seen": 1834680320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044850551654964895, + "loss": 1.2602, + "theoretical_loss": 3.4838272174333387, + "tokens_seen": 1834745856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484854563691073, + "loss": 1.2785, + "theoretical_loss": 3.4838167515578173, + "tokens_seen": 1834811392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484653961885657, + "loss": 1.2475, + "theoretical_loss": 3.4838062861607764, + "tokens_seen": 1834876928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044844533600802404, + "loss": 1.2824, + "theoretical_loss": 3.483795821242177, + "tokens_seen": 1834942464 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044842527582748245, + "loss": 1.2873, + "theoretical_loss": 3.48378535680198, + "tokens_seen": 1835008000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484052156469408, + "loss": 1.3887, + "theoretical_loss": 3.4837748928401466, + "tokens_seen": 1835073536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483851554663992, + "loss": 1.3698, + "theoretical_loss": 3.4837644293566377, + "tokens_seen": 1835139072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044836509528585754, + "loss": 1.3546, + "theoretical_loss": 3.483753966351414, + "tokens_seen": 1835204608 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044834503510531596, + "loss": 1.3905, + "theoretical_loss": 3.4837435038244378, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483249749247743, + "loss": 1.2918, + "theoretical_loss": 3.4837330417756682, + "tokens_seen": 1835335680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483049147442327, + "loss": 1.2334, + "theoretical_loss": 3.483722580205068, + "tokens_seen": 1835401216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828485456369104, + "loss": 1.3341, + "theoretical_loss": 3.4837121191125977, + "tokens_seen": 1835466752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482647943831494, + "loss": 1.3696, + "theoretical_loss": 3.483701658498218, + "tokens_seen": 1835532288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482447342026078, + "loss": 1.4185, + "theoretical_loss": 3.483691198361891, + "tokens_seen": 1835597824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482246740220662, + "loss": 1.371, + "theoretical_loss": 3.483680738703576, + "tokens_seen": 1835663360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044820461384152455, + "loss": 1.2848, + "theoretical_loss": 3.4836702795232357, + "tokens_seen": 1835728896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044818455366098296, + "loss": 1.3714, + "theoretical_loss": 3.4836598208208307, + "tokens_seen": 1835794432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481644934804414, + "loss": 1.3666, + "theoretical_loss": 3.483649362596322, + "tokens_seen": 1835859968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044814443329989974, + "loss": 1.4868, + "theoretical_loss": 3.4836389048496708, + "tokens_seen": 1835925504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481243731193581, + "loss": 1.3598, + "theoretical_loss": 3.4836284475808386, + "tokens_seen": 1835991040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044810431293881647, + "loss": 1.4479, + "theoretical_loss": 3.4836179907897855, + "tokens_seen": 1836056576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480842527582749, + "loss": 1.3072, + "theoretical_loss": 3.4836075344764734, + "tokens_seen": 1836122112 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044806419257773324, + "loss": 1.3202, + "theoretical_loss": 3.483597078640863, + "tokens_seen": 1836187648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480441323971916, + "loss": 1.2778, + "theoretical_loss": 3.4835866232829167, + "tokens_seen": 1836253184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044802407221664997, + "loss": 1.3599, + "theoretical_loss": 3.4835761684025934, + "tokens_seen": 1836318720 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044800401203610833, + "loss": 1.3083, + "theoretical_loss": 3.483565713999856, + "tokens_seen": 1836384256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044798395185556675, + "loss": 1.4001, + "theoretical_loss": 3.483555260074665, + "tokens_seen": 1836449792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479638916750251, + "loss": 1.2928, + "theoretical_loss": 3.4835448066269814, + "tokens_seen": 1836515328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044794383149448347, + "loss": 1.3265, + "theoretical_loss": 3.483534353656767, + "tokens_seen": 1836580864 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044792377131394183, + "loss": 1.3513, + "theoretical_loss": 3.4835239011639825, + "tokens_seen": 1836646400 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044790371113340025, + "loss": 1.3269, + "theoretical_loss": 3.483513449148589, + "tokens_seen": 1836711936 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 669661, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3857011795043945, + "objective/train/theoretical_loss": 3.4835029976105476, + "objective/train/tokens_used": 207236576, + "theoretical_loss": 3.4835029976105476, + "tokens_seen": 1836777472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478836509528586, + "loss": 1.4506, + "theoretical_loss": 3.4835029976105476, + "tokens_seen": 1836777472 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447863590772317, + "loss": 1.3854, + "theoretical_loss": 3.4834925465498197, + "tokens_seen": 1836843008 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044784353059177534, + "loss": 1.2658, + "theoretical_loss": 3.483482095966366, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478234704112337, + "loss": 1.3701, + "theoretical_loss": 3.4834716458601482, + "tokens_seen": 1836974080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478034102306921, + "loss": 1.405, + "theoretical_loss": 3.483461196231128, + "tokens_seen": 1837039616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477833500501505, + "loss": 1.4214, + "theoretical_loss": 3.4834507470792655, + "tokens_seen": 1837105152 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044776328986960884, + "loss": 1.4264, + "theoretical_loss": 3.4834402984045223, + "tokens_seen": 1837170688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477432296890672, + "loss": 1.4037, + "theoretical_loss": 3.4834298502068597, + "tokens_seen": 1837236224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477231695085256, + "loss": 1.3278, + "theoretical_loss": 3.4834194024862386, + "tokens_seen": 1837301760 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447703109327984, + "loss": 1.2432, + "theoretical_loss": 3.483408955242621, + "tokens_seen": 1837367296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044768304914744234, + "loss": 1.3156, + "theoretical_loss": 3.483398508475967, + "tokens_seen": 1837432832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476629889669007, + "loss": 1.3817, + "theoretical_loss": 3.4833880621862385, + "tokens_seen": 1837498368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044764292878635907, + "loss": 1.4581, + "theoretical_loss": 3.4833776163733967, + "tokens_seen": 1837563904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476228686058175, + "loss": 1.4544, + "theoretical_loss": 3.4833671710374023, + "tokens_seen": 1837629440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044760280842527585, + "loss": 1.4868, + "theoretical_loss": 3.4833567261782177, + "tokens_seen": 1837694976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475827482447342, + "loss": 1.2919, + "theoretical_loss": 3.4833462817958027, + "tokens_seen": 1837760512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044756268806419257, + "loss": 1.4497, + "theoretical_loss": 3.4833358378901194, + "tokens_seen": 1837826048 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447542627883651, + "loss": 1.395, + "theoretical_loss": 3.4833253944611284, + "tokens_seen": 1837891584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044752256770310935, + "loss": 1.392, + "theoretical_loss": 3.483314951508792, + "tokens_seen": 1837957120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475025075225677, + "loss": 1.4253, + "theoretical_loss": 3.4833045090330703, + "tokens_seen": 1838022656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474824473420261, + "loss": 1.3788, + "theoretical_loss": 3.483294067033926, + "tokens_seen": 1838088192 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044746238716148444, + "loss": 1.369, + "theoretical_loss": 3.4832836255113184, + "tokens_seen": 1838153728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044744232698094285, + "loss": 1.2772, + "theoretical_loss": 3.48327318446521, + "tokens_seen": 1838219264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474222668004012, + "loss": 1.33, + "theoretical_loss": 3.483262743895562, + "tokens_seen": 1838284800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474022066198596, + "loss": 1.3684, + "theoretical_loss": 3.4832523038023355, + "tokens_seen": 1838350336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044738214643931794, + "loss": 1.3565, + "theoretical_loss": 3.483241864185492, + "tokens_seen": 1838415872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044736208625877636, + "loss": 1.2278, + "theoretical_loss": 3.483231425044993, + "tokens_seen": 1838481408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473420260782347, + "loss": 1.3463, + "theoretical_loss": 3.4832209863807986, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473219658976931, + "loss": 1.3481, + "theoretical_loss": 3.4832105481928712, + "tokens_seen": 1838612480 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044730190571715144, + "loss": 1.3891, + "theoretical_loss": 3.483200110481172, + "tokens_seen": 1838678016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472818455366098, + "loss": 1.3344, + "theoretical_loss": 3.4831896732456613, + "tokens_seen": 1838743552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472617853560682, + "loss": 1.3094, + "theoretical_loss": 3.483179236486302, + "tokens_seen": 1838809088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472417251755266, + "loss": 1.3006, + "theoretical_loss": 3.483168800203054, + "tokens_seen": 1838874624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044722166499498495, + "loss": 1.2804, + "theoretical_loss": 3.4831583643958797, + "tokens_seen": 1838940160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472016048144433, + "loss": 1.4075, + "theoretical_loss": 3.4831479290647396, + "tokens_seen": 1839005696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471815446339017, + "loss": 1.359, + "theoretical_loss": 3.483137494209595, + "tokens_seen": 1839071232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471614844533601, + "loss": 1.4215, + "theoretical_loss": 3.4831270598304087, + "tokens_seen": 1839136768 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044714142427281845, + "loss": 1.4103, + "theoretical_loss": 3.48311662592714, + "tokens_seen": 1839202304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471213640922768, + "loss": 1.3851, + "theoretical_loss": 3.4831061924997515, + "tokens_seen": 1839267840 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044710130391173523, + "loss": 1.3227, + "theoretical_loss": 3.4830957595482035, + "tokens_seen": 1839333376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470812437311936, + "loss": 1.3078, + "theoretical_loss": 3.483085327072459, + "tokens_seen": 1839398912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044706118355065195, + "loss": 1.426, + "theoretical_loss": 3.4830748950724777, + "tokens_seen": 1839464448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470411233701103, + "loss": 1.3378, + "theoretical_loss": 3.4830644635482217, + "tokens_seen": 1839529984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470210631895687, + "loss": 1.3558, + "theoretical_loss": 3.483054032499653, + "tokens_seen": 1839595520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470010030090271, + "loss": 1.3384, + "theoretical_loss": 3.483043601926731, + "tokens_seen": 1839661056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044698094282848546, + "loss": 1.3258, + "theoretical_loss": 3.483033171829419, + "tokens_seen": 1839726592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469608826479438, + "loss": 1.3786, + "theoretical_loss": 3.4830227422076776, + "tokens_seen": 1839792128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469408224674022, + "loss": 1.272, + "theoretical_loss": 3.4830123130614687, + "tokens_seen": 1839857664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469207622868606, + "loss": 1.4065, + "theoretical_loss": 3.4830018843907524, + "tokens_seen": 1839923200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044690070210631896, + "loss": 1.4026, + "theoretical_loss": 3.4829914561954913, + "tokens_seen": 1839988736 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 670853, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3770475387573242, + "objective/train/theoretical_loss": 3.4829810284756464, + "objective/train/tokens_used": 210513376, + "theoretical_loss": 3.4829810284756464, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468806419257773, + "loss": 1.2874, + "theoretical_loss": 3.4829810284756464, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468605817452357, + "loss": 1.2706, + "theoretical_loss": 3.4829706012311794, + "tokens_seen": 1840119808 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044684052156469405, + "loss": 1.4379, + "theoretical_loss": 3.4829601744620513, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044682046138415246, + "loss": 1.4284, + "theoretical_loss": 3.4829497481682234, + "tokens_seen": 1840250880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468004012036108, + "loss": 1.3468, + "theoretical_loss": 3.482939322349657, + "tokens_seen": 1840316416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467803410230692, + "loss": 1.3455, + "theoretical_loss": 3.482928897006315, + "tokens_seen": 1840381952 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044676028084252755, + "loss": 1.2919, + "theoretical_loss": 3.4829184721381568, + "tokens_seen": 1840447488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044674022066198597, + "loss": 1.3297, + "theoretical_loss": 3.482908047745145, + "tokens_seen": 1840513024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044672016048144433, + "loss": 1.3592, + "theoretical_loss": 3.48289762382724, + "tokens_seen": 1840578560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467001003009027, + "loss": 1.3635, + "theoretical_loss": 3.4828872003844045, + "tokens_seen": 1840644096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466800401203611, + "loss": 1.3362, + "theoretical_loss": 3.482876777416599, + "tokens_seen": 1840709632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044665997993981947, + "loss": 1.2639, + "theoretical_loss": 3.482866354923786, + "tokens_seen": 1840775168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466399197592779, + "loss": 1.3608, + "theoretical_loss": 3.4828559329059257, + "tokens_seen": 1840840704 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044661985957873625, + "loss": 1.3963, + "theoretical_loss": 3.48284551136298, + "tokens_seen": 1840906240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465997993981946, + "loss": 1.4256, + "theoretical_loss": 3.482835090294911, + "tokens_seen": 1840971776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044657973921765297, + "loss": 1.307, + "theoretical_loss": 3.482824669701679, + "tokens_seen": 1841037312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465596790371114, + "loss": 1.337, + "theoretical_loss": 3.4828142495832464, + "tokens_seen": 1841102848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044653961885656975, + "loss": 1.3583, + "theoretical_loss": 3.4828038299395745, + "tokens_seen": 1841168384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465195586760281, + "loss": 1.3492, + "theoretical_loss": 3.482793410770624, + "tokens_seen": 1841233920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464994984954865, + "loss": 1.4215, + "theoretical_loss": 3.4827829920763573, + "tokens_seen": 1841299456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044647943831494484, + "loss": 1.321, + "theoretical_loss": 3.482772573856736, + "tokens_seen": 1841364992 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044645937813440325, + "loss": 1.3915, + "theoretical_loss": 3.4827621561117206, + "tokens_seen": 1841430528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464393179538616, + "loss": 1.4487, + "theoretical_loss": 3.4827517388412734, + "tokens_seen": 1841496064 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044641925777332, + "loss": 1.3752, + "theoretical_loss": 3.4827413220453556, + "tokens_seen": 1841561600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044639919759277834, + "loss": 1.4058, + "theoretical_loss": 3.482730905723929, + "tokens_seen": 1841627136 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044637913741223676, + "loss": 1.2613, + "theoretical_loss": 3.482720489876954, + "tokens_seen": 1841692672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463590772316951, + "loss": 1.3246, + "theoretical_loss": 3.4827100745043937, + "tokens_seen": 1841758208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463390170511535, + "loss": 1.444, + "theoretical_loss": 3.4826996596062085, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044631895687061184, + "loss": 1.2866, + "theoretical_loss": 3.4826892451823603, + "tokens_seen": 1841889280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462988966900702, + "loss": 1.2348, + "theoretical_loss": 3.4826788312328105, + "tokens_seen": 1841954816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462788365095286, + "loss": 1.3608, + "theoretical_loss": 3.4826684177575213, + "tokens_seen": 1842020352 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446258776328987, + "loss": 1.3317, + "theoretical_loss": 3.482658004756453, + "tokens_seen": 1842085888 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044623871614844535, + "loss": 1.4414, + "theoretical_loss": 3.4826475922295677, + "tokens_seen": 1842151424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462186559679037, + "loss": 1.3356, + "theoretical_loss": 3.4826371801768277, + "tokens_seen": 1842216960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461985957873621, + "loss": 1.4634, + "theoretical_loss": 3.4826267685981933, + "tokens_seen": 1842282496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461785356068205, + "loss": 1.2762, + "theoretical_loss": 3.482616357493627, + "tokens_seen": 1842348032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044615847542627885, + "loss": 1.4386, + "theoretical_loss": 3.4826059468630897, + "tokens_seen": 1842413568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461384152457372, + "loss": 1.4646, + "theoretical_loss": 3.482595536706543, + "tokens_seen": 1842479104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044611835506519563, + "loss": 1.3202, + "theoretical_loss": 3.482585127023949, + "tokens_seen": 1842544640 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446098294884654, + "loss": 1.3861, + "theoretical_loss": 3.482574717815269, + "tokens_seen": 1842610176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044607823470411235, + "loss": 1.4658, + "theoretical_loss": 3.4825643090804643, + "tokens_seen": 1842675712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460581745235707, + "loss": 1.3338, + "theoretical_loss": 3.4825539008194966, + "tokens_seen": 1842741248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460381143430291, + "loss": 1.3567, + "theoretical_loss": 3.4825434930323276, + "tokens_seen": 1842806784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460180541624875, + "loss": 1.4042, + "theoretical_loss": 3.482533085718919, + "tokens_seen": 1842872320 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044599799398194586, + "loss": 1.337, + "theoretical_loss": 3.4825226788792323, + "tokens_seen": 1842937856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459779338014042, + "loss": 1.3534, + "theoretical_loss": 3.4825122725132287, + "tokens_seen": 1843003392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459578736208626, + "loss": 1.5066, + "theoretical_loss": 3.4825018666208702, + "tokens_seen": 1843068928 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445937813440321, + "loss": 1.383, + "theoretical_loss": 3.4824914612021187, + "tokens_seen": 1843134464 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591775325977936, + "loss": 1.2521, + "theoretical_loss": 3.482481056256935, + "tokens_seen": 1843200000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458976930792377, + "loss": 1.3552, + "theoretical_loss": 3.4824706517852815, + "tokens_seen": 1843265536 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 672182, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2414642572402954, + "objective/train/theoretical_loss": 3.4824602477871194, + "objective/train/tokens_used": 213790176, + "theoretical_loss": 3.4824602477871194, + "tokens_seen": 1843331072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458776328986961, + "loss": 1.3945, + "theoretical_loss": 3.4824602477871194, + "tokens_seen": 1843331072 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044585757271815445, + "loss": 1.4466, + "theoretical_loss": 3.48244984426241, + "tokens_seen": 1843396608 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044583751253761286, + "loss": 1.2829, + "theoretical_loss": 3.482439441211116, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458174523570712, + "loss": 1.2853, + "theoretical_loss": 3.4824290386331977, + "tokens_seen": 1843527680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457973921765296, + "loss": 1.3401, + "theoretical_loss": 3.4824186365286174, + "tokens_seen": 1843593216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044577733199598795, + "loss": 1.3557, + "theoretical_loss": 3.4824082348973366, + "tokens_seen": 1843658752 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575727181544637, + "loss": 1.4321, + "theoretical_loss": 3.4823978337393173, + "tokens_seen": 1843724288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044573721163490473, + "loss": 1.2747, + "theoretical_loss": 3.4823874330545213, + "tokens_seen": 1843789824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457171514543631, + "loss": 1.4164, + "theoretical_loss": 3.4823770328429093, + "tokens_seen": 1843855360 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044569709127382145, + "loss": 1.3941, + "theoretical_loss": 3.4823666331044434, + "tokens_seen": 1843920896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456770310932798, + "loss": 1.4176, + "theoretical_loss": 3.4823562338390857, + "tokens_seen": 1843986432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044565697091273823, + "loss": 1.3957, + "theoretical_loss": 3.4823458350467975, + "tokens_seen": 1844051968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456369107321966, + "loss": 1.4266, + "theoretical_loss": 3.4823354367275403, + "tokens_seen": 1844117504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044561685055165496, + "loss": 1.3711, + "theoretical_loss": 3.482325038881276, + "tokens_seen": 1844183040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455967903711133, + "loss": 1.4531, + "theoretical_loss": 3.482314641507966, + "tokens_seen": 1844248576 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044557673019057173, + "loss": 1.3947, + "theoretical_loss": 3.4823042446075725, + "tokens_seen": 1844314112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455566700100301, + "loss": 1.2882, + "theoretical_loss": 3.4822938481800567, + "tokens_seen": 1844379648 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044553660982948846, + "loss": 1.3367, + "theoretical_loss": 3.4822834522253805, + "tokens_seen": 1844445184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455165496489468, + "loss": 1.3525, + "theoretical_loss": 3.482273056743506, + "tokens_seen": 1844510720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454964894684052, + "loss": 1.3884, + "theoretical_loss": 3.482262661734394, + "tokens_seen": 1844576256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454764292878636, + "loss": 1.2907, + "theoretical_loss": 3.4822522671980067, + "tokens_seen": 1844641792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044545636910732196, + "loss": 1.4096, + "theoretical_loss": 3.4822418731343063, + "tokens_seen": 1844707328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454363089267803, + "loss": 1.4257, + "theoretical_loss": 3.4822314795432536, + "tokens_seen": 1844772864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454162487462387, + "loss": 1.3026, + "theoretical_loss": 3.4822210864248113, + "tokens_seen": 1844838400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453961885656971, + "loss": 1.2793, + "theoretical_loss": 3.4822106937789394, + "tokens_seen": 1844903936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044537612838515547, + "loss": 1.4845, + "theoretical_loss": 3.4822003016056016, + "tokens_seen": 1844969472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044535606820461383, + "loss": 1.2822, + "theoretical_loss": 3.4821899099047586, + "tokens_seen": 1845035008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453360080240722, + "loss": 1.5107, + "theoretical_loss": 3.4821795186763724, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531594784353055, + "loss": 1.2597, + "theoretical_loss": 3.482169127920405, + "tokens_seen": 1845166080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044529588766298897, + "loss": 1.4273, + "theoretical_loss": 3.4821587376368175, + "tokens_seen": 1845231616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527582748244733, + "loss": 1.3833, + "theoretical_loss": 3.482148347825572, + "tokens_seen": 1845297152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452557673019057, + "loss": 1.3197, + "theoretical_loss": 3.4821379584866303, + "tokens_seen": 1845362688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044523570712136406, + "loss": 1.3255, + "theoretical_loss": 3.482127569619954, + "tokens_seen": 1845428224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044521564694082247, + "loss": 1.2742, + "theoretical_loss": 3.482117181225505, + "tokens_seen": 1845493760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044519558676028083, + "loss": 1.3858, + "theoretical_loss": 3.482106793303245, + "tokens_seen": 1845559296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044517552657973925, + "loss": 1.4442, + "theoretical_loss": 3.4820964058531354, + "tokens_seen": 1845624832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451554663991976, + "loss": 1.4046, + "theoretical_loss": 3.482086018875139, + "tokens_seen": 1845690368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044513540621865603, + "loss": 1.3367, + "theoretical_loss": 3.4820756323692166, + "tokens_seen": 1845755904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451153460381144, + "loss": 1.2899, + "theoretical_loss": 3.4820652463353303, + "tokens_seen": 1845821440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044509528585757275, + "loss": 1.4217, + "theoretical_loss": 3.4820548607734425, + "tokens_seen": 1845886976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450752256770311, + "loss": 1.4319, + "theoretical_loss": 3.4820444756835136, + "tokens_seen": 1845952512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450551654964895, + "loss": 1.5386, + "theoretical_loss": 3.4820340910655068, + "tokens_seen": 1846018048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450351053159479, + "loss": 1.3271, + "theoretical_loss": 3.482023706919383, + "tokens_seen": 1846083584 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044501504513540626, + "loss": 1.366, + "theoretical_loss": 3.482013323245104, + "tokens_seen": 1846149120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449949849548646, + "loss": 1.4011, + "theoretical_loss": 3.4820029400426327, + "tokens_seen": 1846214656 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444974924774323, + "loss": 1.277, + "theoretical_loss": 3.481992557311929, + "tokens_seen": 1846280192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449548645937814, + "loss": 1.3212, + "theoretical_loss": 3.481982175052957, + "tokens_seen": 1846345728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044493480441323976, + "loss": 1.5374, + "theoretical_loss": 3.481971793265677, + "tokens_seen": 1846411264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449147442326981, + "loss": 1.3373, + "theoretical_loss": 3.481961411950051, + "tokens_seen": 1846476800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448946840521565, + "loss": 1.3793, + "theoretical_loss": 3.4819510311060418, + "tokens_seen": 1846542336 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 673286, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5119043588638306, + "objective/train/theoretical_loss": 3.4819406507336095, + "objective/train/tokens_used": 217066976, + "theoretical_loss": 3.4819406507336095, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044487462387161485, + "loss": 1.454, + "theoretical_loss": 3.4819406507336095, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044485456369107326, + "loss": 1.3995, + "theoretical_loss": 3.4819302708327173, + "tokens_seen": 1846673408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448345035105316, + "loss": 1.3099, + "theoretical_loss": 3.4819198914033267, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044481444332999, + "loss": 1.3943, + "theoretical_loss": 3.4819095124454, + "tokens_seen": 1846804480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044479438314944835, + "loss": 1.3385, + "theoretical_loss": 3.4818991339588976, + "tokens_seen": 1846870016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044477432296890677, + "loss": 1.2927, + "theoretical_loss": 3.4818887559437828, + "tokens_seen": 1846935552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044475426278836513, + "loss": 1.4671, + "theoretical_loss": 3.481878378400017, + "tokens_seen": 1847001088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447342026078235, + "loss": 1.402, + "theoretical_loss": 3.4818680013275625, + "tokens_seen": 1847066624 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044471414242728185, + "loss": 1.3762, + "theoretical_loss": 3.4818576247263806, + "tokens_seen": 1847132160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446940822467402, + "loss": 1.477, + "theoretical_loss": 3.481847248596433, + "tokens_seen": 1847197696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044467402206619863, + "loss": 1.2732, + "theoretical_loss": 3.4818368729376816, + "tokens_seen": 1847263232 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444653961885657, + "loss": 1.4239, + "theoretical_loss": 3.4818264977500895, + "tokens_seen": 1847328768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044463390170511536, + "loss": 1.3003, + "theoretical_loss": 3.4818161230336173, + "tokens_seen": 1847394304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446138415245737, + "loss": 1.3105, + "theoretical_loss": 3.481805748788227, + "tokens_seen": 1847459840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044459378134403214, + "loss": 1.3835, + "theoretical_loss": 3.4817953750138813, + "tokens_seen": 1847525376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445737211634905, + "loss": 1.3508, + "theoretical_loss": 3.481785001710541, + "tokens_seen": 1847590912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044455366098294886, + "loss": 1.2737, + "theoretical_loss": 3.481774628878169, + "tokens_seen": 1847656448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445336008024072, + "loss": 1.4498, + "theoretical_loss": 3.481764256516726, + "tokens_seen": 1847721984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445135406218656, + "loss": 1.4485, + "theoretical_loss": 3.4817538846261757, + "tokens_seen": 1847787520 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444493480441324, + "loss": 1.422, + "theoretical_loss": 3.481743513206479, + "tokens_seen": 1847853056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044447342026078236, + "loss": 1.3907, + "theoretical_loss": 3.4817331422575974, + "tokens_seen": 1847918592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444533600802407, + "loss": 1.4349, + "theoretical_loss": 3.4817227717794936, + "tokens_seen": 1847984128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444332998996991, + "loss": 1.3739, + "theoretical_loss": 3.4817124017721293, + "tokens_seen": 1848049664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444132397191575, + "loss": 1.3609, + "theoretical_loss": 3.481702032235466, + "tokens_seen": 1848115200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044439317953861587, + "loss": 1.3702, + "theoretical_loss": 3.4816916631694665, + "tokens_seen": 1848180736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044437311935807423, + "loss": 1.3909, + "theoretical_loss": 3.481681294574092, + "tokens_seen": 1848246272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443530591775326, + "loss": 1.339, + "theoretical_loss": 3.4816709264493046, + "tokens_seen": 1848311808 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044433299899699095, + "loss": 1.4149, + "theoretical_loss": 3.481660558795067, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044431293881644937, + "loss": 1.3107, + "theoretical_loss": 3.4816501916113394, + "tokens_seen": 1848442880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044429287863590773, + "loss": 1.386, + "theoretical_loss": 3.4816398248980858, + "tokens_seen": 1848508416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442728184553661, + "loss": 1.4043, + "theoretical_loss": 3.481629458655267, + "tokens_seen": 1848573952 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044425275827482446, + "loss": 1.3636, + "theoretical_loss": 3.4816190928828457, + "tokens_seen": 1848639488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044423269809428287, + "loss": 1.3503, + "theoretical_loss": 3.4816087275807828, + "tokens_seen": 1848705024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044421263791374123, + "loss": 1.4428, + "theoretical_loss": 3.481598362749041, + "tokens_seen": 1848770560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441925777331996, + "loss": 1.4765, + "theoretical_loss": 3.4815879983875826, + "tokens_seen": 1848836096 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044417251755265796, + "loss": 1.331, + "theoretical_loss": 3.481577634496369, + "tokens_seen": 1848901632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441524573721163, + "loss": 1.3547, + "theoretical_loss": 3.4815672710753622, + "tokens_seen": 1848967168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044413239719157474, + "loss": 1.3534, + "theoretical_loss": 3.4815569081245243, + "tokens_seen": 1849032704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441123370110331, + "loss": 1.3678, + "theoretical_loss": 3.481546545643818, + "tokens_seen": 1849098240 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044409227683049146, + "loss": 1.3229, + "theoretical_loss": 3.4815361836332044, + "tokens_seen": 1849163776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440722166499498, + "loss": 1.3476, + "theoretical_loss": 3.481525822092646, + "tokens_seen": 1849229312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044405215646940824, + "loss": 1.4214, + "theoretical_loss": 3.481515461022104, + "tokens_seen": 1849294848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440320962888666, + "loss": 1.3804, + "theoretical_loss": 3.4815051004215416, + "tokens_seen": 1849360384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044401203610832497, + "loss": 1.3375, + "theoretical_loss": 3.4814947402909207, + "tokens_seen": 1849425920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044399197592778333, + "loss": 1.3461, + "theoretical_loss": 3.481484380630202, + "tokens_seen": 1849491456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439719157472417, + "loss": 1.2951, + "theoretical_loss": 3.4814740214393494, + "tokens_seen": 1849556992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439518555667001, + "loss": 1.5267, + "theoretical_loss": 3.4814636627183235, + "tokens_seen": 1849622528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044393179538615847, + "loss": 1.3578, + "theoretical_loss": 3.4814533044670872, + "tokens_seen": 1849688064 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044391173520561683, + "loss": 1.3945, + "theoretical_loss": 3.481442946685602, + "tokens_seen": 1849753600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438916750250752, + "loss": 1.4038, + "theoretical_loss": 3.4814325893738296, + "tokens_seen": 1849819136 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 674404, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4075238704681396, + "objective/train/theoretical_loss": 3.481422232531733, + "objective/train/tokens_used": 220343776, + "theoretical_loss": 3.481422232531733, + "tokens_seen": 1849884672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438716148445336, + "loss": 1.3272, + "theoretical_loss": 3.481422232531733, + "tokens_seen": 1849884672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044385155466399197, + "loss": 1.3789, + "theoretical_loss": 3.4814118761592745, + "tokens_seen": 1849950208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044383149448345033, + "loss": 1.3812, + "theoretical_loss": 3.481401520256415, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438114343029087, + "loss": 1.2449, + "theoretical_loss": 3.4813911648231173, + "tokens_seen": 1850081280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437913741223671, + "loss": 1.5921, + "theoretical_loss": 3.481380809859343, + "tokens_seen": 1850146816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437713139418255, + "loss": 1.5209, + "theoretical_loss": 3.4813704553650546, + "tokens_seen": 1850212352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044375125376128384, + "loss": 1.451, + "theoretical_loss": 3.481360101340214, + "tokens_seen": 1850277888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437311935807422, + "loss": 1.3428, + "theoretical_loss": 3.4813497477847837, + "tokens_seen": 1850343424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044371113340020056, + "loss": 1.337, + "theoretical_loss": 3.4813393946987254, + "tokens_seen": 1850408960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044369107321965903, + "loss": 1.3824, + "theoretical_loss": 3.481329042082001, + "tokens_seen": 1850474496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436710130391174, + "loss": 1.2999, + "theoretical_loss": 3.481318689934573, + "tokens_seen": 1850540032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044365095285857576, + "loss": 1.2647, + "theoretical_loss": 3.481308338256403, + "tokens_seen": 1850605568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436308926780341, + "loss": 1.2919, + "theoretical_loss": 3.481297987047454, + "tokens_seen": 1850671104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044361083249749254, + "loss": 1.3866, + "theoretical_loss": 3.4812876363076874, + "tokens_seen": 1850736640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435907723169509, + "loss": 1.2959, + "theoretical_loss": 3.4812772860370655, + "tokens_seen": 1850802176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044357071213640926, + "loss": 1.4037, + "theoretical_loss": 3.4812669362355506, + "tokens_seen": 1850867712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435506519558676, + "loss": 1.4122, + "theoretical_loss": 3.4812565869031045, + "tokens_seen": 1850933248 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443530591775326, + "loss": 1.3627, + "theoretical_loss": 3.48124623803969, + "tokens_seen": 1850998784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435105315947844, + "loss": 1.3424, + "theoretical_loss": 3.481235889645268, + "tokens_seen": 1851064320 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044349047141424276, + "loss": 1.4296, + "theoretical_loss": 3.4812255417198017, + "tokens_seen": 1851129856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434704112337011, + "loss": 1.3376, + "theoretical_loss": 3.4812151942632523, + "tokens_seen": 1851195392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434503510531595, + "loss": 1.3155, + "theoretical_loss": 3.4812048472755834, + "tokens_seen": 1851260928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434302908726179, + "loss": 1.3321, + "theoretical_loss": 3.4811945007567564, + "tokens_seen": 1851326464 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044341023069207627, + "loss": 1.2365, + "theoretical_loss": 3.4811841547067326, + "tokens_seen": 1851392000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044339017051153463, + "loss": 1.4327, + "theoretical_loss": 3.4811738091254756, + "tokens_seen": 1851457536 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443370110330993, + "loss": 1.3354, + "theoretical_loss": 3.4811634640129467, + "tokens_seen": 1851523072 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044335005015045135, + "loss": 1.2978, + "theoretical_loss": 3.4811531193691083, + "tokens_seen": 1851588608 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044332998996990977, + "loss": 1.2896, + "theoretical_loss": 3.4811427751939226, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044330992978936813, + "loss": 1.3828, + "theoretical_loss": 3.481132431487352, + "tokens_seen": 1851719680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432898696088265, + "loss": 1.3679, + "theoretical_loss": 3.481122088249358, + "tokens_seen": 1851785216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044326980942828486, + "loss": 1.363, + "theoretical_loss": 3.481111745479904, + "tokens_seen": 1851850752 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044324974924774327, + "loss": 1.3166, + "theoretical_loss": 3.4811014031789504, + "tokens_seen": 1851916288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044322968906720163, + "loss": 1.3442, + "theoretical_loss": 3.4810910613464614, + "tokens_seen": 1851981824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044320962888666, + "loss": 1.4, + "theoretical_loss": 3.4810807199823977, + "tokens_seen": 1852047360 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044318956870611836, + "loss": 1.3621, + "theoretical_loss": 3.481070379086722, + "tokens_seen": 1852112896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431695085255767, + "loss": 1.3715, + "theoretical_loss": 3.481060038659397, + "tokens_seen": 1852178432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044314944834503514, + "loss": 1.4101, + "theoretical_loss": 3.4810496987003843, + "tokens_seen": 1852243968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431293881644935, + "loss": 1.2859, + "theoretical_loss": 3.481039359209646, + "tokens_seen": 1852309504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044310932798395186, + "loss": 1.3805, + "theoretical_loss": 3.4810290201871448, + "tokens_seen": 1852375040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430892678034102, + "loss": 1.4143, + "theoretical_loss": 3.481018681632843, + "tokens_seen": 1852440576 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044306920762286864, + "loss": 1.3947, + "theoretical_loss": 3.4810083435467023, + "tokens_seen": 1852506112 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443049147442327, + "loss": 1.4118, + "theoretical_loss": 3.480998005928685, + "tokens_seen": 1852571648 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044302908726178537, + "loss": 1.4132, + "theoretical_loss": 3.480987668778754, + "tokens_seen": 1852637184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044300902708124373, + "loss": 1.3111, + "theoretical_loss": 3.480977332096871, + "tokens_seen": 1852702720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429889669007021, + "loss": 1.3243, + "theoretical_loss": 3.4809669958829983, + "tokens_seen": 1852768256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429689067201605, + "loss": 1.2679, + "theoretical_loss": 3.4809566601370983, + "tokens_seen": 1852833792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294884653961887, + "loss": 1.3461, + "theoretical_loss": 3.4809463248591332, + "tokens_seen": 1852899328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044292878635907723, + "loss": 1.3692, + "theoretical_loss": 3.4809359900490655, + "tokens_seen": 1852964864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429087261785356, + "loss": 1.4018, + "theoretical_loss": 3.480925655706857, + "tokens_seen": 1853030400 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442888665997994, + "loss": 1.4192, + "theoretical_loss": 3.48091532183247, + "tokens_seen": 1853095936 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 675642, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4188053607940674, + "objective/train/theoretical_loss": 3.480904988425867, + "objective/train/tokens_used": 223620576, + "theoretical_loss": 3.480904988425867, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044286860581745237, + "loss": 1.4056, + "theoretical_loss": 3.480904988425867, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044284854563691073, + "loss": 1.3625, + "theoretical_loss": 3.4808946554870106, + "tokens_seen": 1853227008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428284854563691, + "loss": 1.4041, + "theoretical_loss": 3.4808843230158626, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428084252758275, + "loss": 1.3701, + "theoretical_loss": 3.480873991012385, + "tokens_seen": 1853358080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427883650952859, + "loss": 1.3823, + "theoretical_loss": 3.480863659476541, + "tokens_seen": 1853423616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044276830491474424, + "loss": 1.2795, + "theoretical_loss": 3.4808533284082928, + "tokens_seen": 1853489152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427482447342026, + "loss": 1.417, + "theoretical_loss": 3.480842997807602, + "tokens_seen": 1853554688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044272818455366096, + "loss": 1.3195, + "theoretical_loss": 3.480832667674431, + "tokens_seen": 1853620224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427081243731194, + "loss": 1.3827, + "theoretical_loss": 3.4808223380087426, + "tokens_seen": 1853685760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044268806419257774, + "loss": 1.3279, + "theoretical_loss": 3.480812008810499, + "tokens_seen": 1853751296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426680040120361, + "loss": 1.2933, + "theoretical_loss": 3.4808016800796624, + "tokens_seen": 1853816832 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044264794383149447, + "loss": 1.4239, + "theoretical_loss": 3.480791351816195, + "tokens_seen": 1853882368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426278836509529, + "loss": 1.386, + "theoretical_loss": 3.480781024020059, + "tokens_seen": 1853947904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044260782347041124, + "loss": 1.3663, + "theoretical_loss": 3.4807706966912173, + "tokens_seen": 1854013440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425877632898696, + "loss": 1.363, + "theoretical_loss": 3.480760369829632, + "tokens_seen": 1854078976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044256770310932797, + "loss": 1.3051, + "theoretical_loss": 3.4807500434352647, + "tokens_seen": 1854144512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044254764292878633, + "loss": 1.3871, + "theoretical_loss": 3.4807397175080794, + "tokens_seen": 1854210048 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044252758274824475, + "loss": 1.3827, + "theoretical_loss": 3.480729392048037, + "tokens_seen": 1854275584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425075225677031, + "loss": 1.3754, + "theoretical_loss": 3.4807190670551007, + "tokens_seen": 1854341120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044248746238716147, + "loss": 1.3318, + "theoretical_loss": 3.480708742529232, + "tokens_seen": 1854406656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246740220661983, + "loss": 1.3524, + "theoretical_loss": 3.480698418470394, + "tokens_seen": 1854472192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044244734202607825, + "loss": 1.2482, + "theoretical_loss": 3.480688094878549, + "tokens_seen": 1854537728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424272818455366, + "loss": 1.3631, + "theoretical_loss": 3.480677771753659, + "tokens_seen": 1854603264 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442407221664995, + "loss": 1.3305, + "theoretical_loss": 3.4806674490956864, + "tokens_seen": 1854668800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044238716148445334, + "loss": 1.3104, + "theoretical_loss": 3.4806571269045943, + "tokens_seen": 1854734336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423671013039117, + "loss": 1.4065, + "theoretical_loss": 3.4806468051803443, + "tokens_seen": 1854799872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423470411233701, + "loss": 1.2656, + "theoretical_loss": 3.480636483922899, + "tokens_seen": 1854865408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423269809428285, + "loss": 1.3492, + "theoretical_loss": 3.480626163132221, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230692076228684, + "loss": 1.4317, + "theoretical_loss": 3.4806158428082723, + "tokens_seen": 1854996480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422868605817452, + "loss": 1.3879, + "theoretical_loss": 3.480605522951016, + "tokens_seen": 1855062016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422668004012036, + "loss": 1.2738, + "theoretical_loss": 3.480595203560414, + "tokens_seen": 1855127552 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442246740220662, + "loss": 1.3911, + "theoretical_loss": 3.4805848846364285, + "tokens_seen": 1855193088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044222668004012034, + "loss": 1.382, + "theoretical_loss": 3.480574566179022, + "tokens_seen": 1855258624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422066198595787, + "loss": 1.4112, + "theoretical_loss": 3.4805642481881574, + "tokens_seen": 1855324160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421865596790371, + "loss": 1.2419, + "theoretical_loss": 3.480553930663797, + "tokens_seen": 1855389696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044216649949849554, + "loss": 1.3281, + "theoretical_loss": 3.480543613605903, + "tokens_seen": 1855455232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421464393179539, + "loss": 1.4681, + "theoretical_loss": 3.4805332970144383, + "tokens_seen": 1855520768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044212637913741226, + "loss": 1.3449, + "theoretical_loss": 3.480522980889364, + "tokens_seen": 1855586304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421063189568706, + "loss": 1.3734, + "theoretical_loss": 3.4805126652306444, + "tokens_seen": 1855651840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044208625877632904, + "loss": 1.2939, + "theoretical_loss": 3.4805023500382406, + "tokens_seen": 1855717376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420661985957874, + "loss": 1.4606, + "theoretical_loss": 3.480492035312116, + "tokens_seen": 1855782912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044204613841524577, + "loss": 1.287, + "theoretical_loss": 3.480481721052232, + "tokens_seen": 1855848448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044202607823470413, + "loss": 1.2447, + "theoretical_loss": 3.480471407258552, + "tokens_seen": 1855913984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420060180541625, + "loss": 1.2924, + "theoretical_loss": 3.480461093931037, + "tokens_seen": 1855979520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419859578736209, + "loss": 1.4143, + "theoretical_loss": 3.480450781069652, + "tokens_seen": 1856045056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044196589769307927, + "loss": 1.2944, + "theoretical_loss": 3.4804404686743577, + "tokens_seen": 1856110592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044194583751253763, + "loss": 1.315, + "theoretical_loss": 3.4804301567451166, + "tokens_seen": 1856176128 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441925777331996, + "loss": 1.3002, + "theoretical_loss": 3.4804198452818915, + "tokens_seen": 1856241664 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419057171514544, + "loss": 1.3368, + "theoretical_loss": 3.480409534284645, + "tokens_seen": 1856307200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044188565697091277, + "loss": 1.3622, + "theoretical_loss": 3.4803992237533397, + "tokens_seen": 1856372736 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 676855, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.236203908920288, + "objective/train/theoretical_loss": 3.4803889136879373, + "objective/train/tokens_used": 226897376, + "theoretical_loss": 3.4803889136879373, + "tokens_seen": 1856438272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044186559679037113, + "loss": 1.2612, + "theoretical_loss": 3.4803889136879373, + "tokens_seen": 1856438272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418455366098295, + "loss": 1.4207, + "theoretical_loss": 3.4803786040884015, + "tokens_seen": 1856503808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418254764292879, + "loss": 1.4158, + "theoretical_loss": 3.480368294954694, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418054162487463, + "loss": 1.3158, + "theoretical_loss": 3.4803579862867773, + "tokens_seen": 1856634880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044178535606820464, + "loss": 1.3681, + "theoretical_loss": 3.4803476780846143, + "tokens_seen": 1856700416 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441765295887663, + "loss": 1.3378, + "theoretical_loss": 3.4803373703481673, + "tokens_seen": 1856765952 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044174523570712136, + "loss": 1.3602, + "theoretical_loss": 3.4803270630773984, + "tokens_seen": 1856831488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417251755265798, + "loss": 1.3324, + "theoretical_loss": 3.480316756272271, + "tokens_seen": 1856897024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044170511534603814, + "loss": 1.3608, + "theoretical_loss": 3.4803064499327476, + "tokens_seen": 1856962560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416850551654965, + "loss": 1.3962, + "theoretical_loss": 3.48029614405879, + "tokens_seen": 1857028096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044166499498495487, + "loss": 1.3728, + "theoretical_loss": 3.480285838650361, + "tokens_seen": 1857093632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416449348044133, + "loss": 1.3567, + "theoretical_loss": 3.480275533707423, + "tokens_seen": 1857159168 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044162487462387164, + "loss": 1.4274, + "theoretical_loss": 3.480265229229939, + "tokens_seen": 1857224704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044160481444333, + "loss": 1.3359, + "theoretical_loss": 3.4802549252178716, + "tokens_seen": 1857290240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044158475426278837, + "loss": 1.3228, + "theoretical_loss": 3.4802446216711824, + "tokens_seen": 1857355776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044156469408224673, + "loss": 1.3424, + "theoretical_loss": 3.4802343185898352, + "tokens_seen": 1857421312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044154463390170515, + "loss": 1.4352, + "theoretical_loss": 3.480224015973792, + "tokens_seen": 1857486848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415245737211635, + "loss": 1.4175, + "theoretical_loss": 3.480213713823015, + "tokens_seen": 1857552384 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044150451354062187, + "loss": 1.374, + "theoretical_loss": 3.4802034121374676, + "tokens_seen": 1857617920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044148445336008023, + "loss": 1.4289, + "theoretical_loss": 3.480193110917112, + "tokens_seen": 1857683456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044146439317953865, + "loss": 1.4383, + "theoretical_loss": 3.48018281016191, + "tokens_seen": 1857748992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441444332998997, + "loss": 1.2962, + "theoretical_loss": 3.4801725098718257, + "tokens_seen": 1857814528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414242728184554, + "loss": 1.294, + "theoretical_loss": 3.480162210046821, + "tokens_seen": 1857880064 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044140421263791374, + "loss": 1.3548, + "theoretical_loss": 3.4801519106868577, + "tokens_seen": 1857945600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413841524573721, + "loss": 1.3539, + "theoretical_loss": 3.480141611791899, + "tokens_seen": 1858011136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413640922768305, + "loss": 1.3828, + "theoretical_loss": 3.480131313361908, + "tokens_seen": 1858076672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413440320962889, + "loss": 1.3488, + "theoretical_loss": 3.480121015396847, + "tokens_seen": 1858142208 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044132397191574724, + "loss": 1.4505, + "theoretical_loss": 3.480110717896679, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413039117352056, + "loss": 1.3774, + "theoretical_loss": 3.480100420861365, + "tokens_seen": 1858273280 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441283851554664, + "loss": 1.4, + "theoretical_loss": 3.4800901242908693, + "tokens_seen": 1858338816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412637913741224, + "loss": 1.3801, + "theoretical_loss": 3.480079828185154, + "tokens_seen": 1858404352 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044124373119358074, + "loss": 1.3493, + "theoretical_loss": 3.480069532544182, + "tokens_seen": 1858469888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412236710130391, + "loss": 1.4411, + "theoretical_loss": 3.4800592373679153, + "tokens_seen": 1858535424 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044120361083249747, + "loss": 1.3594, + "theoretical_loss": 3.4800489426563166, + "tokens_seen": 1858600960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411835506519559, + "loss": 1.2677, + "theoretical_loss": 3.4800386484093497, + "tokens_seen": 1858666496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044116349047141425, + "loss": 1.2797, + "theoretical_loss": 3.4800283546269757, + "tokens_seen": 1858732032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411434302908726, + "loss": 1.3471, + "theoretical_loss": 3.480018061309158, + "tokens_seen": 1858797568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044112337011033097, + "loss": 1.3357, + "theoretical_loss": 3.4800077684558595, + "tokens_seen": 1858863104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411033099297894, + "loss": 1.4405, + "theoretical_loss": 3.4799974760670427, + "tokens_seen": 1858928640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044108324974924775, + "loss": 1.2756, + "theoretical_loss": 3.4799871841426695, + "tokens_seen": 1858994176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410631895687061, + "loss": 1.2834, + "theoretical_loss": 3.4799768926827035, + "tokens_seen": 1859059712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410431293881645, + "loss": 1.5183, + "theoretical_loss": 3.4799666016871074, + "tokens_seen": 1859125248 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044102306920762284, + "loss": 1.3528, + "theoretical_loss": 3.479956311155843, + "tokens_seen": 1859190784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044100300902708125, + "loss": 1.2681, + "theoretical_loss": 3.4799460210888737, + "tokens_seen": 1859256320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409829488465396, + "loss": 1.2271, + "theoretical_loss": 3.4799357314861625, + "tokens_seen": 1859321856 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440962888665998, + "loss": 1.4465, + "theoretical_loss": 3.479925442347671, + "tokens_seen": 1859387392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094282848545634, + "loss": 1.2832, + "theoretical_loss": 3.479915153673363, + "tokens_seen": 1859452928 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044092276830491476, + "loss": 1.2743, + "theoretical_loss": 3.4799048654632, + "tokens_seen": 1859518464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409027081243731, + "loss": 1.3313, + "theoretical_loss": 3.479894577717146, + "tokens_seen": 1859584000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408826479438315, + "loss": 1.4087, + "theoretical_loss": 3.479884290435163, + "tokens_seen": 1859649536 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 678160, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2900208234786987, + "objective/train/theoretical_loss": 3.4798740036172138, + "objective/train/tokens_used": 230174176, + "theoretical_loss": 3.4798740036172138, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086258776328984, + "loss": 1.308, + "theoretical_loss": 3.4798740036172138, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408425275827482, + "loss": 1.3766, + "theoretical_loss": 3.479863717263261, + "tokens_seen": 1859780608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408224674022066, + "loss": 1.3822, + "theoretical_loss": 3.4798534313732676, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440802407221665, + "loss": 1.3408, + "theoretical_loss": 3.4798431459471963, + "tokens_seen": 1859911680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044078234704112335, + "loss": 1.3509, + "theoretical_loss": 3.4798328609850095, + "tokens_seen": 1859977216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407622868605817, + "loss": 1.3909, + "theoretical_loss": 3.4798225764866704, + "tokens_seen": 1860042752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407422266800401, + "loss": 1.3062, + "theoretical_loss": 3.4798122924521415, + "tokens_seen": 1860108288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407221664994985, + "loss": 1.3854, + "theoretical_loss": 3.4798020088813852, + "tokens_seen": 1860173824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044070210631895685, + "loss": 1.3777, + "theoretical_loss": 3.4797917257743647, + "tokens_seen": 1860239360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044068204613841527, + "loss": 1.3238, + "theoretical_loss": 3.479781443131043, + "tokens_seen": 1860304896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406619859578737, + "loss": 1.3782, + "theoretical_loss": 3.479771160951382, + "tokens_seen": 1860370432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044064192577733204, + "loss": 1.3323, + "theoretical_loss": 3.479760879235345, + "tokens_seen": 1860435968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406218655967904, + "loss": 1.3067, + "theoretical_loss": 3.4797505979828953, + "tokens_seen": 1860501504 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044060180541624877, + "loss": 1.3041, + "theoretical_loss": 3.4797403171939942, + "tokens_seen": 1860567040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044058174523570713, + "loss": 1.4958, + "theoretical_loss": 3.479730036868606, + "tokens_seen": 1860632576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044056168505516555, + "loss": 1.3577, + "theoretical_loss": 3.479719757006693, + "tokens_seen": 1860698112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405416248746239, + "loss": 1.318, + "theoretical_loss": 3.4797094776082176, + "tokens_seen": 1860763648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044052156469408227, + "loss": 1.375, + "theoretical_loss": 3.4796991986731425, + "tokens_seen": 1860829184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044050150451354063, + "loss": 1.304, + "theoretical_loss": 3.479688920201431, + "tokens_seen": 1860894720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044048144433299905, + "loss": 1.4008, + "theoretical_loss": 3.4796786421930457, + "tokens_seen": 1860960256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404613841524574, + "loss": 1.2228, + "theoretical_loss": 3.4796683646479494, + "tokens_seen": 1861025792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404413239719158, + "loss": 1.3021, + "theoretical_loss": 3.479658087566105, + "tokens_seen": 1861091328 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044042126379137414, + "loss": 1.3481, + "theoretical_loss": 3.4796478109474753, + "tokens_seen": 1861156864 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404012036108325, + "loss": 1.4006, + "theoretical_loss": 3.4796375347920225, + "tokens_seen": 1861222400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403811434302909, + "loss": 1.3697, + "theoretical_loss": 3.4796272590997104, + "tokens_seen": 1861287936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403610832497493, + "loss": 1.3966, + "theoretical_loss": 3.4796169838705016, + "tokens_seen": 1861353472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044034102306920764, + "loss": 1.2961, + "theoretical_loss": 3.479606709104358, + "tokens_seen": 1861419008 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440320962888666, + "loss": 1.464, + "theoretical_loss": 3.4795964348012434, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403009027081244, + "loss": 1.3555, + "theoretical_loss": 3.4795861609611203, + "tokens_seen": 1861550080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402808425275828, + "loss": 1.3636, + "theoretical_loss": 3.4795758875839518, + "tokens_seen": 1861615616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044026078234704114, + "loss": 1.3916, + "theoretical_loss": 3.4795656146697, + "tokens_seen": 1861681152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402407221664995, + "loss": 1.39, + "theoretical_loss": 3.4795553422183287, + "tokens_seen": 1861746688 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044022066198595787, + "loss": 1.4329, + "theoretical_loss": 3.4795450702298005, + "tokens_seen": 1861812224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402006018054163, + "loss": 1.3499, + "theoretical_loss": 3.4795347987040777, + "tokens_seen": 1861877760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044018054162487465, + "loss": 1.3222, + "theoretical_loss": 3.4795245276411233, + "tokens_seen": 1861943296 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440160481444333, + "loss": 1.5041, + "theoretical_loss": 3.4795142570409006, + "tokens_seen": 1862008832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044014042126379137, + "loss": 1.4517, + "theoretical_loss": 3.4795039869033726, + "tokens_seen": 1862074368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401203610832498, + "loss": 1.2425, + "theoretical_loss": 3.4794937172285016, + "tokens_seen": 1862139904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044010030090270815, + "loss": 1.423, + "theoretical_loss": 3.479483448016251, + "tokens_seen": 1862205440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400802407221665, + "loss": 1.4235, + "theoretical_loss": 3.479473179266583, + "tokens_seen": 1862270976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400601805416249, + "loss": 1.3433, + "theoretical_loss": 3.4794629109794606, + "tokens_seen": 1862336512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044004012036108324, + "loss": 1.3064, + "theoretical_loss": 3.479452643154848, + "tokens_seen": 1862402048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044002006018054165, + "loss": 1.3943, + "theoretical_loss": 3.479442375792706, + "tokens_seen": 1862467584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044, + "loss": 1.4319, + "theoretical_loss": 3.4794321088929987, + "tokens_seen": 1862533120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399799398194584, + "loss": 1.2236, + "theoretical_loss": 3.4794218424556895, + "tokens_seen": 1862598656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043995987963891674, + "loss": 1.3413, + "theoretical_loss": 3.4794115764807403, + "tokens_seen": 1862664192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043993981945837516, + "loss": 1.37, + "theoretical_loss": 3.479401310968114, + "tokens_seen": 1862729728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399197592778335, + "loss": 1.3581, + "theoretical_loss": 3.479391045917774, + "tokens_seen": 1862795264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398996990972919, + "loss": 1.3499, + "theoretical_loss": 3.4793807813296835, + "tokens_seen": 1862860800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043987963891675024, + "loss": 1.2575, + "theoretical_loss": 3.479370517203805, + "tokens_seen": 1862926336 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 679393, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3746061325073242, + "objective/train/theoretical_loss": 3.4793602535401016, + "objective/train/tokens_used": 233450976, + "theoretical_loss": 3.4793602535401016, + "tokens_seen": 1862991872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398595787362086, + "loss": 1.2845, + "theoretical_loss": 3.4793602535401016, + "tokens_seen": 1862991872 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439839518555667, + "loss": 1.2539, + "theoretical_loss": 3.4793499903385356, + "tokens_seen": 1863057408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398194583751254, + "loss": 1.4335, + "theoretical_loss": 3.4793397275990703, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043979939819458375, + "loss": 1.2686, + "theoretical_loss": 3.479329465321669, + "tokens_seen": 1863188480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397793380140421, + "loss": 1.3696, + "theoretical_loss": 3.4793192035062948, + "tokens_seen": 1863254016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397592778335005, + "loss": 1.309, + "theoretical_loss": 3.47930894215291, + "tokens_seen": 1863319552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397392176529589, + "loss": 1.3723, + "theoretical_loss": 3.4792986812614775, + "tokens_seen": 1863385088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043971915747241725, + "loss": 1.2588, + "theoretical_loss": 3.479288420831961, + "tokens_seen": 1863450624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396990972918756, + "loss": 1.3809, + "theoretical_loss": 3.479278160864323, + "tokens_seen": 1863516160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043967903711133403, + "loss": 1.4304, + "theoretical_loss": 3.4792679013585266, + "tokens_seen": 1863581696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396589769307924, + "loss": 1.3769, + "theoretical_loss": 3.4792576423145345, + "tokens_seen": 1863647232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043963891675025075, + "loss": 1.4156, + "theoretical_loss": 3.4792473837323095, + "tokens_seen": 1863712768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396188565697091, + "loss": 1.3513, + "theoretical_loss": 3.4792371256118155, + "tokens_seen": 1863778304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395987963891675, + "loss": 1.3958, + "theoretical_loss": 3.479226867953014, + "tokens_seen": 1863843840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395787362086259, + "loss": 1.3839, + "theoretical_loss": 3.47921661075587, + "tokens_seen": 1863909376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043955867602808426, + "loss": 1.3333, + "theoretical_loss": 3.4792063540203446, + "tokens_seen": 1863974912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395386158475426, + "loss": 1.3939, + "theoretical_loss": 3.479196097746402, + "tokens_seen": 1864040448 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439518555667001, + "loss": 1.373, + "theoretical_loss": 3.4791858419340045, + "tokens_seen": 1864105984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394984954864594, + "loss": 1.3925, + "theoretical_loss": 3.479175586583116, + "tokens_seen": 1864171520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043947843530591776, + "loss": 1.4448, + "theoretical_loss": 3.4791653316936983, + "tokens_seen": 1864237056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394583751253761, + "loss": 1.3389, + "theoretical_loss": 3.479155077265715, + "tokens_seen": 1864302592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394383149448345, + "loss": 1.3119, + "theoretical_loss": 3.479144823299129, + "tokens_seen": 1864368128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043941825476429285, + "loss": 1.4422, + "theoretical_loss": 3.479134569793904, + "tokens_seen": 1864433664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043939819458375126, + "loss": 1.214, + "theoretical_loss": 3.479124316750002, + "tokens_seen": 1864499200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393781344032096, + "loss": 1.3549, + "theoretical_loss": 3.479114064167387, + "tokens_seen": 1864564736 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439358074222668, + "loss": 1.3359, + "theoretical_loss": 3.479103812046021, + "tokens_seen": 1864630272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933801404212635, + "loss": 1.336, + "theoretical_loss": 3.4790935603858677, + "tokens_seen": 1864695808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043931795386158477, + "loss": 1.4317, + "theoretical_loss": 3.4790833091868905, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043929789368104313, + "loss": 1.3956, + "theoretical_loss": 3.4790730584490515, + "tokens_seen": 1864826880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392778335005015, + "loss": 1.2467, + "theoretical_loss": 3.479062808172314, + "tokens_seen": 1864892416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043925777331995985, + "loss": 1.3515, + "theoretical_loss": 3.479052558356642, + "tokens_seen": 1864957952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392377131394182, + "loss": 1.3135, + "theoretical_loss": 3.479042309001997, + "tokens_seen": 1865023488 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921765295887663, + "loss": 1.4365, + "theoretical_loss": 3.4790320601083433, + "tokens_seen": 1865089024 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439197592778335, + "loss": 1.3232, + "theoretical_loss": 3.4790218116756435, + "tokens_seen": 1865154560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391775325977934, + "loss": 1.3017, + "theoretical_loss": 3.4790115637038608, + "tokens_seen": 1865220096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043915747241725177, + "loss": 1.2733, + "theoretical_loss": 3.4790013161929583, + "tokens_seen": 1865285632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391374122367102, + "loss": 1.3725, + "theoretical_loss": 3.478991069142899, + "tokens_seen": 1865351168 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043911735205616855, + "loss": 1.3736, + "theoretical_loss": 3.4789808225536456, + "tokens_seen": 1865416704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390972918756269, + "loss": 1.3882, + "theoretical_loss": 3.478970576425162, + "tokens_seen": 1865482240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390772316950853, + "loss": 1.3529, + "theoretical_loss": 3.4789603307574106, + "tokens_seen": 1865547776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043905717151454364, + "loss": 1.4199, + "theoretical_loss": 3.478950085550355, + "tokens_seen": 1865613312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043903711133400205, + "loss": 1.3859, + "theoretical_loss": 3.478939840803958, + "tokens_seen": 1865678848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390170511534604, + "loss": 1.3307, + "theoretical_loss": 3.478929596518183, + "tokens_seen": 1865744384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389969909729188, + "loss": 1.3402, + "theoretical_loss": 3.4789193526929925, + "tokens_seen": 1865809920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043897693079237714, + "loss": 1.3475, + "theoretical_loss": 3.4789091093283497, + "tokens_seen": 1865875456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043895687061183556, + "loss": 1.2773, + "theoretical_loss": 3.4788988664242186, + "tokens_seen": 1865940992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389368104312939, + "loss": 1.3979, + "theoretical_loss": 3.478888623980562, + "tokens_seen": 1866006528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389167502507523, + "loss": 1.4352, + "theoretical_loss": 3.478878381997342, + "tokens_seen": 1866072064 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043889669007021064, + "loss": 1.4133, + "theoretical_loss": 3.478868140474523, + "tokens_seen": 1866137600 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438876629889669, + "loss": 1.3842, + "theoretical_loss": 3.478857899412067, + "tokens_seen": 1866203136 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 681054, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4403936862945557, + "objective/train/theoretical_loss": 3.478847658809938, + "objective/train/tokens_used": 236727776, + "theoretical_loss": 3.478847658809938, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388565697091274, + "loss": 1.3943, + "theoretical_loss": 3.478847658809938, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388365095285858, + "loss": 1.262, + "theoretical_loss": 3.4788374186680997, + "tokens_seen": 1866334208 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043881644934804415, + "loss": 1.3017, + "theoretical_loss": 3.4788271789865135, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387963891675025, + "loss": 1.387, + "theoretical_loss": 3.478816939765144, + "tokens_seen": 1866465280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387763289869609, + "loss": 1.3761, + "theoretical_loss": 3.478806701003954, + "tokens_seen": 1866530816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387562688064193, + "loss": 1.2479, + "theoretical_loss": 3.478796462702906, + "tokens_seen": 1866596352 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043873620862587765, + "loss": 1.3339, + "theoretical_loss": 3.4787862248619645, + "tokens_seen": 1866661888 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438716148445336, + "loss": 1.3381, + "theoretical_loss": 3.4787759874810913, + "tokens_seen": 1866727424 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043869608826479443, + "loss": 1.4082, + "theoretical_loss": 3.4787657505602505, + "tokens_seen": 1866792960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386760280842528, + "loss": 1.3474, + "theoretical_loss": 3.478755514099405, + "tokens_seen": 1866858496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043865596790371115, + "loss": 1.4598, + "theoretical_loss": 3.4787452780985175, + "tokens_seen": 1866924032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386359077231695, + "loss": 1.3573, + "theoretical_loss": 3.4787350425575516, + "tokens_seen": 1866989568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386158475426279, + "loss": 1.3373, + "theoretical_loss": 3.478724807476471, + "tokens_seen": 1867055104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385957873620863, + "loss": 1.3677, + "theoretical_loss": 3.478714572855238, + "tokens_seen": 1867120640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043857572718154466, + "loss": 1.3373, + "theoretical_loss": 3.4787043386938166, + "tokens_seen": 1867186176 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438555667001003, + "loss": 1.5246, + "theoretical_loss": 3.4786941049921687, + "tokens_seen": 1867251712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385356068204614, + "loss": 1.3978, + "theoretical_loss": 3.4786838717502593, + "tokens_seen": 1867317248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385155466399198, + "loss": 1.3118, + "theoretical_loss": 3.4786736389680506, + "tokens_seen": 1867382784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043849548645937816, + "loss": 1.4362, + "theoretical_loss": 3.4786634066455058, + "tokens_seen": 1867448320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384754262788365, + "loss": 1.395, + "theoretical_loss": 3.478653174782588, + "tokens_seen": 1867513856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384553660982949, + "loss": 1.3358, + "theoretical_loss": 3.4786429433792607, + "tokens_seen": 1867579392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043843530591775325, + "loss": 1.3989, + "theoretical_loss": 3.4786327124354877, + "tokens_seen": 1867644928 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043841524573721166, + "loss": 1.3759, + "theoretical_loss": 3.4786224819512306, + "tokens_seen": 1867710464 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043839518555667, + "loss": 1.3507, + "theoretical_loss": 3.478612251926455, + "tokens_seen": 1867776000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383751253761284, + "loss": 1.3864, + "theoretical_loss": 3.478602022361122, + "tokens_seen": 1867841536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043835506519558675, + "loss": 1.4664, + "theoretical_loss": 3.4785917932551955, + "tokens_seen": 1867907072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043833500501504517, + "loss": 1.3423, + "theoretical_loss": 3.478581564608639, + "tokens_seen": 1867972608 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043831494483450353, + "loss": 1.4152, + "theoretical_loss": 3.4785713364214166, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382948846539619, + "loss": 1.4306, + "theoretical_loss": 3.47856110869349, + "tokens_seen": 1868103680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043827482447342025, + "loss": 1.3792, + "theoretical_loss": 3.4785508814248227, + "tokens_seen": 1868169216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382547642928786, + "loss": 1.4546, + "theoretical_loss": 3.478540654615379, + "tokens_seen": 1868234752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043823470411233703, + "loss": 1.3498, + "theoretical_loss": 3.4785304282651213, + "tokens_seen": 1868300288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382146439317954, + "loss": 1.2888, + "theoretical_loss": 3.478520202374013, + "tokens_seen": 1868365824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043819458375125376, + "loss": 1.4168, + "theoretical_loss": 3.478509976942018, + "tokens_seen": 1868431360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381745235707121, + "loss": 1.397, + "theoretical_loss": 3.478499751969098, + "tokens_seen": 1868496896 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043815446339017053, + "loss": 1.4325, + "theoretical_loss": 3.4784895274552183, + "tokens_seen": 1868562432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381344032096289, + "loss": 1.4241, + "theoretical_loss": 3.478479303400341, + "tokens_seen": 1868627968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043811434302908726, + "loss": 1.4264, + "theoretical_loss": 3.4784690798044298, + "tokens_seen": 1868693504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380942828485456, + "loss": 1.37, + "theoretical_loss": 3.478458856667448, + "tokens_seen": 1868759040 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438074222668004, + "loss": 1.4807, + "theoretical_loss": 3.478448633989358, + "tokens_seen": 1868824576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380541624874624, + "loss": 1.5314, + "theoretical_loss": 3.4784384117701244, + "tokens_seen": 1868890112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043803410230692076, + "loss": 1.3582, + "theoretical_loss": 3.4784281900097103, + "tokens_seen": 1868955648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380140421263791, + "loss": 1.4331, + "theoretical_loss": 3.4784179687080785, + "tokens_seen": 1869021184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379939819458375, + "loss": 1.316, + "theoretical_loss": 3.4784077478651922, + "tokens_seen": 1869086720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379739217652959, + "loss": 1.4006, + "theoretical_loss": 3.4783975274810155, + "tokens_seen": 1869152256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043795386158475427, + "loss": 1.3135, + "theoretical_loss": 3.478387307555511, + "tokens_seen": 1869217792 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043793380140421263, + "loss": 1.374, + "theoretical_loss": 3.478377088088642, + "tokens_seen": 1869283328 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437913741223671, + "loss": 1.3584, + "theoretical_loss": 3.4783668690803733, + "tokens_seen": 1869348864 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043789368104312935, + "loss": 1.3697, + "theoretical_loss": 3.478356650530666, + "tokens_seen": 1869414400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043787362086258777, + "loss": 1.4704, + "theoretical_loss": 3.4783464324394853, + "tokens_seen": 1869479936 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 682171, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3038382530212402, + "objective/train/theoretical_loss": 3.4783362148067933, + "objective/train/tokens_used": 240004576, + "theoretical_loss": 3.4783362148067933, + "tokens_seen": 1869545472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785356068204613, + "loss": 1.372, + "theoretical_loss": 3.4783362148067933, + "tokens_seen": 1869545472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378335005015045, + "loss": 1.3688, + "theoretical_loss": 3.4783259976325542, + "tokens_seen": 1869611008 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781344032096286, + "loss": 1.4204, + "theoretical_loss": 3.4783157809167307, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043779338014042127, + "loss": 1.5168, + "theoretical_loss": 3.4783055646592866, + "tokens_seen": 1869742080 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777331995987963, + "loss": 1.4738, + "theoretical_loss": 3.4782953488601853, + "tokens_seen": 1869807616 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437753259779338, + "loss": 1.3557, + "theoretical_loss": 3.47828513351939, + "tokens_seen": 1869873152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043773319959879636, + "loss": 1.4439, + "theoretical_loss": 3.478274918636864, + "tokens_seen": 1869938688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377131394182547, + "loss": 1.2031, + "theoretical_loss": 3.478264704212571, + "tokens_seen": 1870004224 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043769307923771314, + "loss": 1.3398, + "theoretical_loss": 3.4782544902464743, + "tokens_seen": 1870069760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043767301905717155, + "loss": 1.3298, + "theoretical_loss": 3.4782442767385366, + "tokens_seen": 1870135296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376529588766299, + "loss": 1.4249, + "theoretical_loss": 3.4782340636887223, + "tokens_seen": 1870200832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376328986960883, + "loss": 1.3177, + "theoretical_loss": 3.4782238510969945, + "tokens_seen": 1870266368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376128385155467, + "loss": 1.4201, + "theoretical_loss": 3.4782136389633163, + "tokens_seen": 1870331904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043759277833500506, + "loss": 1.3452, + "theoretical_loss": 3.478203427287651, + "tokens_seen": 1870397440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375727181544634, + "loss": 1.3268, + "theoretical_loss": 3.4781932160699625, + "tokens_seen": 1870462976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375526579739218, + "loss": 1.261, + "theoretical_loss": 3.478183005310214, + "tokens_seen": 1870528512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375325977933802, + "loss": 1.3436, + "theoretical_loss": 3.478172795008369, + "tokens_seen": 1870594048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043751253761283856, + "loss": 1.2855, + "theoretical_loss": 3.478162585164391, + "tokens_seen": 1870659584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374924774322969, + "loss": 1.3365, + "theoretical_loss": 3.478152375778243, + "tokens_seen": 1870725120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374724172517553, + "loss": 1.5139, + "theoretical_loss": 3.4781421668498886, + "tokens_seen": 1870790656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043745235707121365, + "loss": 1.3465, + "theoretical_loss": 3.4781319583792913, + "tokens_seen": 1870856192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043743229689067206, + "loss": 1.3418, + "theoretical_loss": 3.478121750366415, + "tokens_seen": 1870921728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374122367101304, + "loss": 1.3394, + "theoretical_loss": 3.478111542811222, + "tokens_seen": 1870987264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373921765295888, + "loss": 1.5021, + "theoretical_loss": 3.4781013357136774, + "tokens_seen": 1871052800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043737211634904715, + "loss": 1.4359, + "theoretical_loss": 3.4780911290737433, + "tokens_seen": 1871118336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043735205616850557, + "loss": 1.435, + "theoretical_loss": 3.478080922891383, + "tokens_seen": 1871183872 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043733199598796393, + "loss": 1.4357, + "theoretical_loss": 3.4780707171665615, + "tokens_seen": 1871249408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373119358074223, + "loss": 1.4165, + "theoretical_loss": 3.478060511899241, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043729187562688065, + "loss": 1.4591, + "theoretical_loss": 3.478050307089385, + "tokens_seen": 1871380480 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437271815446339, + "loss": 1.3839, + "theoretical_loss": 3.4780401027369576, + "tokens_seen": 1871446016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043725175526579743, + "loss": 1.4561, + "theoretical_loss": 3.478029898841922, + "tokens_seen": 1871511552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372316950852558, + "loss": 1.3793, + "theoretical_loss": 3.4780196954042415, + "tokens_seen": 1871577088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043721163490471416, + "loss": 1.3833, + "theoretical_loss": 3.4780094924238796, + "tokens_seen": 1871642624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371915747241725, + "loss": 1.4802, + "theoretical_loss": 3.4779992899008, + "tokens_seen": 1871708160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043717151454363093, + "loss": 1.407, + "theoretical_loss": 3.477989087834966, + "tokens_seen": 1871773696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371514543630893, + "loss": 1.2718, + "theoretical_loss": 3.477978886226341, + "tokens_seen": 1871839232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043713139418254766, + "loss": 1.4083, + "theoretical_loss": 3.477968685074889, + "tokens_seen": 1871904768 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437111334002006, + "loss": 1.3741, + "theoretical_loss": 3.477958484380573, + "tokens_seen": 1871970304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370912738214644, + "loss": 1.4443, + "theoretical_loss": 3.477948284143357, + "tokens_seen": 1872035840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370712136409228, + "loss": 1.4117, + "theoretical_loss": 3.477938084363204, + "tokens_seen": 1872101376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043705115346038116, + "loss": 1.3838, + "theoretical_loss": 3.477927885040078, + "tokens_seen": 1872166912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370310932798395, + "loss": 1.346, + "theoretical_loss": 3.477917686173942, + "tokens_seen": 1872232448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370110330992979, + "loss": 1.2985, + "theoretical_loss": 3.4779074877647593, + "tokens_seen": 1872297984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369909729187563, + "loss": 1.3435, + "theoretical_loss": 3.4778972898124945, + "tokens_seen": 1872363520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043697091273821467, + "loss": 1.326, + "theoretical_loss": 3.477887092317111, + "tokens_seen": 1872429056 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043695085255767303, + "loss": 1.3718, + "theoretical_loss": 3.4778768952785715, + "tokens_seen": 1872494592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369307923771314, + "loss": 1.3345, + "theoretical_loss": 3.4778666986968396, + "tokens_seen": 1872560128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043691073219658975, + "loss": 1.4466, + "theoretical_loss": 3.4778565025718797, + "tokens_seen": 1872625664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043689067201604817, + "loss": 1.3711, + "theoretical_loss": 3.477846306903655, + "tokens_seen": 1872691200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043687061183550653, + "loss": 1.3178, + "theoretical_loss": 3.4778361116921284, + "tokens_seen": 1872756736 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 683408, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4890151023864746, + "objective/train/theoretical_loss": 3.477825916937264, + "objective/train/tokens_used": 243281376, + "theoretical_loss": 3.477825916937264, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368505516549649, + "loss": 1.5197, + "theoretical_loss": 3.477825916937264, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043683049147442326, + "loss": 1.4975, + "theoretical_loss": 3.4778157226390256, + "tokens_seen": 1872887808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043681043129388167, + "loss": 1.445, + "theoretical_loss": 3.4778055287973766, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043679037111334003, + "loss": 1.3477, + "theoretical_loss": 3.4777953354122806, + "tokens_seen": 1873018880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367703109327984, + "loss": 1.4355, + "theoretical_loss": 3.4777851424837007, + "tokens_seen": 1873084416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043675025075225676, + "loss": 1.5064, + "theoretical_loss": 3.477774950011601, + "tokens_seen": 1873149952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367301905717151, + "loss": 1.3944, + "theoretical_loss": 3.4777647579959448, + "tokens_seen": 1873215488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043671013039117354, + "loss": 1.4303, + "theoretical_loss": 3.477754566436696, + "tokens_seen": 1873281024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366900702106319, + "loss": 1.3826, + "theoretical_loss": 3.4777443753338178, + "tokens_seen": 1873346560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043667001003009026, + "loss": 1.4101, + "theoretical_loss": 3.4777341846872742, + "tokens_seen": 1873412096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366499498495486, + "loss": 1.3987, + "theoretical_loss": 3.477723994497029, + "tokens_seen": 1873477632 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043662988966900704, + "loss": 1.5339, + "theoretical_loss": 3.477713804763045, + "tokens_seen": 1873543168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366098294884654, + "loss": 1.42, + "theoretical_loss": 3.4777036154852863, + "tokens_seen": 1873608704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043658976930792376, + "loss": 1.3668, + "theoretical_loss": 3.4776934266637167, + "tokens_seen": 1873674240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656970912738213, + "loss": 1.4386, + "theoretical_loss": 3.477683238298299, + "tokens_seen": 1873739776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365496489468405, + "loss": 1.3422, + "theoretical_loss": 3.477673050388998, + "tokens_seen": 1873805312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365295887662989, + "loss": 1.332, + "theoretical_loss": 3.4776628629357766, + "tokens_seen": 1873870848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043650952858575727, + "loss": 1.2139, + "theoretical_loss": 3.4776526759385984, + "tokens_seen": 1873936384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648946840521563, + "loss": 1.3377, + "theoretical_loss": 3.477642489397428, + "tokens_seen": 1874001920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436469408224674, + "loss": 1.4418, + "theoretical_loss": 3.477632303312227, + "tokens_seen": 1874067456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364493480441324, + "loss": 1.3544, + "theoretical_loss": 3.4776221176829614, + "tokens_seen": 1874132992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043642928786359077, + "loss": 1.4147, + "theoretical_loss": 3.477611932509593, + "tokens_seen": 1874198528 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043640922768304913, + "loss": 1.4358, + "theoretical_loss": 3.4776017477920864, + "tokens_seen": 1874264064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363891675025075, + "loss": 1.3215, + "theoretical_loss": 3.477591563530405, + "tokens_seen": 1874329600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363691073219659, + "loss": 1.4032, + "theoretical_loss": 3.477581379724513, + "tokens_seen": 1874395136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363490471414243, + "loss": 1.4525, + "theoretical_loss": 3.477571196374373, + "tokens_seen": 1874460672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632898696088264, + "loss": 1.3883, + "theoretical_loss": 3.4775610134799497, + "tokens_seen": 1874526208 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436308926780341, + "loss": 1.2826, + "theoretical_loss": 3.477550831041206, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043628886659979936, + "loss": 1.4635, + "theoretical_loss": 3.4775406490581062, + "tokens_seen": 1874657280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362688064192578, + "loss": 1.4793, + "theoretical_loss": 3.4775304675306136, + "tokens_seen": 1874722816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043624874623871614, + "loss": 1.3495, + "theoretical_loss": 3.4775202864586916, + "tokens_seen": 1874788352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362286860581745, + "loss": 1.5164, + "theoretical_loss": 3.477510105842305, + "tokens_seen": 1874853888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043620862587763286, + "loss": 1.3681, + "theoretical_loss": 3.4774999256814163, + "tokens_seen": 1874919424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361885656970913, + "loss": 1.3664, + "theoretical_loss": 3.4774897459759897, + "tokens_seen": 1874984960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361685055165497, + "loss": 1.322, + "theoretical_loss": 3.477479566725989, + "tokens_seen": 1875050496 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043614844533600806, + "loss": 1.3255, + "theoretical_loss": 3.477469387931378, + "tokens_seen": 1875116032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361283851554664, + "loss": 1.485, + "theoretical_loss": 3.4774592095921197, + "tokens_seen": 1875181568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361083249749248, + "loss": 1.3523, + "theoretical_loss": 3.4774490317081783, + "tokens_seen": 1875247104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360882647943832, + "loss": 1.3344, + "theoretical_loss": 3.477438854279518, + "tokens_seen": 1875312640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043606820461384156, + "loss": 1.3527, + "theoretical_loss": 3.4774286773061016, + "tokens_seen": 1875378176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360481444332999, + "loss": 1.414, + "theoretical_loss": 3.477418500787894, + "tokens_seen": 1875443712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360280842527583, + "loss": 1.3891, + "theoretical_loss": 3.4774083247248573, + "tokens_seen": 1875509248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360080240722167, + "loss": 1.3888, + "theoretical_loss": 3.477398149116957, + "tokens_seen": 1875574784 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043598796389167507, + "loss": 1.3952, + "theoretical_loss": 3.4773879739641553, + "tokens_seen": 1875640320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043596790371113343, + "loss": 1.3615, + "theoretical_loss": 3.477377799266417, + "tokens_seen": 1875705856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359478435305918, + "loss": 1.3103, + "theoretical_loss": 3.4773676250237058, + "tokens_seen": 1875771392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043592778335005015, + "loss": 1.3074, + "theoretical_loss": 3.477357451235984, + "tokens_seen": 1875836928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043590772316950857, + "loss": 1.4332, + "theoretical_loss": 3.4773472779032177, + "tokens_seen": 1875902464 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043588766298896693, + "loss": 1.3871, + "theoretical_loss": 3.477337105025369, + "tokens_seen": 1875968000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358676028084253, + "loss": 1.3467, + "theoretical_loss": 3.477326932602402, + "tokens_seen": 1876033536 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 684134, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3862347602844238, + "objective/train/theoretical_loss": 3.477316760634281, + "objective/train/tokens_used": 246558176, + "theoretical_loss": 3.477316760634281, + "tokens_seen": 1876099072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043584754262788366, + "loss": 1.3679, + "theoretical_loss": 3.477316760634281, + "tokens_seen": 1876099072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043582748244734207, + "loss": 1.4095, + "theoretical_loss": 3.4773065891209693, + "tokens_seen": 1876164608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043580742226680043, + "loss": 1.3806, + "theoretical_loss": 3.477296418062431, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357873620862588, + "loss": 1.401, + "theoretical_loss": 3.477286247458629, + "tokens_seen": 1876295680 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043576730190571716, + "loss": 1.3478, + "theoretical_loss": 3.477276077309528, + "tokens_seen": 1876361216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357472417251755, + "loss": 1.4373, + "theoretical_loss": 3.4772659076150916, + "tokens_seen": 1876426752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043572718154463394, + "loss": 1.3464, + "theoretical_loss": 3.477255738375283, + "tokens_seen": 1876492288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357071213640923, + "loss": 1.4229, + "theoretical_loss": 3.4772455695900675, + "tokens_seen": 1876557824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043568706118355066, + "loss": 1.2518, + "theoretical_loss": 3.4772354012594073, + "tokens_seen": 1876623360 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435667001003009, + "loss": 1.4456, + "theoretical_loss": 3.4772252333832667, + "tokens_seen": 1876688896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043564694082246744, + "loss": 1.3568, + "theoretical_loss": 3.4772150659616097, + "tokens_seen": 1876754432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356268806419258, + "loss": 1.3992, + "theoretical_loss": 3.4772048989944, + "tokens_seen": 1876819968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043560682046138417, + "loss": 1.4018, + "theoretical_loss": 3.4771947324816024, + "tokens_seen": 1876885504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043558676028084253, + "loss": 1.4062, + "theoretical_loss": 3.477184566423179, + "tokens_seen": 1876951040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355667001003009, + "loss": 1.4356, + "theoretical_loss": 3.477174400819094, + "tokens_seen": 1877016576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355466399197593, + "loss": 1.4477, + "theoretical_loss": 3.4771642356693127, + "tokens_seen": 1877082112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043552657973921767, + "loss": 1.4157, + "theoretical_loss": 3.477154070973797, + "tokens_seen": 1877147648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043550651955867603, + "loss": 1.4087, + "theoretical_loss": 3.477143906732512, + "tokens_seen": 1877213184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354864593781344, + "loss": 1.4263, + "theoretical_loss": 3.477133742945421, + "tokens_seen": 1877278720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354663991975928, + "loss": 1.4682, + "theoretical_loss": 3.4771235796124884, + "tokens_seen": 1877344256 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043544633901705117, + "loss": 1.3881, + "theoretical_loss": 3.4771134167336775, + "tokens_seen": 1877409792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043542627883650953, + "loss": 1.4303, + "theoretical_loss": 3.477103254308952, + "tokens_seen": 1877475328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354062186559679, + "loss": 1.4407, + "theoretical_loss": 3.4770930923382766, + "tokens_seen": 1877540864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353861584754263, + "loss": 1.5102, + "theoretical_loss": 3.4770829308216147, + "tokens_seen": 1877606400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353660982948847, + "loss": 1.4889, + "theoretical_loss": 3.47707276975893, + "tokens_seen": 1877671936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043534603811434304, + "loss": 1.3806, + "theoretical_loss": 3.4770626091501864, + "tokens_seen": 1877737472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353259779338014, + "loss": 1.4294, + "theoretical_loss": 3.477052448995348, + "tokens_seen": 1877803008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043530591775325976, + "loss": 1.4653, + "theoretical_loss": 3.4770422892943786, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352858575727182, + "loss": 1.381, + "theoretical_loss": 3.4770321300472418, + "tokens_seen": 1877934080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043526579739217654, + "loss": 1.4214, + "theoretical_loss": 3.477021971253902, + "tokens_seen": 1877999616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352457372116349, + "loss": 1.4503, + "theoretical_loss": 3.4770118129143226, + "tokens_seen": 1878065152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043522567703109326, + "loss": 1.4693, + "theoretical_loss": 3.4770016550284684, + "tokens_seen": 1878130688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352056168505517, + "loss": 1.3785, + "theoretical_loss": 3.476991497596302, + "tokens_seen": 1878196224 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043518555667001004, + "loss": 1.5234, + "theoretical_loss": 3.476981340617788, + "tokens_seen": 1878261760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351654964894684, + "loss": 1.2875, + "theoretical_loss": 3.47697118409289, + "tokens_seen": 1878327296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043514543630892677, + "loss": 1.3326, + "theoretical_loss": 3.476961028021573, + "tokens_seen": 1878392832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043512537612838513, + "loss": 1.3927, + "theoretical_loss": 3.4769508724038, + "tokens_seen": 1878458368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043510531594784355, + "loss": 1.4017, + "theoretical_loss": 3.4769407172395344, + "tokens_seen": 1878523904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350852557673019, + "loss": 1.4232, + "theoretical_loss": 3.476930562528741, + "tokens_seen": 1878589440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043506519558676027, + "loss": 1.3257, + "theoretical_loss": 3.4769204082713836, + "tokens_seen": 1878654976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043504513540621863, + "loss": 1.3627, + "theoretical_loss": 3.476910254467426, + "tokens_seen": 1878720512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043502507522567705, + "loss": 1.3708, + "theoretical_loss": 3.476900101116832, + "tokens_seen": 1878786048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350050150451354, + "loss": 1.2381, + "theoretical_loss": 3.476889948219566, + "tokens_seen": 1878851584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349849548645938, + "loss": 1.2966, + "theoretical_loss": 3.4768797957755915, + "tokens_seen": 1878917120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043496489468405214, + "loss": 1.4309, + "theoretical_loss": 3.4768696437848723, + "tokens_seen": 1878982656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349448345035105, + "loss": 1.417, + "theoretical_loss": 3.4768594922473732, + "tokens_seen": 1879048192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349247743229689, + "loss": 1.4358, + "theoretical_loss": 3.4768493411630574, + "tokens_seen": 1879113728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349047141424273, + "loss": 1.3979, + "theoretical_loss": 3.476839190531889, + "tokens_seen": 1879179264 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043488465396188564, + "loss": 1.3144, + "theoretical_loss": 3.476829040353832, + "tokens_seen": 1879244800 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434864593781344, + "loss": 1.311, + "theoretical_loss": 3.4768188906288504, + "tokens_seen": 1879310336 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 685672, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.300635576248169, + "objective/train/theoretical_loss": 3.4768087413569084, + "objective/train/tokens_used": 249834976, + "theoretical_loss": 3.4768087413569084, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348445336008024, + "loss": 1.3267, + "theoretical_loss": 3.4768087413569084, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348244734202608, + "loss": 1.5607, + "theoretical_loss": 3.4767985925379703, + "tokens_seen": 1879441408 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480441323971914, + "loss": 1.2839, + "theoretical_loss": 3.4767884441719987, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347843530591775, + "loss": 1.2734, + "theoretical_loss": 3.4767782962589586, + "tokens_seen": 1879572480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476429287863587, + "loss": 1.3421, + "theoretical_loss": 3.476768148798814, + "tokens_seen": 1879638016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347442326980943, + "loss": 1.517, + "theoretical_loss": 3.476758001791529, + "tokens_seen": 1879703552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472417251755265, + "loss": 1.374, + "theoretical_loss": 3.4767478552370674, + "tokens_seen": 1879769088 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434704112337011, + "loss": 1.2775, + "theoretical_loss": 3.4767377091353926, + "tokens_seen": 1879834624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346840521564694, + "loss": 1.416, + "theoretical_loss": 3.4767275634864694, + "tokens_seen": 1879900160 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043466399197592784, + "loss": 1.3652, + "theoretical_loss": 3.476717418290262, + "tokens_seen": 1879965696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346439317953862, + "loss": 1.3781, + "theoretical_loss": 3.476707273546733, + "tokens_seen": 1880031232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043462387161484457, + "loss": 1.3308, + "theoretical_loss": 3.4766971292558484, + "tokens_seen": 1880096768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043460381143430293, + "loss": 1.4244, + "theoretical_loss": 3.476686985417571, + "tokens_seen": 1880162304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345837512537613, + "loss": 1.4142, + "theoretical_loss": 3.476676842031865, + "tokens_seen": 1880227840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345636910732197, + "loss": 1.3023, + "theoretical_loss": 3.4766666990986947, + "tokens_seen": 1880293376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043454363089267807, + "loss": 1.3298, + "theoretical_loss": 3.476656556618024, + "tokens_seen": 1880358912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043452357071213643, + "loss": 1.3911, + "theoretical_loss": 3.4766464145898164, + "tokens_seen": 1880424448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345035105315948, + "loss": 1.4692, + "theoretical_loss": 3.4766362730140368, + "tokens_seen": 1880489984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344834503510532, + "loss": 1.3192, + "theoretical_loss": 3.476626131890649, + "tokens_seen": 1880555520 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043446339017051157, + "loss": 1.291, + "theoretical_loss": 3.476615991219617, + "tokens_seen": 1880621056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043444332998996993, + "loss": 1.37, + "theoretical_loss": 3.4766058510009046, + "tokens_seen": 1880686592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344232698094283, + "loss": 1.3437, + "theoretical_loss": 3.476595711234476, + "tokens_seen": 1880752128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344032096288867, + "loss": 1.3999, + "theoretical_loss": 3.476585571920295, + "tokens_seen": 1880817664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343831494483451, + "loss": 1.4353, + "theoretical_loss": 3.476575433058327, + "tokens_seen": 1880883200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043436308926780344, + "loss": 1.2927, + "theoretical_loss": 3.476565294648535, + "tokens_seen": 1880948736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343430290872618, + "loss": 1.4703, + "theoretical_loss": 3.4765551566908823, + "tokens_seen": 1881014272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043432296890672016, + "loss": 1.3755, + "theoretical_loss": 3.476545019185334, + "tokens_seen": 1881079808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343029087261786, + "loss": 1.328, + "theoretical_loss": 3.4765348821318547, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428284854563694, + "loss": 1.4415, + "theoretical_loss": 3.4765247455304076, + "tokens_seen": 1881210880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342627883650953, + "loss": 1.2975, + "theoretical_loss": 3.476514609380957, + "tokens_seen": 1881276416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043424272818455367, + "loss": 1.3733, + "theoretical_loss": 3.476504473683467, + "tokens_seen": 1881341952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342226680040121, + "loss": 1.4108, + "theoretical_loss": 3.4764943384379015, + "tokens_seen": 1881407488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043420260782347044, + "loss": 1.3032, + "theoretical_loss": 3.476484203644225, + "tokens_seen": 1881473024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341825476429288, + "loss": 1.3831, + "theoretical_loss": 3.4764740693024017, + "tokens_seen": 1881538560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043416248746238717, + "loss": 1.3583, + "theoretical_loss": 3.4764639354123954, + "tokens_seen": 1881604096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043414242728184553, + "loss": 1.2341, + "theoretical_loss": 3.47645380197417, + "tokens_seen": 1881669632 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043412236710130395, + "loss": 1.376, + "theoretical_loss": 3.4764436689876903, + "tokens_seen": 1881735168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341023069207623, + "loss": 1.3477, + "theoretical_loss": 3.47643353645292, + "tokens_seen": 1881800704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043408224674022067, + "loss": 1.485, + "theoretical_loss": 3.476423404369823, + "tokens_seen": 1881866240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043406218655967903, + "loss": 1.3975, + "theoretical_loss": 3.4764132727383643, + "tokens_seen": 1881931776 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043404212637913745, + "loss": 1.3323, + "theoretical_loss": 3.4764031415585066, + "tokens_seen": 1881997312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340220661985958, + "loss": 1.3039, + "theoretical_loss": 3.4763930108302157, + "tokens_seen": 1882062848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340020060180542, + "loss": 1.438, + "theoretical_loss": 3.4763828805534547, + "tokens_seen": 1882128384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043398194583751254, + "loss": 1.446, + "theoretical_loss": 3.4763727507281876, + "tokens_seen": 1882193920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339618856569709, + "loss": 1.5378, + "theoretical_loss": 3.4763626213543795, + "tokens_seen": 1882259456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339418254764293, + "loss": 1.3626, + "theoretical_loss": 3.476352492431994, + "tokens_seen": 1882324992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339217652958877, + "loss": 1.3032, + "theoretical_loss": 3.4763423639609954, + "tokens_seen": 1882390528 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043390170511534604, + "loss": 1.2521, + "theoretical_loss": 3.4763322359413475, + "tokens_seen": 1882456064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338816449348044, + "loss": 1.3163, + "theoretical_loss": 3.4763221083730147, + "tokens_seen": 1882521600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338615847542628, + "loss": 1.4895, + "theoretical_loss": 3.4763119812559613, + "tokens_seen": 1882587136 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 686423, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5192041397094727, + "objective/train/theoretical_loss": 3.4763018545901514, + "objective/train/tokens_used": 253111776, + "theoretical_loss": 3.4763018545901514, + "tokens_seen": 1882652672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338415245737212, + "loss": 1.4678, + "theoretical_loss": 3.4763018545901514, + "tokens_seen": 1882652672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043382146439317954, + "loss": 1.3997, + "theoretical_loss": 3.476291728375549, + "tokens_seen": 1882718208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338014042126379, + "loss": 1.3694, + "theoretical_loss": 3.476281602612119, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043378134403209627, + "loss": 1.4464, + "theoretical_loss": 3.476271477299825, + "tokens_seen": 1882849280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337612838515547, + "loss": 1.4271, + "theoretical_loss": 3.476261352438631, + "tokens_seen": 1882914816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043374122367101305, + "loss": 1.4207, + "theoretical_loss": 3.4762512280285014, + "tokens_seen": 1882980352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337211634904714, + "loss": 1.3115, + "theoretical_loss": 3.476241104069401, + "tokens_seen": 1883045888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043370110330992977, + "loss": 1.39, + "theoretical_loss": 3.4762309805612928, + "tokens_seen": 1883111424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336810431293882, + "loss": 1.4431, + "theoretical_loss": 3.4762208575041424, + "tokens_seen": 1883176960 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043366098294884655, + "loss": 1.4802, + "theoretical_loss": 3.476210734897913, + "tokens_seen": 1883242496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336409227683049, + "loss": 1.4396, + "theoretical_loss": 3.476200612742569, + "tokens_seen": 1883308032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336208625877633, + "loss": 1.4059, + "theoretical_loss": 3.476190491038075, + "tokens_seen": 1883373568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043360080240722164, + "loss": 1.3619, + "theoretical_loss": 3.476180369784395, + "tokens_seen": 1883439104 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043358074222668005, + "loss": 1.4411, + "theoretical_loss": 3.476170248981493, + "tokens_seen": 1883504640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335606820461384, + "loss": 1.4295, + "theoretical_loss": 3.476160128629334, + "tokens_seen": 1883570176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335406218655968, + "loss": 1.3972, + "theoretical_loss": 3.476150008727881, + "tokens_seen": 1883635712 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043352056168505514, + "loss": 1.3767, + "theoretical_loss": 3.4761398892771, + "tokens_seen": 1883701248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043350050150451356, + "loss": 1.2904, + "theoretical_loss": 3.4761297702769536, + "tokens_seen": 1883766784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334804413239719, + "loss": 1.3371, + "theoretical_loss": 3.476119651727407, + "tokens_seen": 1883832320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334603811434303, + "loss": 1.3201, + "theoretical_loss": 3.476109533628424, + "tokens_seen": 1883897856 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043344032096288864, + "loss": 1.4778, + "theoretical_loss": 3.4760994159799687, + "tokens_seen": 1883963392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433420260782347, + "loss": 1.2988, + "theoretical_loss": 3.4760892987820062, + "tokens_seen": 1884028928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334002006018054, + "loss": 1.2729, + "theoretical_loss": 3.4760791820345, + "tokens_seen": 1884094464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333801404212638, + "loss": 1.3424, + "theoretical_loss": 3.4760690657374145, + "tokens_seen": 1884160000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043336008024072215, + "loss": 1.3418, + "theoretical_loss": 3.476058949890714, + "tokens_seen": 1884225536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333400200601805, + "loss": 1.4191, + "theoretical_loss": 3.476048834494363, + "tokens_seen": 1884291072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333199598796389, + "loss": 1.3216, + "theoretical_loss": 3.4760387195483258, + "tokens_seen": 1884356608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332998996990973, + "loss": 1.381, + "theoretical_loss": 3.4760286050525666, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327983951855565, + "loss": 1.3322, + "theoretical_loss": 3.47601849100705, + "tokens_seen": 1884487680 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433259779338014, + "loss": 1.4154, + "theoretical_loss": 3.4760083774117394, + "tokens_seen": 1884553216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323971915747243, + "loss": 1.2989, + "theoretical_loss": 3.4759982642666003, + "tokens_seen": 1884618752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332196589769308, + "loss": 1.3601, + "theoretical_loss": 3.475988151571596, + "tokens_seen": 1884684288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043319959879638915, + "loss": 1.2676, + "theoretical_loss": 3.475978039326691, + "tokens_seen": 1884749824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043317953861584757, + "loss": 1.554, + "theoretical_loss": 3.4759679275318502, + "tokens_seen": 1884815360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043315947843530593, + "loss": 1.3906, + "theoretical_loss": 3.4759578161870373, + "tokens_seen": 1884880896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043313941825476435, + "loss": 1.3675, + "theoretical_loss": 3.475947705292217, + "tokens_seen": 1884946432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331193580742227, + "loss": 1.3803, + "theoretical_loss": 3.4759375948473537, + "tokens_seen": 1885011968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043309929789368107, + "loss": 1.2959, + "theoretical_loss": 3.475927484852411, + "tokens_seen": 1885077504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043307923771313943, + "loss": 1.3708, + "theoretical_loss": 3.475917375307354, + "tokens_seen": 1885143040 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043305917753259785, + "loss": 1.3631, + "theoretical_loss": 3.475907266212147, + "tokens_seen": 1885208576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330391173520562, + "loss": 1.3508, + "theoretical_loss": 3.475897157566754, + "tokens_seen": 1885274112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330190571715146, + "loss": 1.3081, + "theoretical_loss": 3.4758870493711393, + "tokens_seen": 1885339648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043299899699097294, + "loss": 1.4033, + "theoretical_loss": 3.475876941625268, + "tokens_seen": 1885405184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329789368104313, + "loss": 1.4372, + "theoretical_loss": 3.4758668343291035, + "tokens_seen": 1885470720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329588766298897, + "loss": 1.3936, + "theoretical_loss": 3.4758567274826104, + "tokens_seen": 1885536256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329388164493481, + "loss": 1.4228, + "theoretical_loss": 3.475846621085754, + "tokens_seen": 1885601792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043291875626880644, + "loss": 1.4419, + "theoretical_loss": 3.4758365151384973, + "tokens_seen": 1885667328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328986960882648, + "loss": 1.3192, + "theoretical_loss": 3.4758264096408054, + "tokens_seen": 1885732864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328786359077232, + "loss": 1.448, + "theoretical_loss": 3.4758163045926427, + "tokens_seen": 1885798400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328585757271816, + "loss": 1.4919, + "theoretical_loss": 3.475806199993973, + "tokens_seen": 1885863936 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 687569, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3545880317687988, + "objective/train/theoretical_loss": 3.475796095844762, + "objective/train/tokens_used": 256388576, + "theoretical_loss": 3.475796095844762, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043283851554663994, + "loss": 1.3714, + "theoretical_loss": 3.475796095844762, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328184553660983, + "loss": 1.4924, + "theoretical_loss": 3.4757859921449725, + "tokens_seen": 1885995008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043279839518555667, + "loss": 1.4168, + "theoretical_loss": 3.47577588889457, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327783350050151, + "loss": 1.3255, + "theoretical_loss": 3.4757657860935183, + "tokens_seen": 1886126080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043275827482447345, + "loss": 1.4571, + "theoretical_loss": 3.475755683741782, + "tokens_seen": 1886191616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327382146439318, + "loss": 1.3323, + "theoretical_loss": 3.4757455818393255, + "tokens_seen": 1886257152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043271815446339017, + "loss": 1.3504, + "theoretical_loss": 3.4757354803861134, + "tokens_seen": 1886322688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326980942828486, + "loss": 1.4917, + "theoretical_loss": 3.4757253793821095, + "tokens_seen": 1886388224 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043267803410230695, + "loss": 1.4811, + "theoretical_loss": 3.475715278827279, + "tokens_seen": 1886453760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326579739217653, + "loss": 1.3935, + "theoretical_loss": 3.4757051787215865, + "tokens_seen": 1886519296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326379137412237, + "loss": 1.595, + "theoretical_loss": 3.4756950790649954, + "tokens_seen": 1886584832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043261785356068204, + "loss": 1.3704, + "theoretical_loss": 3.4756849798574705, + "tokens_seen": 1886650368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043259779338014045, + "loss": 1.3238, + "theoretical_loss": 3.4756748810989766, + "tokens_seen": 1886715904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325777331995988, + "loss": 1.2713, + "theoretical_loss": 3.4756647827894778, + "tokens_seen": 1886781440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325576730190572, + "loss": 1.4402, + "theoretical_loss": 3.475654684928939, + "tokens_seen": 1886846976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043253761283851554, + "loss": 1.5006, + "theoretical_loss": 3.475644587517324, + "tokens_seen": 1886912512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043251755265797396, + "loss": 1.2872, + "theoretical_loss": 3.4756344905545977, + "tokens_seen": 1886978048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324974924774323, + "loss": 1.5122, + "theoretical_loss": 3.4756243940407243, + "tokens_seen": 1887043584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324774322968907, + "loss": 1.3542, + "theoretical_loss": 3.4756142979756683, + "tokens_seen": 1887109120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043245737211634904, + "loss": 1.3311, + "theoretical_loss": 3.4756042023593947, + "tokens_seen": 1887174656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324373119358074, + "loss": 1.4956, + "theoretical_loss": 3.475594107191867, + "tokens_seen": 1887240192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324172517552658, + "loss": 1.4333, + "theoretical_loss": 3.4755840124730506, + "tokens_seen": 1887305728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323971915747242, + "loss": 1.3688, + "theoretical_loss": 3.475573918202909, + "tokens_seen": 1887371264 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043237713139418255, + "loss": 1.3313, + "theoretical_loss": 3.4755638243814078, + "tokens_seen": 1887436800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323570712136409, + "loss": 1.3202, + "theoretical_loss": 3.475553731008511, + "tokens_seen": 1887502336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323370110330993, + "loss": 1.4015, + "theoretical_loss": 3.475543638084183, + "tokens_seen": 1887567872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323169508525577, + "loss": 1.4399, + "theoretical_loss": 3.4755335456083873, + "tokens_seen": 1887633408 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043229689067201605, + "loss": 1.3367, + "theoretical_loss": 3.4755234535810904, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322768304914744, + "loss": 1.3215, + "theoretical_loss": 3.4755133620022556, + "tokens_seen": 1887764480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043225677031093283, + "loss": 1.3958, + "theoretical_loss": 3.4755032708718474, + "tokens_seen": 1887830016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322367101303912, + "loss": 1.4355, + "theoretical_loss": 3.4754931801898308, + "tokens_seen": 1887895552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043221664994984955, + "loss": 1.3877, + "theoretical_loss": 3.4754830899561697, + "tokens_seen": 1887961088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321965897693079, + "loss": 1.2685, + "theoretical_loss": 3.4754730001708287, + "tokens_seen": 1888026624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321765295887663, + "loss": 1.476, + "theoretical_loss": 3.475462910833773, + "tokens_seen": 1888092160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321564694082247, + "loss": 1.3612, + "theoretical_loss": 3.475452821944967, + "tokens_seen": 1888157696 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043213640922768306, + "loss": 1.2024, + "theoretical_loss": 3.4754427335043743, + "tokens_seen": 1888223232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321163490471414, + "loss": 1.3737, + "theoretical_loss": 3.47543264551196, + "tokens_seen": 1888288768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320962888665998, + "loss": 1.2892, + "theoretical_loss": 3.475422557967689, + "tokens_seen": 1888354304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320762286860582, + "loss": 1.4198, + "theoretical_loss": 3.4754124708715257, + "tokens_seen": 1888419840 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043205616850551656, + "loss": 1.338, + "theoretical_loss": 3.4754023842234343, + "tokens_seen": 1888485376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320361083249749, + "loss": 1.325, + "theoretical_loss": 3.4753922980233796, + "tokens_seen": 1888550912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320160481444333, + "loss": 1.442, + "theoretical_loss": 3.4753822122713256, + "tokens_seen": 1888616448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043199598796389165, + "loss": 1.4417, + "theoretical_loss": 3.4753721269672373, + "tokens_seen": 1888681984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043197592778335006, + "loss": 1.3321, + "theoretical_loss": 3.4753620421110796, + "tokens_seen": 1888747520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319558676028084, + "loss": 1.2704, + "theoretical_loss": 3.4753519577028165, + "tokens_seen": 1888813056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319358074222668, + "loss": 1.354, + "theoretical_loss": 3.475341873742413, + "tokens_seen": 1888878592 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043191574724172515, + "loss": 1.4117, + "theoretical_loss": 3.4753317902298337, + "tokens_seen": 1888944128 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043189568706118357, + "loss": 1.3963, + "theoretical_loss": 3.4753217071650426, + "tokens_seen": 1889009664 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043187562688064193, + "loss": 1.4011, + "theoretical_loss": 3.475311624548005, + "tokens_seen": 1889075200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318555667001003, + "loss": 1.4191, + "theoretical_loss": 3.4753015423786846, + "tokens_seen": 1889140736 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 689100, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.424290418624878, + "objective/train/theoretical_loss": 3.475291460657047, + "objective/train/tokens_used": 259665376, + "theoretical_loss": 3.475291460657047, + "tokens_seen": 1889206272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183550651955865, + "loss": 1.3274, + "theoretical_loss": 3.475291460657047, + "tokens_seen": 1889206272 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431815446339017, + "loss": 1.3744, + "theoretical_loss": 3.475281379383056, + "tokens_seen": 1889271808 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043179538615847543, + "loss": 1.3243, + "theoretical_loss": 3.4752712985566765, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317753259779338, + "loss": 1.2941, + "theoretical_loss": 3.475261218177873, + "tokens_seen": 1889402880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043175526579739215, + "loss": 1.3798, + "theoretical_loss": 3.4752511382466107, + "tokens_seen": 1889468416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317352056168505, + "loss": 1.2578, + "theoretical_loss": 3.4752410587628533, + "tokens_seen": 1889533952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043171514543630893, + "loss": 1.3794, + "theoretical_loss": 3.475230979726566, + "tokens_seen": 1889599488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316950852557673, + "loss": 1.4381, + "theoretical_loss": 3.475220901137713, + "tokens_seen": 1889665024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316750250752257, + "loss": 1.2917, + "theoretical_loss": 3.4752108229962593, + "tokens_seen": 1889730560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316549648946841, + "loss": 1.3257, + "theoretical_loss": 3.4752007453021694, + "tokens_seen": 1889796096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043163490471414244, + "loss": 1.3897, + "theoretical_loss": 3.475190668055408, + "tokens_seen": 1889861632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043161484453360085, + "loss": 1.2737, + "theoretical_loss": 3.4751805912559393, + "tokens_seen": 1889927168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315947843530592, + "loss": 1.415, + "theoretical_loss": 3.475170514903729, + "tokens_seen": 1889992704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315747241725176, + "loss": 1.3807, + "theoretical_loss": 3.4751604389987403, + "tokens_seen": 1890058240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043155466399197594, + "loss": 1.4118, + "theoretical_loss": 3.475150363540939, + "tokens_seen": 1890123776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043153460381143436, + "loss": 1.3708, + "theoretical_loss": 3.475140288530289, + "tokens_seen": 1890189312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315145436308927, + "loss": 1.305, + "theoretical_loss": 3.4751302139667555, + "tokens_seen": 1890254848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314944834503511, + "loss": 1.4094, + "theoretical_loss": 3.4751201398503024, + "tokens_seen": 1890320384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043147442326980944, + "loss": 1.3743, + "theoretical_loss": 3.4751100661808954, + "tokens_seen": 1890385920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314543630892678, + "loss": 1.4374, + "theoretical_loss": 3.4750999929584987, + "tokens_seen": 1890451456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314343029087262, + "loss": 1.4821, + "theoretical_loss": 3.475089920183077, + "tokens_seen": 1890516992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314142427281846, + "loss": 1.3348, + "theoretical_loss": 3.4750798478545946, + "tokens_seen": 1890582528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043139418254764295, + "loss": 1.336, + "theoretical_loss": 3.4750697759730165, + "tokens_seen": 1890648064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313741223671013, + "loss": 1.3409, + "theoretical_loss": 3.4750597045383076, + "tokens_seen": 1890713600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313540621865597, + "loss": 1.3092, + "theoretical_loss": 3.475049633550432, + "tokens_seen": 1890779136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313340020060181, + "loss": 1.3651, + "theoretical_loss": 3.475039563009355, + "tokens_seen": 1890844672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043131394182547645, + "loss": 1.3418, + "theoretical_loss": 3.4750294929150405, + "tokens_seen": 1890910208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312938816449348, + "loss": 1.3331, + "theoretical_loss": 3.4750194232674545, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127382146439323, + "loss": 1.4589, + "theoretical_loss": 3.4750093540665605, + "tokens_seen": 1891041280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312537612838516, + "loss": 1.3404, + "theoretical_loss": 3.4749992853123235, + "tokens_seen": 1891106816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043123370110330995, + "loss": 1.2945, + "theoretical_loss": 3.4749892170047083, + "tokens_seen": 1891172352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312136409227683, + "loss": 1.4336, + "theoretical_loss": 3.47497914914368, + "tokens_seen": 1891237888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311935807422267, + "loss": 1.2902, + "theoretical_loss": 3.474969081729203, + "tokens_seen": 1891303424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311735205616851, + "loss": 1.3762, + "theoretical_loss": 3.4749590147612417, + "tokens_seen": 1891368960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043115346038114346, + "loss": 1.4193, + "theoretical_loss": 3.4749489482397613, + "tokens_seen": 1891434496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311334002006018, + "loss": 1.4225, + "theoretical_loss": 3.474938882164726, + "tokens_seen": 1891500032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311133400200602, + "loss": 1.3227, + "theoretical_loss": 3.474928816536101, + "tokens_seen": 1891565568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310932798395186, + "loss": 1.3073, + "theoretical_loss": 3.4749187513538513, + "tokens_seen": 1891631104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043107321965897696, + "loss": 1.2453, + "theoretical_loss": 3.4749086866179413, + "tokens_seen": 1891696640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310531594784353, + "loss": 1.2515, + "theoretical_loss": 3.474898622328335, + "tokens_seen": 1891762176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310330992978937, + "loss": 1.4127, + "theoretical_loss": 3.4748885584849987, + "tokens_seen": 1891827712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043101303911735205, + "loss": 1.2902, + "theoretical_loss": 3.4748784950878955, + "tokens_seen": 1891893248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043099297893681046, + "loss": 1.4339, + "theoretical_loss": 3.474868432136991, + "tokens_seen": 1891958784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309729187562688, + "loss": 1.294, + "theoretical_loss": 3.4748583696322504, + "tokens_seen": 1892024320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309528585757272, + "loss": 1.3107, + "theoretical_loss": 3.474848307573638, + "tokens_seen": 1892089856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043093279839518555, + "loss": 1.4287, + "theoretical_loss": 3.4748382459611182, + "tokens_seen": 1892155392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043091273821464397, + "loss": 1.2769, + "theoretical_loss": 3.474828184794656, + "tokens_seen": 1892220928 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043089267803410233, + "loss": 1.3889, + "theoretical_loss": 3.4748181240742166, + "tokens_seen": 1892286464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308726178535607, + "loss": 1.3995, + "theoretical_loss": 3.474808063799764, + "tokens_seen": 1892352000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043085255767301905, + "loss": 1.3339, + "theoretical_loss": 3.474798003971264, + "tokens_seen": 1892417536 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 690244, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.412577509880066, + "objective/train/theoretical_loss": 3.474787944588681, + "objective/train/tokens_used": 262942176, + "theoretical_loss": 3.474787944588681, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308324974924774, + "loss": 1.4505, + "theoretical_loss": 3.474787944588681, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043081243731193583, + "loss": 1.3993, + "theoretical_loss": 3.4747778856519793, + "tokens_seen": 1892548608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307923771313942, + "loss": 1.4911, + "theoretical_loss": 3.4747678271611244, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043077231695085256, + "loss": 1.2409, + "theoretical_loss": 3.4747577691160805, + "tokens_seen": 1892679680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307522567703109, + "loss": 1.4645, + "theoretical_loss": 3.4747477115168124, + "tokens_seen": 1892745216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043073219658976933, + "loss": 1.4092, + "theoretical_loss": 3.4747376543632855, + "tokens_seen": 1892810752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307121364092277, + "loss": 1.306, + "theoretical_loss": 3.474727597655464, + "tokens_seen": 1892876288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043069207622868606, + "loss": 1.3555, + "theoretical_loss": 3.4747175413933133, + "tokens_seen": 1892941824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306720160481444, + "loss": 1.3298, + "theoretical_loss": 3.4747074855767974, + "tokens_seen": 1893007360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306519558676028, + "loss": 1.398, + "theoretical_loss": 3.4746974302058824, + "tokens_seen": 1893072896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306318956870612, + "loss": 1.3687, + "theoretical_loss": 3.4746873752805314, + "tokens_seen": 1893138432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043061183550651956, + "loss": 1.4131, + "theoretical_loss": 3.474677320800711, + "tokens_seen": 1893203968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305917753259779, + "loss": 1.3469, + "theoretical_loss": 3.474667266766385, + "tokens_seen": 1893269504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305717151454363, + "loss": 1.2067, + "theoretical_loss": 3.4746572131775184, + "tokens_seen": 1893335040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305516549648947, + "loss": 1.4596, + "theoretical_loss": 3.4746471600340763, + "tokens_seen": 1893400576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043053159478435306, + "loss": 1.3052, + "theoretical_loss": 3.474637107336023, + "tokens_seen": 1893466112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305115346038114, + "loss": 1.3396, + "theoretical_loss": 3.474627055083324, + "tokens_seen": 1893531648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304914744232698, + "loss": 1.4196, + "theoretical_loss": 3.4746170032759442, + "tokens_seen": 1893597184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043047141424272815, + "loss": 1.4317, + "theoretical_loss": 3.474606951913848, + "tokens_seen": 1893662720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043045135406218657, + "loss": 1.4878, + "theoretical_loss": 3.4745969009969997, + "tokens_seen": 1893728256 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043043129388164493, + "loss": 1.3817, + "theoretical_loss": 3.4745868505253656, + "tokens_seen": 1893793792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304112337011033, + "loss": 1.2922, + "theoretical_loss": 3.4745768004989097, + "tokens_seen": 1893859328 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043039117352056165, + "loss": 1.3906, + "theoretical_loss": 3.474566750917597, + "tokens_seen": 1893924864 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043037111334002007, + "loss": 1.3697, + "theoretical_loss": 3.474556701781392, + "tokens_seen": 1893990400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043035105315947843, + "loss": 1.3732, + "theoretical_loss": 3.4745466530902607, + "tokens_seen": 1894055936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303309929789368, + "loss": 1.474, + "theoretical_loss": 3.474536604844167, + "tokens_seen": 1894121472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043031093279839516, + "loss": 1.3413, + "theoretical_loss": 3.4745265570430757, + "tokens_seen": 1894187008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302908726178535, + "loss": 1.4142, + "theoretical_loss": 3.4745165096869526, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043027081243731194, + "loss": 1.3226, + "theoretical_loss": 3.4745064627757616, + "tokens_seen": 1894318080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302507522567703, + "loss": 1.3723, + "theoretical_loss": 3.4744964163094685, + "tokens_seen": 1894383616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043023069207622866, + "loss": 1.3663, + "theoretical_loss": 3.4744863702880378, + "tokens_seen": 1894449152 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430210631895687, + "loss": 1.4078, + "theoretical_loss": 3.474476324711434, + "tokens_seen": 1894514688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043019057171514544, + "loss": 1.3855, + "theoretical_loss": 3.474466279579623, + "tokens_seen": 1894580224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043017051153460386, + "loss": 1.3748, + "theoretical_loss": 3.4744562348925685, + "tokens_seen": 1894645760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301504513540622, + "loss": 1.3102, + "theoretical_loss": 3.4744461906502364, + "tokens_seen": 1894711296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301303911735206, + "loss": 1.3159, + "theoretical_loss": 3.4744361468525913, + "tokens_seen": 1894776832 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430110330992979, + "loss": 1.3533, + "theoretical_loss": 3.474426103499598, + "tokens_seen": 1894842368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043009027081243736, + "loss": 1.4138, + "theoretical_loss": 3.4744160605912215, + "tokens_seen": 1894907904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300702106318957, + "loss": 1.364, + "theoretical_loss": 3.4744060181274268, + "tokens_seen": 1894973440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300501504513541, + "loss": 1.4546, + "theoretical_loss": 3.474395976108179, + "tokens_seen": 1895038976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043003009027081245, + "loss": 1.3582, + "theoretical_loss": 3.474385934533443, + "tokens_seen": 1895104512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043001003009027086, + "loss": 1.3999, + "theoretical_loss": 3.4743758934031836, + "tokens_seen": 1895170048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299899699097292, + "loss": 1.4197, + "theoretical_loss": 3.4743658527173658, + "tokens_seen": 1895235584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299699097291876, + "loss": 1.3912, + "theoretical_loss": 3.474355812475954, + "tokens_seen": 1895301120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042994984954864595, + "loss": 1.3552, + "theoretical_loss": 3.4743457726789146, + "tokens_seen": 1895366656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042992978936810437, + "loss": 1.4356, + "theoretical_loss": 3.4743357333262113, + "tokens_seen": 1895432192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042990972918756273, + "loss": 1.4076, + "theoretical_loss": 3.4743256944178094, + "tokens_seen": 1895497728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298896690070211, + "loss": 1.5283, + "theoretical_loss": 3.4743156559536743, + "tokens_seen": 1895563264 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042986960882647945, + "loss": 1.3122, + "theoretical_loss": 3.47430561793377, + "tokens_seen": 1895628800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298495486459378, + "loss": 1.3786, + "theoretical_loss": 3.4742955803580626, + "tokens_seen": 1895694336 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 691404, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3427211046218872, + "objective/train/theoretical_loss": 3.4742855432265163, + "objective/train/tokens_used": 266218976, + "theoretical_loss": 3.4742855432265163, + "tokens_seen": 1895759872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042982948846539623, + "loss": 1.3634, + "theoretical_loss": 3.4742855432265163, + "tokens_seen": 1895759872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298094282848546, + "loss": 1.3741, + "theoretical_loss": 3.4742755065390964, + "tokens_seen": 1895825408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042978936810431296, + "loss": 1.4185, + "theoretical_loss": 3.4742654702957685, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297693079237713, + "loss": 1.432, + "theoretical_loss": 3.4742554344964964, + "tokens_seen": 1895956480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042974924774322973, + "loss": 1.3078, + "theoretical_loss": 3.474245399141246, + "tokens_seen": 1896022016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297291875626881, + "loss": 1.403, + "theoretical_loss": 3.4742353642299815, + "tokens_seen": 1896087552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042970912738214646, + "loss": 1.3919, + "theoretical_loss": 3.474225329762669, + "tokens_seen": 1896153088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296890672016048, + "loss": 1.3869, + "theoretical_loss": 3.4742152957392722, + "tokens_seen": 1896218624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296690070210632, + "loss": 1.2555, + "theoretical_loss": 3.4742052621597574, + "tokens_seen": 1896284160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296489468405216, + "loss": 1.3245, + "theoretical_loss": 3.4741952290240894, + "tokens_seen": 1896349696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042962888665997996, + "loss": 1.2693, + "theoretical_loss": 3.4741851963322326, + "tokens_seen": 1896415232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296088264794383, + "loss": 1.3253, + "theoretical_loss": 3.4741751640841523, + "tokens_seen": 1896480768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295887662988967, + "loss": 1.4115, + "theoretical_loss": 3.474165132279813, + "tokens_seen": 1896546304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295687061183551, + "loss": 1.3543, + "theoretical_loss": 3.474155100919181, + "tokens_seen": 1896611840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042954864593781347, + "loss": 1.4173, + "theoretical_loss": 3.47414507000222, + "tokens_seen": 1896677376 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042952858575727183, + "loss": 1.3931, + "theoretical_loss": 3.4741350395288966, + "tokens_seen": 1896742912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295085255767302, + "loss": 1.3902, + "theoretical_loss": 3.4741250094991747, + "tokens_seen": 1896808448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042948846539618855, + "loss": 1.3042, + "theoretical_loss": 3.4741149799130193, + "tokens_seen": 1896873984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042946840521564697, + "loss": 1.3126, + "theoretical_loss": 3.474104950770396, + "tokens_seen": 1896939520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042944834503510533, + "loss": 1.3503, + "theoretical_loss": 3.474094922071269, + "tokens_seen": 1897005056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294282848545637, + "loss": 1.3756, + "theoretical_loss": 3.474084893815605, + "tokens_seen": 1897070592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042940822467402206, + "loss": 1.3653, + "theoretical_loss": 3.4740748660033676, + "tokens_seen": 1897136128 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042938816449348047, + "loss": 1.3881, + "theoretical_loss": 3.474064838634522, + "tokens_seen": 1897201664 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042936810431293883, + "loss": 1.4373, + "theoretical_loss": 3.474054811709034, + "tokens_seen": 1897267200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293480441323972, + "loss": 1.3754, + "theoretical_loss": 3.4740447852268685, + "tokens_seen": 1897332736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042932798395185556, + "loss": 1.3489, + "theoretical_loss": 3.4740347591879903, + "tokens_seen": 1897398272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293079237713139, + "loss": 1.4002, + "theoretical_loss": 3.4740247335923646, + "tokens_seen": 1897463808 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042928786359077234, + "loss": 1.2348, + "theoretical_loss": 3.474014708439956, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292678034102307, + "loss": 1.4235, + "theoretical_loss": 3.4740046837307306, + "tokens_seen": 1897594880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042924774322968906, + "loss": 1.3492, + "theoretical_loss": 3.473994659464653, + "tokens_seen": 1897660416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292276830491474, + "loss": 1.3499, + "theoretical_loss": 3.4739846356416875, + "tokens_seen": 1897725952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042920762286860584, + "loss": 1.342, + "theoretical_loss": 3.473974612261801, + "tokens_seen": 1897791488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291875626880642, + "loss": 1.2957, + "theoretical_loss": 3.4739645893249573, + "tokens_seen": 1897857024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042916750250752256, + "loss": 1.349, + "theoretical_loss": 3.4739545668311216, + "tokens_seen": 1897922560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291474423269809, + "loss": 1.3224, + "theoretical_loss": 3.4739445447802595, + "tokens_seen": 1897988096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291273821464393, + "loss": 1.4146, + "theoretical_loss": 3.473934523172336, + "tokens_seen": 1898053632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291073219658977, + "loss": 1.2691, + "theoretical_loss": 3.473924502007316, + "tokens_seen": 1898119168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042908726178535607, + "loss": 1.4653, + "theoretical_loss": 3.4739144812851643, + "tokens_seen": 1898184704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042906720160481443, + "loss": 1.3429, + "theoretical_loss": 3.473904461005847, + "tokens_seen": 1898250240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290471414242728, + "loss": 1.2645, + "theoretical_loss": 3.473894441169329, + "tokens_seen": 1898315776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290270812437312, + "loss": 1.448, + "theoretical_loss": 3.4738844217755744, + "tokens_seen": 1898381312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042900702106318957, + "loss": 1.3709, + "theoretical_loss": 3.47387440282455, + "tokens_seen": 1898446848 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042898696088264793, + "loss": 1.3676, + "theoretical_loss": 3.4738643843162196, + "tokens_seen": 1898512384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289669007021063, + "loss": 1.3592, + "theoretical_loss": 3.4738543662505488, + "tokens_seen": 1898577920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289468405215647, + "loss": 1.3627, + "theoretical_loss": 3.473844348627503, + "tokens_seen": 1898643456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289267803410231, + "loss": 1.4196, + "theoretical_loss": 3.473834331447047, + "tokens_seen": 1898708992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042890672016048144, + "loss": 1.3327, + "theoretical_loss": 3.4738243147091463, + "tokens_seen": 1898774528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288866599799398, + "loss": 1.3173, + "theoretical_loss": 3.4738142984137657, + "tokens_seen": 1898840064 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042886659979939816, + "loss": 1.3653, + "theoretical_loss": 3.473804282560871, + "tokens_seen": 1898905600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288465396188566, + "loss": 1.3874, + "theoretical_loss": 3.473794267150427, + "tokens_seen": 1898971136 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 692658, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4199891090393066, + "objective/train/theoretical_loss": 3.4737842521823983, + "objective/train/tokens_used": 269495776, + "theoretical_loss": 3.4737842521823983, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042882647943831494, + "loss": 1.3691, + "theoretical_loss": 3.4737842521823983, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288064192577733, + "loss": 1.4167, + "theoretical_loss": 3.473774237656751, + "tokens_seen": 1899102208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878635907723166, + "loss": 1.4092, + "theoretical_loss": 3.47376422357345, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287662988966901, + "loss": 1.3841, + "theoretical_loss": 3.4737542099324603, + "tokens_seen": 1899233280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042874623871614844, + "loss": 1.337, + "theoretical_loss": 3.4737441967337475, + "tokens_seen": 1899298816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287261785356068, + "loss": 1.3721, + "theoretical_loss": 3.4737341839772764, + "tokens_seen": 1899364352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042870611835506517, + "loss": 1.3862, + "theoretical_loss": 3.4737241716630125, + "tokens_seen": 1899429888 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042868605817452353, + "loss": 1.3537, + "theoretical_loss": 3.4737141597909207, + "tokens_seen": 1899495424 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428665997993982, + "loss": 1.3901, + "theoretical_loss": 3.4737041483609663, + "tokens_seen": 1899560960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042864593781344036, + "loss": 1.3544, + "theoretical_loss": 3.4736941373731147, + "tokens_seen": 1899626496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286258776328987, + "loss": 1.2165, + "theoretical_loss": 3.4736841268273313, + "tokens_seen": 1899692032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286058174523571, + "loss": 1.3403, + "theoretical_loss": 3.473674116723581, + "tokens_seen": 1899757568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285857572718155, + "loss": 1.3762, + "theoretical_loss": 3.473664107061829, + "tokens_seen": 1899823104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042856569709127387, + "loss": 1.3103, + "theoretical_loss": 3.4736540978420405, + "tokens_seen": 1899888640 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854563691073223, + "loss": 1.3381, + "theoretical_loss": 3.473644089064181, + "tokens_seen": 1899954176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285255767301906, + "loss": 1.2995, + "theoretical_loss": 3.473634080728216, + "tokens_seen": 1900019712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042850551654964895, + "loss": 1.3195, + "theoretical_loss": 3.47362407283411, + "tokens_seen": 1900085248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042848545636910737, + "loss": 1.4056, + "theoretical_loss": 3.4736140653818284, + "tokens_seen": 1900150784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846539618856573, + "loss": 1.362, + "theoretical_loss": 3.473604058371337, + "tokens_seen": 1900216320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284453360080241, + "loss": 1.349, + "theoretical_loss": 3.4735940518026007, + "tokens_seen": 1900281856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042842527582748246, + "loss": 1.3205, + "theoretical_loss": 3.473584045675585, + "tokens_seen": 1900347392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042840521564694087, + "loss": 1.3586, + "theoretical_loss": 3.473574039990255, + "tokens_seen": 1900412928 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042838515546639923, + "loss": 1.3669, + "theoretical_loss": 3.473564034746576, + "tokens_seen": 1900478464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283650952858576, + "loss": 1.2914, + "theoretical_loss": 3.4735540299445127, + "tokens_seen": 1900544000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042834503510531596, + "loss": 1.3597, + "theoretical_loss": 3.4735440255840313, + "tokens_seen": 1900609536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283249749247743, + "loss": 1.4428, + "theoretical_loss": 3.4735340216650967, + "tokens_seen": 1900675072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042830491474423274, + "loss": 1.3922, + "theoretical_loss": 3.4735240181876743, + "tokens_seen": 1900740608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282848545636911, + "loss": 1.3481, + "theoretical_loss": 3.473514015151729, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042826479438314946, + "loss": 1.3261, + "theoretical_loss": 3.4735040125572265, + "tokens_seen": 1900871680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282447342026078, + "loss": 1.2631, + "theoretical_loss": 3.4734940104041323, + "tokens_seen": 1900937216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042822467402206624, + "loss": 1.3716, + "theoretical_loss": 3.473484008692411, + "tokens_seen": 1901002752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282046138415246, + "loss": 1.3907, + "theoretical_loss": 3.4734740074220283, + "tokens_seen": 1901068288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042818455366098296, + "loss": 1.3052, + "theoretical_loss": 3.4734640065929496, + "tokens_seen": 1901133824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281644934804413, + "loss": 1.3997, + "theoretical_loss": 3.47345400620514, + "tokens_seen": 1901199360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281444332998997, + "loss": 1.4308, + "theoretical_loss": 3.473444006258565, + "tokens_seen": 1901264896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281243731193581, + "loss": 1.3591, + "theoretical_loss": 3.47343400675319, + "tokens_seen": 1901330432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042810431293881647, + "loss": 1.4344, + "theoretical_loss": 3.47342400768898, + "tokens_seen": 1901395968 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042808425275827483, + "loss": 1.4546, + "theoretical_loss": 3.473414009065901, + "tokens_seen": 1901461504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280641925777332, + "loss": 1.3709, + "theoretical_loss": 3.4734040108839173, + "tokens_seen": 1901527040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280441323971916, + "loss": 1.317, + "theoretical_loss": 3.473394013142995, + "tokens_seen": 1901592576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802407221664997, + "loss": 1.4808, + "theoretical_loss": 3.4733840158430995, + "tokens_seen": 1901658112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042800401203610833, + "loss": 1.419, + "theoretical_loss": 3.4733740189841953, + "tokens_seen": 1901723648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279839518555667, + "loss": 1.3145, + "theoretical_loss": 3.4733640225662485, + "tokens_seen": 1901789184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279638916750251, + "loss": 1.3979, + "theoretical_loss": 3.4733540265892247, + "tokens_seen": 1901854720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279438314944835, + "loss": 1.3897, + "theoretical_loss": 3.473344031053089, + "tokens_seen": 1901920256 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042792377131394184, + "loss": 1.405, + "theoretical_loss": 3.473334035957806, + "tokens_seen": 1901985792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279037111334002, + "loss": 1.4781, + "theoretical_loss": 3.473324041303342, + "tokens_seen": 1902051328 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042788365095285856, + "loss": 1.3574, + "theoretical_loss": 3.4733140470896617, + "tokens_seen": 1902116864 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427863590772317, + "loss": 1.4382, + "theoretical_loss": 3.473304053316731, + "tokens_seen": 1902182400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042784353059177534, + "loss": 1.3338, + "theoretical_loss": 3.4732940599845152, + "tokens_seen": 1902247936 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 693391, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4137303829193115, + "objective/train/theoretical_loss": 3.4732840670929797, + "objective/train/tokens_used": 272772576, + "theoretical_loss": 3.4732840670929797, + "tokens_seen": 1902313472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278234704112337, + "loss": 1.3666, + "theoretical_loss": 3.4732840670929797, + "tokens_seen": 1902313472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042780341023069206, + "loss": 1.3747, + "theoretical_loss": 3.47327407464209, + "tokens_seen": 1902379008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277833500501505, + "loss": 1.3595, + "theoretical_loss": 3.4732640826318106, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042776328986960884, + "loss": 1.4031, + "theoretical_loss": 3.473254091062108, + "tokens_seen": 1902510080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277432296890672, + "loss": 1.3706, + "theoretical_loss": 3.473244099932947, + "tokens_seen": 1902575616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042772316950852557, + "loss": 1.2708, + "theoretical_loss": 3.4732341092442933, + "tokens_seen": 1902641152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042770310932798393, + "loss": 1.216, + "theoretical_loss": 3.4732241189961117, + "tokens_seen": 1902706688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042768304914744235, + "loss": 1.3736, + "theoretical_loss": 3.473214129188369, + "tokens_seen": 1902772224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276629889669007, + "loss": 1.475, + "theoretical_loss": 3.473204139821029, + "tokens_seen": 1902837760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042764292878635907, + "loss": 1.3918, + "theoretical_loss": 3.473194150894058, + "tokens_seen": 1902903296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042762286860581743, + "loss": 1.3059, + "theoretical_loss": 3.473184162407421, + "tokens_seen": 1902968832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042760280842527585, + "loss": 1.2221, + "theoretical_loss": 3.473174174361084, + "tokens_seen": 1903034368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275827482447342, + "loss": 1.3464, + "theoretical_loss": 3.473164186755012, + "tokens_seen": 1903099904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275626880641926, + "loss": 1.3136, + "theoretical_loss": 3.47315419958917, + "tokens_seen": 1903165440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042754262788365094, + "loss": 1.4245, + "theoretical_loss": 3.4731442128635246, + "tokens_seen": 1903230976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275225677031093, + "loss": 1.3853, + "theoretical_loss": 3.4731342265780407, + "tokens_seen": 1903296512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275025075225677, + "loss": 1.3413, + "theoretical_loss": 3.473124240732683, + "tokens_seen": 1903362048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274824473420261, + "loss": 1.5591, + "theoretical_loss": 3.473114255327418, + "tokens_seen": 1903427584 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042746238716148444, + "loss": 1.324, + "theoretical_loss": 3.473104270362211, + "tokens_seen": 1903493120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274423269809428, + "loss": 1.3618, + "theoretical_loss": 3.4730942858370266, + "tokens_seen": 1903558656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274222668004012, + "loss": 1.3494, + "theoretical_loss": 3.473084301751831, + "tokens_seen": 1903624192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274022066198596, + "loss": 1.2978, + "theoretical_loss": 3.47307431810659, + "tokens_seen": 1903689728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042738214643931794, + "loss": 1.2977, + "theoretical_loss": 3.4730643349012684, + "tokens_seen": 1903755264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273620862587763, + "loss": 1.3044, + "theoretical_loss": 3.4730543521358315, + "tokens_seen": 1903820800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042734202607823467, + "loss": 1.2978, + "theoretical_loss": 3.4730443698102453, + "tokens_seen": 1903886336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273219658976931, + "loss": 1.2835, + "theoretical_loss": 3.473034387924475, + "tokens_seen": 1903951872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042730190571715145, + "loss": 1.4056, + "theoretical_loss": 3.473024406478487, + "tokens_seen": 1904017408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272818455366098, + "loss": 1.3813, + "theoretical_loss": 3.473014425472245, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042726178535606817, + "loss": 1.4001, + "theoretical_loss": 3.4730044449057162, + "tokens_seen": 1904148480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272417251755266, + "loss": 1.2306, + "theoretical_loss": 3.472994464778865, + "tokens_seen": 1904214016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042722166499498495, + "loss": 1.3758, + "theoretical_loss": 3.472984485091657, + "tokens_seen": 1904279552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272016048144433, + "loss": 1.4079, + "theoretical_loss": 3.4729745058440584, + "tokens_seen": 1904345088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271815446339017, + "loss": 1.2416, + "theoretical_loss": 3.4729645270360345, + "tokens_seen": 1904410624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271614844533601, + "loss": 1.2875, + "theoretical_loss": 3.47295454866755, + "tokens_seen": 1904476160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271414242728185, + "loss": 1.3996, + "theoretical_loss": 3.472944570738571, + "tokens_seen": 1904541696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042712136409227687, + "loss": 1.4145, + "theoretical_loss": 3.4729345932490636, + "tokens_seen": 1904607232 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042710130391173523, + "loss": 1.1262, + "theoretical_loss": 3.472924616198993, + "tokens_seen": 1904672768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270812437311936, + "loss": 1.2712, + "theoretical_loss": 3.472914639588324, + "tokens_seen": 1904738304 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427061183550652, + "loss": 1.2907, + "theoretical_loss": 3.4729046634170224, + "tokens_seen": 1904803840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042704112337011037, + "loss": 1.3632, + "theoretical_loss": 3.4728946876850544, + "tokens_seen": 1904869376 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042702106318956873, + "loss": 1.2769, + "theoretical_loss": 3.4728847123923847, + "tokens_seen": 1904934912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270010030090271, + "loss": 1.2793, + "theoretical_loss": 3.4728747375389792, + "tokens_seen": 1905000448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269809428284855, + "loss": 1.4319, + "theoretical_loss": 3.4728647631248037, + "tokens_seen": 1905065984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269608826479439, + "loss": 1.4494, + "theoretical_loss": 3.4728547891498236, + "tokens_seen": 1905131520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042694082246740224, + "loss": 1.4156, + "theoretical_loss": 3.4728448156140046, + "tokens_seen": 1905197056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269207622868606, + "loss": 1.3812, + "theoretical_loss": 3.4728348425173117, + "tokens_seen": 1905262592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042690070210631896, + "loss": 1.3979, + "theoretical_loss": 3.4728248698597106, + "tokens_seen": 1905328128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268806419257774, + "loss": 1.3651, + "theoretical_loss": 3.4728148976411677, + "tokens_seen": 1905393664 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042686058174523574, + "loss": 1.3123, + "theoretical_loss": 3.472804925861648, + "tokens_seen": 1905459200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268405215646941, + "loss": 1.2943, + "theoretical_loss": 3.472794954521116, + "tokens_seen": 1905524736 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 694687, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3013882637023926, + "objective/train/theoretical_loss": 3.472784983619539, + "objective/train/tokens_used": 276049376, + "theoretical_loss": 3.472784983619539, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042682046138415246, + "loss": 1.3291, + "theoretical_loss": 3.472784983619539, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268004012036109, + "loss": 1.4634, + "theoretical_loss": 3.4727750131568818, + "tokens_seen": 1905655808 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042678034102306924, + "loss": 1.2832, + "theoretical_loss": 3.47276504313311, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267602808425276, + "loss": 1.383, + "theoretical_loss": 3.472755073548189, + "tokens_seen": 1905786880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042674022066198597, + "loss": 1.4233, + "theoretical_loss": 3.472745104402085, + "tokens_seen": 1905852416 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042672016048144433, + "loss": 1.3622, + "theoretical_loss": 3.472735135694763, + "tokens_seen": 1905917952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042670010030090275, + "loss": 1.2575, + "theoretical_loss": 3.472725167426189, + "tokens_seen": 1905983488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266800401203611, + "loss": 1.3806, + "theoretical_loss": 3.472715199596329, + "tokens_seen": 1906049024 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042665997993981947, + "loss": 1.4998, + "theoretical_loss": 3.472705232205147, + "tokens_seen": 1906114560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042663991975927783, + "loss": 1.2589, + "theoretical_loss": 3.4726952652526104, + "tokens_seen": 1906180096 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042661985957873625, + "loss": 1.4269, + "theoretical_loss": 3.472685298738684, + "tokens_seen": 1906245632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265997993981946, + "loss": 1.3553, + "theoretical_loss": 3.472675332663333, + "tokens_seen": 1906311168 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426579739217653, + "loss": 1.304, + "theoretical_loss": 3.472665367026524, + "tokens_seen": 1906376704 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042655967903711134, + "loss": 1.2914, + "theoretical_loss": 3.472655401828222, + "tokens_seen": 1906442240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265396188565697, + "loss": 1.3901, + "theoretical_loss": 3.4726454370683926, + "tokens_seen": 1906507776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265195586760281, + "loss": 1.3213, + "theoretical_loss": 3.472635472747002, + "tokens_seen": 1906573312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264994984954865, + "loss": 1.4238, + "theoretical_loss": 3.4726255088640148, + "tokens_seen": 1906638848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042647943831494484, + "loss": 1.4514, + "theoretical_loss": 3.472615545419398, + "tokens_seen": 1906704384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264593781344032, + "loss": 1.4761, + "theoretical_loss": 3.472605582413116, + "tokens_seen": 1906769920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264393179538616, + "loss": 1.3466, + "theoretical_loss": 3.4725956198451353, + "tokens_seen": 1906835456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042641925777332, + "loss": 1.4145, + "theoretical_loss": 3.4725856577154213, + "tokens_seen": 1906900992 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042639919759277834, + "loss": 1.5121, + "theoretical_loss": 3.47257569602394, + "tokens_seen": 1906966528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263791374122367, + "loss": 1.4003, + "theoretical_loss": 3.4725657347706558, + "tokens_seen": 1907032064 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042635907723169507, + "loss": 1.4397, + "theoretical_loss": 3.4725557739555355, + "tokens_seen": 1907097600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263390170511535, + "loss": 1.355, + "theoretical_loss": 3.4725458135785447, + "tokens_seen": 1907163136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042631895687061185, + "loss": 1.3257, + "theoretical_loss": 3.4725358536396485, + "tokens_seen": 1907228672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262988966900702, + "loss": 1.3299, + "theoretical_loss": 3.4725258941388137, + "tokens_seen": 1907294208 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042627883650952857, + "loss": 1.3987, + "theoretical_loss": 3.4725159350760046, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426258776328987, + "loss": 1.3068, + "theoretical_loss": 3.472505976451188, + "tokens_seen": 1907425280 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042623871614844535, + "loss": 1.2822, + "theoretical_loss": 3.472496018264329, + "tokens_seen": 1907490816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262186559679037, + "loss": 1.4213, + "theoretical_loss": 3.4724860605153935, + "tokens_seen": 1907556352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261985957873621, + "loss": 1.375, + "theoretical_loss": 3.4724761032043467, + "tokens_seen": 1907621888 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042617853560682044, + "loss": 1.2884, + "theoretical_loss": 3.472466146331155, + "tokens_seen": 1907687424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042615847542627885, + "loss": 1.3896, + "theoretical_loss": 3.4724561898957838, + "tokens_seen": 1907752960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261384152457372, + "loss": 1.4266, + "theoretical_loss": 3.4724462338981983, + "tokens_seen": 1907818496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261183550651956, + "loss": 1.4355, + "theoretical_loss": 3.4724362783383658, + "tokens_seen": 1907884032 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042609829488465394, + "loss": 1.4033, + "theoretical_loss": 3.47242632321625, + "tokens_seen": 1907949568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042607823470411236, + "loss": 1.3222, + "theoretical_loss": 3.4724163685318183, + "tokens_seen": 1908015104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260581745235707, + "loss": 1.3804, + "theoretical_loss": 3.472406414285035, + "tokens_seen": 1908080640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260381143430291, + "loss": 1.2951, + "theoretical_loss": 3.472396460475867, + "tokens_seen": 1908146176 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042601805416248744, + "loss": 1.4139, + "theoretical_loss": 3.4723865071042797, + "tokens_seen": 1908211712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259979939819458, + "loss": 1.3795, + "theoretical_loss": 3.4723765541702383, + "tokens_seen": 1908277248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259779338014042, + "loss": 1.2519, + "theoretical_loss": 3.472366601673709, + "tokens_seen": 1908342784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259578736208626, + "loss": 1.237, + "theoretical_loss": 3.4723566496146576, + "tokens_seen": 1908408320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042593781344032095, + "loss": 1.3796, + "theoretical_loss": 3.47234669799305, + "tokens_seen": 1908473856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259177532597793, + "loss": 1.4262, + "theoretical_loss": 3.4723367468088515, + "tokens_seen": 1908539392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258976930792377, + "loss": 1.49, + "theoretical_loss": 3.4723267960620277, + "tokens_seen": 1908604928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258776328986961, + "loss": 1.3709, + "theoretical_loss": 3.4723168457525446, + "tokens_seen": 1908670464 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042585757271815445, + "loss": 1.3193, + "theoretical_loss": 3.4723068958803687, + "tokens_seen": 1908736000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258375125376128, + "loss": 1.3829, + "theoretical_loss": 3.4722969464454647, + "tokens_seen": 1908801536 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 695871, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2853118181228638, + "objective/train/theoretical_loss": 3.472286997447799, + "objective/train/tokens_used": 279326176, + "theoretical_loss": 3.472286997447799, + "tokens_seen": 1908867072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258174523570712, + "loss": 1.3275, + "theoretical_loss": 3.472286997447799, + "tokens_seen": 1908867072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257973921765296, + "loss": 1.3316, + "theoretical_loss": 3.472277048887337, + "tokens_seen": 1908932608 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042577733199598795, + "loss": 1.3028, + "theoretical_loss": 3.4722671007640447, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257572718154463, + "loss": 1.389, + "theoretical_loss": 3.472257153077888, + "tokens_seen": 1909063680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257372116349047, + "loss": 1.4206, + "theoretical_loss": 3.4722472058288325, + "tokens_seen": 1909129216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257171514543631, + "loss": 1.455, + "theoretical_loss": 3.472237259016844, + "tokens_seen": 1909194752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042569709127382145, + "loss": 1.3542, + "theoretical_loss": 3.472227312641888, + "tokens_seen": 1909260288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256770310932798, + "loss": 1.3504, + "theoretical_loss": 3.472217366703931, + "tokens_seen": 1909325824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042565697091273823, + "loss": 1.2731, + "theoretical_loss": 3.4722074212029383, + "tokens_seen": 1909391360 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042563691073219665, + "loss": 1.4055, + "theoretical_loss": 3.4721974761388754, + "tokens_seen": 1909456896 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425616850551655, + "loss": 1.3072, + "theoretical_loss": 3.4721875315117092, + "tokens_seen": 1909522432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255967903711134, + "loss": 1.3071, + "theoretical_loss": 3.4721775873214042, + "tokens_seen": 1909587968 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042557673019057174, + "loss": 1.403, + "theoretical_loss": 3.472167643567927, + "tokens_seen": 1909653504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255566700100301, + "loss": 1.3652, + "theoretical_loss": 3.472157700251244, + "tokens_seen": 1909719040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255366098294885, + "loss": 1.3595, + "theoretical_loss": 3.472147757371319, + "tokens_seen": 1909784576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255165496489469, + "loss": 1.2875, + "theoretical_loss": 3.47213781492812, + "tokens_seen": 1909850112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549648946840524, + "loss": 1.2068, + "theoretical_loss": 3.472127872921612, + "tokens_seen": 1909915648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254764292878636, + "loss": 1.3615, + "theoretical_loss": 3.4721179313517605, + "tokens_seen": 1909981184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425456369107322, + "loss": 1.4361, + "theoretical_loss": 3.4721079902185314, + "tokens_seen": 1910046720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254363089267804, + "loss": 1.4119, + "theoretical_loss": 3.4720980495218914, + "tokens_seen": 1910112256 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042541624874623874, + "loss": 1.4695, + "theoretical_loss": 3.4720881092618052, + "tokens_seen": 1910177792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253961885656971, + "loss": 1.3584, + "theoretical_loss": 3.472078169438239, + "tokens_seen": 1910243328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042537612838515547, + "loss": 1.348, + "theoretical_loss": 3.4720682300511596, + "tokens_seen": 1910308864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253560682046139, + "loss": 1.3381, + "theoretical_loss": 3.4720582911005318, + "tokens_seen": 1910374400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533600802407225, + "loss": 1.2519, + "theoretical_loss": 3.4720483525863215, + "tokens_seen": 1910439936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253159478435306, + "loss": 1.4232, + "theoretical_loss": 3.472038414508495, + "tokens_seen": 1910505472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042529588766298897, + "loss": 1.3168, + "theoretical_loss": 3.4720284768670178, + "tokens_seen": 1910571008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252758274824474, + "loss": 1.2948, + "theoretical_loss": 3.4720185396618564, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042525576730190575, + "loss": 1.3708, + "theoretical_loss": 3.472008602892976, + "tokens_seen": 1910702080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252357071213641, + "loss": 1.3437, + "theoretical_loss": 3.4719986665603426, + "tokens_seen": 1910767616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252156469408225, + "loss": 1.3479, + "theoretical_loss": 3.471988730663922, + "tokens_seen": 1910833152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042519558676028084, + "loss": 1.3587, + "theoretical_loss": 3.4719787952036807, + "tokens_seen": 1910898688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042517552657973925, + "loss": 1.3383, + "theoretical_loss": 3.471968860179584, + "tokens_seen": 1910964224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251554663991976, + "loss": 1.4451, + "theoretical_loss": 3.4719589255915984, + "tokens_seen": 1911029760 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425135406218656, + "loss": 1.325, + "theoretical_loss": 3.471948991439689, + "tokens_seen": 1911095296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042511534603811434, + "loss": 1.2898, + "theoretical_loss": 3.471939057723822, + "tokens_seen": 1911160832 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042509528585757276, + "loss": 1.3863, + "theoretical_loss": 3.471929124443964, + "tokens_seen": 1911226368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250752256770311, + "loss": 1.3278, + "theoretical_loss": 3.4719191916000796, + "tokens_seen": 1911291904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250551654964895, + "loss": 1.23, + "theoretical_loss": 3.471909259192136, + "tokens_seen": 1911357440 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042503510531594784, + "loss": 1.3665, + "theoretical_loss": 3.471899327220098, + "tokens_seen": 1911422976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250150451354062, + "loss": 1.3678, + "theoretical_loss": 3.4718893956839327, + "tokens_seen": 1911488512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249949849548646, + "loss": 1.3816, + "theoretical_loss": 3.471879464583605, + "tokens_seen": 1911554048 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424974924774323, + "loss": 1.4047, + "theoretical_loss": 3.4718695339190813, + "tokens_seen": 1911619584 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042495486459378135, + "loss": 1.2681, + "theoretical_loss": 3.4718596036903273, + "tokens_seen": 1911685120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249348044132397, + "loss": 1.345, + "theoretical_loss": 3.4718496738973097, + "tokens_seen": 1911750656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249147442326981, + "loss": 1.377, + "theoretical_loss": 3.4718397445399933, + "tokens_seen": 1911816192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248946840521565, + "loss": 1.4166, + "theoretical_loss": 3.471829815618345, + "tokens_seen": 1911881728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042487462387161485, + "loss": 1.438, + "theoretical_loss": 3.4718198871323303, + "tokens_seen": 1911947264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248545636910732, + "loss": 1.4252, + "theoretical_loss": 3.471809959081915, + "tokens_seen": 1912012800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248345035105316, + "loss": 1.384, + "theoretical_loss": 3.4718000314670654, + "tokens_seen": 1912078336 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 697086, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3212376832962036, + "objective/train/theoretical_loss": 3.4717901042877477, + "objective/train/tokens_used": 282602976, + "theoretical_loss": 3.4717901042877477, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042481444332999, + "loss": 1.3844, + "theoretical_loss": 3.4717901042877477, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042479438314944835, + "loss": 1.4276, + "theoretical_loss": 3.471780177543927, + "tokens_seen": 1912209408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247743229689067, + "loss": 1.4065, + "theoretical_loss": 3.47177025123557, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247542627883651, + "loss": 1.3849, + "theoretical_loss": 3.4717603253626423, + "tokens_seen": 1912340480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247342026078235, + "loss": 1.3053, + "theoretical_loss": 3.47175039992511, + "tokens_seen": 1912406016 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042471414242728186, + "loss": 1.3719, + "theoretical_loss": 3.4717404749229397, + "tokens_seen": 1912471552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246940822467402, + "loss": 1.3186, + "theoretical_loss": 3.471730550356096, + "tokens_seen": 1912537088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246740220661986, + "loss": 1.3231, + "theoretical_loss": 3.4717206262245464, + "tokens_seen": 1912602624 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424653961885657, + "loss": 1.2723, + "theoretical_loss": 3.471710702528256, + "tokens_seen": 1912668160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042463390170511536, + "loss": 1.4056, + "theoretical_loss": 3.4717007792671906, + "tokens_seen": 1912733696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246138415245737, + "loss": 1.3863, + "theoretical_loss": 3.4716908564413167, + "tokens_seen": 1912799232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245937813440321, + "loss": 1.3216, + "theoretical_loss": 3.4716809340506, + "tokens_seen": 1912864768 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042457372116349044, + "loss": 1.4053, + "theoretical_loss": 3.4716710120950074, + "tokens_seen": 1912930304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042455366098294886, + "loss": 1.3346, + "theoretical_loss": 3.4716610905745036, + "tokens_seen": 1912995840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245336008024072, + "loss": 1.3701, + "theoretical_loss": 3.4716511694890553, + "tokens_seen": 1913061376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245135406218656, + "loss": 1.311, + "theoretical_loss": 3.471641248838629, + "tokens_seen": 1913126912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042449348044132395, + "loss": 1.3466, + "theoretical_loss": 3.4716313286231895, + "tokens_seen": 1913192448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042447342026078236, + "loss": 1.4149, + "theoretical_loss": 3.4716214088427035, + "tokens_seen": 1913257984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244533600802407, + "loss": 1.3819, + "theoretical_loss": 3.4716114894971373, + "tokens_seen": 1913323520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244332998996991, + "loss": 1.2682, + "theoretical_loss": 3.471601570586456, + "tokens_seen": 1913389056 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042441323971915745, + "loss": 1.3924, + "theoretical_loss": 3.471591652110627, + "tokens_seen": 1913454592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243931795386158, + "loss": 1.3623, + "theoretical_loss": 3.471581734069616, + "tokens_seen": 1913520128 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042437311935807423, + "loss": 1.3461, + "theoretical_loss": 3.471571816463388, + "tokens_seen": 1913585664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243530591775326, + "loss": 1.3284, + "theoretical_loss": 3.4715618992919097, + "tokens_seen": 1913651200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042433299899699095, + "loss": 1.3253, + "theoretical_loss": 3.4715519825551477, + "tokens_seen": 1913716736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243129388164493, + "loss": 1.2873, + "theoretical_loss": 3.4715420662530674, + "tokens_seen": 1913782272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042429287863590773, + "loss": 1.3215, + "theoretical_loss": 3.471532150385635, + "tokens_seen": 1913847808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242728184553661, + "loss": 1.3909, + "theoretical_loss": 3.4715222349528165, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042425275827482446, + "loss": 1.3988, + "theoretical_loss": 3.471512319954578, + "tokens_seen": 1913978880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242326980942828, + "loss": 1.2377, + "theoretical_loss": 3.4715024053908854, + "tokens_seen": 1914044416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242126379137412, + "loss": 1.3359, + "theoretical_loss": 3.4714924912617056, + "tokens_seen": 1914109952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241925777331996, + "loss": 1.1767, + "theoretical_loss": 3.471482577567004, + "tokens_seen": 1914175488 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424172517552658, + "loss": 1.3653, + "theoretical_loss": 3.4714726643067464, + "tokens_seen": 1914241024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241524573721164, + "loss": 1.3332, + "theoretical_loss": 3.4714627514808996, + "tokens_seen": 1914306560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042413239719157474, + "loss": 1.3313, + "theoretical_loss": 3.4714528390894293, + "tokens_seen": 1914372096 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042411233701103316, + "loss": 1.491, + "theoretical_loss": 3.4714429271323013, + "tokens_seen": 1914437632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240922768304915, + "loss": 1.3816, + "theoretical_loss": 3.471433015609482, + "tokens_seen": 1914503168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240722166499499, + "loss": 1.4036, + "theoretical_loss": 3.4714231045209383, + "tokens_seen": 1914568704 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042405215646940824, + "loss": 1.3297, + "theoretical_loss": 3.471413193866635, + "tokens_seen": 1914634240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240320962888666, + "loss": 1.1832, + "theoretical_loss": 3.4714032836465387, + "tokens_seen": 1914699776 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424012036108325, + "loss": 1.3462, + "theoretical_loss": 3.4713933738606157, + "tokens_seen": 1914765312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239919759277834, + "loss": 1.3834, + "theoretical_loss": 3.4713834645088317, + "tokens_seen": 1914830848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042397191574724175, + "loss": 1.3982, + "theoretical_loss": 3.471373555591154, + "tokens_seen": 1914896384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239518555667001, + "loss": 1.4738, + "theoretical_loss": 3.4713636471075473, + "tokens_seen": 1914961920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239317953861585, + "loss": 1.3269, + "theoretical_loss": 3.471353739057978, + "tokens_seen": 1915027456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239117352056169, + "loss": 1.4749, + "theoretical_loss": 3.4713438314424128, + "tokens_seen": 1915092992 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042389167502507525, + "loss": 1.3876, + "theoretical_loss": 3.4713339242608177, + "tokens_seen": 1915158528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238716148445336, + "loss": 1.4143, + "theoretical_loss": 3.471324017513159, + "tokens_seen": 1915224064 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423851554663992, + "loss": 1.3773, + "theoretical_loss": 3.471314111199402, + "tokens_seen": 1915289600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238314944834504, + "loss": 1.3953, + "theoretical_loss": 3.4713042053195133, + "tokens_seen": 1915355136 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 698281, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3410608768463135, + "objective/train/theoretical_loss": 3.4712942998734597, + "objective/train/tokens_used": 285879776, + "theoretical_loss": 3.4712942998734597, + "tokens_seen": 1915420672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042381143430290875, + "loss": 1.3491, + "theoretical_loss": 3.4712942998734597, + "tokens_seen": 1915420672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237913741223671, + "loss": 1.322, + "theoretical_loss": 3.4712843948612067, + "tokens_seen": 1915486208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237713139418255, + "loss": 1.4388, + "theoretical_loss": 3.47127449028272, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237512537612839, + "loss": 1.297, + "theoretical_loss": 3.4712645861379667, + "tokens_seen": 1915617280 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042373119358074226, + "loss": 1.4093, + "theoretical_loss": 3.4712546824269124, + "tokens_seen": 1915682816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237111334002006, + "loss": 1.3321, + "theoretical_loss": 3.471244779149524, + "tokens_seen": 1915748352 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423691073219659, + "loss": 1.4864, + "theoretical_loss": 3.4712348763057665, + "tokens_seen": 1915813888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236710130391174, + "loss": 1.4675, + "theoretical_loss": 3.471224973895607, + "tokens_seen": 1915879424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042365095285857576, + "loss": 1.3899, + "theoretical_loss": 3.471215071919012, + "tokens_seen": 1915944960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236308926780341, + "loss": 1.4491, + "theoretical_loss": 3.4712051703759466, + "tokens_seen": 1916010496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236108324974925, + "loss": 1.2167, + "theoretical_loss": 3.4711952692663774, + "tokens_seen": 1916076032 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042359077231695085, + "loss": 1.3526, + "theoretical_loss": 3.471185368590271, + "tokens_seen": 1916141568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042357071213640926, + "loss": 1.4873, + "theoretical_loss": 3.4711754683475933, + "tokens_seen": 1916207104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235506519558676, + "loss": 1.2723, + "theoretical_loss": 3.47116556853831, + "tokens_seen": 1916272640 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423530591775326, + "loss": 1.5336, + "theoretical_loss": 3.471155669162388, + "tokens_seen": 1916338176 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042351053159478435, + "loss": 1.3566, + "theoretical_loss": 3.4711457702197936, + "tokens_seen": 1916403712 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042349047141424276, + "loss": 1.2315, + "theoretical_loss": 3.4711358717104925, + "tokens_seen": 1916469248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042347041123370113, + "loss": 1.4048, + "theoretical_loss": 3.471125973634451, + "tokens_seen": 1916534784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234503510531595, + "loss": 1.3125, + "theoretical_loss": 3.4711160759916355, + "tokens_seen": 1916600320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042343029087261785, + "loss": 1.3724, + "theoretical_loss": 3.471106178782012, + "tokens_seen": 1916665856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234102306920762, + "loss": 1.3469, + "theoretical_loss": 3.4710962820055475, + "tokens_seen": 1916731392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042339017051153463, + "loss": 1.301, + "theoretical_loss": 3.4710863856622076, + "tokens_seen": 1916796928 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423370110330993, + "loss": 1.3064, + "theoretical_loss": 3.471076489751958, + "tokens_seen": 1916862464 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042335005015045135, + "loss": 1.3603, + "theoretical_loss": 3.471066594274766, + "tokens_seen": 1916928000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233299899699097, + "loss": 1.4267, + "theoretical_loss": 3.4710566992305973, + "tokens_seen": 1916993536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042330992978936813, + "loss": 1.264, + "theoretical_loss": 3.4710468046194185, + "tokens_seen": 1917059072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232898696088265, + "loss": 1.3519, + "theoretical_loss": 3.471036910441195, + "tokens_seen": 1917124608 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042326980942828486, + "loss": 1.3924, + "theoretical_loss": 3.471027016695894, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232497492477432, + "loss": 1.3352, + "theoretical_loss": 3.471017123383481, + "tokens_seen": 1917255680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232296890672016, + "loss": 1.3834, + "theoretical_loss": 3.4710072305039232, + "tokens_seen": 1917321216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042320962888666, + "loss": 1.2678, + "theoretical_loss": 3.470997338057186, + "tokens_seen": 1917386752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042318956870611836, + "loss": 1.3665, + "theoretical_loss": 3.470987446043236, + "tokens_seen": 1917452288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231695085255767, + "loss": 1.445, + "theoretical_loss": 3.4709775544620394, + "tokens_seen": 1917517824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231494483450351, + "loss": 1.3184, + "theoretical_loss": 3.4709676633135627, + "tokens_seen": 1917583360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231293881644935, + "loss": 1.3992, + "theoretical_loss": 3.470957772597772, + "tokens_seen": 1917648896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042310932798395186, + "loss": 1.3922, + "theoretical_loss": 3.4709478823146336, + "tokens_seen": 1917714432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230892678034102, + "loss": 1.4292, + "theoretical_loss": 3.4709379924641137, + "tokens_seen": 1917779968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230692076228686, + "loss": 1.3808, + "theoretical_loss": 3.470928103046179, + "tokens_seen": 1917845504 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042304914744232695, + "loss": 1.2741, + "theoretical_loss": 3.470918214060795, + "tokens_seen": 1917911040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042302908726178537, + "loss": 1.2935, + "theoretical_loss": 3.4709083255079287, + "tokens_seen": 1917976576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042300902708124373, + "loss": 1.36, + "theoretical_loss": 3.4708984373875467, + "tokens_seen": 1918042112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229889669007021, + "loss": 1.4353, + "theoretical_loss": 3.4708885496996142, + "tokens_seen": 1918107648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042296890672016045, + "loss": 1.3358, + "theoretical_loss": 3.4708786624440986, + "tokens_seen": 1918173184 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042294884653961887, + "loss": 1.3376, + "theoretical_loss": 3.470868775620965, + "tokens_seen": 1918238720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042292878635907723, + "loss": 1.3963, + "theoretical_loss": 3.470858889230181, + "tokens_seen": 1918304256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229087261785356, + "loss": 1.3852, + "theoretical_loss": 3.470849003271712, + "tokens_seen": 1918369792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042288866599799396, + "loss": 1.3698, + "theoretical_loss": 3.470839117745525, + "tokens_seen": 1918435328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228686058174523, + "loss": 1.3956, + "theoretical_loss": 3.470829232651586, + "tokens_seen": 1918500864 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284854563691074, + "loss": 1.2971, + "theoretical_loss": 3.4708193479898615, + "tokens_seen": 1918566400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228284854563691, + "loss": 1.4123, + "theoretical_loss": 3.4708094637603177, + "tokens_seen": 1918631936 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 699453, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3871700763702393, + "objective/train/theoretical_loss": 3.4707995799629208, + "objective/train/tokens_used": 289156576, + "theoretical_loss": 3.4707995799629208, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042280842527582746, + "loss": 1.3768, + "theoretical_loss": 3.4707995799629208, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227883650952858, + "loss": 1.3481, + "theoretical_loss": 3.4707896965976373, + "tokens_seen": 1918763008 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042276830491474424, + "loss": 1.3204, + "theoretical_loss": 3.470779813664434, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227482447342026, + "loss": 1.325, + "theoretical_loss": 3.4707699311632765, + "tokens_seen": 1918894080 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042272818455366096, + "loss": 1.3812, + "theoretical_loss": 3.4707600490941313, + "tokens_seen": 1918959616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227081243731193, + "loss": 1.3722, + "theoretical_loss": 3.470750167456965, + "tokens_seen": 1919025152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226880641925777, + "loss": 1.3694, + "theoretical_loss": 3.4707402862517442, + "tokens_seen": 1919090688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042266800401203616, + "loss": 1.4158, + "theoretical_loss": 3.4707304054784345, + "tokens_seen": 1919156224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226479438314945, + "loss": 1.3443, + "theoretical_loss": 3.470720525137003, + "tokens_seen": 1919221760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226278836509529, + "loss": 1.3175, + "theoretical_loss": 3.4707106452274155, + "tokens_seen": 1919287296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042260782347041125, + "loss": 1.4202, + "theoretical_loss": 3.4707007657496387, + "tokens_seen": 1919352832 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042258776328986966, + "loss": 1.388, + "theoretical_loss": 3.4706908867036397, + "tokens_seen": 1919418368 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422567703109328, + "loss": 1.3134, + "theoretical_loss": 3.4706810080893833, + "tokens_seen": 1919483904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225476429287864, + "loss": 1.2193, + "theoretical_loss": 3.4706711299068376, + "tokens_seen": 1919549440 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252758274824475, + "loss": 1.3715, + "theoretical_loss": 3.4706612521559674, + "tokens_seen": 1919614976 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042250752256770317, + "loss": 1.3441, + "theoretical_loss": 3.47065137483674, + "tokens_seen": 1919680512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248746238716153, + "loss": 1.2731, + "theoretical_loss": 3.4706414979491216, + "tokens_seen": 1919746048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224674022066199, + "loss": 1.4185, + "theoretical_loss": 3.4706316214930792, + "tokens_seen": 1919811584 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042244734202607825, + "loss": 1.2818, + "theoretical_loss": 3.470621745468578, + "tokens_seen": 1919877120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224272818455366, + "loss": 1.374, + "theoretical_loss": 3.4706118698755857, + "tokens_seen": 1919942656 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042240722166499503, + "loss": 1.3041, + "theoretical_loss": 3.470601994714068, + "tokens_seen": 1920008192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223871614844534, + "loss": 1.377, + "theoretical_loss": 3.4705921199839906, + "tokens_seen": 1920073728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042236710130391176, + "loss": 1.4402, + "theoretical_loss": 3.4705822456853213, + "tokens_seen": 1920139264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223470411233701, + "loss": 1.3111, + "theoretical_loss": 3.4705723718180264, + "tokens_seen": 1920204800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042232698094282853, + "loss": 1.3272, + "theoretical_loss": 3.4705624983820718, + "tokens_seen": 1920270336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223069207622869, + "loss": 1.3412, + "theoretical_loss": 3.4705526253774233, + "tokens_seen": 1920335872 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042228686058174526, + "loss": 1.259, + "theoretical_loss": 3.470542752804049, + "tokens_seen": 1920401408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222668004012036, + "loss": 1.1954, + "theoretical_loss": 3.4705328806619136, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422246740220662, + "loss": 1.2623, + "theoretical_loss": 3.470523008950985, + "tokens_seen": 1920532480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222266800401204, + "loss": 1.4743, + "theoretical_loss": 3.470513137671229, + "tokens_seen": 1920598016 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042220661985957876, + "loss": 1.2877, + "theoretical_loss": 3.470503266822612, + "tokens_seen": 1920663552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221865596790371, + "loss": 1.4402, + "theoretical_loss": 3.4704933964051, + "tokens_seen": 1920729088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221664994984955, + "loss": 1.4002, + "theoretical_loss": 3.470483526418661, + "tokens_seen": 1920794624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221464393179539, + "loss": 1.2645, + "theoretical_loss": 3.47047365686326, + "tokens_seen": 1920860160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042212637913741226, + "loss": 1.3769, + "theoretical_loss": 3.470463787738864, + "tokens_seen": 1920925696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221063189568706, + "loss": 1.3736, + "theoretical_loss": 3.470453919045439, + "tokens_seen": 1920991232 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422086258776329, + "loss": 1.2686, + "theoretical_loss": 3.4704440507829526, + "tokens_seen": 1921056768 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042206619859578735, + "loss": 1.3846, + "theoretical_loss": 3.4704341829513696, + "tokens_seen": 1921122304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042204613841524577, + "loss": 1.3452, + "theoretical_loss": 3.4704243155506584, + "tokens_seen": 1921187840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042202607823470413, + "loss": 1.3725, + "theoretical_loss": 3.470414448580784, + "tokens_seen": 1921253376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220060180541625, + "loss": 1.3418, + "theoretical_loss": 3.4704045820417138, + "tokens_seen": 1921318912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042198595787362085, + "loss": 1.4191, + "theoretical_loss": 3.470394715933414, + "tokens_seen": 1921384448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042196589769307927, + "loss": 1.3412, + "theoretical_loss": 3.4703848502558508, + "tokens_seen": 1921449984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042194583751253763, + "loss": 1.366, + "theoretical_loss": 3.4703749850089913, + "tokens_seen": 1921515520 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421925777331996, + "loss": 1.4247, + "theoretical_loss": 3.4703651201928016, + "tokens_seen": 1921581056 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042190571715145436, + "loss": 1.3447, + "theoretical_loss": 3.470355255807248, + "tokens_seen": 1921646592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218856569709127, + "loss": 1.2791, + "theoretical_loss": 3.4703453918522973, + "tokens_seen": 1921712128 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042186559679037114, + "loss": 1.3157, + "theoretical_loss": 3.470335528327916, + "tokens_seen": 1921777664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218455366098295, + "loss": 1.2867, + "theoretical_loss": 3.470325665234071, + "tokens_seen": 1921843200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042182547642928786, + "loss": 1.4806, + "theoretical_loss": 3.4703158025707284, + "tokens_seen": 1921908736 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 700743, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3846075534820557, + "objective/train/theoretical_loss": 3.4703059403378544, + "objective/train/tokens_used": 292433376, + "theoretical_loss": 3.4703059403378544, + "tokens_seen": 1921974272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218054162487462, + "loss": 1.2258, + "theoretical_loss": 3.4703059403378544, + "tokens_seen": 1921974272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042178535606820464, + "loss": 1.4184, + "theoretical_loss": 3.4702960785354158, + "tokens_seen": 1922039808 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421765295887663, + "loss": 1.3426, + "theoretical_loss": 3.4702862171633795, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042174523570712136, + "loss": 1.4345, + "theoretical_loss": 3.4702763562217123, + "tokens_seen": 1922170880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217251755265797, + "loss": 1.2874, + "theoretical_loss": 3.47026649571038, + "tokens_seen": 1922236416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004217051153460381, + "loss": 1.3703, + "theoretical_loss": 3.470256635629349, + "tokens_seen": 1922301952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216850551654965, + "loss": 1.3356, + "theoretical_loss": 3.470246775978586, + "tokens_seen": 1922367488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042166499498495487, + "loss": 1.3424, + "theoretical_loss": 3.4702369167580587, + "tokens_seen": 1922433024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042164493480441323, + "loss": 1.3373, + "theoretical_loss": 3.4702270579677323, + "tokens_seen": 1922498560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216248746238716, + "loss": 1.3344, + "theoretical_loss": 3.470217199607574, + "tokens_seen": 1922564096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042160481444333, + "loss": 1.3912, + "theoretical_loss": 3.4702073416775496, + "tokens_seen": 1922629632 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042158475426278837, + "loss": 1.496, + "theoretical_loss": 3.470197484177627, + "tokens_seen": 1922695168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042156469408224673, + "loss": 1.318, + "theoretical_loss": 3.470187627107772, + "tokens_seen": 1922760704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215446339017051, + "loss": 1.3642, + "theoretical_loss": 3.470177770467951, + "tokens_seen": 1922826240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215245737211635, + "loss": 1.341, + "theoretical_loss": 3.4701679142581305, + "tokens_seen": 1922891776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215045135406219, + "loss": 1.172, + "theoretical_loss": 3.470158058478278, + "tokens_seen": 1922957312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042148445336008024, + "loss": 1.2682, + "theoretical_loss": 3.4701482031283595, + "tokens_seen": 1923022848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214643931795386, + "loss": 1.4813, + "theoretical_loss": 3.4701383482083408, + "tokens_seen": 1923088384 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042144433299899696, + "loss": 1.4482, + "theoretical_loss": 3.4701284937181898, + "tokens_seen": 1923153920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214242728184554, + "loss": 1.3991, + "theoretical_loss": 3.4701186396578727, + "tokens_seen": 1923219456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042140421263791374, + "loss": 1.3467, + "theoretical_loss": 3.470108786027356, + "tokens_seen": 1923284992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213841524573721, + "loss": 1.3614, + "theoretical_loss": 3.4700989328266063, + "tokens_seen": 1923350528 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042136409227683046, + "loss": 1.3238, + "theoretical_loss": 3.4700890800555895, + "tokens_seen": 1923416064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213440320962889, + "loss": 1.2781, + "theoretical_loss": 3.470079227714274, + "tokens_seen": 1923481600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042132397191574724, + "loss": 1.2222, + "theoretical_loss": 3.4700693758026246, + "tokens_seen": 1923547136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213039117352056, + "loss": 1.3536, + "theoretical_loss": 3.470059524320609, + "tokens_seen": 1923612672 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042128385155466397, + "loss": 1.3236, + "theoretical_loss": 3.4700496732681936, + "tokens_seen": 1923678208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042126379137412233, + "loss": 1.2185, + "theoretical_loss": 3.4700398226453446, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042124373119358075, + "loss": 1.356, + "theoretical_loss": 3.470029972452029, + "tokens_seen": 1923809280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212236710130391, + "loss": 1.3047, + "theoretical_loss": 3.4700201226882132, + "tokens_seen": 1923874816 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042120361083249747, + "loss": 1.2991, + "theoretical_loss": 3.4700102733538642, + "tokens_seen": 1923940352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042118355065195583, + "loss": 1.4352, + "theoretical_loss": 3.470000424448949, + "tokens_seen": 1924005888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211634904714143, + "loss": 1.2492, + "theoretical_loss": 3.4699905759734335, + "tokens_seen": 1924071424 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042114343029087266, + "loss": 1.3506, + "theoretical_loss": 3.4699807279272843, + "tokens_seen": 1924136960 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042112337011033103, + "loss": 1.3672, + "theoretical_loss": 3.4699708803104685, + "tokens_seen": 1924202496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211033099297894, + "loss": 1.3594, + "theoretical_loss": 3.469961033122953, + "tokens_seen": 1924268032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042108324974924775, + "loss": 1.3249, + "theoretical_loss": 3.4699511863647032, + "tokens_seen": 1924333568 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042106318956870617, + "loss": 1.3542, + "theoretical_loss": 3.469941340035687, + "tokens_seen": 1924399104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042104312938816453, + "loss": 1.3987, + "theoretical_loss": 3.4699314941358708, + "tokens_seen": 1924464640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210230692076229, + "loss": 1.4009, + "theoretical_loss": 3.4699216486652213, + "tokens_seen": 1924530176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042100300902708125, + "loss": 1.298, + "theoretical_loss": 3.469911803623705, + "tokens_seen": 1924595712 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042098294884653967, + "loss": 1.3296, + "theoretical_loss": 3.469901959011289, + "tokens_seen": 1924661248 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042096288866599803, + "loss": 1.3959, + "theoretical_loss": 3.4698921148279394, + "tokens_seen": 1924726784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209428284854564, + "loss": 1.3748, + "theoretical_loss": 3.4698822710736232, + "tokens_seen": 1924792320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042092276830491476, + "loss": 1.3945, + "theoretical_loss": 3.4698724277483066, + "tokens_seen": 1924857856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209027081243731, + "loss": 1.3703, + "theoretical_loss": 3.469862584851957, + "tokens_seen": 1924923392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042088264794383154, + "loss": 1.2939, + "theoretical_loss": 3.469852742384541, + "tokens_seen": 1924988928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208625877632899, + "loss": 1.359, + "theoretical_loss": 3.469842900346025, + "tokens_seen": 1925054464 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042084252758274826, + "loss": 1.3594, + "theoretical_loss": 3.469833058736376, + "tokens_seen": 1925120000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208224674022066, + "loss": 1.4239, + "theoretical_loss": 3.4698232175555606, + "tokens_seen": 1925185536 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 701973, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5264232158660889, + "objective/train/theoretical_loss": 3.469813376803545, + "objective/train/tokens_used": 295710176, + "theoretical_loss": 3.469813376803545, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042080240722166504, + "loss": 1.4383, + "theoretical_loss": 3.469813376803545, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207823470411234, + "loss": 1.3312, + "theoretical_loss": 3.469803536480297, + "tokens_seen": 1925316608 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042076228686058176, + "loss": 1.2398, + "theoretical_loss": 3.469793696585782, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207422266800401, + "loss": 1.3864, + "theoretical_loss": 3.4697838571199684, + "tokens_seen": 1925447680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207221664994985, + "loss": 1.3397, + "theoretical_loss": 3.469774018082821, + "tokens_seen": 1925513216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207021063189569, + "loss": 1.4218, + "theoretical_loss": 3.4697641794743084, + "tokens_seen": 1925578752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042068204613841527, + "loss": 1.3639, + "theoretical_loss": 3.469754341294396, + "tokens_seen": 1925644288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042066198595787363, + "loss": 1.4777, + "theoretical_loss": 3.469744503543051, + "tokens_seen": 1925709824 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420641925777332, + "loss": 1.4203, + "theoretical_loss": 3.46973466622024, + "tokens_seen": 1925775360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206218655967904, + "loss": 1.4475, + "theoretical_loss": 3.4697248293259304, + "tokens_seen": 1925840896 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042060180541624877, + "loss": 1.4234, + "theoretical_loss": 3.469714992860088, + "tokens_seen": 1925906432 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042058174523570713, + "loss": 1.293, + "theoretical_loss": 3.4697051568226804, + "tokens_seen": 1925971968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205616850551655, + "loss": 1.3534, + "theoretical_loss": 3.4696953212136736, + "tokens_seen": 1926037504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205416248746239, + "loss": 1.3738, + "theoretical_loss": 3.469685486033035, + "tokens_seen": 1926103040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205215646940823, + "loss": 1.3611, + "theoretical_loss": 3.469675651280731, + "tokens_seen": 1926168576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042050150451354064, + "loss": 1.2836, + "theoretical_loss": 3.4696658169567285, + "tokens_seen": 1926234112 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420481444332999, + "loss": 1.3201, + "theoretical_loss": 3.469655983060994, + "tokens_seen": 1926299648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042046138415245736, + "loss": 1.4302, + "theoretical_loss": 3.469646149593495, + "tokens_seen": 1926365184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204413239719158, + "loss": 1.3677, + "theoretical_loss": 3.4696363165541975, + "tokens_seen": 1926430720 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042042126379137414, + "loss": 1.3277, + "theoretical_loss": 3.4696264839430686, + "tokens_seen": 1926496256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204012036108325, + "loss": 1.2716, + "theoretical_loss": 3.4696166517600755, + "tokens_seen": 1926561792 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042038114343029086, + "loss": 1.4919, + "theoretical_loss": 3.4696068200051844, + "tokens_seen": 1926627328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203610832497493, + "loss": 1.3817, + "theoretical_loss": 3.469596988678362, + "tokens_seen": 1926692864 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042034102306920764, + "loss": 1.3356, + "theoretical_loss": 3.469587157779576, + "tokens_seen": 1926758400 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420320962888666, + "loss": 1.3599, + "theoretical_loss": 3.4695773273087918, + "tokens_seen": 1926823936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042030090270812437, + "loss": 1.4011, + "theoretical_loss": 3.4695674972659774, + "tokens_seen": 1926889472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042028084252758273, + "loss": 1.3683, + "theoretical_loss": 3.469557667651099, + "tokens_seen": 1926955008 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042026078234704115, + "loss": 1.4777, + "theoretical_loss": 3.4695478384641243, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202407221664995, + "loss": 1.3343, + "theoretical_loss": 3.469538009705019, + "tokens_seen": 1927086080 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042022066198595787, + "loss": 1.2653, + "theoretical_loss": 3.46952818137375, + "tokens_seen": 1927151616 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042020060180541623, + "loss": 1.498, + "theoretical_loss": 3.4695183534702854, + "tokens_seen": 1927217152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042018054162487465, + "loss": 1.3042, + "theoretical_loss": 3.4695085259945904, + "tokens_seen": 1927282688 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420160481444333, + "loss": 1.4258, + "theoretical_loss": 3.4694986989466328, + "tokens_seen": 1927348224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201404212637914, + "loss": 1.3078, + "theoretical_loss": 3.4694888723263793, + "tokens_seen": 1927413760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042012036108324974, + "loss": 1.3834, + "theoretical_loss": 3.469479046133797, + "tokens_seen": 1927479296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201003009027081, + "loss": 1.4132, + "theoretical_loss": 3.469469220368852, + "tokens_seen": 1927544832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200802407221665, + "loss": 1.4725, + "theoretical_loss": 3.4694593950315116, + "tokens_seen": 1927610368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200601805416249, + "loss": 1.4918, + "theoretical_loss": 3.4694495701217427, + "tokens_seen": 1927675904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042004012036108324, + "loss": 1.3441, + "theoretical_loss": 3.4694397456395123, + "tokens_seen": 1927741440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200200601805416, + "loss": 1.3741, + "theoretical_loss": 3.469429921584787, + "tokens_seen": 1927806976 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042, + "loss": 1.5306, + "theoretical_loss": 3.469420097957533, + "tokens_seen": 1927872512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199799398194584, + "loss": 1.2331, + "theoretical_loss": 3.4694102747577187, + "tokens_seen": 1927938048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041995987963891674, + "loss": 1.3393, + "theoretical_loss": 3.46940045198531, + "tokens_seen": 1928003584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199398194583751, + "loss": 1.3895, + "theoretical_loss": 3.4693906296402734, + "tokens_seen": 1928069120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041991975927783347, + "loss": 1.416, + "theoretical_loss": 3.469380807722577, + "tokens_seen": 1928134656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198996990972919, + "loss": 1.3692, + "theoretical_loss": 3.469370986232186, + "tokens_seen": 1928200192 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041987963891675025, + "loss": 1.3991, + "theoretical_loss": 3.4693611651690692, + "tokens_seen": 1928265728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198595787362086, + "loss": 1.2821, + "theoretical_loss": 3.4693513445331923, + "tokens_seen": 1928331264 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041983951855566697, + "loss": 1.4471, + "theoretical_loss": 3.4693415243245225, + "tokens_seen": 1928396800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198194583751254, + "loss": 1.37, + "theoretical_loss": 3.4693317045430265, + "tokens_seen": 1928462336 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 703126, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3585114479064941, + "objective/train/theoretical_loss": 3.4693218851886716, + "objective/train/tokens_used": 298986976, + "theoretical_loss": 3.4693218851886716, + "tokens_seen": 1928527872 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041979939819458375, + "loss": 1.3773, + "theoretical_loss": 3.4693218851886716, + "tokens_seen": 1928527872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197793380140421, + "loss": 1.3453, + "theoretical_loss": 3.4693120662614243, + "tokens_seen": 1928593408 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041975927783350047, + "loss": 1.295, + "theoretical_loss": 3.4693022477612514, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041973921765295883, + "loss": 1.395, + "theoretical_loss": 3.4692924296881205, + "tokens_seen": 1928724480 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041971915747241725, + "loss": 1.4105, + "theoretical_loss": 3.469282612041998, + "tokens_seen": 1928790016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196990972918756, + "loss": 1.35, + "theoretical_loss": 3.469272794822851, + "tokens_seen": 1928855552 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419679037111334, + "loss": 1.319, + "theoretical_loss": 3.4692629780306463, + "tokens_seen": 1928921088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196589769307924, + "loss": 1.4431, + "theoretical_loss": 3.4692531616653506, + "tokens_seen": 1928986624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196389167502508, + "loss": 1.425, + "theoretical_loss": 3.469243345726931, + "tokens_seen": 1929052160 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041961885656970917, + "loss": 1.3815, + "theoretical_loss": 3.469233530215355, + "tokens_seen": 1929117696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041959879638916753, + "loss": 1.2496, + "theoretical_loss": 3.4692237151305894, + "tokens_seen": 1929183232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195787362086259, + "loss": 1.3619, + "theoretical_loss": 3.4692139004726004, + "tokens_seen": 1929248768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195586760280843, + "loss": 1.275, + "theoretical_loss": 3.469204086241355, + "tokens_seen": 1929314304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195386158475427, + "loss": 1.376, + "theoretical_loss": 3.4691942724368214, + "tokens_seen": 1929379840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041951855566700104, + "loss": 1.359, + "theoretical_loss": 3.469184459058965, + "tokens_seen": 1929445376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194984954864594, + "loss": 1.4429, + "theoretical_loss": 3.4691746461077537, + "tokens_seen": 1929510912 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041947843530591776, + "loss": 1.5078, + "theoretical_loss": 3.4691648335831538, + "tokens_seen": 1929576448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194583751253762, + "loss": 1.3917, + "theoretical_loss": 3.4691550214851334, + "tokens_seen": 1929641984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943831494483454, + "loss": 1.4125, + "theoretical_loss": 3.469145209813658, + "tokens_seen": 1929707520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194182547642929, + "loss": 1.4025, + "theoretical_loss": 3.469135398568696, + "tokens_seen": 1929773056 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041939819458375126, + "loss": 1.363, + "theoretical_loss": 3.469125587750213, + "tokens_seen": 1929838592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193781344032097, + "loss": 1.4099, + "theoretical_loss": 3.469115777358177, + "tokens_seen": 1929904128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041935807422266804, + "loss": 1.4516, + "theoretical_loss": 3.4691059673925544, + "tokens_seen": 1929969664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193380140421264, + "loss": 1.3919, + "theoretical_loss": 3.4690961578533126, + "tokens_seen": 1930035200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041931795386158477, + "loss": 1.3842, + "theoretical_loss": 3.4690863487404187, + "tokens_seen": 1930100736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041929789368104313, + "loss": 1.4357, + "theoretical_loss": 3.4690765400538393, + "tokens_seen": 1930166272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927783350050155, + "loss": 1.3845, + "theoretical_loss": 3.469066731793541, + "tokens_seen": 1930231808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192577733199599, + "loss": 1.3076, + "theoretical_loss": 3.4690569239594917, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041923771313941827, + "loss": 1.381, + "theoretical_loss": 3.469047116551658, + "tokens_seen": 1930362880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041921765295887663, + "loss": 1.3423, + "theoretical_loss": 3.469037309570007, + "tokens_seen": 1930428416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041919759277833505, + "loss": 1.4357, + "theoretical_loss": 3.4690275030145052, + "tokens_seen": 1930493952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191775325977934, + "loss": 1.3848, + "theoretical_loss": 3.4690176968851203, + "tokens_seen": 1930559488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191574724172518, + "loss": 1.4606, + "theoretical_loss": 3.4690078911818194, + "tokens_seen": 1930625024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041913741223671014, + "loss": 1.3832, + "theoretical_loss": 3.468998085904569, + "tokens_seen": 1930690560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191173520561685, + "loss": 1.396, + "theoretical_loss": 3.4689882810533366, + "tokens_seen": 1930756096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190972918756269, + "loss": 1.3276, + "theoretical_loss": 3.4689784766280884, + "tokens_seen": 1930821632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190772316950853, + "loss": 1.5123, + "theoretical_loss": 3.4689686726287925, + "tokens_seen": 1930887168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041905717151454364, + "loss": 1.4276, + "theoretical_loss": 3.468958869055415, + "tokens_seen": 1930952704 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419037111334002, + "loss": 1.3639, + "theoretical_loss": 3.468949065907924, + "tokens_seen": 1931018240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190170511534604, + "loss": 1.4026, + "theoretical_loss": 3.468939263186285, + "tokens_seen": 1931083776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189969909729188, + "loss": 1.2994, + "theoretical_loss": 3.4689294608904664, + "tokens_seen": 1931149312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041897693079237714, + "loss": 1.2572, + "theoretical_loss": 3.468919659020435, + "tokens_seen": 1931214848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189568706118355, + "loss": 1.3661, + "theoretical_loss": 3.4689098575761577, + "tokens_seen": 1931280384 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041893681043129387, + "loss": 1.4139, + "theoretical_loss": 3.4689000565576014, + "tokens_seen": 1931345920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189167502507523, + "loss": 1.3471, + "theoretical_loss": 3.468890255964733, + "tokens_seen": 1931411456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041889669007021065, + "loss": 1.3739, + "theoretical_loss": 3.46888045579752, + "tokens_seen": 1931476992 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418876629889669, + "loss": 1.3559, + "theoretical_loss": 3.46887065605593, + "tokens_seen": 1931542528 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041885656970912737, + "loss": 1.3802, + "theoretical_loss": 3.468860856739929, + "tokens_seen": 1931608064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188365095285858, + "loss": 1.3959, + "theoretical_loss": 3.468851057849484, + "tokens_seen": 1931673600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041881644934804415, + "loss": 1.4677, + "theoretical_loss": 3.468841259384563, + "tokens_seen": 1931739136 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 704289, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5385944843292236, + "objective/train/theoretical_loss": 3.4688314613451325, + "objective/train/tokens_used": 302263776, + "theoretical_loss": 3.4688314613451325, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187963891675025, + "loss": 1.4941, + "theoretical_loss": 3.4688314613451325, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187763289869609, + "loss": 1.4521, + "theoretical_loss": 3.46882166373116, + "tokens_seen": 1931870208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041875626880641924, + "loss": 1.3902, + "theoretical_loss": 3.4688118665426124, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041873620862587765, + "loss": 1.4057, + "theoretical_loss": 3.4688020697794566, + "tokens_seen": 1932001280 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418716148445336, + "loss": 1.272, + "theoretical_loss": 3.46879227344166, + "tokens_seen": 1932066816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186960882647944, + "loss": 1.3272, + "theoretical_loss": 3.4687824775291896, + "tokens_seen": 1932132352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041867602808425274, + "loss": 1.4092, + "theoretical_loss": 3.4687726820420117, + "tokens_seen": 1932197888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041865596790371115, + "loss": 1.344, + "theoretical_loss": 3.4687628869800946, + "tokens_seen": 1932263424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186359077231695, + "loss": 1.1831, + "theoretical_loss": 3.4687530923434053, + "tokens_seen": 1932328960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186158475426279, + "loss": 1.4457, + "theoretical_loss": 3.46874329813191, + "tokens_seen": 1932394496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041859578736208624, + "loss": 1.3865, + "theoretical_loss": 3.468733504345577, + "tokens_seen": 1932460032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185757271815446, + "loss": 1.3186, + "theoretical_loss": 3.468723710984373, + "tokens_seen": 1932525568 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418555667001003, + "loss": 1.3784, + "theoretical_loss": 3.4687139180482647, + "tokens_seen": 1932591104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185356068204614, + "loss": 1.2988, + "theoretical_loss": 3.4687041255372195, + "tokens_seen": 1932656640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041851554663991974, + "loss": 1.314, + "theoretical_loss": 3.4686943334512046, + "tokens_seen": 1932722176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184954864593781, + "loss": 1.3587, + "theoretical_loss": 3.4686845417901866, + "tokens_seen": 1932787712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184754262788365, + "loss": 1.434, + "theoretical_loss": 3.4686747505541335, + "tokens_seen": 1932853248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184553660982949, + "loss": 1.3355, + "theoretical_loss": 3.4686649597430126, + "tokens_seen": 1932918784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041843530591775325, + "loss": 1.3835, + "theoretical_loss": 3.46865516935679, + "tokens_seen": 1932984320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184152457372116, + "loss": 1.3459, + "theoretical_loss": 3.4686453793954337, + "tokens_seen": 1933049856 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041839518555666997, + "loss": 1.3748, + "theoretical_loss": 3.46863558985891, + "tokens_seen": 1933115392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183751253761284, + "loss": 1.3542, + "theoretical_loss": 3.468625800747187, + "tokens_seen": 1933180928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041835506519558675, + "loss": 1.4299, + "theoretical_loss": 3.4686160120602314, + "tokens_seen": 1933246464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183350050150451, + "loss": 1.3459, + "theoretical_loss": 3.4686062237980106, + "tokens_seen": 1933312000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183149448345035, + "loss": 1.4708, + "theoretical_loss": 3.468596435960491, + "tokens_seen": 1933377536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182948846539619, + "loss": 1.2687, + "theoretical_loss": 3.468586648547641, + "tokens_seen": 1933443072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041827482447342025, + "loss": 1.3396, + "theoretical_loss": 3.468576861559427, + "tokens_seen": 1933508608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182547642928786, + "loss": 1.3175, + "theoretical_loss": 3.468567074995817, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418234704112337, + "loss": 1.3236, + "theoretical_loss": 3.4685572888567764, + "tokens_seen": 1933639680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182146439317954, + "loss": 1.3742, + "theoretical_loss": 3.4685475031422746, + "tokens_seen": 1933705216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041819458375125376, + "loss": 1.3708, + "theoretical_loss": 3.468537717852277, + "tokens_seen": 1933770752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181745235707121, + "loss": 1.4306, + "theoretical_loss": 3.4685279329867518, + "tokens_seen": 1933836288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041815446339017054, + "loss": 1.3214, + "theoretical_loss": 3.468518148545666, + "tokens_seen": 1933901824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181344032096289, + "loss": 1.396, + "theoretical_loss": 3.468508364528987, + "tokens_seen": 1933967360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181143430290873, + "loss": 1.4688, + "theoretical_loss": 3.4684985809366813, + "tokens_seen": 1934032896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180942828485457, + "loss": 1.4803, + "theoretical_loss": 3.4684887977687167, + "tokens_seen": 1934098432 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041807422266800404, + "loss": 1.3986, + "theoretical_loss": 3.4684790150250606, + "tokens_seen": 1934163968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180541624874624, + "loss": 1.3921, + "theoretical_loss": 3.4684692327056794, + "tokens_seen": 1934229504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180341023069208, + "loss": 1.2771, + "theoretical_loss": 3.468459450810541, + "tokens_seen": 1934295040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180140421263792, + "loss": 1.322, + "theoretical_loss": 3.4684496693396127, + "tokens_seen": 1934360576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041799398194583754, + "loss": 1.4031, + "theoretical_loss": 3.4684398882928615, + "tokens_seen": 1934426112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179739217652959, + "loss": 1.3434, + "theoretical_loss": 3.4684301076702546, + "tokens_seen": 1934491648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041795386158475427, + "loss": 1.3041, + "theoretical_loss": 3.468420327471759, + "tokens_seen": 1934557184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179338014042127, + "loss": 1.4664, + "theoretical_loss": 3.4684105476973426, + "tokens_seen": 1934622720 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041791374122367105, + "loss": 1.3646, + "theoretical_loss": 3.4684007683469718, + "tokens_seen": 1934688256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178936810431294, + "loss": 1.3553, + "theoretical_loss": 3.468390989420615, + "tokens_seen": 1934753792 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041787362086258777, + "loss": 1.3907, + "theoretical_loss": 3.4683812109182384, + "tokens_seen": 1934819328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178535606820462, + "loss": 1.286, + "theoretical_loss": 3.4683714328398096, + "tokens_seen": 1934884864 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783350050150455, + "loss": 1.314, + "theoretical_loss": 3.4683616551852956, + "tokens_seen": 1934950400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178134403209629, + "loss": 1.4161, + "theoretical_loss": 3.4683518779546647, + "tokens_seen": 1935015936 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 705622, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3978188037872314, + "objective/train/theoretical_loss": 3.4683421011478828, + "objective/train/tokens_used": 305540576, + "theoretical_loss": 3.4683421011478828, + "tokens_seen": 1935081472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177933801404213, + "loss": 1.3751, + "theoretical_loss": 3.4683421011478828, + "tokens_seen": 1935081472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041777331995987964, + "loss": 1.2738, + "theoretical_loss": 3.468332324764918, + "tokens_seen": 1935147008 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041775325977933805, + "loss": 1.3688, + "theoretical_loss": 3.4683225488057374, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177331995987964, + "loss": 1.3081, + "theoretical_loss": 3.468312773270308, + "tokens_seen": 1935278080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177131394182548, + "loss": 1.3663, + "theoretical_loss": 3.4683029981585975, + "tokens_seen": 1935343616 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041769307923771314, + "loss": 1.432, + "theoretical_loss": 3.468293223470573, + "tokens_seen": 1935409152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041767301905717156, + "loss": 1.2619, + "theoretical_loss": 3.4682834492062016, + "tokens_seen": 1935474688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176529588766299, + "loss": 1.2516, + "theoretical_loss": 3.4682736753654515, + "tokens_seen": 1935540224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176328986960883, + "loss": 1.3214, + "theoretical_loss": 3.468263901948289, + "tokens_seen": 1935605760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041761283851554664, + "loss": 1.4344, + "theoretical_loss": 3.468254128954681, + "tokens_seen": 1935671296 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417592778335005, + "loss": 1.344, + "theoretical_loss": 3.468244356384596, + "tokens_seen": 1935736832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175727181544634, + "loss": 1.3614, + "theoretical_loss": 3.468234584238001, + "tokens_seen": 1935802368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175526579739218, + "loss": 1.2512, + "theoretical_loss": 3.468224812514863, + "tokens_seen": 1935867904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041753259779338015, + "loss": 1.3703, + "theoretical_loss": 3.4682150412151493, + "tokens_seen": 1935933440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175125376128385, + "loss": 1.2505, + "theoretical_loss": 3.468205270338828, + "tokens_seen": 1935998976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174924774322969, + "loss": 1.3161, + "theoretical_loss": 3.468195499885865, + "tokens_seen": 1936064512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174724172517553, + "loss": 1.3768, + "theoretical_loss": 3.468185729856229, + "tokens_seen": 1936130048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041745235707121365, + "loss": 1.3987, + "theoretical_loss": 3.4681759602498863, + "tokens_seen": 1936195584 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417432296890672, + "loss": 1.4023, + "theoretical_loss": 3.4681661910668047, + "tokens_seen": 1936261120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041741223671013037, + "loss": 1.3435, + "theoretical_loss": 3.4681564223069516, + "tokens_seen": 1936326656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173921765295888, + "loss": 1.3387, + "theoretical_loss": 3.4681466539702948, + "tokens_seen": 1936392192 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041737211634904715, + "loss": 1.3121, + "theoretical_loss": 3.4681368860568, + "tokens_seen": 1936457728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173520561685055, + "loss": 1.3161, + "theoretical_loss": 3.4681271185664366, + "tokens_seen": 1936523264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173319959879639, + "loss": 1.4292, + "theoretical_loss": 3.4681173514991706, + "tokens_seen": 1936588800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173119358074223, + "loss": 1.3674, + "theoretical_loss": 3.4681075848549696, + "tokens_seen": 1936654336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041729187562688065, + "loss": 1.2904, + "theoretical_loss": 3.4680978186338014, + "tokens_seen": 1936719872 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417271815446339, + "loss": 1.4372, + "theoretical_loss": 3.4680880528356326, + "tokens_seen": 1936785408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172517552657974, + "loss": 1.3775, + "theoretical_loss": 3.4680782874604317, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172316950852558, + "loss": 1.3468, + "theoretical_loss": 3.4680685225081653, + "tokens_seen": 1936916480 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041721163490471416, + "loss": 1.3314, + "theoretical_loss": 3.4680587579788007, + "tokens_seen": 1936982016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171915747241725, + "loss": 1.4066, + "theoretical_loss": 3.468048993872306, + "tokens_seen": 1937047552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171715145436309, + "loss": 1.3649, + "theoretical_loss": 3.468039230188647, + "tokens_seen": 1937113088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041715145436308924, + "loss": 1.4352, + "theoretical_loss": 3.468029466927793, + "tokens_seen": 1937178624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041713139418254766, + "loss": 1.2309, + "theoretical_loss": 3.46801970408971, + "tokens_seen": 1937244160 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417111334002006, + "loss": 1.4011, + "theoretical_loss": 3.468009941674366, + "tokens_seen": 1937309696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170912738214644, + "loss": 1.4143, + "theoretical_loss": 3.468000179681728, + "tokens_seen": 1937375232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041707121364092275, + "loss": 1.3763, + "theoretical_loss": 3.4679904181117642, + "tokens_seen": 1937440768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041705115346038116, + "loss": 1.4496, + "theoretical_loss": 3.467980656964442, + "tokens_seen": 1937506304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170310932798395, + "loss": 1.3204, + "theoretical_loss": 3.467970896239727, + "tokens_seen": 1937571840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170110330992979, + "loss": 1.2882, + "theoretical_loss": 3.467961135937589, + "tokens_seen": 1937637376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041699097291875625, + "loss": 1.4697, + "theoretical_loss": 3.467951376057994, + "tokens_seen": 1937702912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169709127382146, + "loss": 1.4418, + "theoretical_loss": 3.4679416166009096, + "tokens_seen": 1937768448 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041695085255767303, + "loss": 1.3643, + "theoretical_loss": 3.467931857566304, + "tokens_seen": 1937833984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169307923771314, + "loss": 1.277, + "theoretical_loss": 3.4679220989541433, + "tokens_seen": 1937899520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041691073219658975, + "loss": 1.418, + "theoretical_loss": 3.4679123407643955, + "tokens_seen": 1937965056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168906720160481, + "loss": 1.3672, + "theoretical_loss": 3.467902582997029, + "tokens_seen": 1938030592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041687061183550653, + "loss": 1.383, + "theoretical_loss": 3.46789282565201, + "tokens_seen": 1938096128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168505516549649, + "loss": 1.2926, + "theoretical_loss": 3.467883068729306, + "tokens_seen": 1938161664 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041683049147442326, + "loss": 1.4197, + "theoretical_loss": 3.467873312228885, + "tokens_seen": 1938227200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168104312938816, + "loss": 1.2633, + "theoretical_loss": 3.4678635561507143, + "tokens_seen": 1938292736 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 706849, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.241760015487671, + "objective/train/theoretical_loss": 3.467853800494761, + "objective/train/tokens_used": 308817376, + "theoretical_loss": 3.467853800494761, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041679037111334, + "loss": 1.3329, + "theoretical_loss": 3.467853800494761, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167703109327984, + "loss": 1.3804, + "theoretical_loss": 3.467844045260993, + "tokens_seen": 1938423808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041675025075225676, + "loss": 1.4074, + "theoretical_loss": 3.4678342904493777, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167301905717151, + "loss": 1.2663, + "theoretical_loss": 3.467824536059882, + "tokens_seen": 1938554880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167101303911735, + "loss": 1.3218, + "theoretical_loss": 3.4678147820924745, + "tokens_seen": 1938620416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166900702106319, + "loss": 1.425, + "theoretical_loss": 3.4678050285471214, + "tokens_seen": 1938685952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041667001003009026, + "loss": 1.3919, + "theoretical_loss": 3.467795275423791, + "tokens_seen": 1938751488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166499498495487, + "loss": 1.3897, + "theoretical_loss": 3.4677855227224508, + "tokens_seen": 1938817024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041662988966900704, + "loss": 1.3555, + "theoretical_loss": 3.4677757704430676, + "tokens_seen": 1938882560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166098294884654, + "loss": 1.2896, + "theoretical_loss": 3.4677660185856096, + "tokens_seen": 1938948096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165897693079238, + "loss": 1.3669, + "theoretical_loss": 3.467756267150044, + "tokens_seen": 1939013632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165697091273822, + "loss": 1.3284, + "theoretical_loss": 3.467746516136337, + "tokens_seen": 1939079168 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654964894684055, + "loss": 1.4591, + "theoretical_loss": 3.467736765544459, + "tokens_seen": 1939144704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165295887662989, + "loss": 1.2747, + "theoretical_loss": 3.467727015374375, + "tokens_seen": 1939210240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165095285857573, + "loss": 1.3812, + "theoretical_loss": 3.467717265626054, + "tokens_seen": 1939275776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164894684052157, + "loss": 1.2393, + "theoretical_loss": 3.4677075162994617, + "tokens_seen": 1939341312 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041646940822467405, + "loss": 1.2968, + "theoretical_loss": 3.4676977673945677, + "tokens_seen": 1939406848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164493480441324, + "loss": 1.4291, + "theoretical_loss": 3.467688018911338, + "tokens_seen": 1939472384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164292878635908, + "loss": 1.287, + "theoretical_loss": 3.467678270849741, + "tokens_seen": 1939537920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164092276830492, + "loss": 1.3923, + "theoretical_loss": 3.467668523209744, + "tokens_seen": 1939603456 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638916750250755, + "loss": 1.4175, + "theoretical_loss": 3.467658775991315, + "tokens_seen": 1939668992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163691073219659, + "loss": 1.3929, + "theoretical_loss": 3.46764902919442, + "tokens_seen": 1939734528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163490471414243, + "loss": 1.3534, + "theoretical_loss": 3.4676392828190274, + "tokens_seen": 1939800064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163289869608827, + "loss": 1.3307, + "theoretical_loss": 3.4676295368651053, + "tokens_seen": 1939865600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041630892678034105, + "loss": 1.4219, + "theoretical_loss": 3.4676197913326208, + "tokens_seen": 1939931136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162888665997994, + "loss": 1.3602, + "theoretical_loss": 3.467610046221541, + "tokens_seen": 1939996672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162688064192578, + "loss": 1.3338, + "theoretical_loss": 3.467600301531834, + "tokens_seen": 1940062208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162487462387162, + "loss": 1.3719, + "theoretical_loss": 3.467590557263467, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041622868605817456, + "loss": 1.3396, + "theoretical_loss": 3.467580813416408, + "tokens_seen": 1940193280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162086258776329, + "loss": 1.3609, + "theoretical_loss": 3.4675710699906244, + "tokens_seen": 1940258816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161885656970913, + "loss": 1.3998, + "theoretical_loss": 3.4675613269860834, + "tokens_seen": 1940324352 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041616850551654964, + "loss": 1.4225, + "theoretical_loss": 3.4675515844027522, + "tokens_seen": 1940389888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041614844533600806, + "loss": 1.3909, + "theoretical_loss": 3.4675418422405997, + "tokens_seen": 1940455424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161283851554664, + "loss": 1.3074, + "theoretical_loss": 3.467532100499593, + "tokens_seen": 1940520960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161083249749248, + "loss": 1.568, + "theoretical_loss": 3.4675223591796986, + "tokens_seen": 1940586496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041608826479438315, + "loss": 1.416, + "theoretical_loss": 3.4675126182808853, + "tokens_seen": 1940652032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041606820461384156, + "loss": 1.3091, + "theoretical_loss": 3.4675028778031205, + "tokens_seen": 1940717568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160481444332999, + "loss": 1.3716, + "theoretical_loss": 3.467493137746371, + "tokens_seen": 1940783104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160280842527583, + "loss": 1.4581, + "theoretical_loss": 3.4674833981106055, + "tokens_seen": 1940848640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041600802407221665, + "loss": 1.3903, + "theoretical_loss": 3.4674736588957904, + "tokens_seen": 1940914176 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415987963891675, + "loss": 1.3653, + "theoretical_loss": 3.4674639201018946, + "tokens_seen": 1940979712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041596790371113343, + "loss": 1.3429, + "theoretical_loss": 3.4674541817288844, + "tokens_seen": 1941045248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159478435305918, + "loss": 1.2982, + "theoretical_loss": 3.4674444437767282, + "tokens_seen": 1941110784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041592778335005015, + "loss": 1.4436, + "theoretical_loss": 3.4674347062453936, + "tokens_seen": 1941176320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159077231695085, + "loss": 1.2485, + "theoretical_loss": 3.467424969134848, + "tokens_seen": 1941241856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041588766298896693, + "loss": 1.4267, + "theoretical_loss": 3.4674152324450587, + "tokens_seen": 1941307392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158676028084253, + "loss": 1.3311, + "theoretical_loss": 3.467405496175994, + "tokens_seen": 1941372928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041584754262788366, + "loss": 1.281, + "theoretical_loss": 3.467395760327621, + "tokens_seen": 1941438464 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415827482447342, + "loss": 1.3445, + "theoretical_loss": 3.4673860248999073, + "tokens_seen": 1941504000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158074222668004, + "loss": 1.3291, + "theoretical_loss": 3.467376289892821, + "tokens_seen": 1941569536 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 708064, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3450722694396973, + "objective/train/theoretical_loss": 3.4673665553063295, + "objective/train/tokens_used": 312094176, + "theoretical_loss": 3.4673665553063295, + "tokens_seen": 1941635072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157873620862588, + "loss": 1.3436, + "theoretical_loss": 3.4673665553063295, + "tokens_seen": 1941635072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041576730190571716, + "loss": 1.3548, + "theoretical_loss": 3.4673568211404002, + "tokens_seen": 1941700608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157472417251755, + "loss": 1.3427, + "theoretical_loss": 3.4673470873950007, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157271815446339, + "loss": 1.3958, + "theoretical_loss": 3.467337354070099, + "tokens_seen": 1941831680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157071213640923, + "loss": 1.2572, + "theoretical_loss": 3.467327621165663, + "tokens_seen": 1941897216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041568706118355066, + "loss": 1.349, + "theoretical_loss": 3.46731788868166, + "tokens_seen": 1941962752 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415667001003009, + "loss": 1.2885, + "theoretical_loss": 3.4673081566180572, + "tokens_seen": 1942028288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156469408224674, + "loss": 1.3483, + "theoretical_loss": 3.467298424974823, + "tokens_seen": 1942093824 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041562688064192575, + "loss": 1.3482, + "theoretical_loss": 3.4672886937519243, + "tokens_seen": 1942159360 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041560682046138417, + "loss": 1.3247, + "theoretical_loss": 3.467278962949329, + "tokens_seen": 1942224896 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041558676028084253, + "loss": 1.2815, + "theoretical_loss": 3.4672692325670056, + "tokens_seen": 1942290432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155667001003009, + "loss": 1.1718, + "theoretical_loss": 3.4672595026049207, + "tokens_seen": 1942355968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041554663991975925, + "loss": 1.3279, + "theoretical_loss": 3.467249773063042, + "tokens_seen": 1942421504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041552657973921767, + "loss": 1.3205, + "theoretical_loss": 3.467240043941338, + "tokens_seen": 1942487040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041550651955867603, + "loss": 1.3592, + "theoretical_loss": 3.4672303152397763, + "tokens_seen": 1942552576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154864593781344, + "loss": 1.4239, + "theoretical_loss": 3.4672205869583235, + "tokens_seen": 1942618112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041546639919759276, + "loss": 1.422, + "theoretical_loss": 3.467210859096948, + "tokens_seen": 1942683648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154463390170511, + "loss": 1.3735, + "theoretical_loss": 3.467201131655618, + "tokens_seen": 1942749184 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041542627883650954, + "loss": 1.3935, + "theoretical_loss": 3.467191404634301, + "tokens_seen": 1942814720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154062186559679, + "loss": 1.3062, + "theoretical_loss": 3.467181678032963, + "tokens_seen": 1942880256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041538615847542626, + "loss": 1.423, + "theoretical_loss": 3.4671719518515745, + "tokens_seen": 1942945792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153660982948846, + "loss": 1.3051, + "theoretical_loss": 3.4671622260901014, + "tokens_seen": 1943011328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041534603811434304, + "loss": 1.3078, + "theoretical_loss": 3.4671525007485116, + "tokens_seen": 1943076864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153259779338014, + "loss": 1.3397, + "theoretical_loss": 3.4671427758267725, + "tokens_seen": 1943142400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041530591775325976, + "loss": 1.4405, + "theoretical_loss": 3.4671330513248533, + "tokens_seen": 1943207936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152858575727181, + "loss": 1.3526, + "theoretical_loss": 3.46712332724272, + "tokens_seen": 1943273472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152657973921765, + "loss": 1.307, + "theoretical_loss": 3.4671136035803416, + "tokens_seen": 1943339008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152457372116349, + "loss": 1.3822, + "theoretical_loss": 3.4671038803376852, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041522567703109327, + "loss": 1.3264, + "theoretical_loss": 3.4670941575147185, + "tokens_seen": 1943470080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041520561685055163, + "loss": 1.3258, + "theoretical_loss": 3.4670844351114094, + "tokens_seen": 1943535616 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041518555667001, + "loss": 1.3334, + "theoretical_loss": 3.4670747131277255, + "tokens_seen": 1943601152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151654964894684, + "loss": 1.4422, + "theoretical_loss": 3.467064991563635, + "tokens_seen": 1943666688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151454363089268, + "loss": 1.3988, + "theoretical_loss": 3.4670552704191047, + "tokens_seen": 1943732224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151253761283852, + "loss": 1.3525, + "theoretical_loss": 3.4670455496941033, + "tokens_seen": 1943797760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041510531594784355, + "loss": 1.2827, + "theoretical_loss": 3.467035829388598, + "tokens_seen": 1943863296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041508525576730196, + "loss": 1.4013, + "theoretical_loss": 3.467026109502557, + "tokens_seen": 1943928832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150651955867603, + "loss": 1.4096, + "theoretical_loss": 3.4670163900359476, + "tokens_seen": 1943994368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150451354062187, + "loss": 1.3927, + "theoretical_loss": 3.4670066709887375, + "tokens_seen": 1944059904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041502507522567705, + "loss": 1.4027, + "theoretical_loss": 3.466996952360895, + "tokens_seen": 1944125440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150050150451354, + "loss": 1.3256, + "theoretical_loss": 3.4669872341523877, + "tokens_seen": 1944190976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498495486459383, + "loss": 1.3302, + "theoretical_loss": 3.4669775163631833, + "tokens_seen": 1944256512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149648946840522, + "loss": 1.3147, + "theoretical_loss": 3.466967798993249, + "tokens_seen": 1944322048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041494483450351055, + "loss": 1.2997, + "theoretical_loss": 3.466958082042553, + "tokens_seen": 1944387584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149247743229689, + "loss": 1.3472, + "theoretical_loss": 3.466948365511064, + "tokens_seen": 1944453120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041490471414242733, + "loss": 1.3414, + "theoretical_loss": 3.466938649398749, + "tokens_seen": 1944518656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148846539618857, + "loss": 1.3753, + "theoretical_loss": 3.4669289337055753, + "tokens_seen": 1944584192 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041486459378134406, + "loss": 1.3387, + "theoretical_loss": 3.466919218431511, + "tokens_seen": 1944649728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148445336008024, + "loss": 1.333, + "theoretical_loss": 3.4669095035765247, + "tokens_seen": 1944715264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148244734202608, + "loss": 1.4065, + "theoretical_loss": 3.466899789140583, + "tokens_seen": 1944780800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148044132397192, + "loss": 1.4217, + "theoretical_loss": 3.466890075123654, + "tokens_seen": 1944846336 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 709206, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4632246494293213, + "objective/train/theoretical_loss": 3.466880361525707, + "objective/train/tokens_used": 315370976, + "theoretical_loss": 3.466880361525707, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041478435305917756, + "loss": 1.4598, + "theoretical_loss": 3.466880361525707, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147642928786359, + "loss": 1.4358, + "theoretical_loss": 3.466870648346707, + "tokens_seen": 1944977408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147442326980943, + "loss": 1.512, + "theoretical_loss": 3.4668609355866247, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147241725175527, + "loss": 1.3632, + "theoretical_loss": 3.466851223245426, + "tokens_seen": 1945108480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041470411233701106, + "loss": 1.3986, + "theoretical_loss": 3.4668415113230795, + "tokens_seen": 1945174016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146840521564694, + "loss": 1.4201, + "theoretical_loss": 3.466831799819553, + "tokens_seen": 1945239552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146639919759278, + "loss": 1.4438, + "theoretical_loss": 3.466822088734814, + "tokens_seen": 1945305088 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041464393179538615, + "loss": 1.3081, + "theoretical_loss": 3.4668123780688305, + "tokens_seen": 1945370624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041462387161484457, + "loss": 1.4134, + "theoretical_loss": 3.4668026678215704, + "tokens_seen": 1945436160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041460381143430293, + "loss": 1.2866, + "theoretical_loss": 3.4667929579930012, + "tokens_seen": 1945501696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145837512537613, + "loss": 1.3598, + "theoretical_loss": 3.466783248583092, + "tokens_seen": 1945567232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041456369107321965, + "loss": 1.3391, + "theoretical_loss": 3.466773539591809, + "tokens_seen": 1945632768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454363089267807, + "loss": 1.5092, + "theoretical_loss": 3.4667638310191204, + "tokens_seen": 1945698304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041452357071213643, + "loss": 1.2782, + "theoretical_loss": 3.4667541228649954, + "tokens_seen": 1945763840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145035105315948, + "loss": 1.2973, + "theoretical_loss": 3.4667444151294005, + "tokens_seen": 1945829376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041448345035105316, + "loss": 1.4007, + "theoretical_loss": 3.4667347078123036, + "tokens_seen": 1945894912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144633901705115, + "loss": 1.3627, + "theoretical_loss": 3.466725000913673, + "tokens_seen": 1945960448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041444332998996994, + "loss": 1.3403, + "theoretical_loss": 3.4667152944334765, + "tokens_seen": 1946025984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144232698094283, + "loss": 1.3026, + "theoretical_loss": 3.466705588371682, + "tokens_seen": 1946091520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041440320962888666, + "loss": 1.367, + "theoretical_loss": 3.4666958827282572, + "tokens_seen": 1946157056 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414383149448345, + "loss": 1.3843, + "theoretical_loss": 3.46668617750317, + "tokens_seen": 1946222592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041436308926780344, + "loss": 1.425, + "theoretical_loss": 3.4666764726963883, + "tokens_seen": 1946288128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143430290872618, + "loss": 1.3519, + "theoretical_loss": 3.4666667683078805, + "tokens_seen": 1946353664 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041432296890672016, + "loss": 1.4098, + "theoretical_loss": 3.466657064337614, + "tokens_seen": 1946419200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143029087261785, + "loss": 1.415, + "theoretical_loss": 3.466647360785556, + "tokens_seen": 1946484736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142828485456369, + "loss": 1.362, + "theoretical_loss": 3.4666376576516758, + "tokens_seen": 1946550272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142627883650953, + "loss": 1.3715, + "theoretical_loss": 3.4666279549359404, + "tokens_seen": 1946615808 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041424272818455367, + "loss": 1.3768, + "theoretical_loss": 3.4666182526383182, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041422266800401203, + "loss": 1.373, + "theoretical_loss": 3.4666085507587767, + "tokens_seen": 1946746880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142026078234704, + "loss": 1.4301, + "theoretical_loss": 3.466598849297284, + "tokens_seen": 1946812416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141825476429288, + "loss": 1.3681, + "theoretical_loss": 3.466589148253808, + "tokens_seen": 1946877952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041416248746238717, + "loss": 1.333, + "theoretical_loss": 3.466579447628316, + "tokens_seen": 1946943488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041414242728184553, + "loss": 1.4065, + "theoretical_loss": 3.4665697474207775, + "tokens_seen": 1947009024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141223671013039, + "loss": 1.3339, + "theoretical_loss": 3.466560047631159, + "tokens_seen": 1947074560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141023069207623, + "loss": 1.3279, + "theoretical_loss": 3.466550348259428, + "tokens_seen": 1947140096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140822467402207, + "loss": 1.3322, + "theoretical_loss": 3.4665406493055544, + "tokens_seen": 1947205632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041406218655967904, + "loss": 1.3926, + "theoretical_loss": 3.4665309507695046, + "tokens_seen": 1947271168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140421263791374, + "loss": 1.2608, + "theoretical_loss": 3.466521252651247, + "tokens_seen": 1947336704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041402206619859576, + "loss": 1.4738, + "theoretical_loss": 3.46651155495075, + "tokens_seen": 1947402240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140020060180542, + "loss": 1.481, + "theoretical_loss": 3.4665018576679802, + "tokens_seen": 1947467776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041398194583751254, + "loss": 1.4261, + "theoretical_loss": 3.466492160802907, + "tokens_seen": 1947533312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139618856569709, + "loss": 1.4166, + "theoretical_loss": 3.466482464355498, + "tokens_seen": 1947598848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041394182547642926, + "loss": 1.3515, + "theoretical_loss": 3.4664727683257204, + "tokens_seen": 1947664384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139217652958877, + "loss": 1.3183, + "theoretical_loss": 3.466463072713543, + "tokens_seen": 1947729920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041390170511534604, + "loss": 1.4433, + "theoretical_loss": 3.466453377518933, + "tokens_seen": 1947795456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138816449348044, + "loss": 1.4355, + "theoretical_loss": 3.466443682741859, + "tokens_seen": 1947860992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041386158475426277, + "loss": 1.458, + "theoretical_loss": 3.466433988382289, + "tokens_seen": 1947926528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041384152457372113, + "loss": 1.3782, + "theoretical_loss": 3.4664242944401904, + "tokens_seen": 1947992064 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041382146439317954, + "loss": 1.4586, + "theoretical_loss": 3.466414600915532, + "tokens_seen": 1948057600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138014042126379, + "loss": 1.2852, + "theoretical_loss": 3.466404907808281, + "tokens_seen": 1948123136 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 710405, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4715192317962646, + "objective/train/theoretical_loss": 3.466395215118406, + "objective/train/tokens_used": 318647776, + "theoretical_loss": 3.466395215118406, + "tokens_seen": 1948188672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041378134403209627, + "loss": 1.374, + "theoretical_loss": 3.466395215118406, + "tokens_seen": 1948188672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041376128385155463, + "loss": 1.2738, + "theoretical_loss": 3.466385522845874, + "tokens_seen": 1948254208 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041374122367101305, + "loss": 1.4602, + "theoretical_loss": 3.4663758309906543, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137211634904714, + "loss": 1.4301, + "theoretical_loss": 3.466366139552714, + "tokens_seen": 1948385280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041370110330992977, + "loss": 1.337, + "theoretical_loss": 3.4663564485320215, + "tokens_seen": 1948450816 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041368104312938813, + "loss": 1.3774, + "theoretical_loss": 3.466346757928545, + "tokens_seen": 1948516352 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041366098294884655, + "loss": 1.4442, + "theoretical_loss": 3.466337067742252, + "tokens_seen": 1948581888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041364092276830497, + "loss": 1.3464, + "theoretical_loss": 3.4663273779731103, + "tokens_seen": 1948647424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041362086258776333, + "loss": 1.5564, + "theoretical_loss": 3.4663176886210887, + "tokens_seen": 1948712960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136008024072217, + "loss": 1.335, + "theoretical_loss": 3.4663079996861548, + "tokens_seen": 1948778496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041358074222668005, + "loss": 1.3677, + "theoretical_loss": 3.4662983111682766, + "tokens_seen": 1948844032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041356068204613847, + "loss": 1.2431, + "theoretical_loss": 3.466288623067422, + "tokens_seen": 1948909568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041354062186559683, + "loss": 1.352, + "theoretical_loss": 3.4662789353835595, + "tokens_seen": 1948975104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135205616850552, + "loss": 1.4645, + "theoretical_loss": 3.4662692481166566, + "tokens_seen": 1949040640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041350050150451356, + "loss": 1.3282, + "theoretical_loss": 3.466259561266682, + "tokens_seen": 1949106176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134804413239719, + "loss": 1.3414, + "theoretical_loss": 3.466249874833603, + "tokens_seen": 1949171712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041346038114343034, + "loss": 1.3408, + "theoretical_loss": 3.4662401888173875, + "tokens_seen": 1949237248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134403209628887, + "loss": 1.3645, + "theoretical_loss": 3.4662305032180045, + "tokens_seen": 1949302784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041342026078234706, + "loss": 1.4125, + "theoretical_loss": 3.4662208180354215, + "tokens_seen": 1949368320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134002006018054, + "loss": 1.415, + "theoretical_loss": 3.4662111332696064, + "tokens_seen": 1949433856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041338014042126384, + "loss": 1.3519, + "theoretical_loss": 3.4662014489205277, + "tokens_seen": 1949499392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133600802407222, + "loss": 1.3302, + "theoretical_loss": 3.466191764988153, + "tokens_seen": 1949564928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041334002006018056, + "loss": 1.3472, + "theoretical_loss": 3.466182081472451, + "tokens_seen": 1949630464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133199598796389, + "loss": 1.3047, + "theoretical_loss": 3.4661723983733888, + "tokens_seen": 1949696000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132998996990973, + "loss": 1.3471, + "theoretical_loss": 3.4661627156909356, + "tokens_seen": 1949761536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132798395185557, + "loss": 1.2753, + "theoretical_loss": 3.466153033425058, + "tokens_seen": 1949827072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041325977933801407, + "loss": 1.3179, + "theoretical_loss": 3.4661433515757256, + "tokens_seen": 1949892608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041323971915747243, + "loss": 1.3378, + "theoretical_loss": 3.4661336701429057, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132196589769308, + "loss": 1.3095, + "theoretical_loss": 3.4661239891265665, + "tokens_seen": 1950023680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131995987963892, + "loss": 1.3829, + "theoretical_loss": 3.4661143085266763, + "tokens_seen": 1950089216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041317953861584757, + "loss": 1.3426, + "theoretical_loss": 3.466104628343203, + "tokens_seen": 1950154752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041315947843530593, + "loss": 1.4869, + "theoretical_loss": 3.4660949485761146, + "tokens_seen": 1950220288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131394182547643, + "loss": 1.4083, + "theoretical_loss": 3.466085269225379, + "tokens_seen": 1950285824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131193580742227, + "loss": 1.405, + "theoretical_loss": 3.4660755902909646, + "tokens_seen": 1950351360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130992978936811, + "loss": 1.4002, + "theoretical_loss": 3.46606591177284, + "tokens_seen": 1950416896 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041307923771313944, + "loss": 1.3209, + "theoretical_loss": 3.4660562336709724, + "tokens_seen": 1950482432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130591775325978, + "loss": 1.3634, + "theoretical_loss": 3.4660465559853306, + "tokens_seen": 1950547968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041303911735205616, + "loss": 1.3572, + "theoretical_loss": 3.466036878715882, + "tokens_seen": 1950613504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130190571715146, + "loss": 1.4311, + "theoretical_loss": 3.4660272018625955, + "tokens_seen": 1950679040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041299899699097294, + "loss": 1.4459, + "theoretical_loss": 3.466017525425439, + "tokens_seen": 1950744576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129789368104313, + "loss": 1.4468, + "theoretical_loss": 3.4660078494043804, + "tokens_seen": 1950810112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041295887662988966, + "loss": 1.3474, + "theoretical_loss": 3.465998173799388, + "tokens_seen": 1950875648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129388164493481, + "loss": 1.3252, + "theoretical_loss": 3.4659884986104297, + "tokens_seen": 1950941184 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041291875626880644, + "loss": 1.4643, + "theoretical_loss": 3.4659788238374736, + "tokens_seen": 1951006720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128986960882648, + "loss": 1.3873, + "theoretical_loss": 3.4659691494804887, + "tokens_seen": 1951072256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041287863590772317, + "loss": 1.2163, + "theoretical_loss": 3.4659594755394423, + "tokens_seen": 1951137792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041285857572718153, + "loss": 1.3811, + "theoretical_loss": 3.4659498020143023, + "tokens_seen": 1951203328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041283851554663995, + "loss": 1.4208, + "theoretical_loss": 3.4659401289050376, + "tokens_seen": 1951268864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128184553660983, + "loss": 1.4176, + "theoretical_loss": 3.4659304562116158, + "tokens_seen": 1951334400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041279839518555667, + "loss": 1.379, + "theoretical_loss": 3.4659207839340054, + "tokens_seen": 1951399936 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 711575, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4758858680725098, + "objective/train/theoretical_loss": 3.465911112072175, + "objective/train/tokens_used": 321924576, + "theoretical_loss": 3.465911112072175, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041277833500501503, + "loss": 1.3619, + "theoretical_loss": 3.465911112072175, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041275827482447345, + "loss": 1.3404, + "theoretical_loss": 3.4659014406260913, + "tokens_seen": 1951531008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127382146439318, + "loss": 1.4048, + "theoretical_loss": 3.465891769595724, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041271815446339017, + "loss": 1.3414, + "theoretical_loss": 3.465882098981041, + "tokens_seen": 1951662080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041269809428284854, + "loss": 1.4386, + "theoretical_loss": 3.465872428782009, + "tokens_seen": 1951727616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126780341023069, + "loss": 1.308, + "theoretical_loss": 3.465862758998598, + "tokens_seen": 1951793152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126579739217653, + "loss": 1.3508, + "theoretical_loss": 3.4658530896307758, + "tokens_seen": 1951858688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126379137412237, + "loss": 1.2784, + "theoretical_loss": 3.46584342067851, + "tokens_seen": 1951924224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041261785356068204, + "loss": 1.3221, + "theoretical_loss": 3.465833752141769, + "tokens_seen": 1951989760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125977933801404, + "loss": 1.3897, + "theoretical_loss": 3.465824084020521, + "tokens_seen": 1952055296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125777331995988, + "loss": 1.4931, + "theoretical_loss": 3.465814416314734, + "tokens_seen": 1952120832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125576730190572, + "loss": 1.4631, + "theoretical_loss": 3.465804749024377, + "tokens_seen": 1952186368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041253761283851554, + "loss": 1.3334, + "theoretical_loss": 3.465795082149418, + "tokens_seen": 1952251904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125175526579739, + "loss": 1.3649, + "theoretical_loss": 3.4657854156898242, + "tokens_seen": 1952317440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041249749247743227, + "loss": 1.4301, + "theoretical_loss": 3.4657757496455646, + "tokens_seen": 1952382976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124774322968907, + "loss": 1.4002, + "theoretical_loss": 3.4657660840166074, + "tokens_seen": 1952448512 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041245737211634904, + "loss": 1.3878, + "theoretical_loss": 3.4657564188029206, + "tokens_seen": 1952514048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124373119358074, + "loss": 1.3046, + "theoretical_loss": 3.465746754004473, + "tokens_seen": 1952579584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041241725175526577, + "loss": 1.4153, + "theoretical_loss": 3.465737089621232, + "tokens_seen": 1952645120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123971915747242, + "loss": 1.3442, + "theoretical_loss": 3.4657274256531663, + "tokens_seen": 1952710656 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041237713139418255, + "loss": 1.4558, + "theoretical_loss": 3.465717762100244, + "tokens_seen": 1952776192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123570712136409, + "loss": 1.42, + "theoretical_loss": 3.465708098962433, + "tokens_seen": 1952841728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041233701103309927, + "loss": 1.3981, + "theoretical_loss": 3.465698436239702, + "tokens_seen": 1952907264 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041231695085255763, + "loss": 1.3289, + "theoretical_loss": 3.4656887739320195, + "tokens_seen": 1952972800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041229689067201605, + "loss": 1.4061, + "theoretical_loss": 3.465679112039353, + "tokens_seen": 1953038336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122768304914744, + "loss": 1.3814, + "theoretical_loss": 3.4656694505616716, + "tokens_seen": 1953103872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122567703109328, + "loss": 1.4041, + "theoretical_loss": 3.4656597894989427, + "tokens_seen": 1953169408 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041223671013039114, + "loss": 1.2953, + "theoretical_loss": 3.465650128851135, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041221664994984955, + "loss": 1.4445, + "theoretical_loss": 3.4656404686182167, + "tokens_seen": 1953300480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121965897693079, + "loss": 1.3741, + "theoretical_loss": 3.465630808800156, + "tokens_seen": 1953366016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121765295887663, + "loss": 1.446, + "theoretical_loss": 3.4656211493969216, + "tokens_seen": 1953431552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121564694082247, + "loss": 1.409, + "theoretical_loss": 3.465611490408481, + "tokens_seen": 1953497088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121364092276831, + "loss": 1.398, + "theoretical_loss": 3.465601831834803, + "tokens_seen": 1953562624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121163490471415, + "loss": 1.3989, + "theoretical_loss": 3.465592173675856, + "tokens_seen": 1953628160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041209628886659984, + "loss": 1.3836, + "theoretical_loss": 3.465582515931608, + "tokens_seen": 1953693696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120762286860582, + "loss": 1.3167, + "theoretical_loss": 3.4655728586020267, + "tokens_seen": 1953759232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041205616850551656, + "loss": 1.3702, + "theoretical_loss": 3.4655632016870817, + "tokens_seen": 1953824768 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412036108324975, + "loss": 1.2654, + "theoretical_loss": 3.46555354518674, + "tokens_seen": 1953890304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041201604814443334, + "loss": 1.3751, + "theoretical_loss": 3.465543889100971, + "tokens_seen": 1953955840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119959879638917, + "loss": 1.4138, + "theoretical_loss": 3.465534233429742, + "tokens_seen": 1954021376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041197592778335006, + "loss": 1.4064, + "theoretical_loss": 3.4655245781730226, + "tokens_seen": 1954086912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119558676028085, + "loss": 1.3203, + "theoretical_loss": 3.4655149233307796, + "tokens_seen": 1954152448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041193580742226684, + "loss": 1.3949, + "theoretical_loss": 3.465505268902982, + "tokens_seen": 1954217984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119157472417252, + "loss": 1.4469, + "theoretical_loss": 3.4654956148895986, + "tokens_seen": 1954283520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041189568706118357, + "loss": 1.3346, + "theoretical_loss": 3.4654859612905966, + "tokens_seen": 1954349056 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041187562688064193, + "loss": 1.4949, + "theoretical_loss": 3.465476308105945, + "tokens_seen": 1954414592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041185556670010035, + "loss": 1.3463, + "theoretical_loss": 3.4654666553356126, + "tokens_seen": 1954480128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118355065195587, + "loss": 1.3953, + "theoretical_loss": 3.4654570029795666, + "tokens_seen": 1954545664 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041181544633901707, + "loss": 1.4025, + "theoretical_loss": 3.465447351037776, + "tokens_seen": 1954611200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041179538615847543, + "loss": 1.354, + "theoretical_loss": 3.4654376995102094, + "tokens_seen": 1954676736 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 712850, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.313586711883545, + "objective/train/theoretical_loss": 3.4654280483968347, + "objective/train/tokens_used": 325201376, + "theoretical_loss": 3.4654280483968347, + "tokens_seen": 1954742272 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041177532597793385, + "loss": 1.3061, + "theoretical_loss": 3.4654280483968347, + "tokens_seen": 1954742272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117552657973922, + "loss": 1.4814, + "theoretical_loss": 3.4654183976976203, + "tokens_seen": 1954807808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117352056168506, + "loss": 1.5366, + "theoretical_loss": 3.4654087474125346, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041171514543630894, + "loss": 1.3106, + "theoretical_loss": 3.4653990975415456, + "tokens_seen": 1954938880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116950852557673, + "loss": 1.4516, + "theoretical_loss": 3.4653894480846223, + "tokens_seen": 1955004416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116750250752257, + "loss": 1.4531, + "theoretical_loss": 3.4653797990417328, + "tokens_seen": 1955069952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116549648946841, + "loss": 1.4647, + "theoretical_loss": 3.465370150412845, + "tokens_seen": 1955135488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041163490471414244, + "loss": 1.3235, + "theoretical_loss": 3.465360502197928, + "tokens_seen": 1955201024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004116148445336008, + "loss": 1.4262, + "theoretical_loss": 3.4653508543969496, + "tokens_seen": 1955266560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115947843530592, + "loss": 1.3023, + "theoretical_loss": 3.465341207009878, + "tokens_seen": 1955332096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115747241725176, + "loss": 1.4084, + "theoretical_loss": 3.4653315600366827, + "tokens_seen": 1955397632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041155466399197594, + "loss": 1.3525, + "theoretical_loss": 3.465321913477331, + "tokens_seen": 1955463168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115346038114343, + "loss": 1.3024, + "theoretical_loss": 3.4653122673317913, + "tokens_seen": 1955528704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041151454363089267, + "loss": 1.3955, + "theoretical_loss": 3.4653026216000327, + "tokens_seen": 1955594240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114944834503511, + "loss": 1.3814, + "theoretical_loss": 3.465292976282023, + "tokens_seen": 1955659776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041147442326980944, + "loss": 1.4225, + "theoretical_loss": 3.465283331377731, + "tokens_seen": 1955725312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114543630892678, + "loss": 1.3423, + "theoretical_loss": 3.465273686887124, + "tokens_seen": 1955790848 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041143430290872617, + "loss": 1.3667, + "theoretical_loss": 3.4652640428101718, + "tokens_seen": 1955856384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114142427281846, + "loss": 1.3452, + "theoretical_loss": 3.4652543991468425, + "tokens_seen": 1955921920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041139418254764295, + "loss": 1.3615, + "theoretical_loss": 3.4652447558971042, + "tokens_seen": 1955987456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113741223671013, + "loss": 1.3197, + "theoretical_loss": 3.465235113060925, + "tokens_seen": 1956052992 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041135406218655967, + "loss": 1.3394, + "theoretical_loss": 3.4652254706382735, + "tokens_seen": 1956118528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041133400200601803, + "loss": 1.3216, + "theoretical_loss": 3.4652158286291184, + "tokens_seen": 1956184064 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041131394182547645, + "loss": 1.2706, + "theoretical_loss": 3.465206187033428, + "tokens_seen": 1956249600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112938816449348, + "loss": 1.3731, + "theoretical_loss": 3.465196545851171, + "tokens_seen": 1956315136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112738214643932, + "loss": 1.3464, + "theoretical_loss": 3.465186905082315, + "tokens_seen": 1956380672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125376128385154, + "loss": 1.4162, + "theoretical_loss": 3.4651772647268295, + "tokens_seen": 1956446208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041123370110330995, + "loss": 1.2998, + "theoretical_loss": 3.465167624784682, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112136409227683, + "loss": 1.3612, + "theoretical_loss": 3.4651579852558414, + "tokens_seen": 1956577280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111935807422267, + "loss": 1.4036, + "theoretical_loss": 3.465148346140276, + "tokens_seen": 1956642816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041117352056168504, + "loss": 1.3542, + "theoretical_loss": 3.4651387074379545, + "tokens_seen": 1956708352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111534603811434, + "loss": 1.3389, + "theoretical_loss": 3.465129069148845, + "tokens_seen": 1956773888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111334002006018, + "loss": 1.3757, + "theoretical_loss": 3.4651194312729157, + "tokens_seen": 1956839424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111133400200602, + "loss": 1.3944, + "theoretical_loss": 3.465109793810136, + "tokens_seen": 1956904960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041109327983951854, + "loss": 1.4407, + "theoretical_loss": 3.4651001567604736, + "tokens_seen": 1956970496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110732196589769, + "loss": 1.4016, + "theoretical_loss": 3.465090520123897, + "tokens_seen": 1957036032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110531594784353, + "loss": 1.3521, + "theoretical_loss": 3.465080883900375, + "tokens_seen": 1957101568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110330992978937, + "loss": 1.4703, + "theoretical_loss": 3.4650712480898758, + "tokens_seen": 1957167104 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041101303911735205, + "loss": 1.332, + "theoretical_loss": 3.4650616126923675, + "tokens_seen": 1957232640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109929789368104, + "loss": 1.4206, + "theoretical_loss": 3.4650519777078195, + "tokens_seen": 1957298176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041097291875626877, + "loss": 1.408, + "theoretical_loss": 3.4650423431362, + "tokens_seen": 1957363712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109528585757272, + "loss": 1.4464, + "theoretical_loss": 3.4650327089774766, + "tokens_seen": 1957429248 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041093279839518555, + "loss": 1.4103, + "theoretical_loss": 3.4650230752316187, + "tokens_seen": 1957494784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109127382146439, + "loss": 1.3697, + "theoretical_loss": 3.465013441898595, + "tokens_seen": 1957560320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108926780341023, + "loss": 1.3378, + "theoretical_loss": 3.4650038089783726, + "tokens_seen": 1957625856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108726178535607, + "loss": 1.365, + "theoretical_loss": 3.4649941764709213, + "tokens_seen": 1957691392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041085255767301905, + "loss": 1.3233, + "theoretical_loss": 3.4649845443762093, + "tokens_seen": 1957756928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108324974924774, + "loss": 1.3419, + "theoretical_loss": 3.4649749126942053, + "tokens_seen": 1957822464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108124373119358, + "loss": 1.4326, + "theoretical_loss": 3.4649652814248775, + "tokens_seen": 1957888000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107923771313942, + "loss": 1.3894, + "theoretical_loss": 3.4649556505681938, + "tokens_seen": 1957953536 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 713999, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5213258266448975, + "objective/train/theoretical_loss": 3.4649460201241236, + "objective/train/tokens_used": 328478176, + "theoretical_loss": 3.4649460201241236, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041077231695085256, + "loss": 1.4182, + "theoretical_loss": 3.4649460201241236, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107522567703109, + "loss": 1.3096, + "theoretical_loss": 3.4649363900926353, + "tokens_seen": 1958084608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107321965897693, + "loss": 1.4114, + "theoretical_loss": 3.4649267604736966, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041071213640922764, + "loss": 1.4125, + "theoretical_loss": 3.4649171312672777, + "tokens_seen": 1958215680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041069207622868606, + "loss": 1.4225, + "theoretical_loss": 3.4649075024733453, + "tokens_seen": 1958281216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106720160481444, + "loss": 1.4486, + "theoretical_loss": 3.464897874091869, + "tokens_seen": 1958346752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041065195586760284, + "loss": 1.4043, + "theoretical_loss": 3.4648882461228174, + "tokens_seen": 1958412288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106318956870612, + "loss": 1.3946, + "theoretical_loss": 3.464878618566158, + "tokens_seen": 1958477824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106118355065196, + "loss": 1.3537, + "theoretical_loss": 3.4648689914218602, + "tokens_seen": 1958543360 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410591775325978, + "loss": 1.4046, + "theoretical_loss": 3.4648593646898926, + "tokens_seen": 1958608896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041057171514543634, + "loss": 1.253, + "theoretical_loss": 3.4648497383702237, + "tokens_seen": 1958674432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105516549648947, + "loss": 1.4268, + "theoretical_loss": 3.464840112462821, + "tokens_seen": 1958739968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041053159478435307, + "loss": 1.4509, + "theoretical_loss": 3.4648304869676547, + "tokens_seen": 1958805504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105115346038115, + "loss": 1.4759, + "theoretical_loss": 3.464820861884692, + "tokens_seen": 1958871040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041049147442326985, + "loss": 1.4996, + "theoretical_loss": 3.464811237213903, + "tokens_seen": 1958936576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104714142427282, + "loss": 1.3003, + "theoretical_loss": 3.464801612955254, + "tokens_seen": 1959002112 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041045135406218657, + "loss": 1.4419, + "theoretical_loss": 3.4647919891087158, + "tokens_seen": 1959067648 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410431293881645, + "loss": 1.4572, + "theoretical_loss": 3.4647823656742553, + "tokens_seen": 1959133184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041041123370110335, + "loss": 1.3774, + "theoretical_loss": 3.464772742651842, + "tokens_seen": 1959198720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103911735205617, + "loss": 1.386, + "theoretical_loss": 3.464763120041445, + "tokens_seen": 1959264256 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041037111334002007, + "loss": 1.4774, + "theoretical_loss": 3.4647534978430308, + "tokens_seen": 1959329792 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041035105315947844, + "loss": 1.4291, + "theoretical_loss": 3.4647438760565703, + "tokens_seen": 1959395328 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041033099297893685, + "loss": 1.3884, + "theoretical_loss": 3.464734254682031, + "tokens_seen": 1959460864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103109327983952, + "loss": 1.2816, + "theoretical_loss": 3.4647246337193813, + "tokens_seen": 1959526400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102908726178536, + "loss": 1.4086, + "theoretical_loss": 3.4647150131685898, + "tokens_seen": 1959591936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041027081243731194, + "loss": 1.4011, + "theoretical_loss": 3.464705393029626, + "tokens_seen": 1959657472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041025075225677035, + "loss": 1.2854, + "theoretical_loss": 3.4646957733024575, + "tokens_seen": 1959723008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102306920762287, + "loss": 1.419, + "theoretical_loss": 3.4646861539870537, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102106318956871, + "loss": 1.4778, + "theoretical_loss": 3.4646765350833824, + "tokens_seen": 1959854080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041019057171514544, + "loss": 1.3693, + "theoretical_loss": 3.464666916591413, + "tokens_seen": 1959919616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101705115346038, + "loss": 1.351, + "theoretical_loss": 3.4646572985111126, + "tokens_seen": 1959985152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101504513540622, + "loss": 1.2651, + "theoretical_loss": 3.464647680842452, + "tokens_seen": 1960050688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101303911735206, + "loss": 1.3727, + "theoretical_loss": 3.464638063585398, + "tokens_seen": 1960116224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041011033099297894, + "loss": 1.3827, + "theoretical_loss": 3.464628446739921, + "tokens_seen": 1960181760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100902708124373, + "loss": 1.2286, + "theoretical_loss": 3.464618830305988, + "tokens_seen": 1960247296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100702106318957, + "loss": 1.3899, + "theoretical_loss": 3.464609214283568, + "tokens_seen": 1960312832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100501504513541, + "loss": 1.3641, + "theoretical_loss": 3.4645995986726303, + "tokens_seen": 1960378368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041003009027081245, + "loss": 1.3661, + "theoretical_loss": 3.464589983473143, + "tokens_seen": 1960443904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100100300902708, + "loss": 1.417, + "theoretical_loss": 3.4645803686850747, + "tokens_seen": 1960509440 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040998996990972917, + "loss": 1.4208, + "theoretical_loss": 3.464570754308394, + "tokens_seen": 1960574976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099699097291876, + "loss": 1.3454, + "theoretical_loss": 3.4645611403430703, + "tokens_seen": 1960640512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040994984954864595, + "loss": 1.3053, + "theoretical_loss": 3.464551526789071, + "tokens_seen": 1960706048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099297893681043, + "loss": 1.4207, + "theoretical_loss": 3.464541913646366, + "tokens_seen": 1960771584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099097291875627, + "loss": 1.4109, + "theoretical_loss": 3.4645323009149234, + "tokens_seen": 1960837120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098896690070211, + "loss": 1.2338, + "theoretical_loss": 3.4645226885947116, + "tokens_seen": 1960902656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040986960882647945, + "loss": 1.3504, + "theoretical_loss": 3.4645130766856993, + "tokens_seen": 1960968192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098495486459378, + "loss": 1.5134, + "theoretical_loss": 3.464503465187856, + "tokens_seen": 1961033728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098294884653962, + "loss": 1.3625, + "theoretical_loss": 3.464493854101149, + "tokens_seen": 1961099264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098094282848546, + "loss": 1.4069, + "theoretical_loss": 3.464484243425548, + "tokens_seen": 1961164800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040978936810431296, + "loss": 1.3649, + "theoretical_loss": 3.4644746331610214, + "tokens_seen": 1961230336 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 715186, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2633056640625, + "objective/train/theoretical_loss": 3.464465023307538, + "objective/train/tokens_used": 331754976, + "theoretical_loss": 3.464465023307538, + "tokens_seen": 1961295872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097693079237713, + "loss": 1.3257, + "theoretical_loss": 3.464465023307538, + "tokens_seen": 1961295872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097492477432297, + "loss": 1.395, + "theoretical_loss": 3.4644554138650667, + "tokens_seen": 1961361408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040972918756268804, + "loss": 1.3748, + "theoretical_loss": 3.4644458048335753, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040970912738214646, + "loss": 1.4036, + "theoretical_loss": 3.4644361962130334, + "tokens_seen": 1961492480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096890672016048, + "loss": 1.2868, + "theoretical_loss": 3.464426588003409, + "tokens_seen": 1961558016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096690070210632, + "loss": 1.3478, + "theoretical_loss": 3.4644169802046716, + "tokens_seen": 1961623552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040964894684052155, + "loss": 1.3996, + "theoretical_loss": 3.4644073728167895, + "tokens_seen": 1961689088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040962888665997996, + "loss": 1.2375, + "theoretical_loss": 3.4643977658397307, + "tokens_seen": 1961754624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096088264794383, + "loss": 1.4601, + "theoretical_loss": 3.464388159273465, + "tokens_seen": 1961820160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095887662988967, + "loss": 1.367, + "theoretical_loss": 3.4643785531179607, + "tokens_seen": 1961885696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040956870611835505, + "loss": 1.4649, + "theoretical_loss": 3.4643689473731865, + "tokens_seen": 1961951232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095486459378134, + "loss": 1.3531, + "theoretical_loss": 3.464359342039111, + "tokens_seen": 1962016768 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040952858575727183, + "loss": 1.4511, + "theoretical_loss": 3.464349737115703, + "tokens_seen": 1962082304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095085255767302, + "loss": 1.3885, + "theoretical_loss": 3.4643401326029313, + "tokens_seen": 1962147840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040948846539618855, + "loss": 1.438, + "theoretical_loss": 3.464330528500765, + "tokens_seen": 1962213376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094684052156469, + "loss": 1.3779, + "theoretical_loss": 3.464320924809172, + "tokens_seen": 1962278912 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040944834503510533, + "loss": 1.3406, + "theoretical_loss": 3.464311321528122, + "tokens_seen": 1962344448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094282848545637, + "loss": 1.3559, + "theoretical_loss": 3.464301718657582, + "tokens_seen": 1962409984 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040940822467402206, + "loss": 1.3227, + "theoretical_loss": 3.464292116197523, + "tokens_seen": 1962475520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093881644934804, + "loss": 1.4132, + "theoretical_loss": 3.4642825141479126, + "tokens_seen": 1962541056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093681043129388, + "loss": 1.3032, + "theoretical_loss": 3.4642729125087195, + "tokens_seen": 1962606592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093480441323972, + "loss": 1.2996, + "theoretical_loss": 3.464263311279912, + "tokens_seen": 1962672128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040932798395185556, + "loss": 1.2976, + "theoretical_loss": 3.4642537104614606, + "tokens_seen": 1962737664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093079237713139, + "loss": 1.3485, + "theoretical_loss": 3.464244110053332, + "tokens_seen": 1962803200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092878635907723, + "loss": 1.3694, + "theoretical_loss": 3.4642345100554963, + "tokens_seen": 1962868736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092678034102307, + "loss": 1.3855, + "theoretical_loss": 3.4642249104679217, + "tokens_seen": 1962934272 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040924774322968906, + "loss": 1.3285, + "theoretical_loss": 3.464215311290577, + "tokens_seen": 1962999808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092276830491474, + "loss": 1.3954, + "theoretical_loss": 3.4642057125234316, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092076228686058, + "loss": 1.4268, + "theoretical_loss": 3.4641961141664535, + "tokens_seen": 1963130880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040918756268806415, + "loss": 1.3513, + "theoretical_loss": 3.4641865162196117, + "tokens_seen": 1963196416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040916750250752257, + "loss": 1.4266, + "theoretical_loss": 3.464176918682875, + "tokens_seen": 1963261952 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409147442326981, + "loss": 1.3756, + "theoretical_loss": 3.464167321556212, + "tokens_seen": 1963327488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040912738214643934, + "loss": 1.4381, + "theoretical_loss": 3.464157724839592, + "tokens_seen": 1963393024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091073219658977, + "loss": 1.319, + "theoretical_loss": 3.4641481285329836, + "tokens_seen": 1963458560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090872617853561, + "loss": 1.4492, + "theoretical_loss": 3.4641385326363556, + "tokens_seen": 1963524096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090672016048145, + "loss": 1.3999, + "theoretical_loss": 3.4641289371496766, + "tokens_seen": 1963589632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040904714142427285, + "loss": 1.4133, + "theoretical_loss": 3.464119342072915, + "tokens_seen": 1963655168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090270812437312, + "loss": 1.3892, + "theoretical_loss": 3.4641097474060407, + "tokens_seen": 1963720704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040900702106318957, + "loss": 1.2092, + "theoretical_loss": 3.4641001531490216, + "tokens_seen": 1963786240 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408986960882648, + "loss": 1.4021, + "theoretical_loss": 3.464090559301827, + "tokens_seen": 1963851776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040896690070210635, + "loss": 1.2607, + "theoretical_loss": 3.464080965864426, + "tokens_seen": 1963917312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089468405215647, + "loss": 1.3859, + "theoretical_loss": 3.4640713728367865, + "tokens_seen": 1963982848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089267803410231, + "loss": 1.4089, + "theoretical_loss": 3.464061780218878, + "tokens_seen": 1964048384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089067201604815, + "loss": 1.3719, + "theoretical_loss": 3.464052188010669, + "tokens_seen": 1964113920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040888665997993985, + "loss": 1.4005, + "theoretical_loss": 3.4640425962121286, + "tokens_seen": 1964179456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088665997993982, + "loss": 1.3828, + "theoretical_loss": 3.464033004823225, + "tokens_seen": 1964244992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088465396188566, + "loss": 1.3791, + "theoretical_loss": 3.464023413843928, + "tokens_seen": 1964310528 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408826479438315, + "loss": 1.4303, + "theoretical_loss": 3.4640138232742066, + "tokens_seen": 1964376064 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040880641925777336, + "loss": 1.349, + "theoretical_loss": 3.464004233114028, + "tokens_seen": 1964441600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087863590772317, + "loss": 1.419, + "theoretical_loss": 3.4639946433633626, + "tokens_seen": 1964507136 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 716326, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.427863359451294, + "objective/train/theoretical_loss": 3.463985054022179, + "objective/train/tokens_used": 335031776, + "theoretical_loss": 3.463985054022179, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087662988966901, + "loss": 1.3491, + "theoretical_loss": 3.463985054022179, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040874623871614844, + "loss": 1.3889, + "theoretical_loss": 3.4639754650904453, + "tokens_seen": 1964638208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040872617853560686, + "loss": 1.3494, + "theoretical_loss": 3.463965876568131, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087061183550652, + "loss": 1.3962, + "theoretical_loss": 3.463956288455205, + "tokens_seen": 1964769280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086860581745236, + "loss": 1.3086, + "theoretical_loss": 3.463946700751636, + "tokens_seen": 1964834816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040866599799398195, + "loss": 1.3982, + "theoretical_loss": 3.4639371134573924, + "tokens_seen": 1964900352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040864593781344036, + "loss": 1.3411, + "theoretical_loss": 3.463927526572444, + "tokens_seen": 1964965888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086258776328987, + "loss": 1.4402, + "theoretical_loss": 3.463917940096759, + "tokens_seen": 1965031424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086058174523571, + "loss": 1.439, + "theoretical_loss": 3.4639083540303064, + "tokens_seen": 1965096960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040858575727181545, + "loss": 1.3006, + "theoretical_loss": 3.4638987683730553, + "tokens_seen": 1965162496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085656970912738, + "loss": 1.4645, + "theoretical_loss": 3.4638891831249747, + "tokens_seen": 1965228032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040854563691073223, + "loss": 1.3335, + "theoretical_loss": 3.463879598286033, + "tokens_seen": 1965293568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085255767301906, + "loss": 1.3908, + "theoretical_loss": 3.4638700138561993, + "tokens_seen": 1965359104 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040850551654964895, + "loss": 1.4118, + "theoretical_loss": 3.4638604298354427, + "tokens_seen": 1965424640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084854563691073, + "loss": 1.4107, + "theoretical_loss": 3.463850846223732, + "tokens_seen": 1965490176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040846539618856573, + "loss": 1.32, + "theoretical_loss": 3.4638412630210356, + "tokens_seen": 1965555712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084453360080241, + "loss": 1.2415, + "theoretical_loss": 3.4638316802273232, + "tokens_seen": 1965621248 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040842527582748246, + "loss": 1.3294, + "theoretical_loss": 3.463822097842564, + "tokens_seen": 1965686784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084052156469408, + "loss": 1.2951, + "theoretical_loss": 3.4638125158667252, + "tokens_seen": 1965752320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083851554663992, + "loss": 1.3545, + "theoretical_loss": 3.4638029342997774, + "tokens_seen": 1965817856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083650952858576, + "loss": 1.4138, + "theoretical_loss": 3.4637933531416882, + "tokens_seen": 1965883392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040834503510531596, + "loss": 1.3182, + "theoretical_loss": 3.4637837723924276, + "tokens_seen": 1965948928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083249749247743, + "loss": 1.4394, + "theoretical_loss": 3.4637741920519645, + "tokens_seen": 1966014464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083049147442327, + "loss": 1.3229, + "theoretical_loss": 3.4637646121202676, + "tokens_seen": 1966080000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082848545636911, + "loss": 1.2749, + "theoretical_loss": 3.463755032597305, + "tokens_seen": 1966145536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040826479438314946, + "loss": 1.3454, + "theoretical_loss": 3.463745453483047, + "tokens_seen": 1966211072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082447342026078, + "loss": 1.4628, + "theoretical_loss": 3.463735874777462, + "tokens_seen": 1966276608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082246740220662, + "loss": 1.3792, + "theoretical_loss": 3.4637262964805187, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040820461384152455, + "loss": 1.3749, + "theoretical_loss": 3.4637167185921856, + "tokens_seen": 1966407680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040818455366098297, + "loss": 1.4007, + "theoretical_loss": 3.463707141112433, + "tokens_seen": 1966473216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040816449348044133, + "loss": 1.3654, + "theoretical_loss": 3.4636975640412286, + "tokens_seen": 1966538752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081444332998997, + "loss": 1.2707, + "theoretical_loss": 3.4636879873785427, + "tokens_seen": 1966604288 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040812437311935805, + "loss": 1.3707, + "theoretical_loss": 3.4636784111243424, + "tokens_seen": 1966669824 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040810431293881647, + "loss": 1.343, + "theoretical_loss": 3.4636688352785985, + "tokens_seen": 1966735360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040808425275827483, + "loss": 1.3446, + "theoretical_loss": 3.463659259841279, + "tokens_seen": 1966800896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080641925777332, + "loss": 1.4308, + "theoretical_loss": 3.463649684812353, + "tokens_seen": 1966866432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804413239719156, + "loss": 1.3536, + "theoretical_loss": 3.463640110191789, + "tokens_seen": 1966931968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080240722166499, + "loss": 1.4149, + "theoretical_loss": 3.463630535979557, + "tokens_seen": 1966997504 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040800401203610834, + "loss": 1.3658, + "theoretical_loss": 3.463620962175625, + "tokens_seen": 1967063040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079839518555667, + "loss": 1.3509, + "theoretical_loss": 3.4636113887799627, + "tokens_seen": 1967128576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040796389167502506, + "loss": 1.4298, + "theoretical_loss": 3.463601815792539, + "tokens_seen": 1967194112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079438314944834, + "loss": 1.3599, + "theoretical_loss": 3.4635922432133226, + "tokens_seen": 1967259648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040792377131394184, + "loss": 1.3437, + "theoretical_loss": 3.463582671042283, + "tokens_seen": 1967325184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079037111334002, + "loss": 1.3339, + "theoretical_loss": 3.4635730992793885, + "tokens_seen": 1967390720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040788365095285856, + "loss": 1.2887, + "theoretical_loss": 3.463563527924608, + "tokens_seen": 1967456256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078635907723169, + "loss": 1.4638, + "theoretical_loss": 3.4635539569779117, + "tokens_seen": 1967521792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078435305917753, + "loss": 1.4974, + "theoretical_loss": 3.463544386439268, + "tokens_seen": 1967587328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078234704112337, + "loss": 1.3594, + "theoretical_loss": 3.463534816308645, + "tokens_seen": 1967652864 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040780341023069207, + "loss": 1.4089, + "theoretical_loss": 3.4635252465860127, + "tokens_seen": 1967718400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040778335005015043, + "loss": 1.3437, + "theoretical_loss": 3.46351567727134, + "tokens_seen": 1967783936 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 717661, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4225447177886963, + "objective/train/theoretical_loss": 3.4635061083645957, + "objective/train/tokens_used": 338308576, + "theoretical_loss": 3.4635061083645957, + "tokens_seen": 1967849472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077632898696088, + "loss": 1.4754, + "theoretical_loss": 3.4635061083645957, + "tokens_seen": 1967849472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077432296890672, + "loss": 1.3661, + "theoretical_loss": 3.463496539865749, + "tokens_seen": 1967915008 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040772316950852557, + "loss": 1.291, + "theoretical_loss": 3.463486971774769, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040770310932798393, + "loss": 1.3921, + "theoretical_loss": 3.4634774040916243, + "tokens_seen": 1968046080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076830491474423, + "loss": 1.4786, + "theoretical_loss": 3.463467836816285, + "tokens_seen": 1968111616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076629889669007, + "loss": 1.4207, + "theoretical_loss": 3.4634582699487186, + "tokens_seen": 1968177152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076429287863591, + "loss": 1.2721, + "theoretical_loss": 3.463448703488895, + "tokens_seen": 1968242688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076228686058175, + "loss": 1.363, + "theoretical_loss": 3.4634391374367834, + "tokens_seen": 1968308224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040760280842527585, + "loss": 1.3684, + "theoretical_loss": 3.4634295717923527, + "tokens_seen": 1968373760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075827482447342, + "loss": 1.401, + "theoretical_loss": 3.463420006555572, + "tokens_seen": 1968439296 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040756268806419263, + "loss": 1.3656, + "theoretical_loss": 3.4634104417264098, + "tokens_seen": 1968504832 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407542627883651, + "loss": 1.3786, + "theoretical_loss": 3.463400877304836, + "tokens_seen": 1968570368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040752256770310935, + "loss": 1.3912, + "theoretical_loss": 3.4633913132908187, + "tokens_seen": 1968635904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075025075225677, + "loss": 1.3916, + "theoretical_loss": 3.463381749684328, + "tokens_seen": 1968701440 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040748244734202613, + "loss": 1.295, + "theoretical_loss": 3.463372186485332, + "tokens_seen": 1968766976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074623871614845, + "loss": 1.407, + "theoretical_loss": 3.4633626236938007, + "tokens_seen": 1968832512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040744232698094286, + "loss": 1.2804, + "theoretical_loss": 3.4633530613097028, + "tokens_seen": 1968898048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074222668004012, + "loss": 1.3455, + "theoretical_loss": 3.463343499333007, + "tokens_seen": 1968963584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074022066198596, + "loss": 1.2737, + "theoretical_loss": 3.463333937763683, + "tokens_seen": 1969029120 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407382146439318, + "loss": 1.3666, + "theoretical_loss": 3.4633243766016992, + "tokens_seen": 1969094656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040736208625877636, + "loss": 1.3628, + "theoretical_loss": 3.463314815847026, + "tokens_seen": 1969160192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073420260782347, + "loss": 1.3972, + "theoretical_loss": 3.4633052554996304, + "tokens_seen": 1969225728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073219658976931, + "loss": 1.4204, + "theoretical_loss": 3.4632956955594834, + "tokens_seen": 1969291264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073019057171515, + "loss": 1.3792, + "theoretical_loss": 3.4632861360265528, + "tokens_seen": 1969356800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040728184553660986, + "loss": 1.3825, + "theoretical_loss": 3.463276576900809, + "tokens_seen": 1969422336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072617853560682, + "loss": 1.5021, + "theoretical_loss": 3.4632670181822203, + "tokens_seen": 1969487872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072417251755266, + "loss": 1.3361, + "theoretical_loss": 3.4632574598707553, + "tokens_seen": 1969553408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040722166499498495, + "loss": 1.3903, + "theoretical_loss": 3.463247901966384, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040720160481444337, + "loss": 1.3482, + "theoretical_loss": 3.463238344469075, + "tokens_seen": 1969684480 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040718154463390173, + "loss": 1.3917, + "theoretical_loss": 3.463228787378798, + "tokens_seen": 1969750016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071614844533601, + "loss": 1.3821, + "theoretical_loss": 3.463219230695522, + "tokens_seen": 1969815552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040714142427281845, + "loss": 1.2996, + "theoretical_loss": 3.4632096744192156, + "tokens_seen": 1969881088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040712136409227687, + "loss": 1.3301, + "theoretical_loss": 3.4632001185498478, + "tokens_seen": 1969946624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040710130391173523, + "loss": 1.3884, + "theoretical_loss": 3.4631905630873883, + "tokens_seen": 1970012160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070812437311936, + "loss": 1.4168, + "theoretical_loss": 3.4631810080318064, + "tokens_seen": 1970077696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040706118355065196, + "loss": 1.4092, + "theoretical_loss": 3.4631714533830706, + "tokens_seen": 1970143232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070411233701103, + "loss": 1.2583, + "theoretical_loss": 3.4631618991411512, + "tokens_seen": 1970208768 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040702106318956874, + "loss": 1.3538, + "theoretical_loss": 3.463152345306016, + "tokens_seen": 1970274304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070010030090271, + "loss": 1.3519, + "theoretical_loss": 3.463142791877634, + "tokens_seen": 1970339840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040698094282848546, + "loss": 1.2357, + "theoretical_loss": 3.463133238855976, + "tokens_seen": 1970405376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069608826479438, + "loss": 1.3443, + "theoretical_loss": 3.46312368624101, + "tokens_seen": 1970470912 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040694082246740224, + "loss": 1.361, + "theoretical_loss": 3.463114134032705, + "tokens_seen": 1970536448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069207622868606, + "loss": 1.3766, + "theoretical_loss": 3.463104582231031, + "tokens_seen": 1970601984 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040690070210631896, + "loss": 1.3534, + "theoretical_loss": 3.4630950308359565, + "tokens_seen": 1970667520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068806419257773, + "loss": 1.2618, + "theoretical_loss": 3.4630854798474506, + "tokens_seen": 1970733056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068605817452357, + "loss": 1.4153, + "theoretical_loss": 3.463075929265483, + "tokens_seen": 1970798592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068405215646941, + "loss": 1.2814, + "theoretical_loss": 3.4630663790900225, + "tokens_seen": 1970864128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040682046138415247, + "loss": 1.4098, + "theoretical_loss": 3.4630568293210384, + "tokens_seen": 1970929664 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040680040120361083, + "loss": 1.3919, + "theoretical_loss": 3.4630472799585, + "tokens_seen": 1970995200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067803410230692, + "loss": 1.3337, + "theoretical_loss": 3.4630377310023763, + "tokens_seen": 1971060736 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 718798, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.515406608581543, + "objective/train/theoretical_loss": 3.4630281824526365, + "objective/train/tokens_used": 341585376, + "theoretical_loss": 3.4630281824526365, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067602808425276, + "loss": 1.4503, + "theoretical_loss": 3.4630281824526365, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040674022066198597, + "loss": 1.5234, + "theoretical_loss": 3.4630186343092495, + "tokens_seen": 1971191808 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040672016048144433, + "loss": 1.3731, + "theoretical_loss": 3.4630090865721854, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067001003009027, + "loss": 1.3886, + "theoretical_loss": 3.462999539241413, + "tokens_seen": 1971322880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066800401203611, + "loss": 1.4353, + "theoretical_loss": 3.462989992316901, + "tokens_seen": 1971388416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040665997993981947, + "loss": 1.3108, + "theoretical_loss": 3.4629804457986184, + "tokens_seen": 1971453952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040663991975927783, + "loss": 1.3695, + "theoretical_loss": 3.462970899686536, + "tokens_seen": 1971519488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066198595787362, + "loss": 1.4911, + "theoretical_loss": 3.462961353980621, + "tokens_seen": 1971585024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040659979939819456, + "loss": 1.3678, + "theoretical_loss": 3.4629518086808444, + "tokens_seen": 1971650560 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406579739217653, + "loss": 1.2559, + "theoretical_loss": 3.4629422637871743, + "tokens_seen": 1971716096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040655967903711134, + "loss": 1.3756, + "theoretical_loss": 3.4629327192995802, + "tokens_seen": 1971781632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065396188565697, + "loss": 1.3373, + "theoretical_loss": 3.4629231752180316, + "tokens_seen": 1971847168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040651955867602806, + "loss": 1.3767, + "theoretical_loss": 3.4629136315424973, + "tokens_seen": 1971912704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064994984954865, + "loss": 1.3396, + "theoretical_loss": 3.4629040882729467, + "tokens_seen": 1971978240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040647943831494484, + "loss": 1.4023, + "theoretical_loss": 3.462894545409349, + "tokens_seen": 1972043776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064593781344032, + "loss": 1.3312, + "theoretical_loss": 3.462885002951674, + "tokens_seen": 1972109312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040643931795386157, + "loss": 1.3648, + "theoretical_loss": 3.46287546089989, + "tokens_seen": 1972174848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040641925777331993, + "loss": 1.3527, + "theoretical_loss": 3.462865919253967, + "tokens_seen": 1972240384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040639919759277834, + "loss": 1.3348, + "theoretical_loss": 3.462856378013874, + "tokens_seen": 1972305920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063791374122367, + "loss": 1.3202, + "theoretical_loss": 3.4628468371795798, + "tokens_seen": 1972371456 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040635907723169507, + "loss": 1.4713, + "theoretical_loss": 3.4628372967510543, + "tokens_seen": 1972436992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040633901705115343, + "loss": 1.381, + "theoretical_loss": 3.4628277567282666, + "tokens_seen": 1972502528 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040631895687061185, + "loss": 1.4462, + "theoretical_loss": 3.4628182171111863, + "tokens_seen": 1972568064 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062988966900702, + "loss": 1.3506, + "theoretical_loss": 3.4628086778997815, + "tokens_seen": 1972633600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040627883650952857, + "loss": 1.2991, + "theoretical_loss": 3.462799139094023, + "tokens_seen": 1972699136 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040625877632898693, + "loss": 1.454, + "theoretical_loss": 3.462789600693879, + "tokens_seen": 1972764672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062387161484453, + "loss": 1.3945, + "theoretical_loss": 3.462780062699319, + "tokens_seen": 1972830208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062186559679037, + "loss": 1.3899, + "theoretical_loss": 3.4627705251103125, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061985957873621, + "loss": 1.5648, + "theoretical_loss": 3.4627609879268286, + "tokens_seen": 1972961280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040617853560682044, + "loss": 1.2879, + "theoretical_loss": 3.4627514511488364, + "tokens_seen": 1973026816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061584754262788, + "loss": 1.4236, + "theoretical_loss": 3.4627419147763057, + "tokens_seen": 1973092352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040613841524573727, + "loss": 1.3979, + "theoretical_loss": 3.462732378809206, + "tokens_seen": 1973157888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040611835506519563, + "loss": 1.4721, + "theoretical_loss": 3.4627228432475055, + "tokens_seen": 1973223424 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406098294884654, + "loss": 1.4256, + "theoretical_loss": 3.4627133080911743, + "tokens_seen": 1973288960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040607823470411236, + "loss": 1.4593, + "theoretical_loss": 3.4627037733401815, + "tokens_seen": 1973354496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060581745235707, + "loss": 1.2972, + "theoretical_loss": 3.462694238994497, + "tokens_seen": 1973420032 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040603811434302914, + "loss": 1.3536, + "theoretical_loss": 3.462684705054089, + "tokens_seen": 1973485568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060180541624875, + "loss": 1.4176, + "theoretical_loss": 3.4626751715189275, + "tokens_seen": 1973551104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040599799398194586, + "loss": 1.3094, + "theoretical_loss": 3.4626656383889816, + "tokens_seen": 1973616640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059779338014042, + "loss": 1.3243, + "theoretical_loss": 3.462656105664221, + "tokens_seen": 1973682176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040595787362086264, + "loss": 1.3508, + "theoretical_loss": 3.4626465733446143, + "tokens_seen": 1973747712 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405937813440321, + "loss": 1.4177, + "theoretical_loss": 3.4626370414301317, + "tokens_seen": 1973813248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040591775325977936, + "loss": 1.2631, + "theoretical_loss": 3.462627509920742, + "tokens_seen": 1973878784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058976930792377, + "loss": 1.4504, + "theoretical_loss": 3.4626179788164144, + "tokens_seen": 1973944320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058776328986961, + "loss": 1.4411, + "theoretical_loss": 3.462608448117119, + "tokens_seen": 1974009856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058575727181545, + "loss": 1.3834, + "theoretical_loss": 3.4625989178228243, + "tokens_seen": 1974075392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040583751253761287, + "loss": 1.4627, + "theoretical_loss": 3.4625893879335, + "tokens_seen": 1974140928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040581745235707123, + "loss": 1.3651, + "theoretical_loss": 3.462579858449115, + "tokens_seen": 1974206464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057973921765296, + "loss": 1.3855, + "theoretical_loss": 3.4625703293696395, + "tokens_seen": 1974272000 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405777331995988, + "loss": 1.3035, + "theoretical_loss": 3.4625608006950426, + "tokens_seen": 1974337536 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 720061, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3045427799224854, + "objective/train/theoretical_loss": 3.4625512724252934, + "objective/train/tokens_used": 344862176, + "theoretical_loss": 3.4625512724252934, + "tokens_seen": 1974403072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040575727181544637, + "loss": 1.2807, + "theoretical_loss": 3.4625512724252934, + "tokens_seen": 1974403072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040573721163490473, + "loss": 1.458, + "theoretical_loss": 3.4625417445603617, + "tokens_seen": 1974468608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057171514543631, + "loss": 1.3762, + "theoretical_loss": 3.4625322171002155, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040569709127382146, + "loss": 1.4555, + "theoretical_loss": 3.462522690044826, + "tokens_seen": 1974599680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056770310932799, + "loss": 1.522, + "theoretical_loss": 3.4625131633941617, + "tokens_seen": 1974665216 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040565697091273824, + "loss": 1.4294, + "theoretical_loss": 3.462503637148192, + "tokens_seen": 1974730752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056369107321966, + "loss": 1.325, + "theoretical_loss": 3.462494111306886, + "tokens_seen": 1974796288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040561685055165496, + "loss": 1.3833, + "theoretical_loss": 3.4624845858702136, + "tokens_seen": 1974861824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055967903711134, + "loss": 1.3998, + "theoretical_loss": 3.4624750608381434, + "tokens_seen": 1974927360 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040557673019057174, + "loss": 1.3835, + "theoretical_loss": 3.462465536210646, + "tokens_seen": 1974992896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055566700100301, + "loss": 1.347, + "theoretical_loss": 3.4624560119876904, + "tokens_seen": 1975058432 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040553660982948846, + "loss": 1.3459, + "theoretical_loss": 3.462446488169245, + "tokens_seen": 1975123968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055165496489469, + "loss": 1.4374, + "theoretical_loss": 3.4624369647552804, + "tokens_seen": 1975189504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040549648946840524, + "loss": 1.4228, + "theoretical_loss": 3.462427441745765, + "tokens_seen": 1975255040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054764292878636, + "loss": 1.4275, + "theoretical_loss": 3.4624179191406697, + "tokens_seen": 1975320576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040545636910732197, + "loss": 1.3858, + "theoretical_loss": 3.4624083969399626, + "tokens_seen": 1975386112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040543630892678033, + "loss": 1.3259, + "theoretical_loss": 3.462398875143613, + "tokens_seen": 1975451648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040541624874623874, + "loss": 1.4255, + "theoretical_loss": 3.4623893537515915, + "tokens_seen": 1975517184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053961885656971, + "loss": 1.3874, + "theoretical_loss": 3.462379832763866, + "tokens_seen": 1975582720 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040537612838515547, + "loss": 1.3627, + "theoretical_loss": 3.4623703121804077, + "tokens_seen": 1975648256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040535606820461383, + "loss": 1.4221, + "theoretical_loss": 3.462360792001184, + "tokens_seen": 1975713792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040533600802407225, + "loss": 1.377, + "theoretical_loss": 3.462351272226166, + "tokens_seen": 1975779328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053159478435306, + "loss": 1.4155, + "theoretical_loss": 3.4623417528553224, + "tokens_seen": 1975844864 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040529588766298897, + "loss": 1.4925, + "theoretical_loss": 3.462332233888623, + "tokens_seen": 1975910400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040527582748244733, + "loss": 1.3736, + "theoretical_loss": 3.462322715326037, + "tokens_seen": 1975975936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052557673019057, + "loss": 1.2822, + "theoretical_loss": 3.462313197167534, + "tokens_seen": 1976041472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052357071213641, + "loss": 1.4228, + "theoretical_loss": 3.4623036794130826, + "tokens_seen": 1976107008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052156469408225, + "loss": 1.3359, + "theoretical_loss": 3.462294162062653, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040519558676028084, + "loss": 1.3545, + "theoretical_loss": 3.462284645116215, + "tokens_seen": 1976238080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051755265797392, + "loss": 1.3037, + "theoretical_loss": 3.4622751285737374, + "tokens_seen": 1976303616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051554663991976, + "loss": 1.3608, + "theoretical_loss": 3.46226561243519, + "tokens_seen": 1976369152 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405135406218656, + "loss": 1.387, + "theoretical_loss": 3.462256096700542, + "tokens_seen": 1976434688 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040511534603811434, + "loss": 1.3466, + "theoretical_loss": 3.462246581369763, + "tokens_seen": 1976500224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050952858575727, + "loss": 1.2942, + "theoretical_loss": 3.462237066442823, + "tokens_seen": 1976565760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040507522567703107, + "loss": 1.4503, + "theoretical_loss": 3.46222755191969, + "tokens_seen": 1976631296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050551654964895, + "loss": 1.4201, + "theoretical_loss": 3.4622180378003353, + "tokens_seen": 1976696832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040503510531594784, + "loss": 1.4125, + "theoretical_loss": 3.462208524084727, + "tokens_seen": 1976762368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050150451354062, + "loss": 1.2858, + "theoretical_loss": 3.4621990107728355, + "tokens_seen": 1976827904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040499498495486457, + "loss": 1.4234, + "theoretical_loss": 3.46218949786463, + "tokens_seen": 1976893440 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404974924774323, + "loss": 1.3062, + "theoretical_loss": 3.462179985360079, + "tokens_seen": 1976958976 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040495486459378135, + "loss": 1.3107, + "theoretical_loss": 3.4621704732591536, + "tokens_seen": 1977024512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049348044132397, + "loss": 1.3963, + "theoretical_loss": 3.462160961561822, + "tokens_seen": 1977090048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040491474423269807, + "loss": 1.3372, + "theoretical_loss": 3.4621514502680553, + "tokens_seen": 1977155584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040489468405215643, + "loss": 1.2865, + "theoretical_loss": 3.4621419393778208, + "tokens_seen": 1977221120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040487462387161485, + "loss": 1.3857, + "theoretical_loss": 3.4621324288910897, + "tokens_seen": 1977286656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048545636910732, + "loss": 1.2147, + "theoretical_loss": 3.462122918807831, + "tokens_seen": 1977352192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048345035105316, + "loss": 1.3404, + "theoretical_loss": 3.4621134091280137, + "tokens_seen": 1977417728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040481444332998994, + "loss": 1.3318, + "theoretical_loss": 3.4621038998516083, + "tokens_seen": 1977483264 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040479438314944835, + "loss": 1.3173, + "theoretical_loss": 3.4620943909785837, + "tokens_seen": 1977548800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047743229689067, + "loss": 1.3237, + "theoretical_loss": 3.4620848825089094, + "tokens_seen": 1977614336 + }, + { + "debugging/Compilability": 0.9230769230769231, + "debugging/distinct-1-grams": 0.7440830156991918, + "debugging/entropy-1-grams": 5.4710865197057785, + "debugging/length": 464.53846153846155, + "debugging/num_segments": 13, + "debugging/score": 0.008355915970037622, + "debugging/score_std": 0.00655650099508135, + "epoch": 0.2, + "objective/train/docs_used": 721216, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3672314882278442, + "objective/train/theoretical_loss": 3.4620753744425556, + "objective/train/tokens_used": 348138976, + "theoretical_loss": 3.4620753744425556, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047542627883651, + "loss": 1.4309, + "theoretical_loss": 3.4620753744425556, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040473420260782344, + "loss": 1.4119, + "theoretical_loss": 3.462065866779491, + "tokens_seen": 1977745408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047141424272818, + "loss": 1.2317, + "theoretical_loss": 3.4620563595196847, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046940822467402, + "loss": 1.3361, + "theoretical_loss": 3.462046852663108, + "tokens_seen": 1977876480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046740220661986, + "loss": 1.3346, + "theoretical_loss": 3.462037346209729, + "tokens_seen": 1977942016 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404653961885657, + "loss": 1.2395, + "theoretical_loss": 3.462027840159517, + "tokens_seen": 1978007552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040463390170511536, + "loss": 1.3338, + "theoretical_loss": 3.462018334512443, + "tokens_seen": 1978073088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046138415245738, + "loss": 1.3785, + "theoretical_loss": 3.462008829268475, + "tokens_seen": 1978138624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040459378134403214, + "loss": 1.338, + "theoretical_loss": 3.461999324427584, + "tokens_seen": 1978204160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045737211634905, + "loss": 1.378, + "theoretical_loss": 3.4619898199897388, + "tokens_seen": 1978269696 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040455366098294886, + "loss": 1.3553, + "theoretical_loss": 3.461980315954909, + "tokens_seen": 1978335232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045336008024073, + "loss": 1.3256, + "theoretical_loss": 3.461970812323064, + "tokens_seen": 1978400768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040451354062186564, + "loss": 1.4078, + "theoretical_loss": 3.461961309094173, + "tokens_seen": 1978466304 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404493480441324, + "loss": 1.5034, + "theoretical_loss": 3.4619518062682064, + "tokens_seen": 1978531840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040447342026078237, + "loss": 1.4358, + "theoretical_loss": 3.4619423038451336, + "tokens_seen": 1978597376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040445336008024073, + "loss": 1.3451, + "theoretical_loss": 3.461932801824924, + "tokens_seen": 1978662912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040443329989969915, + "loss": 1.3902, + "theoretical_loss": 3.4619233002075473, + "tokens_seen": 1978728448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044132397191575, + "loss": 1.3679, + "theoretical_loss": 3.4619137989929727, + "tokens_seen": 1978793984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040439317953861587, + "loss": 1.3711, + "theoretical_loss": 3.46190429818117, + "tokens_seen": 1978859520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040437311935807423, + "loss": 1.3006, + "theoretical_loss": 3.4618947977721097, + "tokens_seen": 1978925056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040435305917753265, + "loss": 1.3252, + "theoretical_loss": 3.4618852977657593, + "tokens_seen": 1978990592 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404332998996991, + "loss": 1.4415, + "theoretical_loss": 3.4618757981620907, + "tokens_seen": 1979056128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040431293881644937, + "loss": 1.1852, + "theoretical_loss": 3.461866298961072, + "tokens_seen": 1979121664 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040429287863590773, + "loss": 1.379, + "theoretical_loss": 3.461856800162673, + "tokens_seen": 1979187200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042728184553661, + "loss": 1.4844, + "theoretical_loss": 3.461847301766864, + "tokens_seen": 1979252736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042527582748245, + "loss": 1.3181, + "theoretical_loss": 3.461837803773614, + "tokens_seen": 1979318272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042326980942829, + "loss": 1.311, + "theoretical_loss": 3.461828306182893, + "tokens_seen": 1979383808 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040421263791374124, + "loss": 1.365, + "theoretical_loss": 3.4618188089946695, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041925777331996, + "loss": 1.3647, + "theoretical_loss": 3.461809312208915, + "tokens_seen": 1979514880 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404172517552658, + "loss": 1.3756, + "theoretical_loss": 3.4617998158255974, + "tokens_seen": 1979580416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041524573721164, + "loss": 1.2608, + "theoretical_loss": 3.4617903198446878, + "tokens_seen": 1979645952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040413239719157474, + "loss": 1.2831, + "theoretical_loss": 3.4617808242661545, + "tokens_seen": 1979711488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041123370110331, + "loss": 1.298, + "theoretical_loss": 3.461771329089968, + "tokens_seen": 1979777024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040409227683049147, + "loss": 1.4019, + "theoretical_loss": 3.461761834316097, + "tokens_seen": 1979842560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040722166499499, + "loss": 1.4078, + "theoretical_loss": 3.4617523399445123, + "tokens_seen": 1979908096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040405215646940824, + "loss": 1.3002, + "theoretical_loss": 3.4617428459751833, + "tokens_seen": 1979973632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040320962888666, + "loss": 1.3515, + "theoretical_loss": 3.4617333524080784, + "tokens_seen": 1980039168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040401203610832497, + "loss": 1.3781, + "theoretical_loss": 3.461723859243169, + "tokens_seen": 1980104704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039919759277834, + "loss": 1.3454, + "theoretical_loss": 3.461714366480424, + "tokens_seen": 1980170240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040397191574724175, + "loss": 1.2799, + "theoretical_loss": 3.461704874119813, + "tokens_seen": 1980235776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039518555667001, + "loss": 1.4118, + "theoretical_loss": 3.461695382161305, + "tokens_seen": 1980301312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040393179538615847, + "loss": 1.3863, + "theoretical_loss": 3.461685890604871, + "tokens_seen": 1980366848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040391173520561683, + "loss": 1.2998, + "theoretical_loss": 3.4616763994504796, + "tokens_seen": 1980432384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040389167502507525, + "loss": 1.3822, + "theoretical_loss": 3.461666908698101, + "tokens_seen": 1980497920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038716148445336, + "loss": 1.325, + "theoretical_loss": 3.4616574183477047, + "tokens_seen": 1980563456 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403851554663992, + "loss": 1.238, + "theoretical_loss": 3.4616479283992607, + "tokens_seen": 1980628992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040383149448345034, + "loss": 1.4144, + "theoretical_loss": 3.461638438852738, + "tokens_seen": 1980694528 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040381143430290875, + "loss": 1.3013, + "theoretical_loss": 3.461628949708106, + "tokens_seen": 1980760064 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037913741223671, + "loss": 1.3298, + "theoretical_loss": 3.461619460965336, + "tokens_seen": 1980825600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037713139418255, + "loss": 1.4929, + "theoretical_loss": 3.4616099726243963, + "tokens_seen": 1980891136 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 722322, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3439686298370361, + "objective/train/theoretical_loss": 3.4616004846852575, + "objective/train/tokens_used": 351415776, + "theoretical_loss": 3.4616004846852575, + "tokens_seen": 1980956672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040375125376128384, + "loss": 1.3363, + "theoretical_loss": 3.4616004846852575, + "tokens_seen": 1980956672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037311935807422, + "loss": 1.3386, + "theoretical_loss": 3.461590997147889, + "tokens_seen": 1981022208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037111334002006, + "loss": 1.4471, + "theoretical_loss": 3.4615815100122593, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403691073219659, + "loss": 1.4177, + "theoretical_loss": 3.4615720232783396, + "tokens_seen": 1981153280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040367101303911734, + "loss": 1.384, + "theoretical_loss": 3.461562536946099, + "tokens_seen": 1981218816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036509528585757, + "loss": 1.3137, + "theoretical_loss": 3.4615530510155077, + "tokens_seen": 1981284352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036308926780341, + "loss": 1.4487, + "theoretical_loss": 3.4615435654865347, + "tokens_seen": 1981349888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036108324974925, + "loss": 1.2266, + "theoretical_loss": 3.46153408035915, + "tokens_seen": 1981415424 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040359077231695085, + "loss": 1.4123, + "theoretical_loss": 3.4615245956333234, + "tokens_seen": 1981480960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035707121364092, + "loss": 1.3577, + "theoretical_loss": 3.4615151113090246, + "tokens_seen": 1981546496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040355065195586757, + "loss": 1.3255, + "theoretical_loss": 3.4615056273862232, + "tokens_seen": 1981612032 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403530591775326, + "loss": 1.3068, + "theoretical_loss": 3.4614961438648892, + "tokens_seen": 1981677568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040351053159478435, + "loss": 1.3412, + "theoretical_loss": 3.461486660744992, + "tokens_seen": 1981743104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034904714142427, + "loss": 1.3234, + "theoretical_loss": 3.4614771780265015, + "tokens_seen": 1981808640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034704112337011, + "loss": 1.4, + "theoretical_loss": 3.4614676957093873, + "tokens_seen": 1981874176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034503510531595, + "loss": 1.3813, + "theoretical_loss": 3.4614582137936196, + "tokens_seen": 1981939712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040343029087261785, + "loss": 1.3875, + "theoretical_loss": 3.4614487322791674, + "tokens_seen": 1982005248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034102306920762, + "loss": 1.3009, + "theoretical_loss": 3.4614392511660013, + "tokens_seen": 1982070784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033901705115346, + "loss": 1.3277, + "theoretical_loss": 3.46142977045409, + "tokens_seen": 1982136320 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403370110330993, + "loss": 1.3453, + "theoretical_loss": 3.461420290143404, + "tokens_seen": 1982201856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040335005015045136, + "loss": 1.2591, + "theoretical_loss": 3.461410810233913, + "tokens_seen": 1982267392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033299899699097, + "loss": 1.2978, + "theoretical_loss": 3.461401330725587, + "tokens_seen": 1982332928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033099297893681, + "loss": 1.4364, + "theoretical_loss": 3.4613918516183952, + "tokens_seen": 1982398464 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040328986960882644, + "loss": 1.3573, + "theoretical_loss": 3.4613823729123077, + "tokens_seen": 1982464000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040326980942828486, + "loss": 1.4078, + "theoretical_loss": 3.461372894607294, + "tokens_seen": 1982529536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032497492477432, + "loss": 1.3428, + "theoretical_loss": 3.461363416703324, + "tokens_seen": 1982595072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032296890672016, + "loss": 1.3255, + "theoretical_loss": 3.461353939200367, + "tokens_seen": 1982660608 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040320962888665995, + "loss": 1.3935, + "theoretical_loss": 3.4613444620983937, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040318956870611836, + "loss": 1.3975, + "theoretical_loss": 3.461334985397374, + "tokens_seen": 1982791680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031695085255767, + "loss": 1.3443, + "theoretical_loss": 3.4613255090972768, + "tokens_seen": 1982857216 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040314944834503514, + "loss": 1.3687, + "theoretical_loss": 3.461316033198072, + "tokens_seen": 1982922752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031293881644935, + "loss": 1.3576, + "theoretical_loss": 3.4613065576997295, + "tokens_seen": 1982988288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040310932798395187, + "loss": 1.3636, + "theoretical_loss": 3.4612970826022194, + "tokens_seen": 1983053824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030892678034103, + "loss": 1.2994, + "theoretical_loss": 3.4612876079055113, + "tokens_seen": 1983119360 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040306920762286864, + "loss": 1.3355, + "theoretical_loss": 3.461278133609575, + "tokens_seen": 1983184896 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403049147442327, + "loss": 1.2429, + "theoretical_loss": 3.461268659714381, + "tokens_seen": 1983250432 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040302908726178537, + "loss": 1.4229, + "theoretical_loss": 3.4612591862198974, + "tokens_seen": 1983315968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030090270812438, + "loss": 1.3125, + "theoretical_loss": 3.4612497131260955, + "tokens_seen": 1983381504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040298896690070215, + "loss": 1.3685, + "theoretical_loss": 3.4612402404329448, + "tokens_seen": 1983447040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029689067201605, + "loss": 1.4441, + "theoretical_loss": 3.461230768140415, + "tokens_seen": 1983512576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040294884653961887, + "loss": 1.3796, + "theoretical_loss": 3.4612212962484756, + "tokens_seen": 1983578112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040292878635907723, + "loss": 1.2975, + "theoretical_loss": 3.461211824757097, + "tokens_seen": 1983643648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040290872617853565, + "loss": 1.4448, + "theoretical_loss": 3.4612023536662484, + "tokens_seen": 1983709184 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402888665997994, + "loss": 1.3734, + "theoretical_loss": 3.4611928829759004, + "tokens_seen": 1983774720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028686058174524, + "loss": 1.2953, + "theoretical_loss": 3.4611834126860224, + "tokens_seen": 1983840256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040284854563691074, + "loss": 1.3181, + "theoretical_loss": 3.4611739427965835, + "tokens_seen": 1983905792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040282848545636915, + "loss": 1.254, + "theoretical_loss": 3.461164473307555, + "tokens_seen": 1983971328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028084252758275, + "loss": 1.3653, + "theoretical_loss": 3.461155004218906, + "tokens_seen": 1984036864 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027883650952859, + "loss": 1.3605, + "theoretical_loss": 3.4611455355306067, + "tokens_seen": 1984102400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040276830491474424, + "loss": 1.3062, + "theoretical_loss": 3.461136067242626, + "tokens_seen": 1984167936 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 723589, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2202953100204468, + "objective/train/theoretical_loss": 3.4611265993549347, + "objective/train/tokens_used": 354692576, + "theoretical_loss": 3.4611265993549347, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027482447342026, + "loss": 1.2569, + "theoretical_loss": 3.4611265993549347, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402728184553661, + "loss": 1.4007, + "theoretical_loss": 3.461117131867502, + "tokens_seen": 1984299008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027081243731194, + "loss": 1.4271, + "theoretical_loss": 3.4611076647802985, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040268806419257774, + "loss": 1.3595, + "theoretical_loss": 3.4610981980932936, + "tokens_seen": 1984430080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026680040120361, + "loss": 1.4322, + "theoretical_loss": 3.461088731806457, + "tokens_seen": 1984495616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026479438314945, + "loss": 1.3715, + "theoretical_loss": 3.461079265919759, + "tokens_seen": 1984561152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026278836509529, + "loss": 1.3563, + "theoretical_loss": 3.4610698004331693, + "tokens_seen": 1984626688 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040260782347041125, + "loss": 1.425, + "theoretical_loss": 3.461060335346658, + "tokens_seen": 1984692224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025877632898696, + "loss": 1.3892, + "theoretical_loss": 3.461050870660194, + "tokens_seen": 1984757760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040256770310932797, + "loss": 1.3396, + "theoretical_loss": 3.4610414063737487, + "tokens_seen": 1984823296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025476429287864, + "loss": 1.4118, + "theoretical_loss": 3.46103194248729, + "tokens_seen": 1984888832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040252758274824475, + "loss": 1.3685, + "theoretical_loss": 3.4610224790007904, + "tokens_seen": 1984954368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025075225677031, + "loss": 1.3364, + "theoretical_loss": 3.461013015914218, + "tokens_seen": 1985019904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024874623871615, + "loss": 1.3812, + "theoretical_loss": 3.4610035532275427, + "tokens_seen": 1985085440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024674022066199, + "loss": 1.5181, + "theoretical_loss": 3.4609940909407353, + "tokens_seen": 1985150976 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040244734202607825, + "loss": 1.4117, + "theoretical_loss": 3.4609846290537645, + "tokens_seen": 1985216512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024272818455366, + "loss": 1.3317, + "theoretical_loss": 3.4609751675666014, + "tokens_seen": 1985282048 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402407221664995, + "loss": 1.31, + "theoretical_loss": 3.4609657064792154, + "tokens_seen": 1985347584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023871614844534, + "loss": 1.3652, + "theoretical_loss": 3.460956245791576, + "tokens_seen": 1985413120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040236710130391176, + "loss": 1.3886, + "theoretical_loss": 3.460946785503654, + "tokens_seen": 1985478656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023470411233701, + "loss": 1.2739, + "theoretical_loss": 3.4609373256154186, + "tokens_seen": 1985544192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023269809428285, + "loss": 1.3078, + "theoretical_loss": 3.4609278661268403, + "tokens_seen": 1985609728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040230692076228684, + "loss": 1.3446, + "theoretical_loss": 3.4609184070378882, + "tokens_seen": 1985675264 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040228686058174526, + "loss": 1.4043, + "theoretical_loss": 3.460908948348533, + "tokens_seen": 1985740800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022668004012036, + "loss": 1.4045, + "theoretical_loss": 3.460899490058744, + "tokens_seen": 1985806336 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402246740220662, + "loss": 1.2708, + "theoretical_loss": 3.460890032168492, + "tokens_seen": 1985871872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040222668004012035, + "loss": 1.4782, + "theoretical_loss": 3.4608805746777467, + "tokens_seen": 1985937408 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040220661985957876, + "loss": 1.426, + "theoretical_loss": 3.460871117586477, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021865596790371, + "loss": 1.4417, + "theoretical_loss": 3.460861660894654, + "tokens_seen": 1986068480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021664994984955, + "loss": 1.3367, + "theoretical_loss": 3.460852204602247, + "tokens_seen": 1986134016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040214643931795385, + "loss": 1.4435, + "theoretical_loss": 3.4608427487092266, + "tokens_seen": 1986199552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021263791374122, + "loss": 1.4076, + "theoretical_loss": 3.460833293215562, + "tokens_seen": 1986265088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040210631895687063, + "loss": 1.2717, + "theoretical_loss": 3.4608238381212235, + "tokens_seen": 1986330624 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402086258776329, + "loss": 1.351, + "theoretical_loss": 3.4608143834261815, + "tokens_seen": 1986396160 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040206619859578735, + "loss": 1.4113, + "theoretical_loss": 3.4608049291304055, + "tokens_seen": 1986461696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020461384152457, + "loss": 1.3995, + "theoretical_loss": 3.4607954752338657, + "tokens_seen": 1986527232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040202607823470413, + "loss": 1.4118, + "theoretical_loss": 3.4607860217365314, + "tokens_seen": 1986592768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020060180541625, + "loss": 1.2689, + "theoretical_loss": 3.460776568638373, + "tokens_seen": 1986658304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040198595787362086, + "loss": 1.278, + "theoretical_loss": 3.460767115939361, + "tokens_seen": 1986723840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019658976930792, + "loss": 1.3531, + "theoretical_loss": 3.4607576636394644, + "tokens_seen": 1986789376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019458375125376, + "loss": 1.3583, + "theoretical_loss": 3.460748211738654, + "tokens_seen": 1986854912 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401925777331996, + "loss": 1.2634, + "theoretical_loss": 3.4607387602368997, + "tokens_seen": 1986920448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040190571715145436, + "loss": 1.4257, + "theoretical_loss": 3.4607293091341713, + "tokens_seen": 1986985984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018856569709127, + "loss": 1.4302, + "theoretical_loss": 3.4607198584304384, + "tokens_seen": 1987051520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018655967903711, + "loss": 1.3279, + "theoretical_loss": 3.4607104081256717, + "tokens_seen": 1987117056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018455366098295, + "loss": 1.4233, + "theoretical_loss": 3.4607009582198405, + "tokens_seen": 1987182592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040182547642928786, + "loss": 1.3579, + "theoretical_loss": 3.4606915087129155, + "tokens_seen": 1987248128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018054162487462, + "loss": 1.4424, + "theoretical_loss": 3.460682059604866, + "tokens_seen": 1987313664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017853560682046, + "loss": 1.3945, + "theoretical_loss": 3.4606726108956627, + "tokens_seen": 1987379200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040176529588766295, + "loss": 1.3562, + "theoretical_loss": 3.4606631625852757, + "tokens_seen": 1987444736 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 724791, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3534629344940186, + "objective/train/theoretical_loss": 3.460653714673674, + "objective/train/tokens_used": 357969376, + "theoretical_loss": 3.460653714673674, + "tokens_seen": 1987510272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040174523570712137, + "loss": 1.32, + "theoretical_loss": 3.460653714673674, + "tokens_seen": 1987510272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040172517552657973, + "loss": 1.2754, + "theoretical_loss": 3.460644267160829, + "tokens_seen": 1987575808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017051153460381, + "loss": 1.3013, + "theoretical_loss": 3.4606348200467094, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040168505516549645, + "loss": 1.3393, + "theoretical_loss": 3.4606253733312857, + "tokens_seen": 1987706880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040166499498495487, + "loss": 1.3689, + "theoretical_loss": 3.460615927014528, + "tokens_seen": 1987772416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016449348044133, + "loss": 1.2709, + "theoretical_loss": 3.4606064810964066, + "tokens_seen": 1987837952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040162487462387165, + "loss": 1.3312, + "theoretical_loss": 3.4605970355768916, + "tokens_seen": 1987903488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040160481444333, + "loss": 1.4216, + "theoretical_loss": 3.460587590455952, + "tokens_seen": 1987969024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040158475426278837, + "loss": 1.4097, + "theoretical_loss": 3.4605781457335594, + "tokens_seen": 1988034560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015646940822468, + "loss": 1.3176, + "theoretical_loss": 3.4605687014096826, + "tokens_seen": 1988100096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040154463390170515, + "loss": 1.3375, + "theoretical_loss": 3.4605592574842916, + "tokens_seen": 1988165632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015245737211635, + "loss": 1.4103, + "theoretical_loss": 3.460549813957358, + "tokens_seen": 1988231168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015045135406219, + "loss": 1.4437, + "theoretical_loss": 3.46054037082885, + "tokens_seen": 1988296704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014844533600803, + "loss": 1.3659, + "theoretical_loss": 3.4605309280987386, + "tokens_seen": 1988362240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040146439317953865, + "loss": 1.3338, + "theoretical_loss": 3.4605214857669937, + "tokens_seen": 1988427776 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401444332998997, + "loss": 1.4236, + "theoretical_loss": 3.4605120438335852, + "tokens_seen": 1988493312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014242728184554, + "loss": 1.3153, + "theoretical_loss": 3.4605026022984835, + "tokens_seen": 1988558848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014042126379138, + "loss": 1.3989, + "theoretical_loss": 3.4604931611616587, + "tokens_seen": 1988624384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040138415245737216, + "loss": 1.2986, + "theoretical_loss": 3.4604837204230803, + "tokens_seen": 1988689920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013640922768305, + "loss": 1.3591, + "theoretical_loss": 3.460474280082719, + "tokens_seen": 1988755456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013440320962889, + "loss": 1.3271, + "theoretical_loss": 3.4604648401405447, + "tokens_seen": 1988820992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040132397191574724, + "loss": 1.3282, + "theoretical_loss": 3.460455400596527, + "tokens_seen": 1988886528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040130391173520566, + "loss": 1.3491, + "theoretical_loss": 3.460445961450637, + "tokens_seen": 1988952064 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401283851554664, + "loss": 1.3012, + "theoretical_loss": 3.4604365227028437, + "tokens_seen": 1989017600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012637913741224, + "loss": 1.3187, + "theoretical_loss": 3.460427084353118, + "tokens_seen": 1989083136 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040124373119358075, + "loss": 1.4753, + "theoretical_loss": 3.4604176464014293, + "tokens_seen": 1989148672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040122367101303916, + "loss": 1.31, + "theoretical_loss": 3.4604082088477486, + "tokens_seen": 1989214208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012036108324975, + "loss": 1.3481, + "theoretical_loss": 3.460398771692045, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011835506519559, + "loss": 1.3408, + "theoretical_loss": 3.4603893349342894, + "tokens_seen": 1989345280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040116349047141425, + "loss": 1.3436, + "theoretical_loss": 3.460379898574452, + "tokens_seen": 1989410816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011434302908726, + "loss": 1.4249, + "theoretical_loss": 3.4603704626125023, + "tokens_seen": 1989476352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040112337011033103, + "loss": 1.4224, + "theoretical_loss": 3.4603610270484104, + "tokens_seen": 1989541888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011033099297894, + "loss": 1.3064, + "theoretical_loss": 3.4603515918821466, + "tokens_seen": 1989607424 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040108324974924775, + "loss": 1.3215, + "theoretical_loss": 3.460342157113681, + "tokens_seen": 1989672960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010631895687061, + "loss": 1.3517, + "theoretical_loss": 3.4603327227429843, + "tokens_seen": 1989738496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040104312938816453, + "loss": 1.3034, + "theoretical_loss": 3.460323288770026, + "tokens_seen": 1989804032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010230692076229, + "loss": 1.3197, + "theoretical_loss": 3.460313855194776, + "tokens_seen": 1989869568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040100300902708126, + "loss": 1.334, + "theoretical_loss": 3.460304422017205, + "tokens_seen": 1989935104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009829488465396, + "loss": 1.3495, + "theoretical_loss": 3.4602949892372834, + "tokens_seen": 1990000640 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400962888665998, + "loss": 1.3496, + "theoretical_loss": 3.4602855568549806, + "tokens_seen": 1990066176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009428284854564, + "loss": 1.3916, + "theoretical_loss": 3.4602761248702665, + "tokens_seen": 1990131712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040092276830491476, + "loss": 1.3857, + "theoretical_loss": 3.4602666932831125, + "tokens_seen": 1990197248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009027081243731, + "loss": 1.4262, + "theoretical_loss": 3.4602572620934877, + "tokens_seen": 1990262784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008826479438315, + "loss": 1.3856, + "theoretical_loss": 3.4602478313013627, + "tokens_seen": 1990328320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008625877632899, + "loss": 1.392, + "theoretical_loss": 3.4602384009067073, + "tokens_seen": 1990393856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040084252758274826, + "loss": 1.3777, + "theoretical_loss": 3.460228970909492, + "tokens_seen": 1990459392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008224674022066, + "loss": 1.3827, + "theoretical_loss": 3.4602195413096877, + "tokens_seen": 1990524928 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400802407221665, + "loss": 1.3606, + "theoretical_loss": 3.460210112107263, + "tokens_seen": 1990590464 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040078234704112335, + "loss": 1.5201, + "theoretical_loss": 3.4602006833021886, + "tokens_seen": 1990656000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040076228686058177, + "loss": 1.4248, + "theoretical_loss": 3.4601912548944354, + "tokens_seen": 1990721536 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 726036, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3863608837127686, + "objective/train/theoretical_loss": 3.4601818268839732, + "objective/train/tokens_used": 361246176, + "theoretical_loss": 3.4601818268839732, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040074222668004013, + "loss": 1.3912, + "theoretical_loss": 3.4601818268839732, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007221664994985, + "loss": 1.3423, + "theoretical_loss": 3.4601723992707716, + "tokens_seen": 1990852608 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040070210631895685, + "loss": 1.3434, + "theoretical_loss": 3.4601629720548015, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040068204613841527, + "loss": 1.3627, + "theoretical_loss": 3.460153545236033, + "tokens_seen": 1990983680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040066198595787363, + "loss": 1.5416, + "theoretical_loss": 3.4601441188144353, + "tokens_seen": 1991049216 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400641925777332, + "loss": 1.4456, + "theoretical_loss": 3.46013469278998, + "tokens_seen": 1991114752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040062186559679036, + "loss": 1.3619, + "theoretical_loss": 3.4601252671626366, + "tokens_seen": 1991180288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006018054162487, + "loss": 1.3675, + "theoretical_loss": 3.460115841932376, + "tokens_seen": 1991245824 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040058174523570713, + "loss": 1.2165, + "theoretical_loss": 3.4601064170991673, + "tokens_seen": 1991311360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005616850551655, + "loss": 1.5023, + "theoretical_loss": 3.460096992662981, + "tokens_seen": 1991376896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040054162487462386, + "loss": 1.4434, + "theoretical_loss": 3.4600875686237877, + "tokens_seen": 1991442432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005215646940822, + "loss": 1.3902, + "theoretical_loss": 3.4600781449815576, + "tokens_seen": 1991507968 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040050150451354064, + "loss": 1.3706, + "theoretical_loss": 3.4600687217362607, + "tokens_seen": 1991573504 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400481444332999, + "loss": 1.4068, + "theoretical_loss": 3.4600592988878676, + "tokens_seen": 1991639040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040046138415245736, + "loss": 1.3891, + "theoretical_loss": 3.4600498764363476, + "tokens_seen": 1991704576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004413239719157, + "loss": 1.3449, + "theoretical_loss": 3.460040454381672, + "tokens_seen": 1991770112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004212637913741, + "loss": 1.3519, + "theoretical_loss": 3.46003103272381, + "tokens_seen": 1991835648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004012036108325, + "loss": 1.4163, + "theoretical_loss": 3.460021611462733, + "tokens_seen": 1991901184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040038114343029087, + "loss": 1.4043, + "theoretical_loss": 3.4600121905984103, + "tokens_seen": 1991966720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040036108324974923, + "loss": 1.4246, + "theoretical_loss": 3.4600027701308127, + "tokens_seen": 1992032256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003410230692076, + "loss": 1.2987, + "theoretical_loss": 3.45999335005991, + "tokens_seen": 1992097792 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400320962888666, + "loss": 1.3392, + "theoretical_loss": 3.4599839303856728, + "tokens_seen": 1992163328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040030090270812437, + "loss": 1.4051, + "theoretical_loss": 3.4599745111080713, + "tokens_seen": 1992228864 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040028084252758273, + "loss": 1.3026, + "theoretical_loss": 3.4599650922270753, + "tokens_seen": 1992294400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002607823470411, + "loss": 1.4301, + "theoretical_loss": 3.4599556737426553, + "tokens_seen": 1992359936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040024072216649946, + "loss": 1.3771, + "theoretical_loss": 3.459946255654782, + "tokens_seen": 1992425472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040022066198595787, + "loss": 1.364, + "theoretical_loss": 3.4599368379634257, + "tokens_seen": 1992491008 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040020060180541623, + "loss": 1.4313, + "theoretical_loss": 3.4599274206685555, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001805416248746, + "loss": 1.3656, + "theoretical_loss": 3.4599180037701434, + "tokens_seen": 1992622080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040016048144433296, + "loss": 1.3189, + "theoretical_loss": 3.459908587268158, + "tokens_seen": 1992687616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040014042126379143, + "loss": 1.4828, + "theoretical_loss": 3.4598991711625704, + "tokens_seen": 1992753152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001203610832498, + "loss": 1.43, + "theoretical_loss": 3.459889755453351, + "tokens_seen": 1992818688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040010030090270815, + "loss": 1.3663, + "theoretical_loss": 3.45988034014047, + "tokens_seen": 1992884224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000802407221665, + "loss": 1.3134, + "theoretical_loss": 3.459870925223897, + "tokens_seen": 1992949760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040006018054162493, + "loss": 1.4239, + "theoretical_loss": 3.4598615107036035, + "tokens_seen": 1993015296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000401203610833, + "loss": 1.378, + "theoretical_loss": 3.4598520965795587, + "tokens_seen": 1993080832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040002006018054166, + "loss": 1.3894, + "theoretical_loss": 3.4598426828517335, + "tokens_seen": 1993146368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004, + "loss": 1.2587, + "theoretical_loss": 3.4598332695200984, + "tokens_seen": 1993211904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999799398194584, + "loss": 1.483, + "theoretical_loss": 3.459823856584623, + "tokens_seen": 1993277440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999598796389168, + "loss": 1.3825, + "theoretical_loss": 3.4598144440452776, + "tokens_seen": 1993342976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039993981945837516, + "loss": 1.356, + "theoretical_loss": 3.459805031902033, + "tokens_seen": 1993408512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999197592778335, + "loss": 1.3301, + "theoretical_loss": 3.459795620154859, + "tokens_seen": 1993474048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998996990972919, + "loss": 1.4109, + "theoretical_loss": 3.4597862088037266, + "tokens_seen": 1993539584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998796389167503, + "loss": 1.3448, + "theoretical_loss": 3.4597767978486065, + "tokens_seen": 1993605120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039985957873620866, + "loss": 1.3179, + "theoretical_loss": 3.4597673872894674, + "tokens_seen": 1993670656 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399839518555667, + "loss": 1.4707, + "theoretical_loss": 3.459757977126281, + "tokens_seen": 1993736192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998194583751254, + "loss": 1.3044, + "theoretical_loss": 3.4597485673590165, + "tokens_seen": 1993801728 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039979939819458375, + "loss": 1.3809, + "theoretical_loss": 3.459739157987645, + "tokens_seen": 1993867264 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039977933801404217, + "loss": 1.3681, + "theoretical_loss": 3.459729749012137, + "tokens_seen": 1993932800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039975927783350053, + "loss": 1.318, + "theoretical_loss": 3.4597203404324626, + "tokens_seen": 1993998336 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 727257, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3149657249450684, + "objective/train/theoretical_loss": 3.459710932248592, + "objective/train/tokens_used": 364522976, + "theoretical_loss": 3.459710932248592, + "tokens_seen": 1994063872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997392176529589, + "loss": 1.2551, + "theoretical_loss": 3.459710932248592, + "tokens_seen": 1994063872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039971915747241725, + "loss": 1.3166, + "theoretical_loss": 3.4597015244604954, + "tokens_seen": 1994129408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039969909729187567, + "loss": 1.2919, + "theoretical_loss": 3.459692117068144, + "tokens_seen": 1994194944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039967903711133403, + "loss": 1.3725, + "theoretical_loss": 3.459682710071507, + "tokens_seen": 1994260480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996589769307924, + "loss": 1.4415, + "theoretical_loss": 3.459673303470555, + "tokens_seen": 1994326016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039963891675025076, + "loss": 1.3836, + "theoretical_loss": 3.4596638972652594, + "tokens_seen": 1994391552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996188565697091, + "loss": 1.3605, + "theoretical_loss": 3.4596544914555887, + "tokens_seen": 1994457088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039959879638916754, + "loss": 1.3941, + "theoretical_loss": 3.4596450860415153, + "tokens_seen": 1994522624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995787362086259, + "loss": 1.4052, + "theoretical_loss": 3.4596356810230082, + "tokens_seen": 1994588160 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039955867602808426, + "loss": 1.3498, + "theoretical_loss": 3.459626276400038, + "tokens_seen": 1994653696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995386158475426, + "loss": 1.4425, + "theoretical_loss": 3.4596168721725755, + "tokens_seen": 1994719232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039951855566700104, + "loss": 1.2963, + "theoretical_loss": 3.459607468340591, + "tokens_seen": 1994784768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994984954864594, + "loss": 1.2997, + "theoretical_loss": 3.4595980649040543, + "tokens_seen": 1994850304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039947843530591776, + "loss": 1.34, + "theoretical_loss": 3.459588661862936, + "tokens_seen": 1994915840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994583751253761, + "loss": 1.4009, + "theoretical_loss": 3.459579259217207, + "tokens_seen": 1994981376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994383149448345, + "loss": 1.4091, + "theoretical_loss": 3.459569856966837, + "tokens_seen": 1995046912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994182547642929, + "loss": 1.3064, + "theoretical_loss": 3.4595604551117973, + "tokens_seen": 1995112448 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039939819458375127, + "loss": 1.2864, + "theoretical_loss": 3.4595510536520573, + "tokens_seen": 1995177984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039937813440320963, + "loss": 1.3486, + "theoretical_loss": 3.459541652587588, + "tokens_seen": 1995243520 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399358074222668, + "loss": 1.4531, + "theoretical_loss": 3.4595322519183593, + "tokens_seen": 1995309056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993380140421264, + "loss": 1.4836, + "theoretical_loss": 3.4595228516443424, + "tokens_seen": 1995374592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039931795386158477, + "loss": 1.3949, + "theoretical_loss": 3.459513451765507, + "tokens_seen": 1995440128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039929789368104313, + "loss": 1.3453, + "theoretical_loss": 3.4595040522818232, + "tokens_seen": 1995505664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992778335005015, + "loss": 1.2817, + "theoretical_loss": 3.4594946531932624, + "tokens_seen": 1995571200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039925777331995986, + "loss": 1.3728, + "theoretical_loss": 3.459485254499795, + "tokens_seen": 1995636736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039923771313941827, + "loss": 1.3255, + "theoretical_loss": 3.4594758562013905, + "tokens_seen": 1995702272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039921765295887663, + "loss": 1.3223, + "theoretical_loss": 3.45946645829802, + "tokens_seen": 1995767808 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399197592778335, + "loss": 1.3109, + "theoretical_loss": 3.4594570607896538, + "tokens_seen": 1995833344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039917753259779336, + "loss": 1.2799, + "theoretical_loss": 3.4594476636762614, + "tokens_seen": 1995898880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991574724172518, + "loss": 1.3226, + "theoretical_loss": 3.459438266957815, + "tokens_seen": 1995964416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039913741223671014, + "loss": 1.2797, + "theoretical_loss": 3.4594288706342837, + "tokens_seen": 1996029952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991173520561685, + "loss": 1.2252, + "theoretical_loss": 3.459419474705639, + "tokens_seen": 1996095488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039909729187562686, + "loss": 1.2257, + "theoretical_loss": 3.4594100791718496, + "tokens_seen": 1996161024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990772316950853, + "loss": 1.4258, + "theoretical_loss": 3.4594006840328877, + "tokens_seen": 1996226560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039905717151454364, + "loss": 1.3857, + "theoretical_loss": 3.459391289288723, + "tokens_seen": 1996292096 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399037111334002, + "loss": 1.3814, + "theoretical_loss": 3.459381894939326, + "tokens_seen": 1996357632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039901705115346037, + "loss": 1.3452, + "theoretical_loss": 3.459372500984667, + "tokens_seen": 1996423168 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039899699097291873, + "loss": 1.2958, + "theoretical_loss": 3.459363107424717, + "tokens_seen": 1996488704 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039897693079237714, + "loss": 1.4375, + "theoretical_loss": 3.459353714259446, + "tokens_seen": 1996554240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989568706118355, + "loss": 1.4296, + "theoretical_loss": 3.4593443214888246, + "tokens_seen": 1996619776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039893681043129387, + "loss": 1.3543, + "theoretical_loss": 3.4593349291128233, + "tokens_seen": 1996685312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039891675025075223, + "loss": 1.3687, + "theoretical_loss": 3.459325537131412, + "tokens_seen": 1996750848 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039889669007021065, + "loss": 1.3407, + "theoretical_loss": 3.459316145544562, + "tokens_seen": 1996816384 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398876629889669, + "loss": 1.4782, + "theoretical_loss": 3.4593067543522436, + "tokens_seen": 1996881920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039885656970912737, + "loss": 1.3607, + "theoretical_loss": 3.459297363554427, + "tokens_seen": 1996947456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039883650952858573, + "loss": 1.3666, + "theoretical_loss": 3.4592879731510826, + "tokens_seen": 1997012992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988164493480441, + "loss": 1.3864, + "theoretical_loss": 3.459278583142181, + "tokens_seen": 1997078528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987963891675025, + "loss": 1.4695, + "theoretical_loss": 3.4592691935276934, + "tokens_seen": 1997144064 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987763289869609, + "loss": 1.3558, + "theoretical_loss": 3.4592598043075893, + "tokens_seen": 1997209600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039875626880641924, + "loss": 1.367, + "theoretical_loss": 3.4592504154818395, + "tokens_seen": 1997275136 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 728467, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3009185791015625, + "objective/train/theoretical_loss": 3.459241027050415, + "objective/train/tokens_used": 367799776, + "theoretical_loss": 3.459241027050415, + "tokens_seen": 1997340672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987362086258776, + "loss": 1.3176, + "theoretical_loss": 3.459241027050415, + "tokens_seen": 1997340672 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398716148445336, + "loss": 1.3363, + "theoretical_loss": 3.4592316390132853, + "tokens_seen": 1997406208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986960882647944, + "loss": 1.3445, + "theoretical_loss": 3.459222251370422, + "tokens_seen": 1997471744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039867602808425274, + "loss": 1.4535, + "theoretical_loss": 3.4592128641217945, + "tokens_seen": 1997537280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986559679037111, + "loss": 1.3713, + "theoretical_loss": 3.459203477267374, + "tokens_seen": 1997602816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986359077231695, + "loss": 1.4328, + "theoretical_loss": 3.459194090807131, + "tokens_seen": 1997668352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039861584754262794, + "loss": 1.382, + "theoretical_loss": 3.4591847047410362, + "tokens_seen": 1997733888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985957873620863, + "loss": 1.3076, + "theoretical_loss": 3.4591753190690593, + "tokens_seen": 1997799424 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039857572718154466, + "loss": 1.2875, + "theoretical_loss": 3.459165933791172, + "tokens_seen": 1997864960 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398555667001003, + "loss": 1.4044, + "theoretical_loss": 3.4591565489073437, + "tokens_seen": 1997930496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039853560682046144, + "loss": 1.2351, + "theoretical_loss": 3.4591471644175456, + "tokens_seen": 1997996032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985155466399198, + "loss": 1.2581, + "theoretical_loss": 3.4591377803217482, + "tokens_seen": 1998061568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039849548645937816, + "loss": 1.3277, + "theoretical_loss": 3.4591283966199216, + "tokens_seen": 1998127104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984754262788365, + "loss": 1.3737, + "theoretical_loss": 3.4591190133120366, + "tokens_seen": 1998192640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984553660982949, + "loss": 1.2965, + "theoretical_loss": 3.459109630398064, + "tokens_seen": 1998258176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984353059177533, + "loss": 1.4011, + "theoretical_loss": 3.4591002478779744, + "tokens_seen": 1998323712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039841524573721167, + "loss": 1.3383, + "theoretical_loss": 3.4590908657517376, + "tokens_seen": 1998389248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039839518555667003, + "loss": 1.4244, + "theoretical_loss": 3.4590814840193245, + "tokens_seen": 1998454784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983751253761284, + "loss": 1.3942, + "theoretical_loss": 3.459072102680706, + "tokens_seen": 1998520320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983550651955868, + "loss": 1.3323, + "theoretical_loss": 3.4590627217358523, + "tokens_seen": 1998585856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039833500501504517, + "loss": 1.4813, + "theoretical_loss": 3.4590533411847346, + "tokens_seen": 1998651392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039831494483450353, + "loss": 1.2905, + "theoretical_loss": 3.459043961027322, + "tokens_seen": 1998716928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982948846539619, + "loss": 1.3933, + "theoretical_loss": 3.4590345812635865, + "tokens_seen": 1998782464 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039827482447342026, + "loss": 1.2936, + "theoretical_loss": 3.459025201893499, + "tokens_seen": 1998848000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039825476429287867, + "loss": 1.3192, + "theoretical_loss": 3.459015822917028, + "tokens_seen": 1998913536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039823470411233703, + "loss": 1.4075, + "theoretical_loss": 3.459006444334146, + "tokens_seen": 1998979072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982146439317954, + "loss": 1.3472, + "theoretical_loss": 3.4589970661448226, + "tokens_seen": 1999044608 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039819458375125376, + "loss": 1.414, + "theoretical_loss": 3.458987688349029, + "tokens_seen": 1999110144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981745235707122, + "loss": 1.2937, + "theoretical_loss": 3.458978310946735, + "tokens_seen": 1999175680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039815446339017054, + "loss": 1.3359, + "theoretical_loss": 3.4589689339379124, + "tokens_seen": 1999241216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981344032096289, + "loss": 1.3613, + "theoretical_loss": 3.4589595573225305, + "tokens_seen": 1999306752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039811434302908726, + "loss": 1.3896, + "theoretical_loss": 3.458950181100561, + "tokens_seen": 1999372288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980942828485457, + "loss": 1.4136, + "theoretical_loss": 3.4589408052719737, + "tokens_seen": 1999437824 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039807422266800404, + "loss": 1.2264, + "theoretical_loss": 3.4589314298367393, + "tokens_seen": 1999503360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980541624874624, + "loss": 1.3188, + "theoretical_loss": 3.4589220547948285, + "tokens_seen": 1999568896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039803410230692077, + "loss": 1.4333, + "theoretical_loss": 3.4589126801462124, + "tokens_seen": 1999634432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039801404212637913, + "loss": 1.2825, + "theoretical_loss": 3.458903305890861, + "tokens_seen": 1999699968 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039799398194583754, + "loss": 1.5012, + "theoretical_loss": 3.4588939320287455, + "tokens_seen": 1999765504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979739217652959, + "loss": 1.4465, + "theoretical_loss": 3.458884558559835, + "tokens_seen": 1999831040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039795386158475427, + "loss": 1.4439, + "theoretical_loss": 3.4588751854841027, + "tokens_seen": 1999896576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039793380140421263, + "loss": 1.426, + "theoretical_loss": 3.4588658128015166, + "tokens_seen": 1999962112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039791374122367105, + "loss": 1.241, + "theoretical_loss": 3.458856440512049, + "tokens_seen": 2000027648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978936810431294, + "loss": 1.3287, + "theoretical_loss": 3.45884706861567, + "tokens_seen": 2000093184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039787362086258777, + "loss": 1.2514, + "theoretical_loss": 3.4588376971123505, + "tokens_seen": 2000158720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039785356068204613, + "loss": 1.4046, + "theoretical_loss": 3.4588283260020605, + "tokens_seen": 2000224256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978335005015045, + "loss": 1.3508, + "theoretical_loss": 3.4588189552847712, + "tokens_seen": 2000289792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978134403209629, + "loss": 1.3497, + "theoretical_loss": 3.458809584960453, + "tokens_seen": 2000355328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977933801404213, + "loss": 1.3653, + "theoretical_loss": 3.458800215029077, + "tokens_seen": 2000420864 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039777331995987964, + "loss": 1.45, + "theoretical_loss": 3.458790845490613, + "tokens_seen": 2000486400 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397753259779338, + "loss": 1.3146, + "theoretical_loss": 3.458781476345033, + "tokens_seen": 2000551936 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 729542, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.47444748878479, + "objective/train/theoretical_loss": 3.4587721075923055, + "objective/train/tokens_used": 371076576, + "theoretical_loss": 3.4587721075923055, + "tokens_seen": 2000617472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977331995987964, + "loss": 1.3538, + "theoretical_loss": 3.4587721075923055, + "tokens_seen": 2000617472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977131394182548, + "loss": 1.4346, + "theoretical_loss": 3.4587627392324034, + "tokens_seen": 2000683008 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039769307923771314, + "loss": 1.3217, + "theoretical_loss": 3.458753371265296, + "tokens_seen": 2000748544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976730190571715, + "loss": 1.4032, + "theoretical_loss": 3.4587440036909545, + "tokens_seen": 2000814080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039765295887662987, + "loss": 1.2932, + "theoretical_loss": 3.4587346365093494, + "tokens_seen": 2000879616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976328986960883, + "loss": 1.3539, + "theoretical_loss": 3.4587252697204516, + "tokens_seen": 2000945152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039761283851554664, + "loss": 1.3534, + "theoretical_loss": 3.458715903324231, + "tokens_seen": 2001010688 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397592778335005, + "loss": 1.3157, + "theoretical_loss": 3.45870653732066, + "tokens_seen": 2001076224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039757271815446337, + "loss": 1.2682, + "theoretical_loss": 3.4586971717097077, + "tokens_seen": 2001141760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975526579739218, + "loss": 1.4524, + "theoretical_loss": 3.4586878064913447, + "tokens_seen": 2001207296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039753259779338015, + "loss": 1.2751, + "theoretical_loss": 3.4586784416655423, + "tokens_seen": 2001272832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975125376128385, + "loss": 1.4064, + "theoretical_loss": 3.4586690772322717, + "tokens_seen": 2001338368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039749247743229687, + "loss": 1.4282, + "theoretical_loss": 3.4586597131915027, + "tokens_seen": 2001403904 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039747241725175523, + "loss": 1.4682, + "theoretical_loss": 3.4586503495432064, + "tokens_seen": 2001469440 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039745235707121365, + "loss": 1.3423, + "theoretical_loss": 3.458640986287353, + "tokens_seen": 2001534976 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397432296890672, + "loss": 1.3657, + "theoretical_loss": 3.4586316234239143, + "tokens_seen": 2001600512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974122367101304, + "loss": 1.4197, + "theoretical_loss": 3.45862226095286, + "tokens_seen": 2001666048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039739217652958874, + "loss": 1.2928, + "theoretical_loss": 3.458612898874161, + "tokens_seen": 2001731584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039737211634904715, + "loss": 1.4058, + "theoretical_loss": 3.458603537187788, + "tokens_seen": 2001797120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973520561685055, + "loss": 1.4261, + "theoretical_loss": 3.458594175893712, + "tokens_seen": 2001862656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973319959879639, + "loss": 1.3361, + "theoretical_loss": 3.458584814991904, + "tokens_seen": 2001928192 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039731193580742224, + "loss": 1.325, + "theoretical_loss": 3.4585754544823337, + "tokens_seen": 2001993728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972918756268806, + "loss": 1.3129, + "theoretical_loss": 3.458566094364973, + "tokens_seen": 2002059264 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397271815446339, + "loss": 1.3976, + "theoretical_loss": 3.4585567346397914, + "tokens_seen": 2002124800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972517552657974, + "loss": 1.2419, + "theoretical_loss": 3.458547375306761, + "tokens_seen": 2002190336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039723169508525574, + "loss": 1.2932, + "theoretical_loss": 3.4585380163658512, + "tokens_seen": 2002255872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972116349047141, + "loss": 1.4214, + "theoretical_loss": 3.4585286578170336, + "tokens_seen": 2002321408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971915747241725, + "loss": 1.3653, + "theoretical_loss": 3.4585192996602787, + "tokens_seen": 2002386944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971715145436309, + "loss": 1.3961, + "theoretical_loss": 3.4585099418955574, + "tokens_seen": 2002452480 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039715145436308925, + "loss": 1.3751, + "theoretical_loss": 3.45850058452284, + "tokens_seen": 2002518016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039713139418254766, + "loss": 1.3466, + "theoretical_loss": 3.4584912275420976, + "tokens_seen": 2002583552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971113340020061, + "loss": 1.319, + "theoretical_loss": 3.458481870953301, + "tokens_seen": 2002649088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039709127382146444, + "loss": 1.4854, + "theoretical_loss": 3.4584725147564206, + "tokens_seen": 2002714624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970712136409228, + "loss": 1.4173, + "theoretical_loss": 3.4584631589514276, + "tokens_seen": 2002780160 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039705115346038117, + "loss": 1.3369, + "theoretical_loss": 3.4584538035382923, + "tokens_seen": 2002845696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039703109327983953, + "loss": 1.3493, + "theoretical_loss": 3.4584444485169863, + "tokens_seen": 2002911232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039701103309929794, + "loss": 1.2424, + "theoretical_loss": 3.4584350938874797, + "tokens_seen": 2002976768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969909729187563, + "loss": 1.4301, + "theoretical_loss": 3.4584257396497433, + "tokens_seen": 2003042304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039697091273821467, + "loss": 1.3657, + "theoretical_loss": 3.4584163858037478, + "tokens_seen": 2003107840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039695085255767303, + "loss": 1.3571, + "theoretical_loss": 3.458407032349464, + "tokens_seen": 2003173376 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039693079237713145, + "loss": 1.3425, + "theoretical_loss": 3.458397679286863, + "tokens_seen": 2003238912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969107321965898, + "loss": 1.4178, + "theoretical_loss": 3.4583883266159154, + "tokens_seen": 2003304448 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039689067201604817, + "loss": 1.3632, + "theoretical_loss": 3.458378974336592, + "tokens_seen": 2003369984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039687061183550653, + "loss": 1.4131, + "theoretical_loss": 3.4583696224488634, + "tokens_seen": 2003435520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968505516549649, + "loss": 1.4024, + "theoretical_loss": 3.458360270952701, + "tokens_seen": 2003501056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968304914744233, + "loss": 1.3462, + "theoretical_loss": 3.4583509198480744, + "tokens_seen": 2003566592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968104312938817, + "loss": 1.3712, + "theoretical_loss": 3.458341569134956, + "tokens_seen": 2003632128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039679037111334004, + "loss": 1.3217, + "theoretical_loss": 3.4583322188133154, + "tokens_seen": 2003697664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967703109327984, + "loss": 1.3755, + "theoretical_loss": 3.458322868883124, + "tokens_seen": 2003763200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967502507522568, + "loss": 1.3514, + "theoretical_loss": 3.458313519344352, + "tokens_seen": 2003828736 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 730811, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5580706596374512, + "objective/train/theoretical_loss": 3.4583041701969712, + "objective/train/tokens_used": 374353376, + "theoretical_loss": 3.4583041701969712, + "tokens_seen": 2003894272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967301905717152, + "loss": 1.4292, + "theoretical_loss": 3.4583041701969712, + "tokens_seen": 2003894272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039671013039117354, + "loss": 1.5256, + "theoretical_loss": 3.4582948214409512, + "tokens_seen": 2003959808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966900702106319, + "loss": 1.3629, + "theoretical_loss": 3.4582854730762635, + "tokens_seen": 2004025344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039667001003009027, + "loss": 1.3278, + "theoretical_loss": 3.4582761251028793, + "tokens_seen": 2004090880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966499498495487, + "loss": 1.3935, + "theoretical_loss": 3.4582667775207687, + "tokens_seen": 2004156416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039662988966900704, + "loss": 1.3684, + "theoretical_loss": 3.458257430329903, + "tokens_seen": 2004221952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966098294884654, + "loss": 1.3354, + "theoretical_loss": 3.4582480835302527, + "tokens_seen": 2004287488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039658976930792377, + "loss": 1.3821, + "theoretical_loss": 3.458238737121789, + "tokens_seen": 2004353024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965697091273822, + "loss": 1.4094, + "theoretical_loss": 3.4582293911044824, + "tokens_seen": 2004418560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039654964894684055, + "loss": 1.3391, + "theoretical_loss": 3.4582200454783036, + "tokens_seen": 2004484096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965295887662989, + "loss": 1.3748, + "theoretical_loss": 3.4582107002432236, + "tokens_seen": 2004549632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039650952858575727, + "loss": 1.3999, + "theoretical_loss": 3.458201355399214, + "tokens_seen": 2004615168 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039648946840521563, + "loss": 1.3592, + "theoretical_loss": 3.4581920109462443, + "tokens_seen": 2004680704 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039646940822467405, + "loss": 1.3163, + "theoretical_loss": 3.4581826668842863, + "tokens_seen": 2004746240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964493480441324, + "loss": 1.3373, + "theoretical_loss": 3.4581733232133107, + "tokens_seen": 2004811776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964292878635908, + "loss": 1.4029, + "theoretical_loss": 3.4581639799332886, + "tokens_seen": 2004877312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039640922768304914, + "loss": 1.3991, + "theoretical_loss": 3.4581546370441902, + "tokens_seen": 2004942848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039638916750250755, + "loss": 1.3212, + "theoretical_loss": 3.4581452945459867, + "tokens_seen": 2005008384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963691073219659, + "loss": 1.3337, + "theoretical_loss": 3.4581359524386492, + "tokens_seen": 2005073920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963490471414243, + "loss": 1.4318, + "theoretical_loss": 3.4581266107221484, + "tokens_seen": 2005139456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039632898696088264, + "loss": 1.4976, + "theoretical_loss": 3.458117269396455, + "tokens_seen": 2005204992 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396308926780341, + "loss": 1.3582, + "theoretical_loss": 3.4581079284615397, + "tokens_seen": 2005270528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962888665997994, + "loss": 1.2979, + "theoretical_loss": 3.458098587917374, + "tokens_seen": 2005336064 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962688064192578, + "loss": 1.3085, + "theoretical_loss": 3.4580892477639282, + "tokens_seen": 2005401600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039624874623871614, + "loss": 1.4137, + "theoretical_loss": 3.4580799080011735, + "tokens_seen": 2005467136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962286860581745, + "loss": 1.3146, + "theoretical_loss": 3.458070568629081, + "tokens_seen": 2005532672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962086258776329, + "loss": 1.3594, + "theoretical_loss": 3.4580612296476207, + "tokens_seen": 2005598208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961885656970913, + "loss": 1.3144, + "theoretical_loss": 3.458051891056765, + "tokens_seen": 2005663744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039616850551654965, + "loss": 1.4711, + "theoretical_loss": 3.458042552856484, + "tokens_seen": 2005729280 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396148445336008, + "loss": 1.4839, + "theoretical_loss": 3.458033215046748, + "tokens_seen": 2005794816 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039612838515546637, + "loss": 1.1741, + "theoretical_loss": 3.4580238776275283, + "tokens_seen": 2005860352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961083249749248, + "loss": 1.3387, + "theoretical_loss": 3.4580145405987963, + "tokens_seen": 2005925888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039608826479438315, + "loss": 1.4007, + "theoretical_loss": 3.4580052039605222, + "tokens_seen": 2005991424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960682046138415, + "loss": 1.4019, + "theoretical_loss": 3.457995867712678, + "tokens_seen": 2006056960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960481444332999, + "loss": 1.3817, + "theoretical_loss": 3.4579865318552336, + "tokens_seen": 2006122496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960280842527583, + "loss": 1.3628, + "theoretical_loss": 3.45797719638816, + "tokens_seen": 2006188032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039600802407221665, + "loss": 1.2832, + "theoretical_loss": 3.4579678613114284, + "tokens_seen": 2006253568 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395987963891675, + "loss": 1.3931, + "theoretical_loss": 3.45795852662501, + "tokens_seen": 2006319104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959679037111334, + "loss": 1.2503, + "theoretical_loss": 3.457949192328875, + "tokens_seen": 2006384640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959478435305918, + "loss": 1.3403, + "theoretical_loss": 3.4579398584229954, + "tokens_seen": 2006450176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039592778335005016, + "loss": 1.4214, + "theoretical_loss": 3.457930524907341, + "tokens_seen": 2006515712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959077231695085, + "loss": 1.3907, + "theoretical_loss": 3.457921191781883, + "tokens_seen": 2006581248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958876629889669, + "loss": 1.3735, + "theoretical_loss": 3.457911859046593, + "tokens_seen": 2006646784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039586760280842524, + "loss": 1.3382, + "theoretical_loss": 3.4579025267014414, + "tokens_seen": 2006712320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039584754262788366, + "loss": 1.4067, + "theoretical_loss": 3.4578931947464, + "tokens_seen": 2006777856 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395827482447342, + "loss": 1.3581, + "theoretical_loss": 3.4578838631814377, + "tokens_seen": 2006843392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958074222668004, + "loss": 1.1904, + "theoretical_loss": 3.457874532006527, + "tokens_seen": 2006908928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039578736208625875, + "loss": 1.3163, + "theoretical_loss": 3.4578652012216393, + "tokens_seen": 2006974464 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039576730190571716, + "loss": 1.3017, + "theoretical_loss": 3.457855870826745, + "tokens_seen": 2007040000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957472417251755, + "loss": 1.3272, + "theoretical_loss": 3.4578465408218144, + "tokens_seen": 2007105536 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 732034, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3788890838623047, + "objective/train/theoretical_loss": 3.4578372112068196, + "objective/train/tokens_used": 377630176, + "theoretical_loss": 3.4578372112068196, + "tokens_seen": 2007171072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957271815446339, + "loss": 1.4508, + "theoretical_loss": 3.4578372112068196, + "tokens_seen": 2007171072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039570712136409225, + "loss": 1.2763, + "theoretical_loss": 3.4578278819817303, + "tokens_seen": 2007236608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956870611835506, + "loss": 1.5464, + "theoretical_loss": 3.4578185531465184, + "tokens_seen": 2007302144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039566700100300903, + "loss": 1.3669, + "theoretical_loss": 3.457809224701155, + "tokens_seen": 2007367680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956469408224674, + "loss": 1.3768, + "theoretical_loss": 3.4577998966456107, + "tokens_seen": 2007433216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956268806419258, + "loss": 1.2839, + "theoretical_loss": 3.4577905689798563, + "tokens_seen": 2007498752 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039560682046138417, + "loss": 1.4591, + "theoretical_loss": 3.4577812417038634, + "tokens_seen": 2007564288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955867602808426, + "loss": 1.461, + "theoretical_loss": 3.4577719148176023, + "tokens_seen": 2007629824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039556670010030095, + "loss": 1.2743, + "theoretical_loss": 3.457762588321044, + "tokens_seen": 2007695360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955466399197593, + "loss": 1.4023, + "theoretical_loss": 3.4577532622141605, + "tokens_seen": 2007760896 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039552657973921767, + "loss": 1.2974, + "theoretical_loss": 3.4577439364969216, + "tokens_seen": 2007826432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039550651955867603, + "loss": 1.3581, + "theoretical_loss": 3.4577346111692995, + "tokens_seen": 2007891968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039548645937813445, + "loss": 1.2333, + "theoretical_loss": 3.4577252862312635, + "tokens_seen": 2007957504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954663991975928, + "loss": 1.3381, + "theoretical_loss": 3.457715961682786, + "tokens_seen": 2008023040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954463390170512, + "loss": 1.1589, + "theoretical_loss": 3.457706637523838, + "tokens_seen": 2008088576 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039542627883650954, + "loss": 1.4452, + "theoretical_loss": 3.45769731375439, + "tokens_seen": 2008154112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039540621865596795, + "loss": 1.375, + "theoretical_loss": 3.457687990374413, + "tokens_seen": 2008219648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953861584754263, + "loss": 1.2629, + "theoretical_loss": 3.4576786673838784, + "tokens_seen": 2008285184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953660982948847, + "loss": 1.342, + "theoretical_loss": 3.457669344782757, + "tokens_seen": 2008350720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039534603811434304, + "loss": 1.4123, + "theoretical_loss": 3.4576600225710195, + "tokens_seen": 2008416256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953259779338014, + "loss": 1.3774, + "theoretical_loss": 3.457650700748638, + "tokens_seen": 2008481792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953059177532598, + "loss": 1.2621, + "theoretical_loss": 3.4576413793155822, + "tokens_seen": 2008547328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952858575727182, + "loss": 1.4589, + "theoretical_loss": 3.457632058271824, + "tokens_seen": 2008612864 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039526579739217654, + "loss": 1.3776, + "theoretical_loss": 3.457622737617334, + "tokens_seen": 2008678400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952457372116349, + "loss": 1.3602, + "theoretical_loss": 3.4576134173520834, + "tokens_seen": 2008743936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952256770310933, + "loss": 1.301, + "theoretical_loss": 3.457604097476043, + "tokens_seen": 2008809472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952056168505517, + "loss": 1.3629, + "theoretical_loss": 3.457594777989185, + "tokens_seen": 2008875008 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039518555667001005, + "loss": 1.3474, + "theoretical_loss": 3.457585458891479, + "tokens_seen": 2008940544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951654964894684, + "loss": 1.386, + "theoretical_loss": 3.457576140182897, + "tokens_seen": 2009006080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039514543630892677, + "loss": 1.3161, + "theoretical_loss": 3.4575668218634092, + "tokens_seen": 2009071616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951253761283852, + "loss": 1.4063, + "theoretical_loss": 3.4575575039329873, + "tokens_seen": 2009137152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039510531594784355, + "loss": 1.2974, + "theoretical_loss": 3.457548186391602, + "tokens_seen": 2009202688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950852557673019, + "loss": 1.4151, + "theoretical_loss": 3.457538869239225, + "tokens_seen": 2009268224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950651955867603, + "loss": 1.32, + "theoretical_loss": 3.457529552475827, + "tokens_seen": 2009333760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950451354062187, + "loss": 1.4202, + "theoretical_loss": 3.457520236101379, + "tokens_seen": 2009399296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039502507522567705, + "loss": 1.4361, + "theoretical_loss": 3.457510920115852, + "tokens_seen": 2009464832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950050150451354, + "loss": 1.4331, + "theoretical_loss": 3.4575016045192175, + "tokens_seen": 2009530368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949849548645938, + "loss": 1.3594, + "theoretical_loss": 3.457492289311446, + "tokens_seen": 2009595904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949648946840522, + "loss": 1.4339, + "theoretical_loss": 3.4574829744925086, + "tokens_seen": 2009661440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039494483450351056, + "loss": 1.3886, + "theoretical_loss": 3.4574736600623774, + "tokens_seen": 2009726976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949247743229689, + "loss": 1.4655, + "theoretical_loss": 3.457464346021022, + "tokens_seen": 2009792512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949047141424273, + "loss": 1.3693, + "theoretical_loss": 3.4574550323684146, + "tokens_seen": 2009858048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039488465396188564, + "loss": 1.4258, + "theoretical_loss": 3.457445719104526, + "tokens_seen": 2009923584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039486459378134406, + "loss": 1.353, + "theoretical_loss": 3.457436406229327, + "tokens_seen": 2009989120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948445336008024, + "loss": 1.3093, + "theoretical_loss": 3.457427093742789, + "tokens_seen": 2010054656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948244734202608, + "loss": 1.3077, + "theoretical_loss": 3.4574177816448834, + "tokens_seen": 2010120192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039480441323971915, + "loss": 1.3563, + "theoretical_loss": 3.4574084699355807, + "tokens_seen": 2010185728 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039478435305917756, + "loss": 1.3726, + "theoretical_loss": 3.4573991586148525, + "tokens_seen": 2010251264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947642928786359, + "loss": 1.4283, + "theoretical_loss": 3.4573898476826694, + "tokens_seen": 2010316800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947442326980943, + "loss": 1.3414, + "theoretical_loss": 3.457380537139003, + "tokens_seen": 2010382336 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 733267, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.300271987915039, + "objective/train/theoretical_loss": 3.457371226983824, + "objective/train/tokens_used": 380906976, + "theoretical_loss": 3.457371226983824, + "tokens_seen": 2010447872 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039472417251755265, + "loss": 1.3223, + "theoretical_loss": 3.457371226983824, + "tokens_seen": 2010447872 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394704112337011, + "loss": 1.3571, + "theoretical_loss": 3.457361917217104, + "tokens_seen": 2010513408 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039468405215646943, + "loss": 1.3317, + "theoretical_loss": 3.457352607838814, + "tokens_seen": 2010578944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946639919759278, + "loss": 1.3115, + "theoretical_loss": 3.457343298848925, + "tokens_seen": 2010644480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039464393179538615, + "loss": 1.3378, + "theoretical_loss": 3.4573339902474083, + "tokens_seen": 2010710016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946238716148445, + "loss": 1.263, + "theoretical_loss": 3.4573246820342347, + "tokens_seen": 2010775552 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039460381143430293, + "loss": 1.3437, + "theoretical_loss": 3.457315374209376, + "tokens_seen": 2010841088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945837512537613, + "loss": 1.3889, + "theoretical_loss": 3.4573060667728024, + "tokens_seen": 2010906624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039456369107321966, + "loss": 1.3939, + "theoretical_loss": 3.4572967597244855, + "tokens_seen": 2010972160 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394543630892678, + "loss": 1.3829, + "theoretical_loss": 3.4572874530643967, + "tokens_seen": 2011037696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945235707121364, + "loss": 1.4263, + "theoretical_loss": 3.4572781467925067, + "tokens_seen": 2011103232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945035105315948, + "loss": 1.3027, + "theoretical_loss": 3.4572688409087875, + "tokens_seen": 2011168768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039448345035105316, + "loss": 1.3938, + "theoretical_loss": 3.4572595354132094, + "tokens_seen": 2011234304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944633901705115, + "loss": 1.4077, + "theoretical_loss": 3.4572502303057435, + "tokens_seen": 2011299840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944433299899699, + "loss": 1.3055, + "theoretical_loss": 3.457240925586362, + "tokens_seen": 2011365376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944232698094283, + "loss": 1.2892, + "theoretical_loss": 3.457231621255035, + "tokens_seen": 2011430912 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039440320962888666, + "loss": 1.3718, + "theoretical_loss": 3.457222317311734, + "tokens_seen": 2011496448 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394383149448345, + "loss": 1.3397, + "theoretical_loss": 3.45721301375643, + "tokens_seen": 2011561984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943630892678034, + "loss": 1.4151, + "theoretical_loss": 3.457203710589095, + "tokens_seen": 2011627520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039434302908726175, + "loss": 1.2817, + "theoretical_loss": 3.457194407809699, + "tokens_seen": 2011693056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039432296890672017, + "loss": 1.2498, + "theoretical_loss": 3.4571851054182146, + "tokens_seen": 2011758592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039430290872617853, + "loss": 1.3338, + "theoretical_loss": 3.4571758034146116, + "tokens_seen": 2011824128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942828485456369, + "loss": 1.3969, + "theoretical_loss": 3.457166501798862, + "tokens_seen": 2011889664 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039426278836509525, + "loss": 1.3241, + "theoretical_loss": 3.4571572005709363, + "tokens_seen": 2011955200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039424272818455367, + "loss": 1.377, + "theoretical_loss": 3.4571478997308063, + "tokens_seen": 2012020736 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039422266800401203, + "loss": 1.3231, + "theoretical_loss": 3.4571385992784434, + "tokens_seen": 2012086272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942026078234704, + "loss": 1.4517, + "theoretical_loss": 3.4571292992138183, + "tokens_seen": 2012151808 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039418254764292876, + "loss": 1.2457, + "theoretical_loss": 3.457119999536902, + "tokens_seen": 2012217344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941624874623871, + "loss": 1.3541, + "theoretical_loss": 3.4571107002476666, + "tokens_seen": 2012282880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941424272818456, + "loss": 1.3976, + "theoretical_loss": 3.457101401346083, + "tokens_seen": 2012348416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039412236710130395, + "loss": 1.3696, + "theoretical_loss": 3.4570921028321218, + "tokens_seen": 2012413952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941023069207623, + "loss": 1.3038, + "theoretical_loss": 3.4570828047057542, + "tokens_seen": 2012479488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940822467402207, + "loss": 1.2962, + "theoretical_loss": 3.4570735069669523, + "tokens_seen": 2012545024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940621865596791, + "loss": 1.4073, + "theoretical_loss": 3.457064209615687, + "tokens_seen": 2012610560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039404212637913745, + "loss": 1.3706, + "theoretical_loss": 3.4570549126519294, + "tokens_seen": 2012676096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940220661985958, + "loss": 1.3941, + "theoretical_loss": 3.457045616075651, + "tokens_seen": 2012741632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940020060180542, + "loss": 1.3407, + "theoretical_loss": 3.457036319886822, + "tokens_seen": 2012807168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939819458375126, + "loss": 1.3197, + "theoretical_loss": 3.4570270240854146, + "tokens_seen": 2012872704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039396188565697096, + "loss": 1.3686, + "theoretical_loss": 3.4570177286714, + "tokens_seen": 2012938240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939418254764293, + "loss": 1.3787, + "theoretical_loss": 3.4570084336447495, + "tokens_seen": 2013003776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939217652958877, + "loss": 1.2409, + "theoretical_loss": 3.4569991390054335, + "tokens_seen": 2013069312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039390170511534604, + "loss": 1.2971, + "theoretical_loss": 3.4569898447534246, + "tokens_seen": 2013134848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039388164493480446, + "loss": 1.3715, + "theoretical_loss": 3.4569805508886935, + "tokens_seen": 2013200384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938615847542628, + "loss": 1.3844, + "theoretical_loss": 3.456971257411211, + "tokens_seen": 2013265920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938415245737212, + "loss": 1.3091, + "theoretical_loss": 3.456961964320948, + "tokens_seen": 2013331456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039382146439317955, + "loss": 1.3863, + "theoretical_loss": 3.456952671617877, + "tokens_seen": 2013396992 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039380140421263796, + "loss": 1.4279, + "theoretical_loss": 3.4569433793019693, + "tokens_seen": 2013462528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937813440320963, + "loss": 1.2822, + "theoretical_loss": 3.4569340873731944, + "tokens_seen": 2013528064 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937612838515547, + "loss": 1.2731, + "theoretical_loss": 3.4569247958315255, + "tokens_seen": 2013593600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039374122367101305, + "loss": 1.385, + "theoretical_loss": 3.4569155046769326, + "tokens_seen": 2013659136 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 734578, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.427262544631958, + "objective/train/theoretical_loss": 3.456906213909388, + "objective/train/tokens_used": 384183776, + "theoretical_loss": 3.456906213909388, + "tokens_seen": 2013724672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937211634904714, + "loss": 1.4218, + "theoretical_loss": 3.456906213909388, + "tokens_seen": 2013724672 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039370110330992983, + "loss": 1.4594, + "theoretical_loss": 3.4568969235288627, + "tokens_seen": 2013790208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936810431293882, + "loss": 1.2872, + "theoretical_loss": 3.456887633535327, + "tokens_seen": 2013855744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039366098294884655, + "loss": 1.2838, + "theoretical_loss": 3.456878343928753, + "tokens_seen": 2013921280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936409227683049, + "loss": 1.3751, + "theoretical_loss": 3.456869054709112, + "tokens_seen": 2013986816 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039362086258776333, + "loss": 1.3387, + "theoretical_loss": 3.4568597658763753, + "tokens_seen": 2014052352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936008024072217, + "loss": 1.3155, + "theoretical_loss": 3.4568504774305144, + "tokens_seen": 2014117888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039358074222668006, + "loss": 1.3978, + "theoretical_loss": 3.4568411893715, + "tokens_seen": 2014183424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935606820461384, + "loss": 1.4352, + "theoretical_loss": 3.456831901699304, + "tokens_seen": 2014248960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935406218655968, + "loss": 1.3223, + "theoretical_loss": 3.4568226144138974, + "tokens_seen": 2014314496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935205616850552, + "loss": 1.2445, + "theoretical_loss": 3.4568133275152513, + "tokens_seen": 2014380032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039350050150451356, + "loss": 1.4241, + "theoretical_loss": 3.456804041003337, + "tokens_seen": 2014445568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934804413239719, + "loss": 1.286, + "theoretical_loss": 3.4567947548781266, + "tokens_seen": 2014511104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934603811434303, + "loss": 1.4323, + "theoretical_loss": 3.4567854691395903, + "tokens_seen": 2014576640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934403209628887, + "loss": 1.3695, + "theoretical_loss": 3.4567761837877002, + "tokens_seen": 2014642176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039342026078234706, + "loss": 1.3333, + "theoretical_loss": 3.456766898822428, + "tokens_seen": 2014707712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934002006018054, + "loss": 1.3333, + "theoretical_loss": 3.456757614243744, + "tokens_seen": 2014773248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933801404212638, + "loss": 1.2941, + "theoretical_loss": 3.45674833005162, + "tokens_seen": 2014838784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039336008024072215, + "loss": 1.3604, + "theoretical_loss": 3.4567390462460272, + "tokens_seen": 2014904320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039334002006018057, + "loss": 1.2572, + "theoretical_loss": 3.456729762826937, + "tokens_seen": 2014969856 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039331995987963893, + "loss": 1.2464, + "theoretical_loss": 3.456720479794321, + "tokens_seen": 2015035392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932998996990973, + "loss": 1.3362, + "theoretical_loss": 3.45671119714815, + "tokens_seen": 2015100928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039327983951855565, + "loss": 1.4009, + "theoretical_loss": 3.456701914888396, + "tokens_seen": 2015166464 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039325977933801407, + "loss": 1.3729, + "theoretical_loss": 3.45669263301503, + "tokens_seen": 2015232000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039323971915747243, + "loss": 1.2614, + "theoretical_loss": 3.4566833515280235, + "tokens_seen": 2015297536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932196589769308, + "loss": 1.3329, + "theoretical_loss": 3.456674070427347, + "tokens_seen": 2015363072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039319959879638916, + "loss": 1.3501, + "theoretical_loss": 3.456664789712973, + "tokens_seen": 2015428608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931795386158475, + "loss": 1.4081, + "theoretical_loss": 3.4566555093848725, + "tokens_seen": 2015494144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039315947843530593, + "loss": 1.2733, + "theoretical_loss": 3.456646229443017, + "tokens_seen": 2015559680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931394182547643, + "loss": 1.3006, + "theoretical_loss": 3.4566369498873777, + "tokens_seen": 2015625216 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039311935807422266, + "loss": 1.368, + "theoretical_loss": 3.4566276707179258, + "tokens_seen": 2015690752 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393099297893681, + "loss": 1.3525, + "theoretical_loss": 3.456618391934633, + "tokens_seen": 2015756288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039307923771313944, + "loss": 1.295, + "theoretical_loss": 3.45660911353747, + "tokens_seen": 2015821824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930591775325978, + "loss": 1.4187, + "theoretical_loss": 3.4565998355264087, + "tokens_seen": 2015887360 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039303911735205616, + "loss": 1.3145, + "theoretical_loss": 3.456590557901421, + "tokens_seen": 2015952896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930190571715145, + "loss": 1.4118, + "theoretical_loss": 3.456581280662477, + "tokens_seen": 2016018432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929989969909729, + "loss": 1.3591, + "theoretical_loss": 3.4565720038095495, + "tokens_seen": 2016083968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929789368104313, + "loss": 1.3329, + "theoretical_loss": 3.456562727342609, + "tokens_seen": 2016149504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039295887662988967, + "loss": 1.3288, + "theoretical_loss": 3.4565534512616267, + "tokens_seen": 2016215040 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039293881644934803, + "loss": 1.3979, + "theoretical_loss": 3.4565441755665747, + "tokens_seen": 2016280576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929187562688064, + "loss": 1.3076, + "theoretical_loss": 3.4565349002574246, + "tokens_seen": 2016346112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928986960882648, + "loss": 1.2742, + "theoretical_loss": 3.456525625334147, + "tokens_seen": 2016411648 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039287863590772317, + "loss": 1.3801, + "theoretical_loss": 3.456516350796713, + "tokens_seen": 2016477184 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039285857572718153, + "loss": 1.3603, + "theoretical_loss": 3.4565070766450954, + "tokens_seen": 2016542720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928385155466399, + "loss": 1.3577, + "theoretical_loss": 3.4564978028792646, + "tokens_seen": 2016608256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039281845536609825, + "loss": 1.3676, + "theoretical_loss": 3.4564885294991923, + "tokens_seen": 2016673792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039279839518555667, + "loss": 1.3992, + "theoretical_loss": 3.4564792565048497, + "tokens_seen": 2016739328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039277833500501503, + "loss": 1.4252, + "theoretical_loss": 3.4564699838962083, + "tokens_seen": 2016804864 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927582748244734, + "loss": 1.35, + "theoretical_loss": 3.45646071167324, + "tokens_seen": 2016870400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039273821464393176, + "loss": 1.4746, + "theoretical_loss": 3.4564514398359156, + "tokens_seen": 2016935936 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 735776, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.506068468093872, + "objective/train/theoretical_loss": 3.456442168384207, + "objective/train/tokens_used": 387460576, + "theoretical_loss": 3.456442168384207, + "tokens_seen": 2017001472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927181544633902, + "loss": 1.4847, + "theoretical_loss": 3.456442168384207, + "tokens_seen": 2017001472 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039269809428284854, + "loss": 1.2308, + "theoretical_loss": 3.456432897318085, + "tokens_seen": 2017067008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926780341023069, + "loss": 1.3872, + "theoretical_loss": 3.456423626637522, + "tokens_seen": 2017132544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039265797392176526, + "loss": 1.4539, + "theoretical_loss": 3.4564143563424885, + "tokens_seen": 2017198080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039263791374122373, + "loss": 1.3045, + "theoretical_loss": 3.456405086432956, + "tokens_seen": 2017263616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926178535606821, + "loss": 1.3124, + "theoretical_loss": 3.456395816908897, + "tokens_seen": 2017329152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039259779338014046, + "loss": 1.389, + "theoretical_loss": 3.4563865477702818, + "tokens_seen": 2017394688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925777331995988, + "loss": 1.4732, + "theoretical_loss": 3.4563772790170826, + "tokens_seen": 2017460224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925576730190572, + "loss": 1.3558, + "theoretical_loss": 3.4563680106492702, + "tokens_seen": 2017525760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925376128385156, + "loss": 1.2866, + "theoretical_loss": 3.4563587426668168, + "tokens_seen": 2017591296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039251755265797396, + "loss": 1.2783, + "theoretical_loss": 3.456349475069693, + "tokens_seen": 2017656832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924974924774323, + "loss": 1.3588, + "theoretical_loss": 3.4563402078578704, + "tokens_seen": 2017722368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924774322968907, + "loss": 1.3484, + "theoretical_loss": 3.4563309410313217, + "tokens_seen": 2017787904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924573721163491, + "loss": 1.3495, + "theoretical_loss": 3.4563216745900167, + "tokens_seen": 2017853440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039243731193580746, + "loss": 1.2857, + "theoretical_loss": 3.456312408533928, + "tokens_seen": 2017918976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924172517552658, + "loss": 1.4029, + "theoretical_loss": 3.456303142863027, + "tokens_seen": 2017984512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923971915747242, + "loss": 1.2003, + "theoretical_loss": 3.4562938775772842, + "tokens_seen": 2018050048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039237713139418255, + "loss": 1.4004, + "theoretical_loss": 3.456284612676672, + "tokens_seen": 2018115584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039235707121364097, + "loss": 1.2417, + "theoretical_loss": 3.456275348161162, + "tokens_seen": 2018181120 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039233701103309933, + "loss": 1.4594, + "theoretical_loss": 3.456266084030725, + "tokens_seen": 2018246656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923169508525577, + "loss": 1.2536, + "theoretical_loss": 3.4562568202853328, + "tokens_seen": 2018312192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039229689067201605, + "loss": 1.4737, + "theoretical_loss": 3.456247556924957, + "tokens_seen": 2018377728 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039227683049147447, + "loss": 1.3414, + "theoretical_loss": 3.456238293949569, + "tokens_seen": 2018443264 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039225677031093283, + "loss": 1.3431, + "theoretical_loss": 3.4562290313591406, + "tokens_seen": 2018508800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922367101303912, + "loss": 1.4163, + "theoretical_loss": 3.4562197691536425, + "tokens_seen": 2018574336 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039221664994984956, + "loss": 1.3747, + "theoretical_loss": 3.456210507333047, + "tokens_seen": 2018639872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921965897693079, + "loss": 1.4036, + "theoretical_loss": 3.4562012458973257, + "tokens_seen": 2018705408 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039217652958876633, + "loss": 1.3448, + "theoretical_loss": 3.4561919848464493, + "tokens_seen": 2018770944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921564694082247, + "loss": 1.2858, + "theoretical_loss": 3.4561827241803895, + "tokens_seen": 2018836480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039213640922768306, + "loss": 1.3955, + "theoretical_loss": 3.456173463899119, + "tokens_seen": 2018902016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921163490471414, + "loss": 1.3994, + "theoretical_loss": 3.4561642040026075, + "tokens_seen": 2018967552 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039209628886659984, + "loss": 1.3622, + "theoretical_loss": 3.4561549444908275, + "tokens_seen": 2019033088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920762286860582, + "loss": 1.3141, + "theoretical_loss": 3.456145685363751, + "tokens_seen": 2019098624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039205616850551656, + "loss": 1.2607, + "theoretical_loss": 3.456136426621349, + "tokens_seen": 2019164160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920361083249749, + "loss": 1.3596, + "theoretical_loss": 3.456127168263593, + "tokens_seen": 2019229696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920160481444333, + "loss": 1.4247, + "theoretical_loss": 3.4561179102904536, + "tokens_seen": 2019295232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919959879638917, + "loss": 1.331, + "theoretical_loss": 3.4561086527019045, + "tokens_seen": 2019360768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039197592778335007, + "loss": 1.4164, + "theoretical_loss": 3.4560993954979153, + "tokens_seen": 2019426304 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039195586760280843, + "loss": 1.3072, + "theoretical_loss": 3.4560901386784586, + "tokens_seen": 2019491840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919358074222668, + "loss": 1.4174, + "theoretical_loss": 3.456080882243506, + "tokens_seen": 2019557376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919157472417252, + "loss": 1.2632, + "theoretical_loss": 3.456071626193028, + "tokens_seen": 2019622912 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039189568706118357, + "loss": 1.4112, + "theoretical_loss": 3.4560623705269973, + "tokens_seen": 2019688448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039187562688064193, + "loss": 1.3727, + "theoretical_loss": 3.4560531152453846, + "tokens_seen": 2019753984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918555667001003, + "loss": 1.3334, + "theoretical_loss": 3.4560438603481622, + "tokens_seen": 2019819520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039183550651955866, + "loss": 1.3373, + "theoretical_loss": 3.4560346058353018, + "tokens_seen": 2019885056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039181544633901707, + "loss": 1.3559, + "theoretical_loss": 3.4560253517067734, + "tokens_seen": 2019950592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039179538615847543, + "loss": 1.3793, + "theoretical_loss": 3.4560160979625505, + "tokens_seen": 2020016128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917753259779338, + "loss": 1.3522, + "theoretical_loss": 3.4560068446026033, + "tokens_seen": 2020081664 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039175526579739216, + "loss": 1.3314, + "theoretical_loss": 3.455997591626905, + "tokens_seen": 2020147200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917352056168506, + "loss": 1.3064, + "theoretical_loss": 3.455988339035425, + "tokens_seen": 2020212736 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 736878, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3108620643615723, + "objective/train/theoretical_loss": 3.455979086828136, + "objective/train/tokens_used": 390737376, + "theoretical_loss": 3.455979086828136, + "tokens_seen": 2020278272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039171514543630894, + "loss": 1.3245, + "theoretical_loss": 3.455979086828136, + "tokens_seen": 2020278272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916950852557673, + "loss": 1.2786, + "theoretical_loss": 3.45596983500501, + "tokens_seen": 2020343808 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039167502507522566, + "loss": 1.2895, + "theoretical_loss": 3.455960583566018, + "tokens_seen": 2020409344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916549648946841, + "loss": 1.2725, + "theoretical_loss": 3.4559513325111317, + "tokens_seen": 2020474880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039163490471414244, + "loss": 1.4011, + "theoretical_loss": 3.4559420818403233, + "tokens_seen": 2020540416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916148445336008, + "loss": 1.3641, + "theoretical_loss": 3.455932831553563, + "tokens_seen": 2020605952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039159478435305916, + "loss": 1.3793, + "theoretical_loss": 3.455923581650824, + "tokens_seen": 2020671488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915747241725175, + "loss": 1.4393, + "theoretical_loss": 3.4559143321320764, + "tokens_seen": 2020737024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039155466399197594, + "loss": 1.3359, + "theoretical_loss": 3.455905082997293, + "tokens_seen": 2020802560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915346038114343, + "loss": 1.3857, + "theoretical_loss": 3.4558958342464448, + "tokens_seen": 2020868096 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039151454363089267, + "loss": 1.4345, + "theoretical_loss": 3.4558865858795036, + "tokens_seen": 2020933632 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039149448345035103, + "loss": 1.3459, + "theoretical_loss": 3.4558773378964407, + "tokens_seen": 2020999168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039147442326980945, + "loss": 1.4263, + "theoretical_loss": 3.4558680902972285, + "tokens_seen": 2021064704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914543630892678, + "loss": 1.358, + "theoretical_loss": 3.455858843081838, + "tokens_seen": 2021130240 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039143430290872617, + "loss": 1.3971, + "theoretical_loss": 3.455849596250241, + "tokens_seen": 2021195776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039141424272818453, + "loss": 1.3009, + "theoretical_loss": 3.455840349802409, + "tokens_seen": 2021261312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913941825476429, + "loss": 1.3189, + "theoretical_loss": 3.455831103738314, + "tokens_seen": 2021326848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913741223671013, + "loss": 1.3372, + "theoretical_loss": 3.4558218580579263, + "tokens_seen": 2021392384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913540621865597, + "loss": 1.3047, + "theoretical_loss": 3.4558126127612194, + "tokens_seen": 2021457920 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039133400200601804, + "loss": 1.4054, + "theoretical_loss": 3.4558033678481643, + "tokens_seen": 2021523456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913139418254764, + "loss": 1.3177, + "theoretical_loss": 3.455794123318732, + "tokens_seen": 2021588992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912938816449348, + "loss": 1.3129, + "theoretical_loss": 3.455784879172895, + "tokens_seen": 2021654528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912738214643932, + "loss": 1.3085, + "theoretical_loss": 3.4557756354106246, + "tokens_seen": 2021720064 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039125376128385154, + "loss": 1.3978, + "theoretical_loss": 3.455766392031892, + "tokens_seen": 2021785600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912337011033099, + "loss": 1.3599, + "theoretical_loss": 3.4557571490366694, + "tokens_seen": 2021851136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039121364092276826, + "loss": 1.3401, + "theoretical_loss": 3.4557479064249286, + "tokens_seen": 2021916672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911935807422267, + "loss": 1.4223, + "theoretical_loss": 3.45573866419664, + "tokens_seen": 2021982208 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039117352056168504, + "loss": 1.4132, + "theoretical_loss": 3.4557294223517774, + "tokens_seen": 2022047744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911534603811434, + "loss": 1.2706, + "theoretical_loss": 3.455720180890311, + "tokens_seen": 2022113280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911334002006018, + "loss": 1.4531, + "theoretical_loss": 3.455710939812213, + "tokens_seen": 2022178816 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039111334002006024, + "loss": 1.4664, + "theoretical_loss": 3.455701699117454, + "tokens_seen": 2022244352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910932798395186, + "loss": 1.3775, + "theoretical_loss": 3.4556924588060074, + "tokens_seen": 2022309888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039107321965897696, + "loss": 1.3495, + "theoretical_loss": 3.4556832188778435, + "tokens_seen": 2022375424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910531594784353, + "loss": 1.4289, + "theoretical_loss": 3.4556739793329347, + "tokens_seen": 2022440960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910330992978937, + "loss": 1.3323, + "theoretical_loss": 3.455664740171253, + "tokens_seen": 2022506496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910130391173521, + "loss": 1.3608, + "theoretical_loss": 3.4556555013927683, + "tokens_seen": 2022572032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039099297893681047, + "loss": 1.3927, + "theoretical_loss": 3.4556462629974547, + "tokens_seen": 2022637568 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039097291875626883, + "loss": 1.3557, + "theoretical_loss": 3.455637024985282, + "tokens_seen": 2022703104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909528585757272, + "loss": 1.2985, + "theoretical_loss": 3.455627787356223, + "tokens_seen": 2022768640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909327983951856, + "loss": 1.3314, + "theoretical_loss": 3.455618550110249, + "tokens_seen": 2022834176 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039091273821464397, + "loss": 1.4317, + "theoretical_loss": 3.455609313247332, + "tokens_seen": 2022899712 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039089267803410233, + "loss": 1.3545, + "theoretical_loss": 3.4556000767674435, + "tokens_seen": 2022965248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908726178535607, + "loss": 1.3999, + "theoretical_loss": 3.4555908406705544, + "tokens_seen": 2023030784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039085255767301906, + "loss": 1.3433, + "theoretical_loss": 3.455581604956638, + "tokens_seen": 2023096320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039083249749247747, + "loss": 1.2721, + "theoretical_loss": 3.455572369625665, + "tokens_seen": 2023161856 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039081243731193583, + "loss": 1.3659, + "theoretical_loss": 3.455563134677607, + "tokens_seen": 2023227392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907923771313942, + "loss": 1.4036, + "theoretical_loss": 3.4555539001124362, + "tokens_seen": 2023292928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039077231695085256, + "loss": 1.241, + "theoretical_loss": 3.455544665930124, + "tokens_seen": 2023358464 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390752256770311, + "loss": 1.3087, + "theoretical_loss": 3.4555354321306426, + "tokens_seen": 2023424000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039073219658976934, + "loss": 1.1727, + "theoretical_loss": 3.4555261987139634, + "tokens_seen": 2023489536 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 738179, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.421309232711792, + "objective/train/theoretical_loss": 3.455516965680058, + "objective/train/tokens_used": 394014176, + "theoretical_loss": 3.455516965680058, + "tokens_seen": 2023555072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907121364092277, + "loss": 1.3643, + "theoretical_loss": 3.455516965680058, + "tokens_seen": 2023555072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039069207622868606, + "loss": 1.3152, + "theoretical_loss": 3.4555077330288984, + "tokens_seen": 2023620608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906720160481445, + "loss": 1.3485, + "theoretical_loss": 3.455498500760456, + "tokens_seen": 2023686144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039065195586760284, + "loss": 1.3794, + "theoretical_loss": 3.455489268874703, + "tokens_seen": 2023751680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906318956870612, + "loss": 1.3256, + "theoretical_loss": 3.455480037371611, + "tokens_seen": 2023817216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039061183550651957, + "loss": 1.3138, + "theoretical_loss": 3.455470806251151, + "tokens_seen": 2023882752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039059177532597793, + "loss": 1.3413, + "theoretical_loss": 3.4554615755132962, + "tokens_seen": 2023948288 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039057171514543634, + "loss": 1.364, + "theoretical_loss": 3.455452345158017, + "tokens_seen": 2024013824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905516549648947, + "loss": 1.4109, + "theoretical_loss": 3.455443115185286, + "tokens_seen": 2024079360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039053159478435307, + "loss": 1.3155, + "theoretical_loss": 3.455433885595075, + "tokens_seen": 2024144896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039051153460381143, + "loss": 1.3693, + "theoretical_loss": 3.455424656387355, + "tokens_seen": 2024210432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039049147442326985, + "loss": 1.3418, + "theoretical_loss": 3.455415427562098, + "tokens_seen": 2024275968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904714142427282, + "loss": 1.3078, + "theoretical_loss": 3.4554061991192766, + "tokens_seen": 2024341504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039045135406218657, + "loss": 1.3368, + "theoretical_loss": 3.4553969710588612, + "tokens_seen": 2024407040 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039043129388164493, + "loss": 1.2256, + "theoretical_loss": 3.4553877433808253, + "tokens_seen": 2024472576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904112337011033, + "loss": 1.2956, + "theoretical_loss": 3.4553785160851387, + "tokens_seen": 2024538112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903911735205617, + "loss": 1.3184, + "theoretical_loss": 3.4553692891717747, + "tokens_seen": 2024603648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903711133400201, + "loss": 1.3839, + "theoretical_loss": 3.455360062640705, + "tokens_seen": 2024669184 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039035105315947844, + "loss": 1.2909, + "theoretical_loss": 3.4553508364919, + "tokens_seen": 2024734720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903309929789368, + "loss": 1.5578, + "theoretical_loss": 3.4553416107253327, + "tokens_seen": 2024800256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903109327983952, + "loss": 1.3584, + "theoretical_loss": 3.455332385340975, + "tokens_seen": 2024865792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902908726178536, + "loss": 1.296, + "theoretical_loss": 3.455323160338798, + "tokens_seen": 2024931328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039027081243731194, + "loss": 1.4688, + "theoretical_loss": 3.455313935718774, + "tokens_seen": 2024996864 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902507522567703, + "loss": 1.3915, + "theoretical_loss": 3.4553047114808746, + "tokens_seen": 2025062400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039023069207622866, + "loss": 1.3522, + "theoretical_loss": 3.455295487625072, + "tokens_seen": 2025127936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902106318956871, + "loss": 1.4055, + "theoretical_loss": 3.4552862641513373, + "tokens_seen": 2025193472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039019057171514544, + "loss": 1.3314, + "theoretical_loss": 3.4552770410596425, + "tokens_seen": 2025259008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901705115346038, + "loss": 1.3877, + "theoretical_loss": 3.4552678183499594, + "tokens_seen": 2025324544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039015045135406217, + "loss": 1.3633, + "theoretical_loss": 3.4552585960222606, + "tokens_seen": 2025390080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901303911735206, + "loss": 1.3911, + "theoretical_loss": 3.455249374076517, + "tokens_seen": 2025455616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039011033099297895, + "loss": 1.3176, + "theoretical_loss": 3.455240152512701, + "tokens_seen": 2025521152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900902708124373, + "loss": 1.3319, + "theoretical_loss": 3.455230931330784, + "tokens_seen": 2025586688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039007021063189567, + "loss": 1.3708, + "theoretical_loss": 3.4552217105307377, + "tokens_seen": 2025652224 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039005015045135403, + "loss": 1.3965, + "theoretical_loss": 3.4552124901125345, + "tokens_seen": 2025717760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039003009027081245, + "loss": 1.3143, + "theoretical_loss": 3.4552032700761455, + "tokens_seen": 2025783296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900100300902708, + "loss": 1.3551, + "theoretical_loss": 3.455194050421544, + "tokens_seen": 2025848832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899899699097292, + "loss": 1.2757, + "theoretical_loss": 3.4551848311487, + "tokens_seen": 2025914368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038996990972918754, + "loss": 1.4894, + "theoretical_loss": 3.4551756122575865, + "tokens_seen": 2025979904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038994984954864595, + "loss": 1.3522, + "theoretical_loss": 3.4551663937481747, + "tokens_seen": 2026045440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899297893681043, + "loss": 1.304, + "theoretical_loss": 3.455157175620437, + "tokens_seen": 2026110976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899097291875627, + "loss": 1.2434, + "theoretical_loss": 3.455147957874345, + "tokens_seen": 2026176512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038988966900702104, + "loss": 1.3624, + "theoretical_loss": 3.45513874050987, + "tokens_seen": 2026242048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898696088264794, + "loss": 1.4139, + "theoretical_loss": 3.4551295235269857, + "tokens_seen": 2026307584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898495486459378, + "loss": 1.4757, + "theoretical_loss": 3.4551203069256617, + "tokens_seen": 2026373120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898294884653962, + "loss": 1.3204, + "theoretical_loss": 3.4551110907058713, + "tokens_seen": 2026438656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038980942828485454, + "loss": 1.3421, + "theoretical_loss": 3.455101874867586, + "tokens_seen": 2026504192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897893681043129, + "loss": 1.3783, + "theoretical_loss": 3.455092659410777, + "tokens_seen": 2026569728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897693079237713, + "loss": 1.4032, + "theoretical_loss": 3.4550834443354175, + "tokens_seen": 2026635264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897492477432297, + "loss": 1.2936, + "theoretical_loss": 3.4550742296414785, + "tokens_seen": 2026700800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038972918756268805, + "loss": 1.2721, + "theoretical_loss": 3.455065015328932, + "tokens_seen": 2026766336 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 739363, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3435156345367432, + "objective/train/theoretical_loss": 3.4550558013977497, + "objective/train/tokens_used": 397290976, + "theoretical_loss": 3.4550558013977497, + "tokens_seen": 2026831872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897091273821464, + "loss": 1.3073, + "theoretical_loss": 3.4550558013977497, + "tokens_seen": 2026831872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038968906720160477, + "loss": 1.4158, + "theoretical_loss": 3.455046587847904, + "tokens_seen": 2026897408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896690070210632, + "loss": 1.393, + "theoretical_loss": 3.455037374679366, + "tokens_seen": 2026962944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038964894684052155, + "loss": 1.3133, + "theoretical_loss": 3.455028161892108, + "tokens_seen": 2027028480 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038962888665997997, + "loss": 1.4214, + "theoretical_loss": 3.455018949486103, + "tokens_seen": 2027094016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038960882647943833, + "loss": 1.2748, + "theoretical_loss": 3.455009737461321, + "tokens_seen": 2027159552 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038958876629889674, + "loss": 1.2643, + "theoretical_loss": 3.455000525817735, + "tokens_seen": 2027225088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895687061183551, + "loss": 1.3237, + "theoretical_loss": 3.4549913145553166, + "tokens_seen": 2027290624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038954864593781347, + "loss": 1.2969, + "theoretical_loss": 3.4549821036740385, + "tokens_seen": 2027356160 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038952858575727183, + "loss": 1.3471, + "theoretical_loss": 3.454972893173871, + "tokens_seen": 2027421696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038950852557673025, + "loss": 1.3409, + "theoretical_loss": 3.454963683054787, + "tokens_seen": 2027487232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894884653961886, + "loss": 1.3143, + "theoretical_loss": 3.4549544733167585, + "tokens_seen": 2027552768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038946840521564697, + "loss": 1.3953, + "theoretical_loss": 3.4549452639597575, + "tokens_seen": 2027618304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038944834503510533, + "loss": 1.3597, + "theoretical_loss": 3.4549360549837553, + "tokens_seen": 2027683840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894282848545637, + "loss": 1.3207, + "theoretical_loss": 3.4549268463887244, + "tokens_seen": 2027749376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894082246740221, + "loss": 1.3519, + "theoretical_loss": 3.4549176381746367, + "tokens_seen": 2027814912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893881644934805, + "loss": 1.4071, + "theoretical_loss": 3.4549084303414634, + "tokens_seen": 2027880448 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038936810431293884, + "loss": 1.3912, + "theoretical_loss": 3.4548992228891775, + "tokens_seen": 2027945984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893480441323972, + "loss": 1.3927, + "theoretical_loss": 3.45489001581775, + "tokens_seen": 2028011520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893279839518556, + "loss": 1.4808, + "theoretical_loss": 3.4548808091271535, + "tokens_seen": 2028077056 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389307923771314, + "loss": 1.4265, + "theoretical_loss": 3.4548716028173594, + "tokens_seen": 2028142592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038928786359077234, + "loss": 1.4355, + "theoretical_loss": 3.4548623968883403, + "tokens_seen": 2028208128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892678034102307, + "loss": 1.3491, + "theoretical_loss": 3.4548531913400677, + "tokens_seen": 2028273664 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038924774322968906, + "loss": 1.3962, + "theoretical_loss": 3.4548439861725138, + "tokens_seen": 2028339200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892276830491475, + "loss": 1.3312, + "theoretical_loss": 3.45483478138565, + "tokens_seen": 2028404736 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038920762286860584, + "loss": 1.2968, + "theoretical_loss": 3.4548255769794487, + "tokens_seen": 2028470272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891875626880642, + "loss": 1.4201, + "theoretical_loss": 3.454816372953882, + "tokens_seen": 2028535808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038916750250752257, + "loss": 1.2663, + "theoretical_loss": 3.4548071693089217, + "tokens_seen": 2028601344 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389147442326981, + "loss": 1.4309, + "theoretical_loss": 3.4547979660445396, + "tokens_seen": 2028666880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038912738214643935, + "loss": 1.3663, + "theoretical_loss": 3.4547887631607077, + "tokens_seen": 2028732416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891073219658977, + "loss": 1.4298, + "theoretical_loss": 3.4547795606573986, + "tokens_seen": 2028797952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038908726178535607, + "loss": 1.3635, + "theoretical_loss": 3.4547703585345833, + "tokens_seen": 2028863488 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038906720160481443, + "loss": 1.3353, + "theoretical_loss": 3.4547611567922347, + "tokens_seen": 2028929024 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038904714142427285, + "loss": 1.3469, + "theoretical_loss": 3.4547519554303237, + "tokens_seen": 2028994560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890270812437312, + "loss": 1.3301, + "theoretical_loss": 3.454742754448823, + "tokens_seen": 2029060096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890070210631896, + "loss": 1.4622, + "theoretical_loss": 3.4547335538477046, + "tokens_seen": 2029125632 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038898696088264794, + "loss": 1.4134, + "theoretical_loss": 3.4547243536269407, + "tokens_seen": 2029191168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038896690070210635, + "loss": 1.4085, + "theoretical_loss": 3.454715153786503, + "tokens_seen": 2029256704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889468405215647, + "loss": 1.3614, + "theoretical_loss": 3.454705954326363, + "tokens_seen": 2029322240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889267803410231, + "loss": 1.3452, + "theoretical_loss": 3.454696755246493, + "tokens_seen": 2029387776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038890672016048144, + "loss": 1.3949, + "theoretical_loss": 3.4546875565468658, + "tokens_seen": 2029453312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888866599799398, + "loss": 1.3621, + "theoretical_loss": 3.4546783582274525, + "tokens_seen": 2029518848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888665997993982, + "loss": 1.4129, + "theoretical_loss": 3.4546691602882253, + "tokens_seen": 2029584384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888465396188566, + "loss": 1.2995, + "theoretical_loss": 3.4546599627291563, + "tokens_seen": 2029649920 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038882647943831494, + "loss": 1.3142, + "theoretical_loss": 3.454650765550218, + "tokens_seen": 2029715456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888064192577733, + "loss": 1.3571, + "theoretical_loss": 3.4546415687513807, + "tokens_seen": 2029780992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887863590772317, + "loss": 1.2966, + "theoretical_loss": 3.4546323723326187, + "tokens_seen": 2029846528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887662988966901, + "loss": 1.383, + "theoretical_loss": 3.454623176293903, + "tokens_seen": 2029912064 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038874623871614845, + "loss": 1.3505, + "theoretical_loss": 3.4546139806352048, + "tokens_seen": 2029977600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887261785356068, + "loss": 1.4078, + "theoretical_loss": 3.4546047853564974, + "tokens_seen": 2030043136 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 740596, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2953369617462158, + "objective/train/theoretical_loss": 3.4545955904577523, + "objective/train/tokens_used": 400567776, + "theoretical_loss": 3.4545955904577523, + "tokens_seen": 2030108672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038870611835506517, + "loss": 1.3055, + "theoretical_loss": 3.4545955904577523, + "tokens_seen": 2030108672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886860581745236, + "loss": 1.3938, + "theoretical_loss": 3.4545863959389416, + "tokens_seen": 2030174208 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038866599799398195, + "loss": 1.1979, + "theoretical_loss": 3.4545772018000376, + "tokens_seen": 2030239744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886459378134403, + "loss": 1.3141, + "theoretical_loss": 3.4545680080410115, + "tokens_seen": 2030305280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886258776328987, + "loss": 1.4452, + "theoretical_loss": 3.4545588146618362, + "tokens_seen": 2030370816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886058174523571, + "loss": 1.4127, + "theoretical_loss": 3.4545496216624834, + "tokens_seen": 2030436352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038858575727181545, + "loss": 1.3573, + "theoretical_loss": 3.454540429042925, + "tokens_seen": 2030501888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885656970912738, + "loss": 1.2665, + "theoretical_loss": 3.454531236803134, + "tokens_seen": 2030567424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885456369107322, + "loss": 1.2752, + "theoretical_loss": 3.4545220449430807, + "tokens_seen": 2030632960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885255767301906, + "loss": 1.2991, + "theoretical_loss": 3.4545128534627385, + "tokens_seen": 2030698496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038850551654964896, + "loss": 1.3686, + "theoretical_loss": 3.4545036623620793, + "tokens_seen": 2030764032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884854563691073, + "loss": 1.2501, + "theoretical_loss": 3.454494471641075, + "tokens_seen": 2030829568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884653961885657, + "loss": 1.4353, + "theoretical_loss": 3.4544852812996973, + "tokens_seen": 2030895104 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038844533600802404, + "loss": 1.2883, + "theoretical_loss": 3.454476091337919, + "tokens_seen": 2030960640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038842527582748246, + "loss": 1.3937, + "theoretical_loss": 3.4544669017557115, + "tokens_seen": 2031026176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884052156469408, + "loss": 1.3496, + "theoretical_loss": 3.4544577125530473, + "tokens_seen": 2031091712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883851554663992, + "loss": 1.3512, + "theoretical_loss": 3.4544485237298987, + "tokens_seen": 2031157248 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038836509528585755, + "loss": 1.2724, + "theoretical_loss": 3.454439335286237, + "tokens_seen": 2031222784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038834503510531596, + "loss": 1.3478, + "theoretical_loss": 3.454430147222035, + "tokens_seen": 2031288320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883249749247743, + "loss": 1.349, + "theoretical_loss": 3.4544209595372637, + "tokens_seen": 2031353856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883049147442327, + "loss": 1.4137, + "theoretical_loss": 3.4544117722318965, + "tokens_seen": 2031419392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038828485456369105, + "loss": 1.4058, + "theoretical_loss": 3.4544025853059055, + "tokens_seen": 2031484928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882647943831494, + "loss": 1.3437, + "theoretical_loss": 3.4543933987592617, + "tokens_seen": 2031550464 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038824473420260783, + "loss": 1.3349, + "theoretical_loss": 3.454384212591938, + "tokens_seen": 2031616000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882246740220662, + "loss": 1.4231, + "theoretical_loss": 3.4543750268039064, + "tokens_seen": 2031681536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038820461384152455, + "loss": 1.3624, + "theoretical_loss": 3.4543658413951386, + "tokens_seen": 2031747072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881845536609829, + "loss": 1.3414, + "theoretical_loss": 3.454356656365607, + "tokens_seen": 2031812608 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038816449348044133, + "loss": 1.3464, + "theoretical_loss": 3.454347471715284, + "tokens_seen": 2031878144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881444332998997, + "loss": 1.3932, + "theoretical_loss": 3.4543382874441413, + "tokens_seen": 2031943680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881243731193581, + "loss": 1.3818, + "theoretical_loss": 3.454329103552151, + "tokens_seen": 2032009216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038810431293881647, + "loss": 1.2806, + "theoretical_loss": 3.454319920039285, + "tokens_seen": 2032074752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038808425275827483, + "loss": 1.482, + "theoretical_loss": 3.4543107369055166, + "tokens_seen": 2032140288 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038806419257773325, + "loss": 1.4247, + "theoretical_loss": 3.4543015541508164, + "tokens_seen": 2032205824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880441323971916, + "loss": 1.3067, + "theoretical_loss": 3.454292371775158, + "tokens_seen": 2032271360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038802407221665, + "loss": 1.4338, + "theoretical_loss": 3.4542831897785122, + "tokens_seen": 2032336896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038800401203610834, + "loss": 1.2584, + "theoretical_loss": 3.4542740081608514, + "tokens_seen": 2032402432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038798395185556675, + "loss": 1.4509, + "theoretical_loss": 3.4542648269221488, + "tokens_seen": 2032467968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879638916750251, + "loss": 1.2899, + "theoretical_loss": 3.454255646062375, + "tokens_seen": 2032533504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879438314944835, + "loss": 1.4176, + "theoretical_loss": 3.4542464655815035, + "tokens_seen": 2032599040 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038792377131394184, + "loss": 1.4586, + "theoretical_loss": 3.454237285479506, + "tokens_seen": 2032664576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879037111334002, + "loss": 1.3193, + "theoretical_loss": 3.454228105756354, + "tokens_seen": 2032730112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878836509528586, + "loss": 1.2534, + "theoretical_loss": 3.4542189264120204, + "tokens_seen": 2032795648 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387863590772317, + "loss": 1.3168, + "theoretical_loss": 3.4542097474464772, + "tokens_seen": 2032861184 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038784353059177534, + "loss": 1.4072, + "theoretical_loss": 3.4542005688596964, + "tokens_seen": 2032926720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878234704112337, + "loss": 1.3353, + "theoretical_loss": 3.4541913906516504, + "tokens_seen": 2032992256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878034102306921, + "loss": 1.3053, + "theoretical_loss": 3.454182212822311, + "tokens_seen": 2033057792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877833500501505, + "loss": 1.4108, + "theoretical_loss": 3.4541730353716504, + "tokens_seen": 2033123328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038776328986960885, + "loss": 1.4252, + "theoretical_loss": 3.4541638582996415, + "tokens_seen": 2033188864 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877432296890672, + "loss": 1.3502, + "theoretical_loss": 3.454154681606256, + "tokens_seen": 2033254400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038772316950852557, + "loss": 1.4077, + "theoretical_loss": 3.454145505291465, + "tokens_seen": 2033319936 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 741743, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2943028211593628, + "objective/train/theoretical_loss": 3.4541363293552427, + "objective/train/tokens_used": 403844576, + "theoretical_loss": 3.4541363293552427, + "tokens_seen": 2033385472 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387703109327984, + "loss": 1.2981, + "theoretical_loss": 3.4541363293552427, + "tokens_seen": 2033385472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038768304914744235, + "loss": 1.4162, + "theoretical_loss": 3.4541271537975593, + "tokens_seen": 2033451008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876629889669007, + "loss": 1.3723, + "theoretical_loss": 3.4541179786183887, + "tokens_seen": 2033516544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876429287863591, + "loss": 1.3366, + "theoretical_loss": 3.454108803817702, + "tokens_seen": 2033582080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876228686058175, + "loss": 1.3827, + "theoretical_loss": 3.454099629395472, + "tokens_seen": 2033647616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038760280842527585, + "loss": 1.301, + "theoretical_loss": 3.45409045535167, + "tokens_seen": 2033713152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875827482447342, + "loss": 1.3125, + "theoretical_loss": 3.4540812816862694, + "tokens_seen": 2033778688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875626880641926, + "loss": 1.3894, + "theoretical_loss": 3.4540721083992416, + "tokens_seen": 2033844224 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387542627883651, + "loss": 1.3829, + "theoretical_loss": 3.454062935490559, + "tokens_seen": 2033909760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038752256770310936, + "loss": 1.2911, + "theoretical_loss": 3.454053762960194, + "tokens_seen": 2033975296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875025075225677, + "loss": 1.3809, + "theoretical_loss": 3.4540445908081185, + "tokens_seen": 2034040832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874824473420261, + "loss": 1.2672, + "theoretical_loss": 3.454035419034305, + "tokens_seen": 2034106368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038746238716148444, + "loss": 1.3431, + "theoretical_loss": 3.4540262476387253, + "tokens_seen": 2034171904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038744232698094286, + "loss": 1.3566, + "theoretical_loss": 3.4540170766213523, + "tokens_seen": 2034237440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874222668004012, + "loss": 1.4055, + "theoretical_loss": 3.4540079059821576, + "tokens_seen": 2034302976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874022066198596, + "loss": 1.3575, + "theoretical_loss": 3.453998735721113, + "tokens_seen": 2034368512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038738214643931795, + "loss": 1.2981, + "theoretical_loss": 3.453989565838192, + "tokens_seen": 2034434048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038736208625877636, + "loss": 1.3379, + "theoretical_loss": 3.4539803963333657, + "tokens_seen": 2034499584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873420260782347, + "loss": 1.3378, + "theoretical_loss": 3.4539712272066074, + "tokens_seen": 2034565120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873219658976931, + "loss": 1.3163, + "theoretical_loss": 3.453962058457888, + "tokens_seen": 2034630656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038730190571715145, + "loss": 1.3456, + "theoretical_loss": 3.453952890087181, + "tokens_seen": 2034696192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872818455366098, + "loss": 1.356, + "theoretical_loss": 3.4539437220944578, + "tokens_seen": 2034761728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038726178535606823, + "loss": 1.2547, + "theoretical_loss": 3.453934554479691, + "tokens_seen": 2034827264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872417251755266, + "loss": 1.3025, + "theoretical_loss": 3.453925387242853, + "tokens_seen": 2034892800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038722166499498495, + "loss": 1.2914, + "theoretical_loss": 3.453916220383916, + "tokens_seen": 2034958336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872016048144433, + "loss": 1.4451, + "theoretical_loss": 3.4539070539028516, + "tokens_seen": 2035023872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038718154463390173, + "loss": 1.3704, + "theoretical_loss": 3.4538978877996325, + "tokens_seen": 2035089408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871614844533601, + "loss": 1.3373, + "theoretical_loss": 3.4538887220742316, + "tokens_seen": 2035154944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038714142427281846, + "loss": 1.2474, + "theoretical_loss": 3.45387955672662, + "tokens_seen": 2035220480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871213640922768, + "loss": 1.3338, + "theoretical_loss": 3.4538703917567712, + "tokens_seen": 2035286016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871013039117352, + "loss": 1.3395, + "theoretical_loss": 3.4538612271646563, + "tokens_seen": 2035351552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870812437311936, + "loss": 1.3663, + "theoretical_loss": 3.453852062950248, + "tokens_seen": 2035417088 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038706118355065196, + "loss": 1.2895, + "theoretical_loss": 3.453842899113519, + "tokens_seen": 2035482624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870411233701103, + "loss": 1.3297, + "theoretical_loss": 3.453833735654441, + "tokens_seen": 2035548160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870210631895687, + "loss": 1.2956, + "theoretical_loss": 3.453824572572987, + "tokens_seen": 2035613696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870010030090271, + "loss": 1.3892, + "theoretical_loss": 3.453815409869128, + "tokens_seen": 2035679232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038698094282848546, + "loss": 1.3754, + "theoretical_loss": 3.4538062475428375, + "tokens_seen": 2035744768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869608826479438, + "loss": 1.3424, + "theoretical_loss": 3.453797085594087, + "tokens_seen": 2035810304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869408224674022, + "loss": 1.4326, + "theoretical_loss": 3.4537879240228495, + "tokens_seen": 2035875840 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038692076228686055, + "loss": 1.3881, + "theoretical_loss": 3.453778762829097, + "tokens_seen": 2035941376 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038690070210631896, + "loss": 1.4186, + "theoretical_loss": 3.4537696020128017, + "tokens_seen": 2036006912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038688064192577733, + "loss": 1.3043, + "theoretical_loss": 3.4537604415739356, + "tokens_seen": 2036072448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868605817452357, + "loss": 1.2976, + "theoretical_loss": 3.4537512815124716, + "tokens_seen": 2036137984 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038684052156469405, + "loss": 1.3379, + "theoretical_loss": 3.4537421218283817, + "tokens_seen": 2036203520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038682046138415247, + "loss": 1.3641, + "theoretical_loss": 3.4537329625216384, + "tokens_seen": 2036269056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038680040120361083, + "loss": 1.3004, + "theoretical_loss": 3.4537238035922133, + "tokens_seen": 2036334592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867803410230692, + "loss": 1.4283, + "theoretical_loss": 3.4537146450400797, + "tokens_seen": 2036400128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038676028084252755, + "loss": 1.3937, + "theoretical_loss": 3.4537054868652097, + "tokens_seen": 2036465664 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867402206619859, + "loss": 1.3937, + "theoretical_loss": 3.4536963290675753, + "tokens_seen": 2036531200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038672016048144433, + "loss": 1.3632, + "theoretical_loss": 3.453687171647149, + "tokens_seen": 2036596736 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 742794, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5007350444793701, + "objective/train/theoretical_loss": 3.4536780146039026, + "objective/train/tokens_used": 407121376, + "theoretical_loss": 3.4536780146039026, + "tokens_seen": 2036662272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867001003009027, + "loss": 1.4683, + "theoretical_loss": 3.4536780146039026, + "tokens_seen": 2036662272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038668004012036106, + "loss": 1.5579, + "theoretical_loss": 3.453668857937809, + "tokens_seen": 2036727808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866599799398194, + "loss": 1.3813, + "theoretical_loss": 3.4536597016488404, + "tokens_seen": 2036793344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038663991975927784, + "loss": 1.326, + "theoretical_loss": 3.4536505457369695, + "tokens_seen": 2036858880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038661985957873625, + "loss": 1.3433, + "theoretical_loss": 3.453641390202168, + "tokens_seen": 2036924416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865997993981946, + "loss": 1.4184, + "theoretical_loss": 3.4536322350444086, + "tokens_seen": 2036989952 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386579739217653, + "loss": 1.2716, + "theoretical_loss": 3.4536230802636636, + "tokens_seen": 2037055488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865596790371114, + "loss": 1.3808, + "theoretical_loss": 3.4536139258599055, + "tokens_seen": 2037121024 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038653961885656976, + "loss": 1.3594, + "theoretical_loss": 3.453604771833106, + "tokens_seen": 2037186560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865195586760281, + "loss": 1.211, + "theoretical_loss": 3.4535956181832383, + "tokens_seen": 2037252096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864994984954865, + "loss": 1.3527, + "theoretical_loss": 3.4535864649102743, + "tokens_seen": 2037317632 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038647943831494484, + "loss": 1.3579, + "theoretical_loss": 3.453577312014186, + "tokens_seen": 2037383168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038645937813440326, + "loss": 1.404, + "theoretical_loss": 3.4535681594949468, + "tokens_seen": 2037448704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864393179538616, + "loss": 1.3965, + "theoretical_loss": 3.4535590073525286, + "tokens_seen": 2037514240 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038641925777332, + "loss": 1.3719, + "theoretical_loss": 3.4535498555869033, + "tokens_seen": 2037579776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038639919759277835, + "loss": 1.4499, + "theoretical_loss": 3.4535407041980433, + "tokens_seen": 2037645312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038637913741223676, + "loss": 1.3947, + "theoretical_loss": 3.453531553185922, + "tokens_seen": 2037710848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863590772316951, + "loss": 1.2748, + "theoretical_loss": 3.45352240255051, + "tokens_seen": 2037776384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863390170511535, + "loss": 1.2704, + "theoretical_loss": 3.4535132522917813, + "tokens_seen": 2037841920 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038631895687061185, + "loss": 1.4036, + "theoretical_loss": 3.4535041024097075, + "tokens_seen": 2037907456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862988966900702, + "loss": 1.343, + "theoretical_loss": 3.4534949529042613, + "tokens_seen": 2037972992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038627883650952863, + "loss": 1.285, + "theoretical_loss": 3.4534858037754153, + "tokens_seen": 2038038528 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386258776328987, + "loss": 1.2852, + "theoretical_loss": 3.453476655023141, + "tokens_seen": 2038104064 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038623871614844535, + "loss": 1.4242, + "theoretical_loss": 3.453467506647412, + "tokens_seen": 2038169600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862186559679037, + "loss": 1.2899, + "theoretical_loss": 3.4534583586481995, + "tokens_seen": 2038235136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038619859578736213, + "loss": 1.2833, + "theoretical_loss": 3.453449211025476, + "tokens_seen": 2038300672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861785356068205, + "loss": 1.333, + "theoretical_loss": 3.4534400637792153, + "tokens_seen": 2038366208 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038615847542627886, + "loss": 1.2974, + "theoretical_loss": 3.4534309169093884, + "tokens_seen": 2038431744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861384152457372, + "loss": 1.3435, + "theoretical_loss": 3.453421770415968, + "tokens_seen": 2038497280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861183550651956, + "loss": 1.4038, + "theoretical_loss": 3.453412624298927, + "tokens_seen": 2038562816 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386098294884654, + "loss": 1.3596, + "theoretical_loss": 3.4534034785582373, + "tokens_seen": 2038628352 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038607823470411236, + "loss": 1.2759, + "theoretical_loss": 3.4533943331938715, + "tokens_seen": 2038693888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860581745235707, + "loss": 1.3478, + "theoretical_loss": 3.453385188205802, + "tokens_seen": 2038759424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860381143430291, + "loss": 1.3647, + "theoretical_loss": 3.4533760435940013, + "tokens_seen": 2038824960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860180541624875, + "loss": 1.4712, + "theoretical_loss": 3.453366899358442, + "tokens_seen": 2038890496 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038599799398194586, + "loss": 1.4224, + "theoretical_loss": 3.4533577554990957, + "tokens_seen": 2038956032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859779338014042, + "loss": 1.2756, + "theoretical_loss": 3.453348612015936, + "tokens_seen": 2039021568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859578736208626, + "loss": 1.3375, + "theoretical_loss": 3.453339468908934, + "tokens_seen": 2039087104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038593781344032095, + "loss": 1.3705, + "theoretical_loss": 3.4533303261780635, + "tokens_seen": 2039152640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038591775325977937, + "loss": 1.4297, + "theoretical_loss": 3.453321183823296, + "tokens_seen": 2039218176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038589769307923773, + "loss": 1.3233, + "theoretical_loss": 3.4533120418446046, + "tokens_seen": 2039283712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858776328986961, + "loss": 1.5151, + "theoretical_loss": 3.453302900241961, + "tokens_seen": 2039349248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038585757271815445, + "loss": 1.2903, + "theoretical_loss": 3.453293759015338, + "tokens_seen": 2039414784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038583751253761287, + "loss": 1.3717, + "theoretical_loss": 3.4532846181647088, + "tokens_seen": 2039480320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038581745235707123, + "loss": 1.4175, + "theoretical_loss": 3.4532754776900445, + "tokens_seen": 2039545856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857973921765296, + "loss": 1.3789, + "theoretical_loss": 3.4532663375913186, + "tokens_seen": 2039611392 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038577733199598796, + "loss": 1.3848, + "theoretical_loss": 3.4532571978685027, + "tokens_seen": 2039676928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857572718154463, + "loss": 1.4254, + "theoretical_loss": 3.45324805852157, + "tokens_seen": 2039742464 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038573721163490473, + "loss": 1.2984, + "theoretical_loss": 3.4532389195504933, + "tokens_seen": 2039808000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857171514543631, + "loss": 1.3998, + "theoretical_loss": 3.453229780955244, + "tokens_seen": 2039873536 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 744036, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4734983444213867, + "objective/train/theoretical_loss": 3.4532206427357948, + "objective/train/tokens_used": 410398176, + "theoretical_loss": 3.4532206427357948, + "tokens_seen": 2039939072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038569709127382146, + "loss": 1.4085, + "theoretical_loss": 3.4532206427357948, + "tokens_seen": 2039939072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856770310932798, + "loss": 1.3001, + "theoretical_loss": 3.4532115048921184, + "tokens_seen": 2040004608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038565697091273824, + "loss": 1.4096, + "theoretical_loss": 3.4532023674241876, + "tokens_seen": 2040070144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856369107321966, + "loss": 1.3953, + "theoretical_loss": 3.4531932303319746, + "tokens_seen": 2040135680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038561685055165496, + "loss": 1.3066, + "theoretical_loss": 3.4531840936154516, + "tokens_seen": 2040201216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855967903711133, + "loss": 1.4007, + "theoretical_loss": 3.4531749572745913, + "tokens_seen": 2040266752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855767301905717, + "loss": 1.3585, + "theoretical_loss": 3.4531658213093666, + "tokens_seen": 2040332288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855566700100301, + "loss": 1.3067, + "theoretical_loss": 3.4531566857197493, + "tokens_seen": 2040397824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038553660982948846, + "loss": 1.2366, + "theoretical_loss": 3.4531475505057125, + "tokens_seen": 2040463360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855165496489468, + "loss": 1.4365, + "theoretical_loss": 3.453138415667228, + "tokens_seen": 2040528896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854964894684052, + "loss": 1.3368, + "theoretical_loss": 3.453129281204269, + "tokens_seen": 2040594432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854764292878636, + "loss": 1.4471, + "theoretical_loss": 3.4531201471168074, + "tokens_seen": 2040659968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038545636910732197, + "loss": 1.3365, + "theoretical_loss": 3.4531110134048166, + "tokens_seen": 2040725504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038543630892678033, + "loss": 1.4399, + "theoretical_loss": 3.453101880068268, + "tokens_seen": 2040791040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854162487462387, + "loss": 1.3237, + "theoretical_loss": 3.4530927471071347, + "tokens_seen": 2040856576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038539618856569705, + "loss": 1.3338, + "theoretical_loss": 3.4530836145213892, + "tokens_seen": 2040922112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038537612838515547, + "loss": 1.4383, + "theoretical_loss": 3.453074482311004, + "tokens_seen": 2040987648 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038535606820461383, + "loss": 1.2489, + "theoretical_loss": 3.453065350475952, + "tokens_seen": 2041053184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853360080240722, + "loss": 1.327, + "theoretical_loss": 3.4530562190162044, + "tokens_seen": 2041118720 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038531594784353056, + "loss": 1.3274, + "theoretical_loss": 3.4530470879317354, + "tokens_seen": 2041184256 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385295887662989, + "loss": 1.2954, + "theoretical_loss": 3.4530379572225165, + "tokens_seen": 2041249792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038527582748244734, + "loss": 1.3721, + "theoretical_loss": 3.4530288268885205, + "tokens_seen": 2041315328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852557673019057, + "loss": 1.334, + "theoretical_loss": 3.45301969692972, + "tokens_seen": 2041380864 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038523570712136406, + "loss": 1.3752, + "theoretical_loss": 3.4530105673460874, + "tokens_seen": 2041446400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852156469408225, + "loss": 1.3378, + "theoretical_loss": 3.4530014381375955, + "tokens_seen": 2041511936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038519558676028084, + "loss": 1.4225, + "theoretical_loss": 3.4529923093042165, + "tokens_seen": 2041577472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851755265797392, + "loss": 1.4502, + "theoretical_loss": 3.452983180845923, + "tokens_seen": 2041643008 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038515546639919756, + "loss": 1.4751, + "theoretical_loss": 3.4529740527626878, + "tokens_seen": 2041708544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851354062186559, + "loss": 1.3939, + "theoretical_loss": 3.4529649250544834, + "tokens_seen": 2041774080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851153460381144, + "loss": 1.2775, + "theoretical_loss": 3.452955797721282, + "tokens_seen": 2041839616 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038509528585757276, + "loss": 1.4563, + "theoretical_loss": 3.4529466707630565, + "tokens_seen": 2041905152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850752256770311, + "loss": 1.4465, + "theoretical_loss": 3.45293754417978, + "tokens_seen": 2041970688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850551654964895, + "loss": 1.2924, + "theoretical_loss": 3.4529284179714237, + "tokens_seen": 2042036224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850351053159479, + "loss": 1.3668, + "theoretical_loss": 3.4529192921379606, + "tokens_seen": 2042101760 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038501504513540626, + "loss": 1.3468, + "theoretical_loss": 3.4529101666793642, + "tokens_seen": 2042167296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849949849548646, + "loss": 1.2854, + "theoretical_loss": 3.4529010415956063, + "tokens_seen": 2042232832 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384974924774323, + "loss": 1.3504, + "theoretical_loss": 3.4528919168866596, + "tokens_seen": 2042298368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038495486459378135, + "loss": 1.3947, + "theoretical_loss": 3.4528827925524967, + "tokens_seen": 2042363904 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038493480441323977, + "loss": 1.3875, + "theoretical_loss": 3.4528736685930905, + "tokens_seen": 2042429440 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038491474423269813, + "loss": 1.4093, + "theoretical_loss": 3.4528645450084126, + "tokens_seen": 2042494976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848946840521565, + "loss": 1.4187, + "theoretical_loss": 3.4528554217984366, + "tokens_seen": 2042560512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038487462387161485, + "loss": 1.3936, + "theoretical_loss": 3.4528462989631348, + "tokens_seen": 2042626048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038485456369107327, + "loss": 1.3232, + "theoretical_loss": 3.45283717650248, + "tokens_seen": 2042691584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038483450351053163, + "loss": 1.3368, + "theoretical_loss": 3.452828054416444, + "tokens_seen": 2042757120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038481444332999, + "loss": 1.4154, + "theoretical_loss": 3.452818932705, + "tokens_seen": 2042822656 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038479438314944836, + "loss": 1.3316, + "theoretical_loss": 3.452809811368121, + "tokens_seen": 2042888192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847743229689067, + "loss": 1.418, + "theoretical_loss": 3.4528006904057786, + "tokens_seen": 2042953728 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038475426278836513, + "loss": 1.4383, + "theoretical_loss": 3.452791569817946, + "tokens_seen": 2043019264 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847342026078235, + "loss": 1.353, + "theoretical_loss": 3.4527824496045962, + "tokens_seen": 2043084800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038471414242728186, + "loss": 1.2809, + "theoretical_loss": 3.452773329765701, + "tokens_seen": 2043150336 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 745253, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.362492561340332, + "objective/train/theoretical_loss": 3.4527642103012335, + "objective/train/tokens_used": 413674976, + "theoretical_loss": 3.4527642103012335, + "tokens_seen": 2043215872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846940822467402, + "loss": 1.3692, + "theoretical_loss": 3.4527642103012335, + "tokens_seen": 2043215872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038467402206619864, + "loss": 1.2931, + "theoretical_loss": 3.452755091211166, + "tokens_seen": 2043281408 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384653961885657, + "loss": 1.2935, + "theoretical_loss": 3.452745972495472, + "tokens_seen": 2043346944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038463390170511536, + "loss": 1.2623, + "theoretical_loss": 3.4527368541541223, + "tokens_seen": 2043412480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846138415245737, + "loss": 1.3338, + "theoretical_loss": 3.452727736187091, + "tokens_seen": 2043478016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845937813440321, + "loss": 1.3515, + "theoretical_loss": 3.452718618594351, + "tokens_seen": 2043543552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845737211634905, + "loss": 1.3427, + "theoretical_loss": 3.4527095013758737, + "tokens_seen": 2043609088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038455366098294886, + "loss": 1.2894, + "theoretical_loss": 3.452700384531633, + "tokens_seen": 2043674624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038453360080240723, + "loss": 1.2934, + "theoretical_loss": 3.4526912680616006, + "tokens_seen": 2043740160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845135406218656, + "loss": 1.2907, + "theoretical_loss": 3.4526821519657496, + "tokens_seen": 2043805696 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384493480441324, + "loss": 1.297, + "theoretical_loss": 3.452673036244052, + "tokens_seen": 2043871232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038447342026078237, + "loss": 1.305, + "theoretical_loss": 3.452663920896481, + "tokens_seen": 2043936768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038445336008024073, + "loss": 1.3429, + "theoretical_loss": 3.45265480592301, + "tokens_seen": 2044002304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844332998996991, + "loss": 1.3216, + "theoretical_loss": 3.4526456913236103, + "tokens_seen": 2044067840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038441323971915745, + "loss": 1.3142, + "theoretical_loss": 3.4526365770982546, + "tokens_seen": 2044133376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038439317953861587, + "loss": 1.3304, + "theoretical_loss": 3.452627463246917, + "tokens_seen": 2044198912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038437311935807423, + "loss": 1.3744, + "theoretical_loss": 3.452618349769568, + "tokens_seen": 2044264448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843530591775326, + "loss": 1.3371, + "theoretical_loss": 3.4526092366661825, + "tokens_seen": 2044329984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038433299899699096, + "loss": 1.2915, + "theoretical_loss": 3.4526001239367323, + "tokens_seen": 2044395520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843129388164494, + "loss": 1.3101, + "theoretical_loss": 3.4525910115811893, + "tokens_seen": 2044461056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038429287863590774, + "loss": 1.2463, + "theoretical_loss": 3.4525818995995268, + "tokens_seen": 2044526592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842728184553661, + "loss": 1.3502, + "theoretical_loss": 3.452572787991718, + "tokens_seen": 2044592128 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038425275827482446, + "loss": 1.505, + "theoretical_loss": 3.452563676757735, + "tokens_seen": 2044657664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842326980942829, + "loss": 1.3697, + "theoretical_loss": 3.4525545658975494, + "tokens_seen": 2044723200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038421263791374124, + "loss": 1.3999, + "theoretical_loss": 3.452545455411136, + "tokens_seen": 2044788736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841925777331996, + "loss": 1.3617, + "theoretical_loss": 3.4525363452984665, + "tokens_seen": 2044854272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038417251755265796, + "loss": 1.2997, + "theoretical_loss": 3.452527235559513, + "tokens_seen": 2044919808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841524573721163, + "loss": 1.1854, + "theoretical_loss": 3.4525181261942492, + "tokens_seen": 2044985344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038413239719157474, + "loss": 1.3809, + "theoretical_loss": 3.452509017202648, + "tokens_seen": 2045050880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841123370110331, + "loss": 1.3263, + "theoretical_loss": 3.452499908584681, + "tokens_seen": 2045116416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038409227683049147, + "loss": 1.3863, + "theoretical_loss": 3.4524908003403207, + "tokens_seen": 2045181952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038407221664994983, + "loss": 1.3317, + "theoretical_loss": 3.452481692469541, + "tokens_seen": 2045247488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038405215646940825, + "loss": 1.3132, + "theoretical_loss": 3.4524725849723144, + "tokens_seen": 2045313024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840320962888666, + "loss": 1.2879, + "theoretical_loss": 3.452463477848613, + "tokens_seen": 2045378560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038401203610832497, + "loss": 1.3786, + "theoretical_loss": 3.4524543710984097, + "tokens_seen": 2045444096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038399197592778333, + "loss": 1.2848, + "theoretical_loss": 3.4524452647216775, + "tokens_seen": 2045509632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839719157472417, + "loss": 1.3435, + "theoretical_loss": 3.452436158718389, + "tokens_seen": 2045575168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839518555667001, + "loss": 1.2941, + "theoretical_loss": 3.452427053088517, + "tokens_seen": 2045640704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839317953861585, + "loss": 1.2937, + "theoretical_loss": 3.452417947832034, + "tokens_seen": 2045706240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038391173520561684, + "loss": 1.304, + "theoretical_loss": 3.452408842948912, + "tokens_seen": 2045771776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838916750250752, + "loss": 1.3078, + "theoretical_loss": 3.4523997384391256, + "tokens_seen": 2045837312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838716148445336, + "loss": 1.2936, + "theoretical_loss": 3.4523906343026463, + "tokens_seen": 2045902848 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383851554663992, + "loss": 1.3888, + "theoretical_loss": 3.4523815305394465, + "tokens_seen": 2045968384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038383149448345034, + "loss": 1.372, + "theoretical_loss": 3.4523724271495, + "tokens_seen": 2046033920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838114343029087, + "loss": 1.466, + "theoretical_loss": 3.452363324132779, + "tokens_seen": 2046099456 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038379137412236706, + "loss": 1.3502, + "theoretical_loss": 3.452354221489256, + "tokens_seen": 2046164992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837713139418255, + "loss": 1.4677, + "theoretical_loss": 3.452345119218904, + "tokens_seen": 2046230528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038375125376128384, + "loss": 1.419, + "theoretical_loss": 3.452336017321696, + "tokens_seen": 2046296064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837311935807422, + "loss": 1.3453, + "theoretical_loss": 3.452326915797604, + "tokens_seen": 2046361600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038371113340020057, + "loss": 1.322, + "theoretical_loss": 3.4523178146466016, + "tokens_seen": 2046427136 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 746480, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2623852491378784, + "objective/train/theoretical_loss": 3.4523087138686614, + "objective/train/tokens_used": 416951776, + "theoretical_loss": 3.4523087138686614, + "tokens_seen": 2046492672 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383691073219659, + "loss": 1.2568, + "theoretical_loss": 3.4523087138686614, + "tokens_seen": 2046492672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038367101303911735, + "loss": 1.3009, + "theoretical_loss": 3.452299613463756, + "tokens_seen": 2046558208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836509528585757, + "loss": 1.3756, + "theoretical_loss": 3.4522905134318576, + "tokens_seen": 2046623744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836308926780341, + "loss": 1.389, + "theoretical_loss": 3.4522814137729396, + "tokens_seen": 2046689280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836108324974925, + "loss": 1.2688, + "theoretical_loss": 3.4522723144869754, + "tokens_seen": 2046754816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835907723169509, + "loss": 1.457, + "theoretical_loss": 3.4522632155739363, + "tokens_seen": 2046820352 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038357071213640927, + "loss": 1.4059, + "theoretical_loss": 3.4522541170337964, + "tokens_seen": 2046885888 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038355065195586763, + "loss": 1.2877, + "theoretical_loss": 3.4522450188665275, + "tokens_seen": 2046951424 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383530591775326, + "loss": 1.4025, + "theoretical_loss": 3.452235921072103, + "tokens_seen": 2047016960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835105315947844, + "loss": 1.2992, + "theoretical_loss": 3.452226823650495, + "tokens_seen": 2047082496 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038349047141424277, + "loss": 1.3467, + "theoretical_loss": 3.4522177266016776, + "tokens_seen": 2047148032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038347041123370113, + "loss": 1.3536, + "theoretical_loss": 3.452208629925622, + "tokens_seen": 2047213568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834503510531595, + "loss": 1.4178, + "theoretical_loss": 3.452199533622302, + "tokens_seen": 2047279104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038343029087261786, + "loss": 1.3349, + "theoretical_loss": 3.4521904376916903, + "tokens_seen": 2047344640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038341023069207627, + "loss": 1.3552, + "theoretical_loss": 3.4521813421337595, + "tokens_seen": 2047410176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038339017051153463, + "loss": 1.3091, + "theoretical_loss": 3.4521722469484826, + "tokens_seen": 2047475712 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383370110330993, + "loss": 1.3126, + "theoretical_loss": 3.4521631521358316, + "tokens_seen": 2047541248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038335005015045136, + "loss": 1.3336, + "theoretical_loss": 3.4521540576957808, + "tokens_seen": 2047606784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833299899699098, + "loss": 1.3191, + "theoretical_loss": 3.4521449636283013, + "tokens_seen": 2047672320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038330992978936814, + "loss": 1.2763, + "theoretical_loss": 3.4521358699333677, + "tokens_seen": 2047737856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832898696088265, + "loss": 1.4055, + "theoretical_loss": 3.4521267766109514, + "tokens_seen": 2047803392 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038326980942828486, + "loss": 1.4188, + "theoretical_loss": 3.452117683661026, + "tokens_seen": 2047868928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832497492477433, + "loss": 1.3475, + "theoretical_loss": 3.4521085910835634, + "tokens_seen": 2047934464 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038322968906720164, + "loss": 1.3026, + "theoretical_loss": 3.4520994988785376, + "tokens_seen": 2048000000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038320962888666, + "loss": 1.3318, + "theoretical_loss": 3.4520904070459206, + "tokens_seen": 2048065536 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038318956870611836, + "loss": 1.3502, + "theoretical_loss": 3.4520813155856858, + "tokens_seen": 2048131072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831695085255767, + "loss": 1.4059, + "theoretical_loss": 3.4520722244978055, + "tokens_seen": 2048196608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038314944834503514, + "loss": 1.4382, + "theoretical_loss": 3.4520631337822527, + "tokens_seen": 2048262144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831293881644935, + "loss": 1.3304, + "theoretical_loss": 3.4520540434390012, + "tokens_seen": 2048327680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038310932798395187, + "loss": 1.3094, + "theoretical_loss": 3.4520449534680218, + "tokens_seen": 2048393216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038308926780341023, + "loss": 1.3647, + "theoretical_loss": 3.4520358638692894, + "tokens_seen": 2048458752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038306920762286865, + "loss": 1.2854, + "theoretical_loss": 3.4520267746427757, + "tokens_seen": 2048524288 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383049147442327, + "loss": 1.3308, + "theoretical_loss": 3.4520176857884532, + "tokens_seen": 2048589824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038302908726178537, + "loss": 1.3543, + "theoretical_loss": 3.452008597306296, + "tokens_seen": 2048655360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038300902708124373, + "loss": 1.3534, + "theoretical_loss": 3.451999509196276, + "tokens_seen": 2048720896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829889669007021, + "loss": 1.3722, + "theoretical_loss": 3.451990421458367, + "tokens_seen": 2048786432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829689067201605, + "loss": 1.3481, + "theoretical_loss": 3.4519813340925407, + "tokens_seen": 2048851968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829488465396189, + "loss": 1.3788, + "theoretical_loss": 3.451972247098771, + "tokens_seen": 2048917504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038292878635907724, + "loss": 1.3879, + "theoretical_loss": 3.4519631604770296, + "tokens_seen": 2048983040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829087261785356, + "loss": 1.4259, + "theoretical_loss": 3.4519540742272907, + "tokens_seen": 2049048576 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382888665997994, + "loss": 1.2431, + "theoretical_loss": 3.451944988349526, + "tokens_seen": 2049114112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828686058174524, + "loss": 1.3701, + "theoretical_loss": 3.451935902843709, + "tokens_seen": 2049179648 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038284854563691074, + "loss": 1.275, + "theoretical_loss": 3.4519268177098126, + "tokens_seen": 2049245184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828284854563691, + "loss": 1.3213, + "theoretical_loss": 3.4519177329478095, + "tokens_seen": 2049310720 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038280842527582746, + "loss": 1.4012, + "theoretical_loss": 3.451908648557673, + "tokens_seen": 2049376256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827883650952859, + "loss": 1.3776, + "theoretical_loss": 3.451899564539375, + "tokens_seen": 2049441792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038276830491474424, + "loss": 1.4168, + "theoretical_loss": 3.451890480892889, + "tokens_seen": 2049507328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827482447342026, + "loss": 1.3523, + "theoretical_loss": 3.4518813976181884, + "tokens_seen": 2049572864 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038272818455366097, + "loss": 1.3579, + "theoretical_loss": 3.4518723147152457, + "tokens_seen": 2049638400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827081243731194, + "loss": 1.3042, + "theoretical_loss": 3.4518632321840332, + "tokens_seen": 2049703936 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 747702, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.376419186592102, + "objective/train/theoretical_loss": 3.4518541500245243, + "objective/train/tokens_used": 420228576, + "theoretical_loss": 3.4518541500245243, + "tokens_seen": 2049769472 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038268806419257775, + "loss": 1.3937, + "theoretical_loss": 3.4518541500245243, + "tokens_seen": 2049769472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826680040120361, + "loss": 1.3062, + "theoretical_loss": 3.4518450682366923, + "tokens_seen": 2049835008 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038264794383149447, + "loss": 1.3337, + "theoretical_loss": 3.4518359868205097, + "tokens_seen": 2049900544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038262788365095283, + "loss": 1.3448, + "theoretical_loss": 3.4518269057759494, + "tokens_seen": 2049966080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038260782347041125, + "loss": 1.2981, + "theoretical_loss": 3.4518178251029843, + "tokens_seen": 2050031616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825877632898696, + "loss": 1.4342, + "theoretical_loss": 3.4518087448015873, + "tokens_seen": 2050097152 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382567703109328, + "loss": 1.3958, + "theoretical_loss": 3.4517996648717313, + "tokens_seen": 2050162688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038254764292878634, + "loss": 1.436, + "theoretical_loss": 3.4517905853133897, + "tokens_seen": 2050228224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038252758274824475, + "loss": 1.4748, + "theoretical_loss": 3.4517815061265344, + "tokens_seen": 2050293760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825075225677031, + "loss": 1.2471, + "theoretical_loss": 3.4517724273111394, + "tokens_seen": 2050359296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824874623871615, + "loss": 1.423, + "theoretical_loss": 3.4517633488671766, + "tokens_seen": 2050424832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038246740220661984, + "loss": 1.2773, + "theoretical_loss": 3.4517542707946203, + "tokens_seen": 2050490368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824473420260782, + "loss": 1.3578, + "theoretical_loss": 3.4517451930934424, + "tokens_seen": 2050555904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824272818455366, + "loss": 1.3225, + "theoretical_loss": 3.451736115763616, + "tokens_seen": 2050621440 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382407221664995, + "loss": 1.4584, + "theoretical_loss": 3.451727038805114, + "tokens_seen": 2050686976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038238716148445334, + "loss": 1.3118, + "theoretical_loss": 3.4517179622179093, + "tokens_seen": 2050752512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823671013039117, + "loss": 1.3664, + "theoretical_loss": 3.451708886001976, + "tokens_seen": 2050818048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823470411233701, + "loss": 1.2905, + "theoretical_loss": 3.451699810157285, + "tokens_seen": 2050883584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823269809428285, + "loss": 1.3224, + "theoretical_loss": 3.451690734683811, + "tokens_seen": 2050949120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038230692076228685, + "loss": 1.2581, + "theoretical_loss": 3.4516816595815256, + "tokens_seen": 2051014656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822868605817452, + "loss": 1.5565, + "theoretical_loss": 3.451672584850403, + "tokens_seen": 2051080192 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038226680040120357, + "loss": 1.4085, + "theoretical_loss": 3.451663510490415, + "tokens_seen": 2051145728 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382246740220662, + "loss": 1.3184, + "theoretical_loss": 3.451654436501536, + "tokens_seen": 2051211264 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038222668004012035, + "loss": 1.3814, + "theoretical_loss": 3.451645362883738, + "tokens_seen": 2051276800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822066198595787, + "loss": 1.4038, + "theoretical_loss": 3.451636289636993, + "tokens_seen": 2051342336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821865596790371, + "loss": 1.3888, + "theoretical_loss": 3.4516272167612767, + "tokens_seen": 2051407872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821664994984955, + "loss": 1.3621, + "theoretical_loss": 3.4516181442565594, + "tokens_seen": 2051473408 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038214643931795385, + "loss": 1.3239, + "theoretical_loss": 3.4516090721228156, + "tokens_seen": 2051538944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038212637913741227, + "loss": 1.3418, + "theoretical_loss": 3.451600000360017, + "tokens_seen": 2051604480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038210631895687063, + "loss": 1.41, + "theoretical_loss": 3.4515909289681383, + "tokens_seen": 2051670016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038208625877632905, + "loss": 1.2409, + "theoretical_loss": 3.451581857947151, + "tokens_seen": 2051735552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820661985957874, + "loss": 1.3528, + "theoretical_loss": 3.451572787297029, + "tokens_seen": 2051801088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038204613841524577, + "loss": 1.3039, + "theoretical_loss": 3.451563717017745, + "tokens_seen": 2051866624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038202607823470413, + "loss": 1.3142, + "theoretical_loss": 3.4515546471092717, + "tokens_seen": 2051932160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820060180541625, + "loss": 1.387, + "theoretical_loss": 3.4515455775715824, + "tokens_seen": 2051997696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819859578736209, + "loss": 1.3922, + "theoretical_loss": 3.4515365084046503, + "tokens_seen": 2052063232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819658976930793, + "loss": 1.4019, + "theoretical_loss": 3.451527439608448, + "tokens_seen": 2052128768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038194583751253764, + "loss": 1.2926, + "theoretical_loss": 3.4515183711829485, + "tokens_seen": 2052194304 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381925777331996, + "loss": 1.3609, + "theoretical_loss": 3.451509303128125, + "tokens_seen": 2052259840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819057171514544, + "loss": 1.3647, + "theoretical_loss": 3.451500235443951, + "tokens_seen": 2052325376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818856569709128, + "loss": 1.2831, + "theoretical_loss": 3.4514911681303984, + "tokens_seen": 2052390912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038186559679037114, + "loss": 1.4023, + "theoretical_loss": 3.451482101187441, + "tokens_seen": 2052456448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818455366098295, + "loss": 1.3395, + "theoretical_loss": 3.4514730346150517, + "tokens_seen": 2052521984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038182547642928786, + "loss": 1.3827, + "theoretical_loss": 3.4514639684132034, + "tokens_seen": 2052587520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818054162487463, + "loss": 1.355, + "theoretical_loss": 3.4514549025818693, + "tokens_seen": 2052653056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038178535606820464, + "loss": 1.3568, + "theoretical_loss": 3.451445837121022, + "tokens_seen": 2052718592 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381765295887663, + "loss": 1.4609, + "theoretical_loss": 3.451436772030635, + "tokens_seen": 2052784128 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038174523570712137, + "loss": 1.3821, + "theoretical_loss": 3.451427707310682, + "tokens_seen": 2052849664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817251755265798, + "loss": 1.3841, + "theoretical_loss": 3.451418642961134, + "tokens_seen": 2052915200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038170511534603815, + "loss": 1.3775, + "theoretical_loss": 3.4514095789819654, + "tokens_seen": 2052980736 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 748910, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4846608638763428, + "objective/train/theoretical_loss": 3.45140051537315, + "objective/train/tokens_used": 423505376, + "theoretical_loss": 3.45140051537315, + "tokens_seen": 2053046272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816850551654965, + "loss": 1.3551, + "theoretical_loss": 3.45140051537315, + "tokens_seen": 2053046272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038166499498495487, + "loss": 1.4236, + "theoretical_loss": 3.451391452134659, + "tokens_seen": 2053111808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038164493480441323, + "loss": 1.379, + "theoretical_loss": 3.451382389266467, + "tokens_seen": 2053177344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038162487462387165, + "loss": 1.3736, + "theoretical_loss": 3.4513733267685462, + "tokens_seen": 2053242880 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038160481444333, + "loss": 1.3852, + "theoretical_loss": 3.45136426464087, + "tokens_seen": 2053308416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815847542627884, + "loss": 1.416, + "theoretical_loss": 3.451355202883411, + "tokens_seen": 2053373952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038156469408224674, + "loss": 1.332, + "theoretical_loss": 3.4513461414961433, + "tokens_seen": 2053439488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038154463390170515, + "loss": 1.446, + "theoretical_loss": 3.451337080479039, + "tokens_seen": 2053505024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815245737211635, + "loss": 1.36, + "theoretical_loss": 3.4513280198320713, + "tokens_seen": 2053570560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815045135406219, + "loss": 1.3645, + "theoretical_loss": 3.4513189595552136, + "tokens_seen": 2053636096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038148445336008024, + "loss": 1.4119, + "theoretical_loss": 3.4513098996484386, + "tokens_seen": 2053701632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814643931795386, + "loss": 1.3962, + "theoretical_loss": 3.4513008401117196, + "tokens_seen": 2053767168 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381444332998997, + "loss": 1.2842, + "theoretical_loss": 3.45129178094503, + "tokens_seen": 2053832704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814242728184554, + "loss": 1.3114, + "theoretical_loss": 3.451282722148342, + "tokens_seen": 2053898240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038140421263791374, + "loss": 1.3962, + "theoretical_loss": 3.4512736637216292, + "tokens_seen": 2053963776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813841524573721, + "loss": 1.2303, + "theoretical_loss": 3.451264605664865, + "tokens_seen": 2054029312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813640922768305, + "loss": 1.3652, + "theoretical_loss": 3.4512555479780223, + "tokens_seen": 2054094848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813440320962889, + "loss": 1.3699, + "theoretical_loss": 3.451246490661074, + "tokens_seen": 2054160384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038132397191574725, + "loss": 1.3429, + "theoretical_loss": 3.451237433713993, + "tokens_seen": 2054225920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813039117352056, + "loss": 1.3754, + "theoretical_loss": 3.451228377136753, + "tokens_seen": 2054291456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038128385155466397, + "loss": 1.4331, + "theoretical_loss": 3.4512193209293267, + "tokens_seen": 2054356992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812637913741224, + "loss": 1.3229, + "theoretical_loss": 3.451210265091687, + "tokens_seen": 2054422528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038124373119358075, + "loss": 1.394, + "theoretical_loss": 3.4512012096238074, + "tokens_seen": 2054488064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812236710130391, + "loss": 1.3932, + "theoretical_loss": 3.4511921545256605, + "tokens_seen": 2054553600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812036108324975, + "loss": 1.3018, + "theoretical_loss": 3.4511830997972206, + "tokens_seen": 2054619136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811835506519559, + "loss": 1.2877, + "theoretical_loss": 3.451174045438459, + "tokens_seen": 2054684672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038116349047141425, + "loss": 1.41, + "theoretical_loss": 3.4511649914493505, + "tokens_seen": 2054750208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811434302908726, + "loss": 1.4521, + "theoretical_loss": 3.451155937829868, + "tokens_seen": 2054815744 + }, + { + "epoch": 0.25, + "learning_rate": 0.000381123370110331, + "loss": 1.4979, + "theoretical_loss": 3.451146884579983, + "tokens_seen": 2054881280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038110330992978934, + "loss": 1.4527, + "theoretical_loss": 3.451137831699671, + "tokens_seen": 2054946816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038108324974924776, + "loss": 1.3333, + "theoretical_loss": 3.451128779188903, + "tokens_seen": 2055012352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810631895687061, + "loss": 1.3992, + "theoretical_loss": 3.4511197270476535, + "tokens_seen": 2055077888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810431293881645, + "loss": 1.3989, + "theoretical_loss": 3.4511106752758947, + "tokens_seen": 2055143424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038102306920762284, + "loss": 1.3372, + "theoretical_loss": 3.4511016238736008, + "tokens_seen": 2055208960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038100300902708126, + "loss": 1.3027, + "theoretical_loss": 3.4510925728407438, + "tokens_seen": 2055274496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809829488465396, + "loss": 1.355, + "theoretical_loss": 3.4510835221772975, + "tokens_seen": 2055340032 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380962888665998, + "loss": 1.3683, + "theoretical_loss": 3.451074471883236, + "tokens_seen": 2055405568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038094282848545635, + "loss": 1.3575, + "theoretical_loss": 3.45106542195853, + "tokens_seen": 2055471104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038092276830491476, + "loss": 1.383, + "theoretical_loss": 3.4510563724031544, + "tokens_seen": 2055536640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809027081243731, + "loss": 1.2824, + "theoretical_loss": 3.4510473232170824, + "tokens_seen": 2055602176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808826479438315, + "loss": 1.4672, + "theoretical_loss": 3.451038274400286, + "tokens_seen": 2055667712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038086258776328985, + "loss": 1.4254, + "theoretical_loss": 3.4510292259527398, + "tokens_seen": 2055733248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808425275827482, + "loss": 1.3263, + "theoretical_loss": 3.4510201778744163, + "tokens_seen": 2055798784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808224674022066, + "loss": 1.4186, + "theoretical_loss": 3.451011130165288, + "tokens_seen": 2055864320 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380802407221665, + "loss": 1.3537, + "theoretical_loss": 3.451002082825329, + "tokens_seen": 2055929856 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038078234704112335, + "loss": 1.3471, + "theoretical_loss": 3.4509930358545127, + "tokens_seen": 2055995392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807622868605817, + "loss": 1.4668, + "theoretical_loss": 3.450983989252811, + "tokens_seen": 2056060928 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038074222668004013, + "loss": 1.4103, + "theoretical_loss": 3.4509749430201984, + "tokens_seen": 2056126464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807221664994985, + "loss": 1.3253, + "theoretical_loss": 3.450965897156647, + "tokens_seen": 2056192000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038070210631895685, + "loss": 1.3438, + "theoretical_loss": 3.4509568516621307, + "tokens_seen": 2056257536 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 750166, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2493493556976318, + "objective/train/theoretical_loss": 3.450947806536622, + "objective/train/tokens_used": 426782176, + "theoretical_loss": 3.450947806536622, + "tokens_seen": 2056323072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806820461384152, + "loss": 1.3098, + "theoretical_loss": 3.450947806536622, + "tokens_seen": 2056323072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806619859578736, + "loss": 1.4908, + "theoretical_loss": 3.4509387617800953, + "tokens_seen": 2056388608 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380641925777332, + "loss": 1.3482, + "theoretical_loss": 3.4509297173925226, + "tokens_seen": 2056454144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806218655967904, + "loss": 1.316, + "theoretical_loss": 3.4509206733738775, + "tokens_seen": 2056519680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806018054162488, + "loss": 1.3271, + "theoretical_loss": 3.4509116297241333, + "tokens_seen": 2056585216 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038058174523570714, + "loss": 1.3884, + "theoretical_loss": 3.450902586443263, + "tokens_seen": 2056650752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038056168505516555, + "loss": 1.2964, + "theoretical_loss": 3.4508935435312402, + "tokens_seen": 2056716288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805416248746239, + "loss": 1.2602, + "theoretical_loss": 3.450884500988038, + "tokens_seen": 2056781824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805215646940823, + "loss": 1.3469, + "theoretical_loss": 3.450875458813629, + "tokens_seen": 2056847360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038050150451354064, + "loss": 1.3771, + "theoretical_loss": 3.450866417007987, + "tokens_seen": 2056912896 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380481444332999, + "loss": 1.3656, + "theoretical_loss": 3.450857375571085, + "tokens_seen": 2056978432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804613841524574, + "loss": 1.2998, + "theoretical_loss": 3.450848334502896, + "tokens_seen": 2057043968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804413239719158, + "loss": 1.2712, + "theoretical_loss": 3.4508392938033943, + "tokens_seen": 2057109504 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038042126379137414, + "loss": 1.3993, + "theoretical_loss": 3.4508302534725512, + "tokens_seen": 2057175040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804012036108325, + "loss": 1.3898, + "theoretical_loss": 3.4508212135103418, + "tokens_seen": 2057240576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803811434302909, + "loss": 1.2993, + "theoretical_loss": 3.4508121739167383, + "tokens_seen": 2057306112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803610832497493, + "loss": 1.4177, + "theoretical_loss": 3.4508031346917143, + "tokens_seen": 2057371648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038034102306920765, + "loss": 1.3663, + "theoretical_loss": 3.450794095835243, + "tokens_seen": 2057437184 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380320962888666, + "loss": 1.3182, + "theoretical_loss": 3.4507850573472973, + "tokens_seen": 2057502720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038030090270812437, + "loss": 1.1937, + "theoretical_loss": 3.4507760192278507, + "tokens_seen": 2057568256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802808425275828, + "loss": 1.2975, + "theoretical_loss": 3.4507669814768764, + "tokens_seen": 2057633792 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038026078234704115, + "loss": 1.4395, + "theoretical_loss": 3.450757944094348, + "tokens_seen": 2057699328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802407221664995, + "loss": 1.3469, + "theoretical_loss": 3.450748907080238, + "tokens_seen": 2057764864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802206619859579, + "loss": 1.2901, + "theoretical_loss": 3.4507398704345205, + "tokens_seen": 2057830400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802006018054163, + "loss": 1.2503, + "theoretical_loss": 3.4507308341571674, + "tokens_seen": 2057895936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038018054162487465, + "loss": 1.3031, + "theoretical_loss": 3.4507217982481535, + "tokens_seen": 2057961472 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380160481444333, + "loss": 1.467, + "theoretical_loss": 3.450712762707452, + "tokens_seen": 2058027008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801404212637914, + "loss": 1.3075, + "theoretical_loss": 3.4507037275350347, + "tokens_seen": 2058092544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038012036108324974, + "loss": 1.3497, + "theoretical_loss": 3.450694692730876, + "tokens_seen": 2058158080 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038010030090270816, + "loss": 1.3783, + "theoretical_loss": 3.450685658294949, + "tokens_seen": 2058223616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800802407221665, + "loss": 1.4522, + "theoretical_loss": 3.450676624227227, + "tokens_seen": 2058289152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800601805416249, + "loss": 1.3264, + "theoretical_loss": 3.450667590527683, + "tokens_seen": 2058354688 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038004012036108324, + "loss": 1.2982, + "theoretical_loss": 3.45065855719629, + "tokens_seen": 2058420224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038002006018054166, + "loss": 1.3798, + "theoretical_loss": 3.4506495242330226, + "tokens_seen": 2058485760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038, + "loss": 1.275, + "theoretical_loss": 3.4506404916378526, + "tokens_seen": 2058551296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799799398194584, + "loss": 1.2402, + "theoretical_loss": 3.450631459410754, + "tokens_seen": 2058616832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037995987963891675, + "loss": 1.4422, + "theoretical_loss": 3.4506224275517, + "tokens_seen": 2058682368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037993981945837516, + "loss": 1.2565, + "theoretical_loss": 3.4506133960606635, + "tokens_seen": 2058747904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799197592778335, + "loss": 1.3837, + "theoretical_loss": 3.450604364937618, + "tokens_seen": 2058813440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798996990972919, + "loss": 1.4496, + "theoretical_loss": 3.4505953341825375, + "tokens_seen": 2058878976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037987963891675025, + "loss": 1.4201, + "theoretical_loss": 3.4505863037953945, + "tokens_seen": 2058944512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798595787362086, + "loss": 1.2644, + "theoretical_loss": 3.4505772737761626, + "tokens_seen": 2059010048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037983951855566703, + "loss": 1.292, + "theoretical_loss": 3.4505682441248147, + "tokens_seen": 2059075584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798194583751254, + "loss": 1.2584, + "theoretical_loss": 3.4505592148413244, + "tokens_seen": 2059141120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037979939819458375, + "loss": 1.2263, + "theoretical_loss": 3.4505501859256653, + "tokens_seen": 2059206656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797793380140421, + "loss": 1.3503, + "theoretical_loss": 3.45054115737781, + "tokens_seen": 2059272192 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037975927783350053, + "loss": 1.3637, + "theoretical_loss": 3.450532129197733, + "tokens_seen": 2059337728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797392176529589, + "loss": 1.3186, + "theoretical_loss": 3.4505231013854063, + "tokens_seen": 2059403264 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037971915747241725, + "loss": 1.4067, + "theoretical_loss": 3.450514073940804, + "tokens_seen": 2059468800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796990972918756, + "loss": 1.3664, + "theoretical_loss": 3.450505046863899, + "tokens_seen": 2059534336 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 751330, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3781014680862427, + "objective/train/theoretical_loss": 3.450496020154665, + "objective/train/tokens_used": 430058976, + "theoretical_loss": 3.450496020154665, + "tokens_seen": 2059599872 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379679037111334, + "loss": 1.4111, + "theoretical_loss": 3.450496020154665, + "tokens_seen": 2059599872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796589769307924, + "loss": 1.3682, + "theoretical_loss": 3.450486993813075, + "tokens_seen": 2059665408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037963891675025076, + "loss": 1.222, + "theoretical_loss": 3.4504779678391024, + "tokens_seen": 2059730944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796188565697091, + "loss": 1.4279, + "theoretical_loss": 3.4504689422327206, + "tokens_seen": 2059796480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795987963891675, + "loss": 1.3905, + "theoretical_loss": 3.4504599169939034, + "tokens_seen": 2059862016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795787362086259, + "loss": 1.3263, + "theoretical_loss": 3.450450892122623, + "tokens_seen": 2059927552 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037955867602808426, + "loss": 1.43, + "theoretical_loss": 3.4504418676188537, + "tokens_seen": 2059993088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795386158475426, + "loss": 1.2826, + "theoretical_loss": 3.4504328434825684, + "tokens_seen": 2060058624 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379518555667001, + "loss": 1.3222, + "theoretical_loss": 3.450423819713741, + "tokens_seen": 2060124160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037949849548645935, + "loss": 1.3427, + "theoretical_loss": 3.4504147963123444, + "tokens_seen": 2060189696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037947843530591776, + "loss": 1.3051, + "theoretical_loss": 3.450405773278352, + "tokens_seen": 2060255232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794583751253761, + "loss": 1.3728, + "theoretical_loss": 3.450396750611737, + "tokens_seen": 2060320768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794383149448345, + "loss": 1.3736, + "theoretical_loss": 3.4503877283124726, + "tokens_seen": 2060386304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037941825476429285, + "loss": 1.3498, + "theoretical_loss": 3.4503787063805333, + "tokens_seen": 2060451840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037939819458375127, + "loss": 1.4216, + "theoretical_loss": 3.450369684815891, + "tokens_seen": 2060517376 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037937813440320963, + "loss": 1.4388, + "theoretical_loss": 3.45036066361852, + "tokens_seen": 2060582912 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379358074222668, + "loss": 1.3142, + "theoretical_loss": 3.450351642788393, + "tokens_seen": 2060648448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037933801404212635, + "loss": 1.3774, + "theoretical_loss": 3.4503426223254845, + "tokens_seen": 2060713984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793179538615847, + "loss": 1.2908, + "theoretical_loss": 3.450333602229766, + "tokens_seen": 2060779520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037929789368104313, + "loss": 1.376, + "theoretical_loss": 3.4503245825012128, + "tokens_seen": 2060845056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792778335005015, + "loss": 1.3131, + "theoretical_loss": 3.4503155631397977, + "tokens_seen": 2060910592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037925777331995986, + "loss": 1.3065, + "theoretical_loss": 3.4503065441454934, + "tokens_seen": 2060976128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792377131394182, + "loss": 1.3306, + "theoretical_loss": 3.4502975255182733, + "tokens_seen": 2061041664 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037921765295887664, + "loss": 1.449, + "theoretical_loss": 3.450288507258112, + "tokens_seen": 2061107200 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379197592778335, + "loss": 1.3525, + "theoretical_loss": 3.4502794893649815, + "tokens_seen": 2061172736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037917753259779336, + "loss": 1.3103, + "theoretical_loss": 3.4502704718388566, + "tokens_seen": 2061238272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791574724172517, + "loss": 1.2614, + "theoretical_loss": 3.4502614546797092, + "tokens_seen": 2061303808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791374122367101, + "loss": 1.3807, + "theoretical_loss": 3.4502524378875137, + "tokens_seen": 2061369344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037911735205616856, + "loss": 1.3671, + "theoretical_loss": 3.4502434214622433, + "tokens_seen": 2061434880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790972918756269, + "loss": 1.3784, + "theoretical_loss": 3.4502344054038714, + "tokens_seen": 2061500416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790772316950853, + "loss": 1.3529, + "theoretical_loss": 3.4502253897123705, + "tokens_seen": 2061565952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037905717151454364, + "loss": 1.249, + "theoretical_loss": 3.4502163743877157, + "tokens_seen": 2061631488 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037903711133400206, + "loss": 1.3148, + "theoretical_loss": 3.4502073594298794, + "tokens_seen": 2061697024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790170511534604, + "loss": 1.3856, + "theoretical_loss": 3.450198344838835, + "tokens_seen": 2061762560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789969909729188, + "loss": 1.3319, + "theoretical_loss": 3.450189330614556, + "tokens_seen": 2061828096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037897693079237715, + "loss": 1.3713, + "theoretical_loss": 3.4501803167570158, + "tokens_seen": 2061893632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037895687061183556, + "loss": 1.3886, + "theoretical_loss": 3.4501713032661883, + "tokens_seen": 2061959168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789368104312939, + "loss": 1.3708, + "theoretical_loss": 3.450162290142046, + "tokens_seen": 2062024704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789167502507523, + "loss": 1.4222, + "theoretical_loss": 3.4501532773845636, + "tokens_seen": 2062090240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037889669007021065, + "loss": 1.3278, + "theoretical_loss": 3.4501442649937135, + "tokens_seen": 2062155776 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378876629889669, + "loss": 1.3538, + "theoretical_loss": 3.4501352529694693, + "tokens_seen": 2062221312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037885656970912743, + "loss": 1.3442, + "theoretical_loss": 3.450126241311805, + "tokens_seen": 2062286848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788365095285858, + "loss": 1.3701, + "theoretical_loss": 3.450117230020693, + "tokens_seen": 2062352384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037881644934804415, + "loss": 1.4017, + "theoretical_loss": 3.4501082190961077, + "tokens_seen": 2062417920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787963891675025, + "loss": 1.3094, + "theoretical_loss": 3.450099208538022, + "tokens_seen": 2062483456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037877632898696093, + "loss": 1.2367, + "theoretical_loss": 3.45009019834641, + "tokens_seen": 2062548992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787562688064193, + "loss": 1.3556, + "theoretical_loss": 3.450081188521244, + "tokens_seen": 2062614528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037873620862587766, + "loss": 1.2267, + "theoretical_loss": 3.4500721790624986, + "tokens_seen": 2062680064 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378716148445336, + "loss": 1.3159, + "theoretical_loss": 3.4500631699701465, + "tokens_seen": 2062745600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786960882647944, + "loss": 1.3928, + "theoretical_loss": 3.450054161244162, + "tokens_seen": 2062811136 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 752534, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.330669641494751, + "objective/train/theoretical_loss": 3.4500451528845177, + "objective/train/tokens_used": 433335776, + "theoretical_loss": 3.4500451528845177, + "tokens_seen": 2062876672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786760280842528, + "loss": 1.3501, + "theoretical_loss": 3.4500451528845177, + "tokens_seen": 2062876672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037865596790371116, + "loss": 1.3431, + "theoretical_loss": 3.450036144891188, + "tokens_seen": 2062942208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786359077231695, + "loss": 1.3088, + "theoretical_loss": 3.450027137264145, + "tokens_seen": 2063007744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786158475426279, + "loss": 1.3189, + "theoretical_loss": 3.450018130003363, + "tokens_seen": 2063073280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785957873620863, + "loss": 1.4687, + "theoretical_loss": 3.4500091231088157, + "tokens_seen": 2063138816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037857572718154466, + "loss": 1.3607, + "theoretical_loss": 3.450000116580476, + "tokens_seen": 2063204352 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378555667001003, + "loss": 1.3475, + "theoretical_loss": 3.449991110418318, + "tokens_seen": 2063269888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785356068204614, + "loss": 1.4041, + "theoretical_loss": 3.4499821046223147, + "tokens_seen": 2063335424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037851554663991975, + "loss": 1.3203, + "theoretical_loss": 3.44997309919244, + "tokens_seen": 2063400960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037849548645937816, + "loss": 1.3866, + "theoretical_loss": 3.449964094128667, + "tokens_seen": 2063466496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784754262788365, + "loss": 1.3617, + "theoretical_loss": 3.4499550894309694, + "tokens_seen": 2063532032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784553660982949, + "loss": 1.2786, + "theoretical_loss": 3.44994608509932, + "tokens_seen": 2063597568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037843530591775325, + "loss": 1.3252, + "theoretical_loss": 3.449937081133694, + "tokens_seen": 2063663104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037841524573721167, + "loss": 1.3894, + "theoretical_loss": 3.4499280775340626, + "tokens_seen": 2063728640 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037839518555667003, + "loss": 1.3072, + "theoretical_loss": 3.449919074300401, + "tokens_seen": 2063794176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783751253761284, + "loss": 1.2985, + "theoretical_loss": 3.449910071432682, + "tokens_seen": 2063859712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037835506519558675, + "loss": 1.3177, + "theoretical_loss": 3.4499010689308798, + "tokens_seen": 2063925248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783350050150451, + "loss": 1.3237, + "theoretical_loss": 3.449892066794967, + "tokens_seen": 2063990784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037831494483450353, + "loss": 1.3256, + "theoretical_loss": 3.4498830650249177, + "tokens_seen": 2064056320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782948846539619, + "loss": 1.3122, + "theoretical_loss": 3.4498740636207055, + "tokens_seen": 2064121856 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037827482447342026, + "loss": 1.3388, + "theoretical_loss": 3.4498650625823033, + "tokens_seen": 2064187392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782547642928786, + "loss": 1.4253, + "theoretical_loss": 3.449856061909685, + "tokens_seen": 2064252928 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037823470411233704, + "loss": 1.3886, + "theoretical_loss": 3.4498470616028243, + "tokens_seen": 2064318464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782146439317954, + "loss": 1.3155, + "theoretical_loss": 3.4498380616616946, + "tokens_seen": 2064384000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037819458375125376, + "loss": 1.3096, + "theoretical_loss": 3.449829062086269, + "tokens_seen": 2064449536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781745235707121, + "loss": 1.3848, + "theoretical_loss": 3.4498200628765217, + "tokens_seen": 2064515072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781544633901705, + "loss": 1.2935, + "theoretical_loss": 3.449811064032426, + "tokens_seen": 2064580608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781344032096289, + "loss": 1.3963, + "theoretical_loss": 3.449802065553955, + "tokens_seen": 2064646144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037811434302908726, + "loss": 1.4688, + "theoretical_loss": 3.4497930674410826, + "tokens_seen": 2064711680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780942828485456, + "loss": 1.3633, + "theoretical_loss": 3.4497840696937825, + "tokens_seen": 2064777216 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378074222668004, + "loss": 1.4294, + "theoretical_loss": 3.449775072312028, + "tokens_seen": 2064842752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780541624874624, + "loss": 1.4046, + "theoretical_loss": 3.449766075295793, + "tokens_seen": 2064908288 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037803410230692077, + "loss": 1.3639, + "theoretical_loss": 3.449757078645051, + "tokens_seen": 2064973824 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037801404212637913, + "loss": 1.3637, + "theoretical_loss": 3.4497480823597746, + "tokens_seen": 2065039360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779939819458375, + "loss": 1.3935, + "theoretical_loss": 3.449739086439939, + "tokens_seen": 2065104896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037797392176529585, + "loss": 1.4495, + "theoretical_loss": 3.449730090885516, + "tokens_seen": 2065170432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037795386158475427, + "loss": 1.3631, + "theoretical_loss": 3.4497210956964803, + "tokens_seen": 2065235968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037793380140421263, + "loss": 1.4188, + "theoretical_loss": 3.4497121008728056, + "tokens_seen": 2065301504 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377913741223671, + "loss": 1.2886, + "theoretical_loss": 3.4497031064144648, + "tokens_seen": 2065367040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037789368104312936, + "loss": 1.3587, + "theoretical_loss": 3.4496941123214313, + "tokens_seen": 2065432576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778736208625878, + "loss": 1.2964, + "theoretical_loss": 3.44968511859368, + "tokens_seen": 2065498112 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037785356068204614, + "loss": 1.2949, + "theoretical_loss": 3.449676125231183, + "tokens_seen": 2065563648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778335005015045, + "loss": 1.3425, + "theoretical_loss": 3.4496671322339143, + "tokens_seen": 2065629184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037781344032096286, + "loss": 1.3075, + "theoretical_loss": 3.4496581396018478, + "tokens_seen": 2065694720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777933801404213, + "loss": 1.3222, + "theoretical_loss": 3.4496491473349566, + "tokens_seen": 2065760256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037777331995987964, + "loss": 1.5782, + "theoretical_loss": 3.4496401554332152, + "tokens_seen": 2065825792 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377753259779338, + "loss": 1.31, + "theoretical_loss": 3.449631163896596, + "tokens_seen": 2065891328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037773319959879636, + "loss": 1.245, + "theoretical_loss": 3.449622172725074, + "tokens_seen": 2065956864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777131394182547, + "loss": 1.3825, + "theoretical_loss": 3.4496131819186213, + "tokens_seen": 2066022400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037769307923771314, + "loss": 1.3257, + "theoretical_loss": 3.449604191477212, + "tokens_seen": 2066087936 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 753515, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3310260772705078, + "objective/train/theoretical_loss": 3.4495952014008204, + "objective/train/tokens_used": 436612576, + "theoretical_loss": 3.4495952014008204, + "tokens_seen": 2066153472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776730190571715, + "loss": 1.2863, + "theoretical_loss": 3.4495952014008204, + "tokens_seen": 2066153472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037765295887662987, + "loss": 1.2429, + "theoretical_loss": 3.44958621168942, + "tokens_seen": 2066219008 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037763289869608823, + "loss": 1.3256, + "theoretical_loss": 3.449577222342983, + "tokens_seen": 2066284544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776128385155467, + "loss": 1.3724, + "theoretical_loss": 3.449568233361484, + "tokens_seen": 2066350080 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037759277833500506, + "loss": 1.3123, + "theoretical_loss": 3.449559244744897, + "tokens_seen": 2066415616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775727181544634, + "loss": 1.3765, + "theoretical_loss": 3.449550256493195, + "tokens_seen": 2066481152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775526579739218, + "loss": 1.4511, + "theoretical_loss": 3.4495412686063522, + "tokens_seen": 2066546688 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037753259779338015, + "loss": 1.3386, + "theoretical_loss": 3.449532281084342, + "tokens_seen": 2066612224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037751253761283857, + "loss": 1.3284, + "theoretical_loss": 3.449523293927137, + "tokens_seen": 2066677760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037749247743229693, + "loss": 1.3404, + "theoretical_loss": 3.4495143071347125, + "tokens_seen": 2066743296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774724172517553, + "loss": 1.2295, + "theoretical_loss": 3.449505320707041, + "tokens_seen": 2066808832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037745235707121365, + "loss": 1.305, + "theoretical_loss": 3.4494963346440968, + "tokens_seen": 2066874368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037743229689067207, + "loss": 1.3815, + "theoretical_loss": 3.449487348945853, + "tokens_seen": 2066939904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037741223671013043, + "loss": 1.462, + "theoretical_loss": 3.449478363612283, + "tokens_seen": 2067005440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773921765295888, + "loss": 1.3252, + "theoretical_loss": 3.4494693786433612, + "tokens_seen": 2067070976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037737211634904715, + "loss": 1.3532, + "theoretical_loss": 3.449460394039061, + "tokens_seen": 2067136512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773520561685055, + "loss": 1.3121, + "theoretical_loss": 3.449451409799356, + "tokens_seen": 2067202048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037733199598796393, + "loss": 1.2725, + "theoretical_loss": 3.4494424259242193, + "tokens_seen": 2067267584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773119358074223, + "loss": 1.3043, + "theoretical_loss": 3.4494334424136257, + "tokens_seen": 2067333120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037729187562688066, + "loss": 1.2655, + "theoretical_loss": 3.4494244592675476, + "tokens_seen": 2067398656 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377271815446339, + "loss": 1.3997, + "theoretical_loss": 3.4494154764859593, + "tokens_seen": 2067464192 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037725175526579744, + "loss": 1.3476, + "theoretical_loss": 3.449406494068835, + "tokens_seen": 2067529728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772316950852558, + "loss": 1.4015, + "theoretical_loss": 3.449397512016147, + "tokens_seen": 2067595264 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037721163490471416, + "loss": 1.37, + "theoretical_loss": 3.4493885303278704, + "tokens_seen": 2067660800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771915747241725, + "loss": 1.3407, + "theoretical_loss": 3.449379549003978, + "tokens_seen": 2067726336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771715145436309, + "loss": 1.3115, + "theoretical_loss": 3.4493705680444435, + "tokens_seen": 2067791872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771514543630893, + "loss": 1.332, + "theoretical_loss": 3.449361587449241, + "tokens_seen": 2067857408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037713139418254766, + "loss": 1.2769, + "theoretical_loss": 3.4493526072183434, + "tokens_seen": 2067922944 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377111334002006, + "loss": 1.4331, + "theoretical_loss": 3.4493436273517255, + "tokens_seen": 2067988480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770912738214644, + "loss": 1.368, + "theoretical_loss": 3.4493346478493603, + "tokens_seen": 2068054016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770712136409228, + "loss": 1.3091, + "theoretical_loss": 3.4493256687112215, + "tokens_seen": 2068119552 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037705115346038117, + "loss": 1.3052, + "theoretical_loss": 3.449316689937282, + "tokens_seen": 2068185088 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037703109327983953, + "loss": 1.3963, + "theoretical_loss": 3.4493077115275175, + "tokens_seen": 2068250624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770110330992979, + "loss": 1.4061, + "theoretical_loss": 3.4492987334819, + "tokens_seen": 2068316160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037699097291875625, + "loss": 1.3042, + "theoretical_loss": 3.449289755800404, + "tokens_seen": 2068381696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037697091273821467, + "loss": 1.4105, + "theoretical_loss": 3.4492807784830024, + "tokens_seen": 2068447232 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037695085255767303, + "loss": 1.278, + "theoretical_loss": 3.4492718015296697, + "tokens_seen": 2068512768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769307923771314, + "loss": 1.4581, + "theoretical_loss": 3.449262824940379, + "tokens_seen": 2068578304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037691073219658976, + "loss": 1.3948, + "theoretical_loss": 3.449253848715105, + "tokens_seen": 2068643840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768906720160482, + "loss": 1.3494, + "theoretical_loss": 3.4492448728538196, + "tokens_seen": 2068709376 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037687061183550654, + "loss": 1.3224, + "theoretical_loss": 3.4492358973564987, + "tokens_seen": 2068774912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768505516549649, + "loss": 1.4357, + "theoretical_loss": 3.4492269222231147, + "tokens_seen": 2068840448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037683049147442326, + "loss": 1.2935, + "theoretical_loss": 3.4492179474536413, + "tokens_seen": 2068905984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768104312938817, + "loss": 1.3937, + "theoretical_loss": 3.4492089730480524, + "tokens_seen": 2068971520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037679037111334004, + "loss": 1.3955, + "theoretical_loss": 3.4491999990063222, + "tokens_seen": 2069037056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767703109327984, + "loss": 1.4024, + "theoretical_loss": 3.4491910253284237, + "tokens_seen": 2069102592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037675025075225676, + "loss": 1.3335, + "theoretical_loss": 3.449182052014331, + "tokens_seen": 2069168128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767301905717151, + "loss": 1.293, + "theoretical_loss": 3.4491730790640176, + "tokens_seen": 2069233664 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037671013039117354, + "loss": 1.3413, + "theoretical_loss": 3.4491641064774576, + "tokens_seen": 2069299200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766900702106319, + "loss": 1.4542, + "theoretical_loss": 3.449155134254625, + "tokens_seen": 2069364736 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 754612, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3888747692108154, + "objective/train/theoretical_loss": 3.4491461623954924, + "objective/train/tokens_used": 439889376, + "theoretical_loss": 3.4491461623954924, + "tokens_seen": 2069430272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037667001003009027, + "loss": 1.3643, + "theoretical_loss": 3.4491461623954924, + "tokens_seen": 2069430272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037664994984954863, + "loss": 1.2622, + "theoretical_loss": 3.449137190900035, + "tokens_seen": 2069495808 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037662988966900705, + "loss": 1.3799, + "theoretical_loss": 3.449128219768225, + "tokens_seen": 2069561344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766098294884654, + "loss": 1.3806, + "theoretical_loss": 3.449119249000037, + "tokens_seen": 2069626880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037658976930792377, + "loss": 1.4275, + "theoretical_loss": 3.4491102785954446, + "tokens_seen": 2069692416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037656970912738213, + "loss": 1.3538, + "theoretical_loss": 3.4491013085544218, + "tokens_seen": 2069757952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765496489468405, + "loss": 1.3409, + "theoretical_loss": 3.449092338876942, + "tokens_seen": 2069823488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765295887662989, + "loss": 1.4112, + "theoretical_loss": 3.4490833695629792, + "tokens_seen": 2069889024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765095285857573, + "loss": 1.3635, + "theoretical_loss": 3.449074400612507, + "tokens_seen": 2069954560 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037648946840521564, + "loss": 1.3504, + "theoretical_loss": 3.449065432025499, + "tokens_seen": 2070020096 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376469408224674, + "loss": 1.3353, + "theoretical_loss": 3.44905646380193, + "tokens_seen": 2070085632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764493480441324, + "loss": 1.3992, + "theoretical_loss": 3.4490474959417723, + "tokens_seen": 2070151168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764292878635908, + "loss": 1.3999, + "theoretical_loss": 3.4490385284450005, + "tokens_seen": 2070216704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037640922768304914, + "loss": 1.287, + "theoretical_loss": 3.4490295613115878, + "tokens_seen": 2070282240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763891675025075, + "loss": 1.4165, + "theoretical_loss": 3.449020594541509, + "tokens_seen": 2070347776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037636910732196586, + "loss": 1.3939, + "theoretical_loss": 3.4490116281347367, + "tokens_seen": 2070413312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763490471414243, + "loss": 1.352, + "theoretical_loss": 3.449002662091246, + "tokens_seen": 2070478848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037632898696088264, + "loss": 1.3397, + "theoretical_loss": 3.4489936964110095, + "tokens_seen": 2070544384 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376308926780341, + "loss": 1.3467, + "theoretical_loss": 3.448984731094001, + "tokens_seen": 2070609920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037628886659979937, + "loss": 1.4159, + "theoretical_loss": 3.448975766140195, + "tokens_seen": 2070675456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003762688064192578, + "loss": 1.3443, + "theoretical_loss": 3.448966801549565, + "tokens_seen": 2070740992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037624874623871615, + "loss": 1.4185, + "theoretical_loss": 3.448957837322085, + "tokens_seen": 2070806528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762286860581745, + "loss": 1.2558, + "theoretical_loss": 3.448948873457728, + "tokens_seen": 2070872064 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037620862587763287, + "loss": 1.376, + "theoretical_loss": 3.4489399099564695, + "tokens_seen": 2070937600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037618856569709123, + "loss": 1.2335, + "theoretical_loss": 3.4489309468182814, + "tokens_seen": 2071003136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037616850551654965, + "loss": 1.345, + "theoretical_loss": 3.448921984043138, + "tokens_seen": 2071068672 + }, + { + "epoch": 0.26, + "learning_rate": 0.000376148445336008, + "loss": 1.4028, + "theoretical_loss": 3.4489130216310135, + "tokens_seen": 2071134208 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037612838515546637, + "loss": 1.4193, + "theoretical_loss": 3.4489040595818823, + "tokens_seen": 2071199744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761083249749248, + "loss": 1.4051, + "theoretical_loss": 3.4488950978957167, + "tokens_seen": 2071265280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760882647943832, + "loss": 1.4004, + "theoretical_loss": 3.448886136572492, + "tokens_seen": 2071330816 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037606820461384157, + "loss": 1.4277, + "theoretical_loss": 3.448877175612181, + "tokens_seen": 2071396352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037604814443329993, + "loss": 1.3254, + "theoretical_loss": 3.448868215014758, + "tokens_seen": 2071461888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760280842527583, + "loss": 1.251, + "theoretical_loss": 3.4488592547801966, + "tokens_seen": 2071527424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037600802407221665, + "loss": 1.4546, + "theoretical_loss": 3.4488502949084703, + "tokens_seen": 2071592960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037598796389167507, + "loss": 1.3047, + "theoretical_loss": 3.448841335399554, + "tokens_seen": 2071658496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037596790371113343, + "loss": 1.3826, + "theoretical_loss": 3.4488323762534208, + "tokens_seen": 2071724032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759478435305918, + "loss": 1.2144, + "theoretical_loss": 3.4488234174700443, + "tokens_seen": 2071789568 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037592778335005016, + "loss": 1.3958, + "theoretical_loss": 3.448814459049399, + "tokens_seen": 2071855104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759077231695086, + "loss": 1.2211, + "theoretical_loss": 3.448805500991458, + "tokens_seen": 2071920640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037588766298896694, + "loss": 1.4389, + "theoretical_loss": 3.4487965432961962, + "tokens_seen": 2071986176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758676028084253, + "loss": 1.2737, + "theoretical_loss": 3.4487875859635864, + "tokens_seen": 2072051712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037584754262788366, + "loss": 1.2849, + "theoretical_loss": 3.4487786289936024, + "tokens_seen": 2072117248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758274824473421, + "loss": 1.3343, + "theoretical_loss": 3.448769672386219, + "tokens_seen": 2072182784 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037580742226680044, + "loss": 1.3565, + "theoretical_loss": 3.4487607161414093, + "tokens_seen": 2072248320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757873620862588, + "loss": 1.2022, + "theoretical_loss": 3.4487517602591478, + "tokens_seen": 2072313856 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037576730190571716, + "loss": 1.364, + "theoretical_loss": 3.4487428047394073, + "tokens_seen": 2072379392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757472417251755, + "loss": 1.4593, + "theoretical_loss": 3.4487338495821627, + "tokens_seen": 2072444928 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037572718154463394, + "loss": 1.4096, + "theoretical_loss": 3.4487248947873876, + "tokens_seen": 2072510464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757071213640923, + "loss": 1.3664, + "theoretical_loss": 3.4487159403550556, + "tokens_seen": 2072576000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037568706118355067, + "loss": 1.215, + "theoretical_loss": 3.4487069862851407, + "tokens_seen": 2072641536 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 755744, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3763511180877686, + "objective/train/theoretical_loss": 3.448698032577617, + "objective/train/tokens_used": 443166176, + "theoretical_loss": 3.448698032577617, + "tokens_seen": 2072707072 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037566700100300903, + "loss": 1.3839, + "theoretical_loss": 3.448698032577617, + "tokens_seen": 2072707072 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037564694082246745, + "loss": 1.3018, + "theoretical_loss": 3.448689079232458, + "tokens_seen": 2072772608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756268806419258, + "loss": 1.3844, + "theoretical_loss": 3.448680126249638, + "tokens_seen": 2072838144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037560682046138417, + "loss": 1.4293, + "theoretical_loss": 3.44867117362913, + "tokens_seen": 2072903680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037558676028084253, + "loss": 1.4653, + "theoretical_loss": 3.448662221370909, + "tokens_seen": 2072969216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755667001003009, + "loss": 1.3939, + "theoretical_loss": 3.4486532694749483, + "tokens_seen": 2073034752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755466399197593, + "loss": 1.3372, + "theoretical_loss": 3.4486443179412216, + "tokens_seen": 2073100288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755265797392177, + "loss": 1.2907, + "theoretical_loss": 3.4486353667697034, + "tokens_seen": 2073165824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037550651955867604, + "loss": 1.4508, + "theoretical_loss": 3.4486264159603675, + "tokens_seen": 2073231360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754864593781344, + "loss": 1.418, + "theoretical_loss": 3.448617465513187, + "tokens_seen": 2073296896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754663991975928, + "loss": 1.3827, + "theoretical_loss": 3.4486085154281367, + "tokens_seen": 2073362432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754463390170512, + "loss": 1.3121, + "theoretical_loss": 3.4485995657051904, + "tokens_seen": 2073427968 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037542627883650954, + "loss": 1.2852, + "theoretical_loss": 3.4485906163443216, + "tokens_seen": 2073493504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754062186559679, + "loss": 1.2771, + "theoretical_loss": 3.448581667345504, + "tokens_seen": 2073559040 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037538615847542626, + "loss": 1.3048, + "theoretical_loss": 3.4485727187087125, + "tokens_seen": 2073624576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753660982948847, + "loss": 1.3124, + "theoretical_loss": 3.4485637704339203, + "tokens_seen": 2073690112 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037534603811434304, + "loss": 1.2474, + "theoretical_loss": 3.4485548225211007, + "tokens_seen": 2073755648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753259779338014, + "loss": 1.4349, + "theoretical_loss": 3.448545874970229, + "tokens_seen": 2073821184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037530591775325977, + "loss": 1.2436, + "theoretical_loss": 3.4485369277812783, + "tokens_seen": 2073886720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752858575727182, + "loss": 1.2962, + "theoretical_loss": 3.448527980954223, + "tokens_seen": 2073952256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037526579739217655, + "loss": 1.3145, + "theoretical_loss": 3.4485190344890366, + "tokens_seen": 2074017792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752457372116349, + "loss": 1.3318, + "theoretical_loss": 3.448510088385693, + "tokens_seen": 2074083328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037522567703109327, + "loss": 1.2096, + "theoretical_loss": 3.448501142644166, + "tokens_seen": 2074148864 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037520561685055163, + "loss": 1.3782, + "theoretical_loss": 3.4484921972644305, + "tokens_seen": 2074214400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037518555667001005, + "loss": 1.3167, + "theoretical_loss": 3.4484832522464597, + "tokens_seen": 2074279936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751654964894684, + "loss": 1.3409, + "theoretical_loss": 3.448474307590227, + "tokens_seen": 2074345472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751454363089268, + "loss": 1.4329, + "theoretical_loss": 3.448465363295707, + "tokens_seen": 2074411008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037512537612838514, + "loss": 1.4182, + "theoretical_loss": 3.4484564193628744, + "tokens_seen": 2074476544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037510531594784355, + "loss": 1.3918, + "theoretical_loss": 3.4484474757917014, + "tokens_seen": 2074542080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750852557673019, + "loss": 1.4584, + "theoretical_loss": 3.4484385325821636, + "tokens_seen": 2074607616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750651955867603, + "loss": 1.3237, + "theoretical_loss": 3.4484295897342334, + "tokens_seen": 2074673152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037504513540621864, + "loss": 1.298, + "theoretical_loss": 3.448420647247886, + "tokens_seen": 2074738688 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375025075225677, + "loss": 1.217, + "theoretical_loss": 3.448411705123095, + "tokens_seen": 2074804224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750050150451354, + "loss": 1.4113, + "theoretical_loss": 3.4484027633598346, + "tokens_seen": 2074869760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749849548645938, + "loss": 1.3677, + "theoretical_loss": 3.4483938219580783, + "tokens_seen": 2074935296 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037496489468405214, + "loss": 1.2873, + "theoretical_loss": 3.4483848809178, + "tokens_seen": 2075000832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749448345035105, + "loss": 1.2899, + "theoretical_loss": 3.448375940238974, + "tokens_seen": 2075066368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749247743229689, + "loss": 1.2029, + "theoretical_loss": 3.448366999921575, + "tokens_seen": 2075131904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749047141424273, + "loss": 1.388, + "theoretical_loss": 3.448358059965575, + "tokens_seen": 2075197440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037488465396188564, + "loss": 1.4259, + "theoretical_loss": 3.44834912037095, + "tokens_seen": 2075262976 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374864593781344, + "loss": 1.3073, + "theoretical_loss": 3.4483401811376724, + "tokens_seen": 2075328512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037484453360080237, + "loss": 1.3622, + "theoretical_loss": 3.4483312422657173, + "tokens_seen": 2075394048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748244734202608, + "loss": 1.2799, + "theoretical_loss": 3.4483223037550585, + "tokens_seen": 2075459584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037480441323971915, + "loss": 1.2492, + "theoretical_loss": 3.4483133656056695, + "tokens_seen": 2075525120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747843530591775, + "loss": 1.4148, + "theoretical_loss": 3.4483044278175248, + "tokens_seen": 2075590656 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037476429287863587, + "loss": 1.2438, + "theoretical_loss": 3.448295490390598, + "tokens_seen": 2075656192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747442326980943, + "loss": 1.4106, + "theoretical_loss": 3.448286553324863, + "tokens_seen": 2075721728 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037472417251755265, + "loss": 1.3273, + "theoretical_loss": 3.4482776166202944, + "tokens_seen": 2075787264 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374704112337011, + "loss": 1.3045, + "theoretical_loss": 3.4482686802768656, + "tokens_seen": 2075852800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746840521564694, + "loss": 1.3206, + "theoretical_loss": 3.4482597442945515, + "tokens_seen": 2075918336 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 756913, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1429805755615234, + "objective/train/theoretical_loss": 3.4482508086733246, + "objective/train/tokens_used": 446442976, + "theoretical_loss": 3.4482508086733246, + "tokens_seen": 2075983872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037466399197592774, + "loss": 1.2074, + "theoretical_loss": 3.4482508086733246, + "tokens_seen": 2075983872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037464393179538615, + "loss": 1.4924, + "theoretical_loss": 3.448241873413161, + "tokens_seen": 2076049408 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037462387161484457, + "loss": 1.2696, + "theoretical_loss": 3.4482329385140327, + "tokens_seen": 2076114944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037460381143430293, + "loss": 1.3054, + "theoretical_loss": 3.4482240039759144, + "tokens_seen": 2076180480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745837512537613, + "loss": 1.3742, + "theoretical_loss": 3.4482150697987803, + "tokens_seen": 2076246016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745636910732197, + "loss": 1.3688, + "theoretical_loss": 3.4482061359826046, + "tokens_seen": 2076311552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745436308926781, + "loss": 1.4298, + "theoretical_loss": 3.448197202527361, + "tokens_seen": 2076377088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037452357071213644, + "loss": 1.3149, + "theoretical_loss": 3.4481882694330235, + "tokens_seen": 2076442624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745035105315948, + "loss": 1.2994, + "theoretical_loss": 3.4481793366995666, + "tokens_seen": 2076508160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744834503510532, + "loss": 1.3241, + "theoretical_loss": 3.448170404326964, + "tokens_seen": 2076573696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744633901705116, + "loss": 1.2989, + "theoretical_loss": 3.4481614723151894, + "tokens_seen": 2076639232 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037444332998996994, + "loss": 1.3164, + "theoretical_loss": 3.448152540664217, + "tokens_seen": 2076704768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744232698094283, + "loss": 1.4324, + "theoretical_loss": 3.4481436093740214, + "tokens_seen": 2076770304 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037440320962888666, + "loss": 1.2686, + "theoretical_loss": 3.448134678444576, + "tokens_seen": 2076835840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743831494483451, + "loss": 1.3958, + "theoretical_loss": 3.4481257478758556, + "tokens_seen": 2076901376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037436308926780344, + "loss": 1.3549, + "theoretical_loss": 3.4481168176678327, + "tokens_seen": 2076966912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743430290872618, + "loss": 1.3391, + "theoretical_loss": 3.448107887820483, + "tokens_seen": 2077032448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037432296890672017, + "loss": 1.3458, + "theoretical_loss": 3.44809895833378, + "tokens_seen": 2077097984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743029087261786, + "loss": 1.2878, + "theoretical_loss": 3.4480900292076977, + "tokens_seen": 2077163520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037428284854563695, + "loss": 1.2474, + "theoretical_loss": 3.4480811004422103, + "tokens_seen": 2077229056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742627883650953, + "loss": 1.3133, + "theoretical_loss": 3.4480721720372913, + "tokens_seen": 2077294592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037424272818455367, + "loss": 1.2687, + "theoretical_loss": 3.448063243992915, + "tokens_seen": 2077360128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037422266800401203, + "loss": 1.4019, + "theoretical_loss": 3.4480543163090562, + "tokens_seen": 2077425664 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037420260782347045, + "loss": 1.2623, + "theoretical_loss": 3.4480453889856886, + "tokens_seen": 2077491200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741825476429288, + "loss": 1.3361, + "theoretical_loss": 3.4480364620227855, + "tokens_seen": 2077556736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741624874623872, + "loss": 1.3704, + "theoretical_loss": 3.448027535420322, + "tokens_seen": 2077622272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037414242728184554, + "loss": 1.3427, + "theoretical_loss": 3.448018609178271, + "tokens_seen": 2077687808 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037412236710130395, + "loss": 1.315, + "theoretical_loss": 3.448009683296608, + "tokens_seen": 2077753344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741023069207623, + "loss": 1.2976, + "theoretical_loss": 3.448000757775306, + "tokens_seen": 2077818880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740822467402207, + "loss": 1.2403, + "theoretical_loss": 3.4479918326143397, + "tokens_seen": 2077884416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037406218655967904, + "loss": 1.3891, + "theoretical_loss": 3.447982907813683, + "tokens_seen": 2077949952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740421263791374, + "loss": 1.3874, + "theoretical_loss": 3.44797398337331, + "tokens_seen": 2078015488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740220661985958, + "loss": 1.3062, + "theoretical_loss": 3.4479650592931947, + "tokens_seen": 2078081024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740020060180542, + "loss": 1.4022, + "theoretical_loss": 3.4479561355733113, + "tokens_seen": 2078146560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037398194583751254, + "loss": 1.3615, + "theoretical_loss": 3.447947212213634, + "tokens_seen": 2078212096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739618856569709, + "loss": 1.3038, + "theoretical_loss": 3.447938289214137, + "tokens_seen": 2078277632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739418254764293, + "loss": 1.3652, + "theoretical_loss": 3.447929366574793, + "tokens_seen": 2078343168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739217652958877, + "loss": 1.4028, + "theoretical_loss": 3.447920444295578, + "tokens_seen": 2078408704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037390170511534605, + "loss": 1.2863, + "theoretical_loss": 3.4479115223764656, + "tokens_seen": 2078474240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738816449348044, + "loss": 1.3141, + "theoretical_loss": 3.4479026008174296, + "tokens_seen": 2078539776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037386158475426277, + "loss": 1.3018, + "theoretical_loss": 3.447893679618444, + "tokens_seen": 2078605312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738415245737212, + "loss": 1.3015, + "theoretical_loss": 3.447884758779483, + "tokens_seen": 2078670848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037382146439317955, + "loss": 1.3282, + "theoretical_loss": 3.4478758383005212, + "tokens_seen": 2078736384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738014042126379, + "loss": 1.2612, + "theoretical_loss": 3.447866918181532, + "tokens_seen": 2078801920 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037378134403209627, + "loss": 1.337, + "theoretical_loss": 3.4478579984224904, + "tokens_seen": 2078867456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737612838515547, + "loss": 1.3796, + "theoretical_loss": 3.44784907902337, + "tokens_seen": 2078932992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037374122367101305, + "loss": 1.3387, + "theoretical_loss": 3.4478401599841444, + "tokens_seen": 2078998528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737211634904714, + "loss": 1.3103, + "theoretical_loss": 3.447831241304789, + "tokens_seen": 2079064064 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737011033099298, + "loss": 1.3103, + "theoretical_loss": 3.4478223229852767, + "tokens_seen": 2079129600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037368104312938814, + "loss": 1.4519, + "theoretical_loss": 3.447813405025582, + "tokens_seen": 2079195136 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 758165, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.335489273071289, + "objective/train/theoretical_loss": 3.44780448742568, + "objective/train/tokens_used": 449719776, + "theoretical_loss": 3.44780448742568, + "tokens_seen": 2079260672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037366098294884655, + "loss": 1.3579, + "theoretical_loss": 3.44780448742568, + "tokens_seen": 2079260672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736409227683049, + "loss": 1.2901, + "theoretical_loss": 3.4477955701855434, + "tokens_seen": 2079326208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736208625877633, + "loss": 1.4038, + "theoretical_loss": 3.447786653305147, + "tokens_seen": 2079391744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037360080240722164, + "loss": 1.2924, + "theoretical_loss": 3.4477777367844653, + "tokens_seen": 2079457280 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037358074222668006, + "loss": 1.4341, + "theoretical_loss": 3.4477688206234722, + "tokens_seen": 2079522816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735606820461384, + "loss": 1.4142, + "theoretical_loss": 3.4477599048221412, + "tokens_seen": 2079588352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735406218655968, + "loss": 1.3315, + "theoretical_loss": 3.4477509893804474, + "tokens_seen": 2079653888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037352056168505514, + "loss": 1.4694, + "theoretical_loss": 3.4477420742983647, + "tokens_seen": 2079719424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037350050150451356, + "loss": 1.2809, + "theoretical_loss": 3.4477331595758667, + "tokens_seen": 2079784960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734804413239719, + "loss": 1.3458, + "theoretical_loss": 3.447724245212928, + "tokens_seen": 2079850496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734603811434303, + "loss": 1.427, + "theoretical_loss": 3.4477153312095234, + "tokens_seen": 2079916032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037344032096288865, + "loss": 1.3898, + "theoretical_loss": 3.447706417565626, + "tokens_seen": 2079981568 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373420260782347, + "loss": 1.3154, + "theoretical_loss": 3.4476975042812104, + "tokens_seen": 2080047104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734002006018054, + "loss": 1.3298, + "theoretical_loss": 3.4476885913562514, + "tokens_seen": 2080112640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733801404212638, + "loss": 1.3451, + "theoretical_loss": 3.447679678790722, + "tokens_seen": 2080178176 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037336008024072215, + "loss": 1.3329, + "theoretical_loss": 3.4476707665845967, + "tokens_seen": 2080243712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733400200601805, + "loss": 1.3843, + "theoretical_loss": 3.4476618547378504, + "tokens_seen": 2080309248 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037331995987963893, + "loss": 1.3055, + "theoretical_loss": 3.447652943250457, + "tokens_seen": 2080374784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732998996990973, + "loss": 1.4442, + "theoretical_loss": 3.44764403212239, + "tokens_seen": 2080440320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037327983951855565, + "loss": 1.3153, + "theoretical_loss": 3.4476351213536245, + "tokens_seen": 2080505856 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373259779338014, + "loss": 1.2781, + "theoretical_loss": 3.447626210944134, + "tokens_seen": 2080571392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732397191574724, + "loss": 1.4366, + "theoretical_loss": 3.447617300893893, + "tokens_seen": 2080636928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732196589769308, + "loss": 1.337, + "theoretical_loss": 3.447608391202876, + "tokens_seen": 2080702464 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037319959879638916, + "loss": 1.3664, + "theoretical_loss": 3.447599481871057, + "tokens_seen": 2080768000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731795386158475, + "loss": 1.2406, + "theoretical_loss": 3.44759057289841, + "tokens_seen": 2080833536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731594784353059, + "loss": 1.2714, + "theoretical_loss": 3.4475816642849093, + "tokens_seen": 2080899072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731394182547643, + "loss": 1.3488, + "theoretical_loss": 3.447572756030529, + "tokens_seen": 2080964608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731193580742227, + "loss": 1.3154, + "theoretical_loss": 3.4475638481352435, + "tokens_seen": 2081030144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730992978936811, + "loss": 1.2924, + "theoretical_loss": 3.4475549405990273, + "tokens_seen": 2081095680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037307923771313944, + "loss": 1.4311, + "theoretical_loss": 3.4475460334218537, + "tokens_seen": 2081161216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730591775325978, + "loss": 1.4018, + "theoretical_loss": 3.447537126603698, + "tokens_seen": 2081226752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730391173520562, + "loss": 1.3416, + "theoretical_loss": 3.447528220144534, + "tokens_seen": 2081292288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730190571715146, + "loss": 1.3829, + "theoretical_loss": 3.447519314044335, + "tokens_seen": 2081357824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037299899699097294, + "loss": 1.3314, + "theoretical_loss": 3.447510408303077, + "tokens_seen": 2081423360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729789368104313, + "loss": 1.4558, + "theoretical_loss": 3.4475015029207334, + "tokens_seen": 2081488896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729588766298897, + "loss": 1.3249, + "theoretical_loss": 3.447492597897278, + "tokens_seen": 2081554432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729388164493481, + "loss": 1.3054, + "theoretical_loss": 3.4474836932326856, + "tokens_seen": 2081619968 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037291875626880645, + "loss": 1.1993, + "theoretical_loss": 3.4474747889269297, + "tokens_seen": 2081685504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728986960882648, + "loss": 1.3733, + "theoretical_loss": 3.4474658849799855, + "tokens_seen": 2081751040 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037287863590772317, + "loss": 1.2718, + "theoretical_loss": 3.447456981391827, + "tokens_seen": 2081816576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728585757271816, + "loss": 1.39, + "theoretical_loss": 3.447448078162428, + "tokens_seen": 2081882112 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037283851554663995, + "loss": 1.3114, + "theoretical_loss": 3.4474391752917635, + "tokens_seen": 2081947648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728184553660983, + "loss": 1.3593, + "theoretical_loss": 3.4474302727798065, + "tokens_seen": 2082013184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727983951855567, + "loss": 1.3305, + "theoretical_loss": 3.4474213706265324, + "tokens_seen": 2082078720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727783350050151, + "loss": 1.4255, + "theoretical_loss": 3.4474124688319154, + "tokens_seen": 2082144256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037275827482447345, + "loss": 1.3578, + "theoretical_loss": 3.4474035673959293, + "tokens_seen": 2082209792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727382146439318, + "loss": 1.3362, + "theoretical_loss": 3.4473946663185484, + "tokens_seen": 2082275328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727181544633902, + "loss": 1.3871, + "theoretical_loss": 3.4473857655997473, + "tokens_seen": 2082340864 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037269809428284854, + "loss": 1.3939, + "theoretical_loss": 3.4473768652395, + "tokens_seen": 2082406400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037267803410230696, + "loss": 1.3154, + "theoretical_loss": 3.4473679652377807, + "tokens_seen": 2082471936 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 759184, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.221189022064209, + "objective/train/theoretical_loss": 3.447359065594564, + "objective/train/tokens_used": 452996576, + "theoretical_loss": 3.447359065594564, + "tokens_seen": 2082537472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726579739217653, + "loss": 1.2887, + "theoretical_loss": 3.447359065594564, + "tokens_seen": 2082537472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726379137412237, + "loss": 1.4102, + "theoretical_loss": 3.447350166309824, + "tokens_seen": 2082603008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037261785356068204, + "loss": 1.2748, + "theoretical_loss": 3.4473412673835346, + "tokens_seen": 2082668544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037259779338014046, + "loss": 1.3869, + "theoretical_loss": 3.4473323688156707, + "tokens_seen": 2082734080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725777331995988, + "loss": 1.3816, + "theoretical_loss": 3.4473234706062064, + "tokens_seen": 2082799616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725576730190572, + "loss": 1.2924, + "theoretical_loss": 3.4473145727551158, + "tokens_seen": 2082865152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037253761283851554, + "loss": 1.3734, + "theoretical_loss": 3.447305675262373, + "tokens_seen": 2082930688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037251755265797396, + "loss": 1.2656, + "theoretical_loss": 3.4472967781279533, + "tokens_seen": 2082996224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724974924774323, + "loss": 1.2967, + "theoretical_loss": 3.44728788135183, + "tokens_seen": 2083061760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724774322968907, + "loss": 1.3389, + "theoretical_loss": 3.4472789849339778, + "tokens_seen": 2083127296 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037245737211634905, + "loss": 1.4214, + "theoretical_loss": 3.447270088874371, + "tokens_seen": 2083192832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724373119358074, + "loss": 1.4837, + "theoretical_loss": 3.4472611931729835, + "tokens_seen": 2083258368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724172517552658, + "loss": 1.3538, + "theoretical_loss": 3.4472522978297904, + "tokens_seen": 2083323904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723971915747242, + "loss": 1.3755, + "theoretical_loss": 3.447243402844765, + "tokens_seen": 2083389440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037237713139418255, + "loss": 1.3115, + "theoretical_loss": 3.447234508217883, + "tokens_seen": 2083454976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723570712136409, + "loss": 1.3346, + "theoretical_loss": 3.447225613949117, + "tokens_seen": 2083520512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037233701103309933, + "loss": 1.3728, + "theoretical_loss": 3.4472167200384423, + "tokens_seen": 2083586048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723169508525577, + "loss": 1.3903, + "theoretical_loss": 3.4472078264858332, + "tokens_seen": 2083651584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037229689067201605, + "loss": 1.3117, + "theoretical_loss": 3.447198933291264, + "tokens_seen": 2083717120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722768304914744, + "loss": 1.3741, + "theoretical_loss": 3.447190040454709, + "tokens_seen": 2083782656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722567703109328, + "loss": 1.329, + "theoretical_loss": 3.447181147976142, + "tokens_seen": 2083848192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722367101303912, + "loss": 1.3859, + "theoretical_loss": 3.4471722558555387, + "tokens_seen": 2083913728 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037221664994984956, + "loss": 1.4165, + "theoretical_loss": 3.447163364092872, + "tokens_seen": 2083979264 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721965897693079, + "loss": 1.296, + "theoretical_loss": 3.4471544726881165, + "tokens_seen": 2084044800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721765295887663, + "loss": 1.2957, + "theoretical_loss": 3.4471455816412475, + "tokens_seen": 2084110336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721564694082247, + "loss": 1.3047, + "theoretical_loss": 3.447136690952238, + "tokens_seen": 2084175872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037213640922768306, + "loss": 1.3856, + "theoretical_loss": 3.4471278006210637, + "tokens_seen": 2084241408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721163490471414, + "loss": 1.3583, + "theoretical_loss": 3.447118910647698, + "tokens_seen": 2084306944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720962888665998, + "loss": 1.3197, + "theoretical_loss": 3.447110021032115, + "tokens_seen": 2084372480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037207622868605815, + "loss": 1.3628, + "theoretical_loss": 3.44710113177429, + "tokens_seen": 2084438016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037205616850551656, + "loss": 1.3225, + "theoretical_loss": 3.447092242874197, + "tokens_seen": 2084503552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720361083249749, + "loss": 1.3943, + "theoretical_loss": 3.4470833543318102, + "tokens_seen": 2084569088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720160481444333, + "loss": 1.4609, + "theoretical_loss": 3.4470744661471038, + "tokens_seen": 2084634624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037199598796389165, + "loss": 1.3049, + "theoretical_loss": 3.4470655783200526, + "tokens_seen": 2084700160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037197592778335007, + "loss": 1.2727, + "theoretical_loss": 3.447056690850631, + "tokens_seen": 2084765696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037195586760280843, + "loss": 1.3587, + "theoretical_loss": 3.447047803738812, + "tokens_seen": 2084831232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719358074222668, + "loss": 1.4363, + "theoretical_loss": 3.447038916984572, + "tokens_seen": 2084896768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037191574724172515, + "loss": 1.4422, + "theoretical_loss": 3.4470300305878845, + "tokens_seen": 2084962304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718956870611835, + "loss": 1.2346, + "theoretical_loss": 3.4470211445487235, + "tokens_seen": 2085027840 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037187562688064193, + "loss": 1.3941, + "theoretical_loss": 3.447012258867064, + "tokens_seen": 2085093376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718555667001003, + "loss": 1.2277, + "theoretical_loss": 3.44700337354288, + "tokens_seen": 2085158912 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037183550651955866, + "loss": 1.4075, + "theoretical_loss": 3.446994488576146, + "tokens_seen": 2085224448 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371815446339017, + "loss": 1.2745, + "theoretical_loss": 3.4469856039668363, + "tokens_seen": 2085289984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037179538615847544, + "loss": 1.2611, + "theoretical_loss": 3.4469767197149253, + "tokens_seen": 2085355520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717753259779338, + "loss": 1.3076, + "theoretical_loss": 3.446967835820388, + "tokens_seen": 2085421056 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037175526579739216, + "loss": 1.279, + "theoretical_loss": 3.4469589522831976, + "tokens_seen": 2085486592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717352056168505, + "loss": 1.3766, + "theoretical_loss": 3.4469500691033295, + "tokens_seen": 2085552128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717151454363089, + "loss": 1.4607, + "theoretical_loss": 3.4469411862807577, + "tokens_seen": 2085617664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716950852557673, + "loss": 1.3921, + "theoretical_loss": 3.4469323038154567, + "tokens_seen": 2085683200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037167502507522566, + "loss": 1.4168, + "theoretical_loss": 3.4469234217074005, + "tokens_seen": 2085748736 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 760445, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3558768033981323, + "objective/train/theoretical_loss": 3.4469145399565644, + "objective/train/tokens_used": 456273376, + "theoretical_loss": 3.4469145399565644, + "tokens_seen": 2085814272 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371654964894684, + "loss": 1.3861, + "theoretical_loss": 3.4469145399565644, + "tokens_seen": 2085814272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716349047141424, + "loss": 1.2515, + "theoretical_loss": 3.4469056585629216, + "tokens_seen": 2085879808 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037161484453360086, + "loss": 1.3498, + "theoretical_loss": 3.4468967775264474, + "tokens_seen": 2085945344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715947843530592, + "loss": 1.2705, + "theoretical_loss": 3.4468878968471164, + "tokens_seen": 2086010880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715747241725176, + "loss": 1.5168, + "theoretical_loss": 3.4468790165249024, + "tokens_seen": 2086076416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037155466399197595, + "loss": 1.3204, + "theoretical_loss": 3.44687013655978, + "tokens_seen": 2086141952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037153460381143436, + "loss": 1.4237, + "theoretical_loss": 3.4468612569517236, + "tokens_seen": 2086207488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715145436308927, + "loss": 1.3912, + "theoretical_loss": 3.446852377700708, + "tokens_seen": 2086273024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714944834503511, + "loss": 1.4818, + "theoretical_loss": 3.4468434988067065, + "tokens_seen": 2086338560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037147442326980945, + "loss": 1.4445, + "theoretical_loss": 3.446834620269695, + "tokens_seen": 2086404096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714543630892678, + "loss": 1.4011, + "theoretical_loss": 3.446825742089647, + "tokens_seen": 2086469632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037143430290872623, + "loss": 1.2855, + "theoretical_loss": 3.4468168642665376, + "tokens_seen": 2086535168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714142427281846, + "loss": 1.3122, + "theoretical_loss": 3.4468079868003407, + "tokens_seen": 2086600704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037139418254764295, + "loss": 1.3643, + "theoretical_loss": 3.446799109691031, + "tokens_seen": 2086666240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713741223671013, + "loss": 1.1952, + "theoretical_loss": 3.446790232938582, + "tokens_seen": 2086731776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037135406218655973, + "loss": 1.377, + "theoretical_loss": 3.44678135654297, + "tokens_seen": 2086797312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713340020060181, + "loss": 1.3972, + "theoretical_loss": 3.446772480504168, + "tokens_seen": 2086862848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037131394182547645, + "loss": 1.3611, + "theoretical_loss": 3.4467636048221513, + "tokens_seen": 2086928384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712938816449348, + "loss": 1.4277, + "theoretical_loss": 3.4467547294968934, + "tokens_seen": 2086993920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712738214643932, + "loss": 1.385, + "theoretical_loss": 3.44674585452837, + "tokens_seen": 2087059456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712537612838516, + "loss": 1.2567, + "theoretical_loss": 3.446736979916554, + "tokens_seen": 2087124992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037123370110330996, + "loss": 1.3637, + "theoretical_loss": 3.446728105661421, + "tokens_seen": 2087190528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712136409227683, + "loss": 1.262, + "theoretical_loss": 3.4467192317629456, + "tokens_seen": 2087256064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711935807422267, + "loss": 1.3156, + "theoretical_loss": 3.446710358221102, + "tokens_seen": 2087321600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711735205616851, + "loss": 1.3109, + "theoretical_loss": 3.446701485035864, + "tokens_seen": 2087387136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037115346038114346, + "loss": 1.2819, + "theoretical_loss": 3.4466926122072064, + "tokens_seen": 2087452672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711334002006018, + "loss": 1.3675, + "theoretical_loss": 3.4466837397351044, + "tokens_seen": 2087518208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711133400200602, + "loss": 1.4508, + "theoretical_loss": 3.4466748676195316, + "tokens_seen": 2087583744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037109327983951855, + "loss": 1.3272, + "theoretical_loss": 3.4466659958604633, + "tokens_seen": 2087649280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037107321965897696, + "loss": 1.3219, + "theoretical_loss": 3.4466571244578734, + "tokens_seen": 2087714816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710531594784353, + "loss": 1.4201, + "theoretical_loss": 3.446648253411736, + "tokens_seen": 2087780352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710330992978937, + "loss": 1.3165, + "theoretical_loss": 3.4466393827220267, + "tokens_seen": 2087845888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037101303911735205, + "loss": 1.3381, + "theoretical_loss": 3.4466305123887193, + "tokens_seen": 2087911424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037099297893681047, + "loss": 1.3481, + "theoretical_loss": 3.446621642411788, + "tokens_seen": 2087976960 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037097291875626883, + "loss": 1.2969, + "theoretical_loss": 3.446612772791208, + "tokens_seen": 2088042496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709528585757272, + "loss": 1.3704, + "theoretical_loss": 3.4466039035269533, + "tokens_seen": 2088108032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037093279839518555, + "loss": 1.3183, + "theoretical_loss": 3.4465950346189986, + "tokens_seen": 2088173568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709127382146439, + "loss": 1.3428, + "theoretical_loss": 3.446586166067319, + "tokens_seen": 2088239104 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037089267803410233, + "loss": 1.3269, + "theoretical_loss": 3.4465772978718876, + "tokens_seen": 2088304640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708726178535607, + "loss": 1.3119, + "theoretical_loss": 3.44656843003268, + "tokens_seen": 2088370176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037085255767301906, + "loss": 1.3168, + "theoretical_loss": 3.4465595625496706, + "tokens_seen": 2088435712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708324974924774, + "loss": 1.2566, + "theoretical_loss": 3.4465506954228338, + "tokens_seen": 2088501248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037081243731193584, + "loss": 1.3596, + "theoretical_loss": 3.4465418286521436, + "tokens_seen": 2088566784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707923771313942, + "loss": 1.4234, + "theoretical_loss": 3.4465329622375753, + "tokens_seen": 2088632320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037077231695085256, + "loss": 1.2814, + "theoretical_loss": 3.446524096179103, + "tokens_seen": 2088697856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707522567703109, + "loss": 1.2835, + "theoretical_loss": 3.4465152304767015, + "tokens_seen": 2088763392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707321965897693, + "loss": 1.3687, + "theoretical_loss": 3.446506365130345, + "tokens_seen": 2088828928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707121364092277, + "loss": 1.3366, + "theoretical_loss": 3.446497500140008, + "tokens_seen": 2088894464 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037069207622868606, + "loss": 1.4386, + "theoretical_loss": 3.4464886355056654, + "tokens_seen": 2088960000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706720160481444, + "loss": 1.3665, + "theoretical_loss": 3.446479771227292, + "tokens_seen": 2089025536 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 761616, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3948633670806885, + "objective/train/theoretical_loss": 3.446470907304861, + "objective/train/tokens_used": 459550176, + "theoretical_loss": 3.446470907304861, + "tokens_seen": 2089091072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706519558676028, + "loss": 1.3112, + "theoretical_loss": 3.446470907304861, + "tokens_seen": 2089091072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706318956870612, + "loss": 1.3293, + "theoretical_loss": 3.4464620437383484, + "tokens_seen": 2089156608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037061183550651957, + "loss": 1.293, + "theoretical_loss": 3.446453180527728, + "tokens_seen": 2089222144 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037059177532597793, + "loss": 1.413, + "theoretical_loss": 3.4464443176729747, + "tokens_seen": 2089287680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705717151454363, + "loss": 1.3553, + "theoretical_loss": 3.446435455174063, + "tokens_seen": 2089353216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037055165496489465, + "loss": 1.3972, + "theoretical_loss": 3.446426593030967, + "tokens_seen": 2089418752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037053159478435307, + "loss": 1.3368, + "theoretical_loss": 3.4464177312436615, + "tokens_seen": 2089484288 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037051153460381143, + "loss": 1.3972, + "theoretical_loss": 3.446408869812122, + "tokens_seen": 2089549824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704914744232698, + "loss": 1.2802, + "theoretical_loss": 3.446400008736321, + "tokens_seen": 2089615360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037047141424272816, + "loss": 1.3065, + "theoretical_loss": 3.446391148016235, + "tokens_seen": 2089680896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704513540621866, + "loss": 1.2517, + "theoretical_loss": 3.4463822876518377, + "tokens_seen": 2089746432 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037043129388164494, + "loss": 1.293, + "theoretical_loss": 3.4463734276431035, + "tokens_seen": 2089811968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704112337011033, + "loss": 1.3577, + "theoretical_loss": 3.446364567990008, + "tokens_seen": 2089877504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037039117352056166, + "loss": 1.3469, + "theoretical_loss": 3.4463557086925243, + "tokens_seen": 2089943040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703711133400201, + "loss": 1.3559, + "theoretical_loss": 3.446346849750628, + "tokens_seen": 2090008576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037035105315947844, + "loss": 1.266, + "theoretical_loss": 3.4463379911642935, + "tokens_seen": 2090074112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703309929789368, + "loss": 1.34, + "theoretical_loss": 3.446329132933495, + "tokens_seen": 2090139648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037031093279839516, + "loss": 1.4095, + "theoretical_loss": 3.4463202750582074, + "tokens_seen": 2090205184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702908726178535, + "loss": 1.2532, + "theoretical_loss": 3.4463114175384053, + "tokens_seen": 2090270720 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037027081243731194, + "loss": 1.3557, + "theoretical_loss": 3.446302560374064, + "tokens_seen": 2090336256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702507522567703, + "loss": 1.3104, + "theoretical_loss": 3.446293703565156, + "tokens_seen": 2090401792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037023069207622867, + "loss": 1.3731, + "theoretical_loss": 3.446284847111658, + "tokens_seen": 2090467328 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037021063189568703, + "loss": 1.3606, + "theoretical_loss": 3.4462759910135436, + "tokens_seen": 2090532864 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037019057171514544, + "loss": 1.1928, + "theoretical_loss": 3.4462671352707877, + "tokens_seen": 2090598400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701705115346038, + "loss": 1.3777, + "theoretical_loss": 3.4462582798833648, + "tokens_seen": 2090663936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037015045135406217, + "loss": 1.4143, + "theoretical_loss": 3.4462494248512496, + "tokens_seen": 2090729472 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037013039117352053, + "loss": 1.3055, + "theoretical_loss": 3.4462405701744165, + "tokens_seen": 2090795008 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037011033099297895, + "loss": 1.3743, + "theoretical_loss": 3.44623171585284, + "tokens_seen": 2090860544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037009027081243736, + "loss": 1.3612, + "theoretical_loss": 3.4462228618864956, + "tokens_seen": 2090926080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700702106318957, + "loss": 1.4749, + "theoretical_loss": 3.4462140082753567, + "tokens_seen": 2090991616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700501504513541, + "loss": 1.2325, + "theoretical_loss": 3.446205155019399, + "tokens_seen": 2091057152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037003009027081245, + "loss": 1.3897, + "theoretical_loss": 3.4461963021185964, + "tokens_seen": 2091122688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037001003009027087, + "loss": 1.3854, + "theoretical_loss": 3.4461874495729234, + "tokens_seen": 2091188224 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036998996990972923, + "loss": 1.3896, + "theoretical_loss": 3.4461785973823553, + "tokens_seen": 2091253760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699699097291876, + "loss": 1.3562, + "theoretical_loss": 3.4461697455468663, + "tokens_seen": 2091319296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036994984954864595, + "loss": 1.393, + "theoretical_loss": 3.4461608940664314, + "tokens_seen": 2091384832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699297893681043, + "loss": 1.347, + "theoretical_loss": 3.446152042941024, + "tokens_seen": 2091450368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036990972918756273, + "loss": 1.5436, + "theoretical_loss": 3.446143192170621, + "tokens_seen": 2091515904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698896690070211, + "loss": 1.363, + "theoretical_loss": 3.4461343417551946, + "tokens_seen": 2091581440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036986960882647946, + "loss": 1.2586, + "theoretical_loss": 3.446125491694721, + "tokens_seen": 2091646976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698495486459378, + "loss": 1.292, + "theoretical_loss": 3.4461166419891747, + "tokens_seen": 2091712512 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036982948846539624, + "loss": 1.3086, + "theoretical_loss": 3.4461077926385295, + "tokens_seen": 2091778048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698094282848546, + "loss": 1.4264, + "theoretical_loss": 3.446098943642761, + "tokens_seen": 2091843584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036978936810431296, + "loss": 1.3687, + "theoretical_loss": 3.446090095001843, + "tokens_seen": 2091909120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697693079237713, + "loss": 1.367, + "theoretical_loss": 3.4460812467157513, + "tokens_seen": 2091974656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697492477432297, + "loss": 1.3264, + "theoretical_loss": 3.4460723987844593, + "tokens_seen": 2092040192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697291875626881, + "loss": 1.3454, + "theoretical_loss": 3.446063551207942, + "tokens_seen": 2092105728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036970912738214646, + "loss": 1.3447, + "theoretical_loss": 3.446054703986175, + "tokens_seen": 2092171264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696890672016048, + "loss": 1.3832, + "theoretical_loss": 3.446045857119132, + "tokens_seen": 2092236800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696690070210632, + "loss": 1.3116, + "theoretical_loss": 3.446037010606788, + "tokens_seen": 2092302336 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 762773, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.473339319229126, + "objective/train/theoretical_loss": 3.446028164449117, + "objective/train/tokens_used": 462826976, + "theoretical_loss": 3.446028164449117, + "tokens_seen": 2092367872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696489468405216, + "loss": 1.4383, + "theoretical_loss": 3.446028164449117, + "tokens_seen": 2092367872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036962888665997997, + "loss": 1.293, + "theoretical_loss": 3.446019318646095, + "tokens_seen": 2092433408 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036960882647943833, + "loss": 1.3879, + "theoretical_loss": 3.4460104731976955, + "tokens_seen": 2092498944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695887662988967, + "loss": 1.4025, + "theoretical_loss": 3.4460016281038937, + "tokens_seen": 2092564480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036956870611835505, + "loss": 1.2483, + "theoretical_loss": 3.445992783364664, + "tokens_seen": 2092630016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036954864593781347, + "loss": 1.4044, + "theoretical_loss": 3.445983938979982, + "tokens_seen": 2092695552 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036952858575727183, + "loss": 1.3442, + "theoretical_loss": 3.4459750949498207, + "tokens_seen": 2092761088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695085255767302, + "loss": 1.4085, + "theoretical_loss": 3.4459662512741565, + "tokens_seen": 2092826624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036948846539618856, + "loss": 1.3335, + "theoretical_loss": 3.445957407952963, + "tokens_seen": 2092892160 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369468405215647, + "loss": 1.3896, + "theoretical_loss": 3.4459485649862147, + "tokens_seen": 2092957696 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036944834503510534, + "loss": 1.3336, + "theoretical_loss": 3.4459397223738875, + "tokens_seen": 2093023232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694282848545637, + "loss": 1.3698, + "theoretical_loss": 3.445930880115955, + "tokens_seen": 2093088768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036940822467402206, + "loss": 1.3368, + "theoretical_loss": 3.4459220382123927, + "tokens_seen": 2093154304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693881644934805, + "loss": 1.3133, + "theoretical_loss": 3.445913196663175, + "tokens_seen": 2093219840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036936810431293884, + "loss": 1.316, + "theoretical_loss": 3.4459043554682762, + "tokens_seen": 2093285376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693480441323972, + "loss": 1.4123, + "theoretical_loss": 3.4458955146276713, + "tokens_seen": 2093350912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036932798395185556, + "loss": 1.2945, + "theoretical_loss": 3.4458866741413354, + "tokens_seen": 2093416448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693079237713139, + "loss": 1.3152, + "theoretical_loss": 3.445877834009243, + "tokens_seen": 2093481984 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036928786359077234, + "loss": 1.2877, + "theoretical_loss": 3.4458689942313683, + "tokens_seen": 2093547520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692678034102307, + "loss": 1.3279, + "theoretical_loss": 3.4458601548076864, + "tokens_seen": 2093613056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036924774322968907, + "loss": 1.3856, + "theoretical_loss": 3.445851315738172, + "tokens_seen": 2093678592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036922768304914743, + "loss": 1.2926, + "theoretical_loss": 3.4458424770227998, + "tokens_seen": 2093744128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036920762286860585, + "loss": 1.339, + "theoretical_loss": 3.445833638661545, + "tokens_seen": 2093809664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691875626880642, + "loss": 1.3602, + "theoretical_loss": 3.4458248006543815, + "tokens_seen": 2093875200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036916750250752257, + "loss": 1.3063, + "theoretical_loss": 3.4458159630012846, + "tokens_seen": 2093940736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036914744232698093, + "loss": 1.2788, + "theoretical_loss": 3.445807125702229, + "tokens_seen": 2094006272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691273821464393, + "loss": 1.2991, + "theoretical_loss": 3.4457982887571887, + "tokens_seen": 2094071808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691073219658977, + "loss": 1.4432, + "theoretical_loss": 3.4457894521661396, + "tokens_seen": 2094137344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690872617853561, + "loss": 1.4004, + "theoretical_loss": 3.445780615929056, + "tokens_seen": 2094202880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036906720160481444, + "loss": 1.27, + "theoretical_loss": 3.445771780045912, + "tokens_seen": 2094268416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690471414242728, + "loss": 1.2412, + "theoretical_loss": 3.4457629445166833, + "tokens_seen": 2094333952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690270812437312, + "loss": 1.3104, + "theoretical_loss": 3.445754109341344, + "tokens_seen": 2094399488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690070210631896, + "loss": 1.4074, + "theoretical_loss": 3.4457452745198696, + "tokens_seen": 2094465024 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036898696088264794, + "loss": 1.3001, + "theoretical_loss": 3.445736440052234, + "tokens_seen": 2094530560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689669007021063, + "loss": 1.4295, + "theoretical_loss": 3.445727605938412, + "tokens_seen": 2094596096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036894684052156466, + "loss": 1.3814, + "theoretical_loss": 3.445718772178379, + "tokens_seen": 2094661632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689267803410231, + "loss": 1.2488, + "theoretical_loss": 3.445709938772109, + "tokens_seen": 2094727168 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036890672016048144, + "loss": 1.3465, + "theoretical_loss": 3.4457011057195777, + "tokens_seen": 2094792704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688866599799398, + "loss": 1.3506, + "theoretical_loss": 3.445692273020759, + "tokens_seen": 2094858240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036886659979939817, + "loss": 1.3613, + "theoretical_loss": 3.4456834406756283, + "tokens_seen": 2094923776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688465396188566, + "loss": 1.3898, + "theoretical_loss": 3.44567460868416, + "tokens_seen": 2094989312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036882647943831494, + "loss": 1.3197, + "theoretical_loss": 3.4456657770463286, + "tokens_seen": 2095054848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688064192577733, + "loss": 1.4066, + "theoretical_loss": 3.44565694576211, + "tokens_seen": 2095120384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036878635907723167, + "loss": 1.3436, + "theoretical_loss": 3.4456481148314775, + "tokens_seen": 2095185920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036876629889669003, + "loss": 1.3218, + "theoretical_loss": 3.445639284254407, + "tokens_seen": 2095251456 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036874623871614845, + "loss": 1.4083, + "theoretical_loss": 3.445630454030873, + "tokens_seen": 2095316992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687261785356068, + "loss": 1.3455, + "theoretical_loss": 3.44562162416085, + "tokens_seen": 2095382528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036870611835506517, + "loss": 1.3063, + "theoretical_loss": 3.4456127946443127, + "tokens_seen": 2095448064 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036868605817452353, + "loss": 1.2633, + "theoretical_loss": 3.4456039654812365, + "tokens_seen": 2095513600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036866599799398195, + "loss": 1.3725, + "theoretical_loss": 3.445595136671596, + "tokens_seen": 2095579136 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 764034, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1738439798355103, + "objective/train/theoretical_loss": 3.4455863082153657, + "objective/train/tokens_used": 466103776, + "theoretical_loss": 3.4455863082153657, + "tokens_seen": 2095644672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686459378134403, + "loss": 1.2526, + "theoretical_loss": 3.4455863082153657, + "tokens_seen": 2095644672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686258776328987, + "loss": 1.3044, + "theoretical_loss": 3.4455774801125205, + "tokens_seen": 2095710208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686058174523571, + "loss": 1.5151, + "theoretical_loss": 3.4455686523630358, + "tokens_seen": 2095775744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036858575727181545, + "loss": 1.3669, + "theoretical_loss": 3.4455598249668853, + "tokens_seen": 2095841280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036856569709127387, + "loss": 1.2847, + "theoretical_loss": 3.4455509979240446, + "tokens_seen": 2095906816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036854563691073223, + "loss": 1.3165, + "theoretical_loss": 3.4455421712344885, + "tokens_seen": 2095972352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685255767301906, + "loss": 1.3753, + "theoretical_loss": 3.445533344898191, + "tokens_seen": 2096037888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036850551654964896, + "loss": 1.3569, + "theoretical_loss": 3.4455245189151285, + "tokens_seen": 2096103424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684854563691074, + "loss": 1.4557, + "theoretical_loss": 3.445515693285274, + "tokens_seen": 2096168960 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036846539618856574, + "loss": 1.3958, + "theoretical_loss": 3.4455068680086036, + "tokens_seen": 2096234496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684453360080241, + "loss": 1.3968, + "theoretical_loss": 3.4454980430850917, + "tokens_seen": 2096300032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036842527582748246, + "loss": 1.3377, + "theoretical_loss": 3.4454892185147132, + "tokens_seen": 2096365568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684052156469409, + "loss": 1.4239, + "theoretical_loss": 3.4454803942974426, + "tokens_seen": 2096431104 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036838515546639924, + "loss": 1.4622, + "theoretical_loss": 3.4454715704332552, + "tokens_seen": 2096496640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683650952858576, + "loss": 1.4109, + "theoretical_loss": 3.445462746922126, + "tokens_seen": 2096562176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036834503510531596, + "loss": 1.2503, + "theoretical_loss": 3.4454539237640285, + "tokens_seen": 2096627712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683249749247743, + "loss": 1.403, + "theoretical_loss": 3.4454451009589393, + "tokens_seen": 2096693248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036830491474423274, + "loss": 1.385, + "theoretical_loss": 3.445436278506832, + "tokens_seen": 2096758784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682848545636911, + "loss": 1.4668, + "theoretical_loss": 3.4454274564076823, + "tokens_seen": 2096824320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036826479438314947, + "loss": 1.3409, + "theoretical_loss": 3.445418634661465, + "tokens_seen": 2096889856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036824473420260783, + "loss": 1.3594, + "theoretical_loss": 3.445409813268154, + "tokens_seen": 2096955392 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036822467402206625, + "loss": 1.4659, + "theoretical_loss": 3.4454009922277242, + "tokens_seen": 2097020928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682046138415246, + "loss": 1.3343, + "theoretical_loss": 3.445392171540152, + "tokens_seen": 2097086464 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036818455366098297, + "loss": 1.4127, + "theoretical_loss": 3.445383351205411, + "tokens_seen": 2097152000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036816449348044133, + "loss": 1.3335, + "theoretical_loss": 3.445374531223476, + "tokens_seen": 2097217536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681444332998997, + "loss": 1.5249, + "theoretical_loss": 3.445365711594323, + "tokens_seen": 2097283072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681243731193581, + "loss": 1.3209, + "theoretical_loss": 3.445356892317925, + "tokens_seen": 2097348608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681043129388165, + "loss": 1.3393, + "theoretical_loss": 3.4453480733942587, + "tokens_seen": 2097414144 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036808425275827484, + "loss": 1.311, + "theoretical_loss": 3.445339254823298, + "tokens_seen": 2097479680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680641925777332, + "loss": 1.226, + "theoretical_loss": 3.445330436605018, + "tokens_seen": 2097545216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680441323971916, + "loss": 1.2744, + "theoretical_loss": 3.4453216187393934, + "tokens_seen": 2097610752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036802407221665, + "loss": 1.3542, + "theoretical_loss": 3.4453128012263994, + "tokens_seen": 2097676288 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036800401203610834, + "loss": 1.3595, + "theoretical_loss": 3.445303984066011, + "tokens_seen": 2097741824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679839518555667, + "loss": 1.3351, + "theoretical_loss": 3.445295167258202, + "tokens_seen": 2097807360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036796389167502506, + "loss": 1.2429, + "theoretical_loss": 3.4452863508029488, + "tokens_seen": 2097872896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679438314944835, + "loss": 1.3506, + "theoretical_loss": 3.4452775347002254, + "tokens_seen": 2097938432 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036792377131394184, + "loss": 1.3581, + "theoretical_loss": 3.4452687189500066, + "tokens_seen": 2098003968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679037111334002, + "loss": 1.4119, + "theoretical_loss": 3.4452599035522677, + "tokens_seen": 2098069504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036788365095285857, + "loss": 1.3218, + "theoretical_loss": 3.445251088506984, + "tokens_seen": 2098135040 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367863590772317, + "loss": 1.3215, + "theoretical_loss": 3.445242273814129, + "tokens_seen": 2098200576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036784353059177535, + "loss": 1.3167, + "theoretical_loss": 3.4452334594736786, + "tokens_seen": 2098266112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678234704112337, + "loss": 1.3798, + "theoretical_loss": 3.4452246454856077, + "tokens_seen": 2098331648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036780341023069207, + "loss": 1.5176, + "theoretical_loss": 3.4452158318498913, + "tokens_seen": 2098397184 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036778335005015043, + "loss": 1.2312, + "theoretical_loss": 3.4452070185665042, + "tokens_seen": 2098462720 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036776328986960885, + "loss": 1.3267, + "theoretical_loss": 3.445198205635421, + "tokens_seen": 2098528256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677432296890672, + "loss": 1.3297, + "theoretical_loss": 3.4451893930566166, + "tokens_seen": 2098593792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036772316950852557, + "loss": 1.3517, + "theoretical_loss": 3.4451805808300664, + "tokens_seen": 2098659328 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036770310932798393, + "loss": 1.3699, + "theoretical_loss": 3.4451717689557446, + "tokens_seen": 2098724864 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036768304914744235, + "loss": 1.3563, + "theoretical_loss": 3.4451629574336273, + "tokens_seen": 2098790400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676629889669007, + "loss": 1.2697, + "theoretical_loss": 3.4451541462636883, + "tokens_seen": 2098855936 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 765110, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5750041007995605, + "objective/train/theoretical_loss": 3.445145335445903, + "objective/train/tokens_used": 469380576, + "theoretical_loss": 3.445145335445903, + "tokens_seen": 2098921472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676429287863591, + "loss": 1.4734, + "theoretical_loss": 3.445145335445903, + "tokens_seen": 2098921472 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036762286860581744, + "loss": 1.3044, + "theoretical_loss": 3.4451365249802466, + "tokens_seen": 2098987008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676028084252758, + "loss": 1.3037, + "theoretical_loss": 3.4451277148666932, + "tokens_seen": 2099052544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675827482447342, + "loss": 1.4516, + "theoretical_loss": 3.4451189051052182, + "tokens_seen": 2099118080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675626880641926, + "loss": 1.2558, + "theoretical_loss": 3.4451100956957967, + "tokens_seen": 2099183616 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036754262788365094, + "loss": 1.431, + "theoretical_loss": 3.4451012866384034, + "tokens_seen": 2099249152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675225677031093, + "loss": 1.2659, + "theoretical_loss": 3.445092477933014, + "tokens_seen": 2099314688 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675025075225677, + "loss": 1.381, + "theoretical_loss": 3.445083669579602, + "tokens_seen": 2099380224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674824473420261, + "loss": 1.3665, + "theoretical_loss": 3.4450748615781435, + "tokens_seen": 2099445760 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036746238716148444, + "loss": 1.2901, + "theoretical_loss": 3.445066053928613, + "tokens_seen": 2099511296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674423269809428, + "loss": 1.3324, + "theoretical_loss": 3.4450572466309852, + "tokens_seen": 2099576832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036742226680040117, + "loss": 1.2707, + "theoretical_loss": 3.445048439685236, + "tokens_seen": 2099642368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674022066198596, + "loss": 1.3485, + "theoretical_loss": 3.4450396330913398, + "tokens_seen": 2099707904 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036738214643931795, + "loss": 1.4169, + "theoretical_loss": 3.4450308268492713, + "tokens_seen": 2099773440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673620862587763, + "loss": 1.3343, + "theoretical_loss": 3.4450220209590054, + "tokens_seen": 2099838976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036734202607823467, + "loss": 1.2902, + "theoretical_loss": 3.4450132154205177, + "tokens_seen": 2099904512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673219658976931, + "loss": 1.3017, + "theoretical_loss": 3.445004410233783, + "tokens_seen": 2099970048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036730190571715145, + "loss": 1.3733, + "theoretical_loss": 3.444995605398776, + "tokens_seen": 2100035584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672818455366098, + "loss": 1.2952, + "theoretical_loss": 3.4449868009154714, + "tokens_seen": 2100101120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672617853560682, + "loss": 1.2873, + "theoretical_loss": 3.444977996783845, + "tokens_seen": 2100166656 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036724172517552654, + "loss": 1.3068, + "theoretical_loss": 3.444969193003871, + "tokens_seen": 2100232192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036722166499498495, + "loss": 1.3285, + "theoretical_loss": 3.4449603895755248, + "tokens_seen": 2100297728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672016048144433, + "loss": 1.307, + "theoretical_loss": 3.4449515864987816, + "tokens_seen": 2100363264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671815446339017, + "loss": 1.4674, + "theoretical_loss": 3.4449427837736155, + "tokens_seen": 2100428800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036716148445336004, + "loss": 1.3359, + "theoretical_loss": 3.4449339814000024, + "tokens_seen": 2100494336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036714142427281846, + "loss": 1.3569, + "theoretical_loss": 3.4449251793779174, + "tokens_seen": 2100559872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671213640922768, + "loss": 1.3685, + "theoretical_loss": 3.4449163777073344, + "tokens_seen": 2100625408 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036710130391173524, + "loss": 1.2736, + "theoretical_loss": 3.4449075763882293, + "tokens_seen": 2100690944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670812437311936, + "loss": 1.3554, + "theoretical_loss": 3.444898775420577, + "tokens_seen": 2100756480 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367061183550652, + "loss": 1.2767, + "theoretical_loss": 3.4448899748043518, + "tokens_seen": 2100822016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670411233701104, + "loss": 1.3741, + "theoretical_loss": 3.44488117453953, + "tokens_seen": 2100887552 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036702106318956874, + "loss": 1.259, + "theoretical_loss": 3.4448723746260854, + "tokens_seen": 2100953088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670010030090271, + "loss": 1.3969, + "theoretical_loss": 3.4448635750639935, + "tokens_seen": 2101018624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036698094282848546, + "loss": 1.3561, + "theoretical_loss": 3.44485477585323, + "tokens_seen": 2101084160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669608826479439, + "loss": 1.257, + "theoretical_loss": 3.4448459769937685, + "tokens_seen": 2101149696 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036694082246740224, + "loss": 1.2743, + "theoretical_loss": 3.4448371784855847, + "tokens_seen": 2101215232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669207622868606, + "loss": 1.3324, + "theoretical_loss": 3.4448283803286537, + "tokens_seen": 2101280768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036690070210631897, + "loss": 1.3854, + "theoretical_loss": 3.4448195825229506, + "tokens_seen": 2101346304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668806419257774, + "loss": 1.2842, + "theoretical_loss": 3.4448107850684506, + "tokens_seen": 2101411840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036686058174523575, + "loss": 1.3194, + "theoretical_loss": 3.444801987965128, + "tokens_seen": 2101477376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668405215646941, + "loss": 1.3103, + "theoretical_loss": 3.444793191212958, + "tokens_seen": 2101542912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036682046138415247, + "loss": 1.3467, + "theoretical_loss": 3.444784394811917, + "tokens_seen": 2101608448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036680040120361083, + "loss": 1.3795, + "theoretical_loss": 3.4447755987619777, + "tokens_seen": 2101673984 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036678034102306925, + "loss": 1.4101, + "theoretical_loss": 3.444766803063117, + "tokens_seen": 2101739520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667602808425276, + "loss": 1.4029, + "theoretical_loss": 3.4447580077153095, + "tokens_seen": 2101805056 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366740220661986, + "loss": 1.3449, + "theoretical_loss": 3.4447492127185293, + "tokens_seen": 2101870592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036672016048144434, + "loss": 1.3296, + "theoretical_loss": 3.4447404180727528, + "tokens_seen": 2101936128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036670010030090275, + "loss": 1.262, + "theoretical_loss": 3.444731623777954, + "tokens_seen": 2102001664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666800401203611, + "loss": 1.2664, + "theoretical_loss": 3.4447228298341086, + "tokens_seen": 2102067200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666599799398195, + "loss": 1.2667, + "theoretical_loss": 3.4447140362411917, + "tokens_seen": 2102132736 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 765966, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3137551546096802, + "objective/train/theoretical_loss": 3.444705242999178, + "objective/train/tokens_used": 472657376, + "theoretical_loss": 3.444705242999178, + "tokens_seen": 2102198272 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036663991975927784, + "loss": 1.3257, + "theoretical_loss": 3.444705242999178, + "tokens_seen": 2102198272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666198595787362, + "loss": 1.3463, + "theoretical_loss": 3.4446964501080424, + "tokens_seen": 2102263808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665997993981946, + "loss": 1.2422, + "theoretical_loss": 3.4446876575677603, + "tokens_seen": 2102329344 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366579739217653, + "loss": 1.2923, + "theoretical_loss": 3.444678865378307, + "tokens_seen": 2102394880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036655967903711134, + "loss": 1.3261, + "theoretical_loss": 3.444670073539657, + "tokens_seen": 2102460416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665396188565697, + "loss": 1.3142, + "theoretical_loss": 3.4446612820517855, + "tokens_seen": 2102525952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665195586760281, + "loss": 1.3094, + "theoretical_loss": 3.4446524909146676, + "tokens_seen": 2102591488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664994984954865, + "loss": 1.2831, + "theoretical_loss": 3.444643700128279, + "tokens_seen": 2102657024 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036647943831494484, + "loss": 1.2635, + "theoretical_loss": 3.444634909692594, + "tokens_seen": 2102722560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664593781344032, + "loss": 1.3214, + "theoretical_loss": 3.444626119607588, + "tokens_seen": 2102788096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036643931795386157, + "loss": 1.3377, + "theoretical_loss": 3.4446173298732354, + "tokens_seen": 2102853632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036641925777332, + "loss": 1.3006, + "theoretical_loss": 3.4446085404895124, + "tokens_seen": 2102919168 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036639919759277835, + "loss": 1.4046, + "theoretical_loss": 3.4445997514563937, + "tokens_seen": 2102984704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663791374122367, + "loss": 1.3554, + "theoretical_loss": 3.4445909627738542, + "tokens_seen": 2103050240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036635907723169507, + "loss": 1.3723, + "theoretical_loss": 3.444582174441869, + "tokens_seen": 2103115776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663390170511535, + "loss": 1.3524, + "theoretical_loss": 3.444573386460413, + "tokens_seen": 2103181312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036631895687061185, + "loss": 1.2296, + "theoretical_loss": 3.444564598829462, + "tokens_seen": 2103246848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662988966900702, + "loss": 1.4416, + "theoretical_loss": 3.4445558115489905, + "tokens_seen": 2103312384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662788365095286, + "loss": 1.3546, + "theoretical_loss": 3.4445470246189736, + "tokens_seen": 2103377920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036625877632898694, + "loss": 1.2454, + "theoretical_loss": 3.4445382380393865, + "tokens_seen": 2103443456 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036623871614844535, + "loss": 1.3167, + "theoretical_loss": 3.4445294518102045, + "tokens_seen": 2103508992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662186559679037, + "loss": 1.3641, + "theoretical_loss": 3.4445206659314027, + "tokens_seen": 2103574528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003661985957873621, + "loss": 1.2864, + "theoretical_loss": 3.4445118804029558, + "tokens_seen": 2103640064 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036617853560682044, + "loss": 1.3506, + "theoretical_loss": 3.444503095224839, + "tokens_seen": 2103705600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036615847542627886, + "loss": 1.4125, + "theoretical_loss": 3.4444943103970282, + "tokens_seen": 2103771136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661384152457372, + "loss": 1.3836, + "theoretical_loss": 3.4444855259194975, + "tokens_seen": 2103836672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661183550651956, + "loss": 1.26, + "theoretical_loss": 3.444476741792223, + "tokens_seen": 2103902208 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036609829488465394, + "loss": 1.3657, + "theoretical_loss": 3.4444679580151787, + "tokens_seen": 2103967744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036607823470411236, + "loss": 1.2863, + "theoretical_loss": 3.4444591745883404, + "tokens_seen": 2104033280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660581745235707, + "loss": 1.3529, + "theoretical_loss": 3.4444503915116833, + "tokens_seen": 2104098816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660381143430291, + "loss": 1.2918, + "theoretical_loss": 3.4444416087851826, + "tokens_seen": 2104164352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036601805416248745, + "loss": 1.3565, + "theoretical_loss": 3.4444328264088124, + "tokens_seen": 2104229888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659979939819458, + "loss": 1.3884, + "theoretical_loss": 3.44442404438255, + "tokens_seen": 2104295424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659779338014042, + "loss": 1.3525, + "theoretical_loss": 3.444415262706368, + "tokens_seen": 2104360960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659578736208626, + "loss": 1.2538, + "theoretical_loss": 3.444406481380243, + "tokens_seen": 2104426496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036593781344032095, + "loss": 1.2556, + "theoretical_loss": 3.44439770040415, + "tokens_seen": 2104492032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659177532597793, + "loss": 1.2113, + "theoretical_loss": 3.444388919778064, + "tokens_seen": 2104557568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036589769307923773, + "loss": 1.3034, + "theoretical_loss": 3.44438013950196, + "tokens_seen": 2104623104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658776328986961, + "loss": 1.3635, + "theoretical_loss": 3.4443713595758134, + "tokens_seen": 2104688640 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036585757271815445, + "loss": 1.27, + "theoretical_loss": 3.4443625799995994, + "tokens_seen": 2104754176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658375125376128, + "loss": 1.353, + "theoretical_loss": 3.444353800773293, + "tokens_seen": 2104819712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658174523570712, + "loss": 1.3352, + "theoretical_loss": 3.4443450218968694, + "tokens_seen": 2104885248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657973921765296, + "loss": 1.3609, + "theoretical_loss": 3.4443362433703038, + "tokens_seen": 2104950784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036577733199598796, + "loss": 1.3424, + "theoretical_loss": 3.444327465193571, + "tokens_seen": 2105016320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657572718154463, + "loss": 1.2589, + "theoretical_loss": 3.444318687366647, + "tokens_seen": 2105081856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657372116349047, + "loss": 1.3467, + "theoretical_loss": 3.4443099098895065, + "tokens_seen": 2105147392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657171514543631, + "loss": 1.2555, + "theoretical_loss": 3.4443011327621242, + "tokens_seen": 2105212928 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036569709127382146, + "loss": 1.3104, + "theoretical_loss": 3.4442923559844756, + "tokens_seen": 2105278464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656770310932798, + "loss": 1.347, + "theoretical_loss": 3.4442835795565365, + "tokens_seen": 2105344000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656569709127382, + "loss": 1.3144, + "theoretical_loss": 3.4442748034782813, + "tokens_seen": 2105409536 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 767157, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3022453784942627, + "objective/train/theoretical_loss": 3.444266027749685, + "objective/train/tokens_used": 475934176, + "theoretical_loss": 3.444266027749685, + "tokens_seen": 2105475072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036563691073219655, + "loss": 1.2919, + "theoretical_loss": 3.444266027749685, + "tokens_seen": 2105475072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036561685055165496, + "loss": 1.4169, + "theoretical_loss": 3.4442572523707238, + "tokens_seen": 2105540608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655967903711134, + "loss": 1.2953, + "theoretical_loss": 3.4442484773413726, + "tokens_seen": 2105606144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036557673019057174, + "loss": 1.292, + "theoretical_loss": 3.444239702661606, + "tokens_seen": 2105671680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655566700100301, + "loss": 1.3519, + "theoretical_loss": 3.4442309283313994, + "tokens_seen": 2105737216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655366098294885, + "loss": 1.2543, + "theoretical_loss": 3.444222154350728, + "tokens_seen": 2105802752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655165496489469, + "loss": 1.3603, + "theoretical_loss": 3.4442133807195674, + "tokens_seen": 2105868288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036549648946840525, + "loss": 1.3932, + "theoretical_loss": 3.444204607437892, + "tokens_seen": 2105933824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654764292878636, + "loss": 1.3252, + "theoretical_loss": 3.4441958345056785, + "tokens_seen": 2105999360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036545636910732197, + "loss": 1.3187, + "theoretical_loss": 3.4441870619229, + "tokens_seen": 2106064896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654363089267804, + "loss": 1.3562, + "theoretical_loss": 3.4441782896895337, + "tokens_seen": 2106130432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036541624874623875, + "loss": 1.4001, + "theoretical_loss": 3.4441695178055536, + "tokens_seen": 2106195968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653961885656971, + "loss": 1.3228, + "theoretical_loss": 3.444160746270935, + "tokens_seen": 2106261504 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036537612838515547, + "loss": 1.3213, + "theoretical_loss": 3.4441519750856537, + "tokens_seen": 2106327040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653560682046139, + "loss": 1.2848, + "theoretical_loss": 3.4441432042496842, + "tokens_seen": 2106392576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036533600802407225, + "loss": 1.4016, + "theoretical_loss": 3.4441344337630024, + "tokens_seen": 2106458112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653159478435306, + "loss": 1.3188, + "theoretical_loss": 3.4441256636255835, + "tokens_seen": 2106523648 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365295887662989, + "loss": 1.3766, + "theoretical_loss": 3.444116893837402, + "tokens_seen": 2106589184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036527582748244734, + "loss": 1.3528, + "theoretical_loss": 3.4441081243984337, + "tokens_seen": 2106654720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036525576730190575, + "loss": 1.2933, + "theoretical_loss": 3.444099355308654, + "tokens_seen": 2106720256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652357071213641, + "loss": 1.284, + "theoretical_loss": 3.444090586568038, + "tokens_seen": 2106785792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652156469408225, + "loss": 1.368, + "theoretical_loss": 3.44408181817656, + "tokens_seen": 2106851328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036519558676028084, + "loss": 1.2779, + "theoretical_loss": 3.4440730501341967, + "tokens_seen": 2106916864 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036517552657973926, + "loss": 1.3118, + "theoretical_loss": 3.444064282440922, + "tokens_seen": 2106982400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651554663991976, + "loss": 1.2364, + "theoretical_loss": 3.4440555150967125, + "tokens_seen": 2107047936 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365135406218656, + "loss": 1.2787, + "theoretical_loss": 3.444046748101542, + "tokens_seen": 2107113472 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036511534603811434, + "loss": 1.2568, + "theoretical_loss": 3.444037981455387, + "tokens_seen": 2107179008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036509528585757276, + "loss": 1.2529, + "theoretical_loss": 3.4440292151582224, + "tokens_seen": 2107244544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650752256770311, + "loss": 1.2324, + "theoretical_loss": 3.444020449210023, + "tokens_seen": 2107310080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650551654964895, + "loss": 1.3414, + "theoretical_loss": 3.4440116836107646, + "tokens_seen": 2107375616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036503510531594785, + "loss": 1.201, + "theoretical_loss": 3.444002918360422, + "tokens_seen": 2107441152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650150451354062, + "loss": 1.3474, + "theoretical_loss": 3.443994153458971, + "tokens_seen": 2107506688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649949849548646, + "loss": 1.3961, + "theoretical_loss": 3.4439853889063863, + "tokens_seen": 2107572224 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364974924774323, + "loss": 1.4234, + "theoretical_loss": 3.4439766247026435, + "tokens_seen": 2107637760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036495486459378135, + "loss": 1.2678, + "theoretical_loss": 3.4439678608477178, + "tokens_seen": 2107703296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649348044132397, + "loss": 1.3538, + "theoretical_loss": 3.443959097341584, + "tokens_seen": 2107768832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036491474423269813, + "loss": 1.2892, + "theoretical_loss": 3.4439503341842186, + "tokens_seen": 2107834368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648946840521565, + "loss": 1.3419, + "theoretical_loss": 3.443941571375596, + "tokens_seen": 2107899904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036487462387161485, + "loss": 1.2762, + "theoretical_loss": 3.443932808915691, + "tokens_seen": 2107965440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648545636910732, + "loss": 1.3632, + "theoretical_loss": 3.4439240468044803, + "tokens_seen": 2108030976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648345035105316, + "loss": 1.2554, + "theoretical_loss": 3.4439152850419377, + "tokens_seen": 2108096512 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036481444332999, + "loss": 1.361, + "theoretical_loss": 3.4439065236280393, + "tokens_seen": 2108162048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036479438314944836, + "loss": 1.2867, + "theoretical_loss": 3.4438977625627603, + "tokens_seen": 2108227584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647743229689067, + "loss": 1.3119, + "theoretical_loss": 3.4438890018460757, + "tokens_seen": 2108293120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647542627883651, + "loss": 1.3828, + "theoretical_loss": 3.4438802414779612, + "tokens_seen": 2108358656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647342026078235, + "loss": 1.294, + "theoretical_loss": 3.443871481458392, + "tokens_seen": 2108424192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036471414242728186, + "loss": 1.2974, + "theoretical_loss": 3.4438627217873434, + "tokens_seen": 2108489728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646940822467402, + "loss": 1.3563, + "theoretical_loss": 3.4438539624647904, + "tokens_seen": 2108555264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646740220661986, + "loss": 1.4333, + "theoretical_loss": 3.4438452034907083, + "tokens_seen": 2108620800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036465396188565695, + "loss": 1.3752, + "theoretical_loss": 3.443836444865073, + "tokens_seen": 2108686336 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 768317, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.153150200843811, + "objective/train/theoretical_loss": 3.443827686587859, + "objective/train/tokens_used": 479210976, + "theoretical_loss": 3.443827686587859, + "tokens_seen": 2108751872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036463390170511536, + "loss": 1.1949, + "theoretical_loss": 3.443827686587859, + "tokens_seen": 2108751872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646138415245737, + "loss": 1.3138, + "theoretical_loss": 3.443818928659043, + "tokens_seen": 2108817408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645937813440321, + "loss": 1.3458, + "theoretical_loss": 3.4438101710785984, + "tokens_seen": 2108882944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036457372116349045, + "loss": 1.3948, + "theoretical_loss": 3.443801413846502, + "tokens_seen": 2108948480 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036455366098294887, + "loss": 1.3119, + "theoretical_loss": 3.4437926569627284, + "tokens_seen": 2109014016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036453360080240723, + "loss": 1.3953, + "theoretical_loss": 3.4437839004272535, + "tokens_seen": 2109079552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645135406218656, + "loss": 1.3311, + "theoretical_loss": 3.4437751442400515, + "tokens_seen": 2109145088 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036449348044132395, + "loss": 1.3462, + "theoretical_loss": 3.443766388401099, + "tokens_seen": 2109210624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644734202607823, + "loss": 1.2598, + "theoretical_loss": 3.443757632910371, + "tokens_seen": 2109276160 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036445336008024073, + "loss": 1.3711, + "theoretical_loss": 3.4437488777678418, + "tokens_seen": 2109341696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644332998996991, + "loss": 1.3975, + "theoretical_loss": 3.4437401229734883, + "tokens_seen": 2109407232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036441323971915746, + "loss": 1.4033, + "theoretical_loss": 3.443731368527285, + "tokens_seen": 2109472768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643931795386158, + "loss": 1.3222, + "theoretical_loss": 3.4437226144292072, + "tokens_seen": 2109538304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036437311935807424, + "loss": 1.3333, + "theoretical_loss": 3.4437138606792304, + "tokens_seen": 2109603840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643530591775326, + "loss": 1.272, + "theoretical_loss": 3.4437051072773297, + "tokens_seen": 2109669376 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036433299899699096, + "loss": 1.2041, + "theoretical_loss": 3.4436963542234813, + "tokens_seen": 2109734912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643129388164493, + "loss": 1.3683, + "theoretical_loss": 3.4436876015176594, + "tokens_seen": 2109800448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642928786359077, + "loss": 1.393, + "theoretical_loss": 3.4436788491598405, + "tokens_seen": 2109865984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642728184553661, + "loss": 1.2238, + "theoretical_loss": 3.4436700971499983, + "tokens_seen": 2109931520 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036425275827482446, + "loss": 1.1717, + "theoretical_loss": 3.4436613454881098, + "tokens_seen": 2109997056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642326980942828, + "loss": 1.3435, + "theoretical_loss": 3.44365259417415, + "tokens_seen": 2110062592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642126379137412, + "loss": 1.3071, + "theoretical_loss": 3.443643843208094, + "tokens_seen": 2110128128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641925777331996, + "loss": 1.3552, + "theoretical_loss": 3.4436350925899166, + "tokens_seen": 2110193664 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036417251755265797, + "loss": 1.3927, + "theoretical_loss": 3.443626342319594, + "tokens_seen": 2110259200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036415245737211633, + "loss": 1.2876, + "theoretical_loss": 3.4436175923971017, + "tokens_seen": 2110324736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641323971915747, + "loss": 1.2379, + "theoretical_loss": 3.4436088428224143, + "tokens_seen": 2110390272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036411233701103316, + "loss": 1.3604, + "theoretical_loss": 3.443600093595508, + "tokens_seen": 2110455808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640922768304915, + "loss": 1.3995, + "theoretical_loss": 3.4435913447163573, + "tokens_seen": 2110521344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640722166499499, + "loss": 1.2756, + "theoretical_loss": 3.443582596184938, + "tokens_seen": 2110586880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036405215646940825, + "loss": 1.2481, + "theoretical_loss": 3.4435738480012255, + "tokens_seen": 2110652416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640320962888666, + "loss": 1.291, + "theoretical_loss": 3.4435651001651957, + "tokens_seen": 2110717952 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364012036108325, + "loss": 1.3307, + "theoretical_loss": 3.443556352676823, + "tokens_seen": 2110783488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639919759277834, + "loss": 1.2903, + "theoretical_loss": 3.443547605536083, + "tokens_seen": 2110849024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036397191574724175, + "loss": 1.1547, + "theoretical_loss": 3.443538858742952, + "tokens_seen": 2110914560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639518555667001, + "loss": 1.3908, + "theoretical_loss": 3.4435301122974047, + "tokens_seen": 2110980096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036393179538615853, + "loss": 1.2112, + "theoretical_loss": 3.443521366199416, + "tokens_seen": 2111045632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639117352056169, + "loss": 1.1677, + "theoretical_loss": 3.4435126204489626, + "tokens_seen": 2111111168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036389167502507525, + "loss": 1.3342, + "theoretical_loss": 3.4435038750460185, + "tokens_seen": 2111176704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638716148445336, + "loss": 1.4041, + "theoretical_loss": 3.44349512999056, + "tokens_seen": 2111242240 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363851554663992, + "loss": 1.2708, + "theoretical_loss": 3.4434863852825623, + "tokens_seen": 2111307776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638314944834504, + "loss": 1.3325, + "theoretical_loss": 3.443477640922001, + "tokens_seen": 2111373312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036381143430290876, + "loss": 1.1847, + "theoretical_loss": 3.4434688969088514, + "tokens_seen": 2111438848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637913741223671, + "loss": 1.4129, + "theoretical_loss": 3.4434601532430884, + "tokens_seen": 2111504384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637713139418255, + "loss": 1.2798, + "theoretical_loss": 3.4434514099246876, + "tokens_seen": 2111569920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637512537612839, + "loss": 1.1966, + "theoretical_loss": 3.443442666953625, + "tokens_seen": 2111635456 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036373119358074226, + "loss": 1.3453, + "theoretical_loss": 3.4434339243298755, + "tokens_seen": 2111700992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637111334002006, + "loss": 1.305, + "theoretical_loss": 3.4434251820534145, + "tokens_seen": 2111766528 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363691073219659, + "loss": 1.2496, + "theoretical_loss": 3.443416440124218, + "tokens_seen": 2111832064 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036367101303911735, + "loss": 1.2774, + "theoretical_loss": 3.443407698542261, + "tokens_seen": 2111897600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036365095285857576, + "loss": 1.3294, + "theoretical_loss": 3.443398957307519, + "tokens_seen": 2111963136 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 769436, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3213162422180176, + "objective/train/theoretical_loss": 3.4433902164199672, + "objective/train/tokens_used": 482487776, + "theoretical_loss": 3.4433902164199672, + "tokens_seen": 2112028672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636308926780341, + "loss": 1.4059, + "theoretical_loss": 3.4433902164199672, + "tokens_seen": 2112028672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636108324974925, + "loss": 1.2794, + "theoretical_loss": 3.4433814758795815, + "tokens_seen": 2112094208 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036359077231695085, + "loss": 1.3745, + "theoretical_loss": 3.443372735686337, + "tokens_seen": 2112159744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036357071213640927, + "loss": 1.3908, + "theoretical_loss": 3.443363995840209, + "tokens_seen": 2112225280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036355065195586763, + "loss": 1.2363, + "theoretical_loss": 3.4433552563411736, + "tokens_seen": 2112290816 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363530591775326, + "loss": 1.2649, + "theoretical_loss": 3.4433465171892057, + "tokens_seen": 2112356352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036351053159478435, + "loss": 1.3192, + "theoretical_loss": 3.443337778384281, + "tokens_seen": 2112421888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634904714142427, + "loss": 1.3019, + "theoretical_loss": 3.4433290399263745, + "tokens_seen": 2112487424 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036347041123370113, + "loss": 1.3039, + "theoretical_loss": 3.4433203018154623, + "tokens_seen": 2112552960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634503510531595, + "loss": 1.2723, + "theoretical_loss": 3.443311564051519, + "tokens_seen": 2112618496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036343029087261786, + "loss": 1.3675, + "theoretical_loss": 3.443302826634521, + "tokens_seen": 2112684032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634102306920762, + "loss": 1.4277, + "theoretical_loss": 3.4432940895644433, + "tokens_seen": 2112749568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036339017051153464, + "loss": 1.2983, + "theoretical_loss": 3.4432853528412615, + "tokens_seen": 2112815104 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363370110330993, + "loss": 1.3462, + "theoretical_loss": 3.443276616464951, + "tokens_seen": 2112880640 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036335005015045136, + "loss": 1.4393, + "theoretical_loss": 3.443267880435487, + "tokens_seen": 2112946176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633299899699097, + "loss": 1.4435, + "theoretical_loss": 3.4432591447528456, + "tokens_seen": 2113011712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633099297893681, + "loss": 1.2632, + "theoretical_loss": 3.4432504094170016, + "tokens_seen": 2113077248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632898696088265, + "loss": 1.2447, + "theoretical_loss": 3.443241674427931, + "tokens_seen": 2113142784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036326980942828486, + "loss": 1.237, + "theoretical_loss": 3.443232939785609, + "tokens_seen": 2113208320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632497492477432, + "loss": 1.2437, + "theoretical_loss": 3.443224205490011, + "tokens_seen": 2113273856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632296890672016, + "loss": 1.2972, + "theoretical_loss": 3.443215471541113, + "tokens_seen": 2113339392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036320962888666, + "loss": 1.3624, + "theoretical_loss": 3.44320673793889, + "tokens_seen": 2113404928 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036318956870611837, + "loss": 1.3551, + "theoretical_loss": 3.4431980046833175, + "tokens_seen": 2113470464 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036316950852557673, + "loss": 1.3455, + "theoretical_loss": 3.4431892717743713, + "tokens_seen": 2113536000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631494483450351, + "loss": 1.2969, + "theoretical_loss": 3.443180539212026, + "tokens_seen": 2113601536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036312938816449345, + "loss": 1.2587, + "theoretical_loss": 3.4431718069962587, + "tokens_seen": 2113667072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036310932798395187, + "loss": 1.3012, + "theoretical_loss": 3.4431630751270434, + "tokens_seen": 2113732608 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036308926780341023, + "loss": 1.2223, + "theoretical_loss": 3.4431543436043563, + "tokens_seen": 2113798144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630692076228686, + "loss": 1.2994, + "theoretical_loss": 3.4431456124281734, + "tokens_seen": 2113863680 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036304914744232696, + "loss": 1.3054, + "theoretical_loss": 3.443136881598469, + "tokens_seen": 2113929216 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036302908726178537, + "loss": 1.5396, + "theoretical_loss": 3.443128151115219, + "tokens_seen": 2113994752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036300902708124374, + "loss": 1.3605, + "theoretical_loss": 3.4431194209784, + "tokens_seen": 2114060288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629889669007021, + "loss": 1.3526, + "theoretical_loss": 3.443110691187986, + "tokens_seen": 2114125824 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036296890672016046, + "loss": 1.3501, + "theoretical_loss": 3.4431019617439533, + "tokens_seen": 2114191360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629488465396188, + "loss": 1.3248, + "theoretical_loss": 3.4430932326462775, + "tokens_seen": 2114256896 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036292878635907724, + "loss": 1.3472, + "theoretical_loss": 3.4430845038949336, + "tokens_seen": 2114322432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629087261785356, + "loss": 1.3485, + "theoretical_loss": 3.4430757754898975, + "tokens_seen": 2114387968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036288866599799396, + "loss": 1.3616, + "theoretical_loss": 3.4430670474311444, + "tokens_seen": 2114453504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628686058174523, + "loss": 1.2865, + "theoretical_loss": 3.4430583197186504, + "tokens_seen": 2114519040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036284854563691074, + "loss": 1.3653, + "theoretical_loss": 3.443049592352391, + "tokens_seen": 2114584576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628284854563691, + "loss": 1.3586, + "theoretical_loss": 3.443040865332341, + "tokens_seen": 2114650112 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036280842527582747, + "loss": 1.2638, + "theoretical_loss": 3.443032138658477, + "tokens_seen": 2114715648 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036278836509528583, + "loss": 1.2767, + "theoretical_loss": 3.4430234123307732, + "tokens_seen": 2114781184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036276830491474424, + "loss": 1.2595, + "theoretical_loss": 3.443014686349206, + "tokens_seen": 2114846720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627482447342026, + "loss": 1.3639, + "theoretical_loss": 3.4430059607137515, + "tokens_seen": 2114912256 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036272818455366097, + "loss": 1.3061, + "theoretical_loss": 3.442997235424384, + "tokens_seen": 2114977792 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036270812437311933, + "loss": 1.2461, + "theoretical_loss": 3.4429885104810793, + "tokens_seen": 2115043328 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626880641925777, + "loss": 1.2092, + "theoretical_loss": 3.442979785883814, + "tokens_seen": 2115108864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626680040120361, + "loss": 1.3049, + "theoretical_loss": 3.442971061632562, + "tokens_seen": 2115174400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036264794383149447, + "loss": 1.3942, + "theoretical_loss": 3.442962337727301, + "tokens_seen": 2115239936 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 770743, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.205157995223999, + "objective/train/theoretical_loss": 3.4429536141680046, + "objective/train/tokens_used": 485764576, + "theoretical_loss": 3.4429536141680046, + "tokens_seen": 2115305472 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036262788365095283, + "loss": 1.2541, + "theoretical_loss": 3.4429536141680046, + "tokens_seen": 2115305472 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036260782347041125, + "loss": 1.3759, + "theoretical_loss": 3.442944890954649, + "tokens_seen": 2115371008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036258776328986967, + "loss": 1.3069, + "theoretical_loss": 3.4429361680872104, + "tokens_seen": 2115436544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036256770310932803, + "loss": 1.3426, + "theoretical_loss": 3.442927445565663, + "tokens_seen": 2115502080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625476429287864, + "loss": 1.286, + "theoretical_loss": 3.442918723389984, + "tokens_seen": 2115567616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036252758274824475, + "loss": 1.2622, + "theoretical_loss": 3.442910001560148, + "tokens_seen": 2115633152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625075225677031, + "loss": 1.2757, + "theoretical_loss": 3.4429012800761303, + "tokens_seen": 2115698688 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036248746238716153, + "loss": 1.401, + "theoretical_loss": 3.442892558937907, + "tokens_seen": 2115764224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624674022066199, + "loss": 1.2647, + "theoretical_loss": 3.4428838381454545, + "tokens_seen": 2115829760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036244734202607826, + "loss": 1.27, + "theoretical_loss": 3.4428751176987467, + "tokens_seen": 2115895296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624272818455366, + "loss": 1.328, + "theoretical_loss": 3.4428663975977596, + "tokens_seen": 2115960832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036240722166499504, + "loss": 1.388, + "theoretical_loss": 3.44285767784247, + "tokens_seen": 2116026368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623871614844534, + "loss": 1.273, + "theoretical_loss": 3.4428489584328523, + "tokens_seen": 2116091904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036236710130391176, + "loss": 1.3418, + "theoretical_loss": 3.4428402393688824, + "tokens_seen": 2116157440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623470411233701, + "loss": 1.3098, + "theoretical_loss": 3.442831520650536, + "tokens_seen": 2116222976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623269809428285, + "loss": 1.3199, + "theoretical_loss": 3.4428228022777887, + "tokens_seen": 2116288512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623069207622869, + "loss": 1.2965, + "theoretical_loss": 3.442814084250616, + "tokens_seen": 2116354048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036228686058174526, + "loss": 1.313, + "theoretical_loss": 3.4428053665689937, + "tokens_seen": 2116419584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622668004012036, + "loss": 1.4513, + "theoretical_loss": 3.4427966492328967, + "tokens_seen": 2116485120 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362246740220662, + "loss": 1.2546, + "theoretical_loss": 3.4427879322423016, + "tokens_seen": 2116550656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622266800401204, + "loss": 1.2784, + "theoretical_loss": 3.4427792155971835, + "tokens_seen": 2116616192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036220661985957877, + "loss": 1.461, + "theoretical_loss": 3.442770499297518, + "tokens_seen": 2116681728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036218655967903713, + "loss": 1.2706, + "theoretical_loss": 3.442761783343281, + "tokens_seen": 2116747264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621664994984955, + "loss": 1.3393, + "theoretical_loss": 3.442753067734447, + "tokens_seen": 2116812800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036214643931795385, + "loss": 1.3943, + "theoretical_loss": 3.4427443524709935, + "tokens_seen": 2116878336 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036212637913741227, + "loss": 1.3089, + "theoretical_loss": 3.442735637552895, + "tokens_seen": 2116943872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036210631895687063, + "loss": 1.3366, + "theoretical_loss": 3.442726922980127, + "tokens_seen": 2117009408 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362086258776329, + "loss": 1.2695, + "theoretical_loss": 3.442718208752665, + "tokens_seen": 2117074944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036206619859578736, + "loss": 1.2996, + "theoretical_loss": 3.4427094948704857, + "tokens_seen": 2117140480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620461384152458, + "loss": 1.3239, + "theoretical_loss": 3.442700781333564, + "tokens_seen": 2117206016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036202607823470414, + "loss": 1.3329, + "theoretical_loss": 3.4426920681418753, + "tokens_seen": 2117271552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620060180541625, + "loss": 1.2319, + "theoretical_loss": 3.4426833552953955, + "tokens_seen": 2117337088 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036198595787362086, + "loss": 1.3202, + "theoretical_loss": 3.4426746427941004, + "tokens_seen": 2117402624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619658976930792, + "loss": 1.2641, + "theoretical_loss": 3.4426659306379657, + "tokens_seen": 2117468160 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036194583751253764, + "loss": 1.384, + "theoretical_loss": 3.4426572188269664, + "tokens_seen": 2117533696 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361925777331996, + "loss": 1.3536, + "theoretical_loss": 3.4426485073610786, + "tokens_seen": 2117599232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036190571715145436, + "loss": 1.189, + "theoretical_loss": 3.442639796240278, + "tokens_seen": 2117664768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618856569709127, + "loss": 1.2704, + "theoretical_loss": 3.4426310854645408, + "tokens_seen": 2117730304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036186559679037114, + "loss": 1.3323, + "theoretical_loss": 3.4426223750338414, + "tokens_seen": 2117795840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618455366098295, + "loss": 1.3183, + "theoretical_loss": 3.442613664948156, + "tokens_seen": 2117861376 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036182547642928787, + "loss": 1.2426, + "theoretical_loss": 3.442604955207461, + "tokens_seen": 2117926912 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036180541624874623, + "loss": 1.303, + "theoretical_loss": 3.442596245811731, + "tokens_seen": 2117992448 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036178535606820464, + "loss": 1.2611, + "theoretical_loss": 3.442587536760942, + "tokens_seen": 2118057984 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361765295887663, + "loss": 1.3111, + "theoretical_loss": 3.4425788280550695, + "tokens_seen": 2118123520 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036174523570712137, + "loss": 1.3992, + "theoretical_loss": 3.44257011969409, + "tokens_seen": 2118189056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036172517552657973, + "loss": 1.3823, + "theoretical_loss": 3.4425614116779784, + "tokens_seen": 2118254592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617051153460381, + "loss": 1.2894, + "theoretical_loss": 3.44255270400671, + "tokens_seen": 2118320128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616850551654965, + "loss": 1.3736, + "theoretical_loss": 3.4425439966802616, + "tokens_seen": 2118385664 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036166499498495487, + "loss": 1.2861, + "theoretical_loss": 3.4425352896986086, + "tokens_seen": 2118451200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036164493480441323, + "loss": 1.3303, + "theoretical_loss": 3.442526583061726, + "tokens_seen": 2118516736 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 771858, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.377901554107666, + "objective/train/theoretical_loss": 3.44251787676959, + "objective/train/tokens_used": 489041376, + "theoretical_loss": 3.44251787676959, + "tokens_seen": 2118582272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616248746238716, + "loss": 1.4125, + "theoretical_loss": 3.44251787676959, + "tokens_seen": 2118582272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036160481444333, + "loss": 1.3668, + "theoretical_loss": 3.442509170822176, + "tokens_seen": 2118647808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615847542627884, + "loss": 1.3339, + "theoretical_loss": 3.4425004652194593, + "tokens_seen": 2118713344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036156469408224674, + "loss": 1.2589, + "theoretical_loss": 3.442491759961417, + "tokens_seen": 2118778880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615446339017051, + "loss": 1.372, + "theoretical_loss": 3.4424830550480237, + "tokens_seen": 2118844416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036152457372116346, + "loss": 1.3164, + "theoretical_loss": 3.442474350479255, + "tokens_seen": 2118909952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615045135406219, + "loss": 1.2664, + "theoretical_loss": 3.4424656462550876, + "tokens_seen": 2118975488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036148445336008024, + "loss": 1.3556, + "theoretical_loss": 3.442456942375496, + "tokens_seen": 2119041024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614643931795386, + "loss": 1.3522, + "theoretical_loss": 3.4424482388404565, + "tokens_seen": 2119106560 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036144433299899697, + "loss": 1.3515, + "theoretical_loss": 3.442439535649945, + "tokens_seen": 2119172096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614242728184554, + "loss": 1.3032, + "theoretical_loss": 3.442430832803937, + "tokens_seen": 2119237632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036140421263791374, + "loss": 1.3835, + "theoretical_loss": 3.4424221303024076, + "tokens_seen": 2119303168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613841524573721, + "loss": 1.3999, + "theoretical_loss": 3.4424134281453336, + "tokens_seen": 2119368704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036136409227683047, + "loss": 1.3163, + "theoretical_loss": 3.44240472633269, + "tokens_seen": 2119434240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036134403209628883, + "loss": 1.3365, + "theoretical_loss": 3.4423960248644523, + "tokens_seen": 2119499776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036132397191574725, + "loss": 1.3551, + "theoretical_loss": 3.442387323740597, + "tokens_seen": 2119565312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613039117352056, + "loss": 1.4044, + "theoretical_loss": 3.4423786229610998, + "tokens_seen": 2119630848 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036128385155466397, + "loss": 1.3283, + "theoretical_loss": 3.442369922525936, + "tokens_seen": 2119696384 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036126379137412233, + "loss": 1.3172, + "theoretical_loss": 3.442361222435081, + "tokens_seen": 2119761920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036124373119358075, + "loss": 1.3253, + "theoretical_loss": 3.4423525226885108, + "tokens_seen": 2119827456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612236710130391, + "loss": 1.332, + "theoretical_loss": 3.442343823286201, + "tokens_seen": 2119892992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612036108324975, + "loss": 1.306, + "theoretical_loss": 3.4423351242281286, + "tokens_seen": 2119958528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036118355065195584, + "loss": 1.3389, + "theoretical_loss": 3.442326425514268, + "tokens_seen": 2120024064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611634904714142, + "loss": 1.3884, + "theoretical_loss": 3.442317727144595, + "tokens_seen": 2120089600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611434302908726, + "loss": 1.3722, + "theoretical_loss": 3.4423090291190857, + "tokens_seen": 2120155136 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361123370110331, + "loss": 1.3955, + "theoretical_loss": 3.442300331437716, + "tokens_seen": 2120220672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611033099297894, + "loss": 1.3419, + "theoretical_loss": 3.442291634100461, + "tokens_seen": 2120286208 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036108324974924776, + "loss": 1.3343, + "theoretical_loss": 3.4422829371072967, + "tokens_seen": 2120351744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610631895687062, + "loss": 1.2999, + "theoretical_loss": 3.4422742404581994, + "tokens_seen": 2120417280 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036104312938816454, + "loss": 1.3733, + "theoretical_loss": 3.4422655441531447, + "tokens_seen": 2120482816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610230692076229, + "loss": 1.1956, + "theoretical_loss": 3.442256848192107, + "tokens_seen": 2120548352 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036100300902708126, + "loss": 1.3165, + "theoretical_loss": 3.442248152575065, + "tokens_seen": 2120613888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609829488465396, + "loss": 1.1961, + "theoretical_loss": 3.4422394573019908, + "tokens_seen": 2120679424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036096288866599804, + "loss": 1.3948, + "theoretical_loss": 3.442230762372863, + "tokens_seen": 2120744960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609428284854564, + "loss": 1.2374, + "theoretical_loss": 3.4422220677876565, + "tokens_seen": 2120810496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036092276830491476, + "loss": 1.4987, + "theoretical_loss": 3.4422133735463465, + "tokens_seen": 2120876032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609027081243731, + "loss": 1.3857, + "theoretical_loss": 3.4422046796489094, + "tokens_seen": 2120941568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036088264794383154, + "loss": 1.2896, + "theoretical_loss": 3.442195986095321, + "tokens_seen": 2121007104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608625877632899, + "loss": 1.237, + "theoretical_loss": 3.4421872928855564, + "tokens_seen": 2121072640 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036084252758274827, + "loss": 1.2575, + "theoretical_loss": 3.442178600019592, + "tokens_seen": 2121138176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036082246740220663, + "loss": 1.2289, + "theoretical_loss": 3.4421699074974033, + "tokens_seen": 2121203712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036080240722166505, + "loss": 1.2649, + "theoretical_loss": 3.4421612153189667, + "tokens_seen": 2121269248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607823470411234, + "loss": 1.3219, + "theoretical_loss": 3.442152523484257, + "tokens_seen": 2121334784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036076228686058177, + "loss": 1.3498, + "theoretical_loss": 3.442143831993251, + "tokens_seen": 2121400320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036074222668004013, + "loss": 1.3002, + "theoretical_loss": 3.4421351408459238, + "tokens_seen": 2121465856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607221664994985, + "loss": 1.3041, + "theoretical_loss": 3.442126450042251, + "tokens_seen": 2121531392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607021063189569, + "loss": 1.335, + "theoretical_loss": 3.4421177595822092, + "tokens_seen": 2121596928 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036068204613841527, + "loss": 1.3382, + "theoretical_loss": 3.4421090694657734, + "tokens_seen": 2121662464 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036066198595787364, + "loss": 1.2393, + "theoretical_loss": 3.4421003796929197, + "tokens_seen": 2121728000 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360641925777332, + "loss": 1.2346, + "theoretical_loss": 3.4420916902636245, + "tokens_seen": 2121793536 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 773131, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.180802345275879, + "objective/train/theoretical_loss": 3.442083001177863, + "objective/train/tokens_used": 492318176, + "theoretical_loss": 3.442083001177863, + "tokens_seen": 2121859072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606218655967904, + "loss": 1.2983, + "theoretical_loss": 3.442083001177863, + "tokens_seen": 2121859072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606018054162488, + "loss": 1.4459, + "theoretical_loss": 3.4420743124356106, + "tokens_seen": 2121924608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036058174523570714, + "loss": 1.2797, + "theoretical_loss": 3.442065624036844, + "tokens_seen": 2121990144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605616850551655, + "loss": 1.357, + "theoretical_loss": 3.4420569359815385, + "tokens_seen": 2122055680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036054162487462386, + "loss": 1.3843, + "theoretical_loss": 3.4420482482696704, + "tokens_seen": 2122121216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605215646940823, + "loss": 1.3349, + "theoretical_loss": 3.4420395609012147, + "tokens_seen": 2122186752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036050150451354064, + "loss": 1.2876, + "theoretical_loss": 3.4420308738761474, + "tokens_seen": 2122252288 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360481444332999, + "loss": 1.2774, + "theoretical_loss": 3.4420221871944454, + "tokens_seen": 2122317824 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036046138415245737, + "loss": 1.3869, + "theoretical_loss": 3.442013500856083, + "tokens_seen": 2122383360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604413239719158, + "loss": 1.311, + "theoretical_loss": 3.442004814861037, + "tokens_seen": 2122448896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036042126379137414, + "loss": 1.4405, + "theoretical_loss": 3.441996129209283, + "tokens_seen": 2122514432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604012036108325, + "loss": 1.3202, + "theoretical_loss": 3.4419874439007967, + "tokens_seen": 2122579968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036038114343029087, + "loss": 1.414, + "theoretical_loss": 3.441978758935554, + "tokens_seen": 2122645504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036036108324974923, + "loss": 1.4109, + "theoretical_loss": 3.441970074313531, + "tokens_seen": 2122711040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036034102306920765, + "loss": 1.3051, + "theoretical_loss": 3.4419613900347032, + "tokens_seen": 2122776576 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360320962888666, + "loss": 1.3899, + "theoretical_loss": 3.4419527060990465, + "tokens_seen": 2122842112 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036030090270812437, + "loss": 1.3934, + "theoretical_loss": 3.4419440225065374, + "tokens_seen": 2122907648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036028084252758273, + "loss": 1.39, + "theoretical_loss": 3.4419353392571503, + "tokens_seen": 2122973184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036026078234704115, + "loss": 1.3739, + "theoretical_loss": 3.441926656350862, + "tokens_seen": 2123038720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602407221664995, + "loss": 1.1971, + "theoretical_loss": 3.441917973787649, + "tokens_seen": 2123104256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602206619859579, + "loss": 1.3694, + "theoretical_loss": 3.441909291567486, + "tokens_seen": 2123169792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036020060180541624, + "loss": 1.4121, + "theoretical_loss": 3.4419006096903493, + "tokens_seen": 2123235328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601805416248746, + "loss": 1.2774, + "theoretical_loss": 3.441891928156215, + "tokens_seen": 2123300864 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360160481444333, + "loss": 1.3135, + "theoretical_loss": 3.441883246965058, + "tokens_seen": 2123366400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601404212637914, + "loss": 1.382, + "theoretical_loss": 3.441874566116855, + "tokens_seen": 2123431936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036012036108324974, + "loss": 1.3478, + "theoretical_loss": 3.4418658856115822, + "tokens_seen": 2123497472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601003009027081, + "loss": 1.3478, + "theoretical_loss": 3.4418572054492147, + "tokens_seen": 2123563008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600802407221665, + "loss": 1.2753, + "theoretical_loss": 3.441848525629729, + "tokens_seen": 2123628544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600601805416249, + "loss": 1.3944, + "theoretical_loss": 3.4418398461531003, + "tokens_seen": 2123694080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036004012036108324, + "loss": 1.3863, + "theoretical_loss": 3.4418311670193047, + "tokens_seen": 2123759616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600200601805416, + "loss": 1.2979, + "theoretical_loss": 3.441822488228319, + "tokens_seen": 2123825152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035999999999999997, + "loss": 1.2667, + "theoretical_loss": 3.4418138097801174, + "tokens_seen": 2123890688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599799398194584, + "loss": 1.3518, + "theoretical_loss": 3.441805131674677, + "tokens_seen": 2123956224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035995987963891675, + "loss": 1.3924, + "theoretical_loss": 3.4417964539119734, + "tokens_seen": 2124021760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599398194583751, + "loss": 1.2942, + "theoretical_loss": 3.4417877764919824, + "tokens_seen": 2124087296 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035991975927783347, + "loss": 1.3718, + "theoretical_loss": 3.44177909941468, + "tokens_seen": 2124152832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598996990972919, + "loss": 1.2736, + "theoretical_loss": 3.441770422680042, + "tokens_seen": 2124218368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035987963891675025, + "loss": 1.4003, + "theoretical_loss": 3.4417617462880443, + "tokens_seen": 2124283904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598595787362086, + "loss": 1.2612, + "theoretical_loss": 3.4417530702386627, + "tokens_seen": 2124349440 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359839518555667, + "loss": 1.3643, + "theoretical_loss": 3.4417443945318738, + "tokens_seen": 2124414976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035981945837512534, + "loss": 1.5319, + "theoretical_loss": 3.441735719167652, + "tokens_seen": 2124480512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035979939819458375, + "loss": 1.2494, + "theoretical_loss": 3.441727044145975, + "tokens_seen": 2124546048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597793380140421, + "loss": 1.2923, + "theoretical_loss": 3.441718369466817, + "tokens_seen": 2124611584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597592778335005, + "loss": 1.2791, + "theoretical_loss": 3.4417096951301556, + "tokens_seen": 2124677120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035973921765295884, + "loss": 1.3473, + "theoretical_loss": 3.4417010211359655, + "tokens_seen": 2124742656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035971915747241726, + "loss": 1.3139, + "theoretical_loss": 3.4416923474842234, + "tokens_seen": 2124808192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596990972918756, + "loss": 1.3076, + "theoretical_loss": 3.441683674174904, + "tokens_seen": 2124873728 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359679037111334, + "loss": 1.2904, + "theoretical_loss": 3.441675001207985, + "tokens_seen": 2124939264 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035965897693079234, + "loss": 1.3804, + "theoretical_loss": 3.44166632858344, + "tokens_seen": 2125004800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035963891675025076, + "loss": 1.3686, + "theoretical_loss": 3.4416576563012473, + "tokens_seen": 2125070336 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 774338, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4190452098846436, + "objective/train/theoretical_loss": 3.4416489843613816, + "objective/train/tokens_used": 495594976, + "theoretical_loss": 3.4416489843613816, + "tokens_seen": 2125135872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596188565697091, + "loss": 1.376, + "theoretical_loss": 3.4416489843613816, + "tokens_seen": 2125135872 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035959879638916754, + "loss": 1.3828, + "theoretical_loss": 3.4416403127638193, + "tokens_seen": 2125201408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595787362086259, + "loss": 1.3238, + "theoretical_loss": 3.441631641508536, + "tokens_seen": 2125266944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035955867602808426, + "loss": 1.4609, + "theoretical_loss": 3.4416229705955077, + "tokens_seen": 2125332480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595386158475427, + "loss": 1.3779, + "theoretical_loss": 3.44161430002471, + "tokens_seen": 2125398016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035951855566700104, + "loss": 1.2614, + "theoretical_loss": 3.4416056297961193, + "tokens_seen": 2125463552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594984954864594, + "loss": 1.2702, + "theoretical_loss": 3.4415969599097114, + "tokens_seen": 2125529088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035947843530591777, + "loss": 1.3084, + "theoretical_loss": 3.4415882903654627, + "tokens_seen": 2125594624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594583751253762, + "loss": 1.3342, + "theoretical_loss": 3.4415796211633483, + "tokens_seen": 2125660160 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035943831494483454, + "loss": 1.3428, + "theoretical_loss": 3.4415709523033446, + "tokens_seen": 2125725696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594182547642929, + "loss": 1.3003, + "theoretical_loss": 3.441562283785428, + "tokens_seen": 2125791232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035939819458375127, + "loss": 1.4025, + "theoretical_loss": 3.4415536156095734, + "tokens_seen": 2125856768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035937813440320963, + "loss": 1.4132, + "theoretical_loss": 3.4415449477757574, + "tokens_seen": 2125922304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035935807422266805, + "loss": 1.3226, + "theoretical_loss": 3.441536280283956, + "tokens_seen": 2125987840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593380140421264, + "loss": 1.34, + "theoretical_loss": 3.441527613134145, + "tokens_seen": 2126053376 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035931795386158477, + "loss": 1.3548, + "theoretical_loss": 3.4415189463263007, + "tokens_seen": 2126118912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035929789368104313, + "loss": 1.277, + "theoretical_loss": 3.441510279860399, + "tokens_seen": 2126184448 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035927783350050155, + "loss": 1.3075, + "theoretical_loss": 3.441501613736415, + "tokens_seen": 2126249984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592577733199599, + "loss": 1.3694, + "theoretical_loss": 3.441492947954326, + "tokens_seen": 2126315520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592377131394183, + "loss": 1.2697, + "theoretical_loss": 3.4414842825141063, + "tokens_seen": 2126381056 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035921765295887664, + "loss": 1.304, + "theoretical_loss": 3.4414756174157337, + "tokens_seen": 2126446592 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359197592778335, + "loss": 1.3831, + "theoretical_loss": 3.4414669526591832, + "tokens_seen": 2126512128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591775325977934, + "loss": 1.3481, + "theoretical_loss": 3.441458288244431, + "tokens_seen": 2126577664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591574724172518, + "loss": 1.3012, + "theoretical_loss": 3.441449624171453, + "tokens_seen": 2126643200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035913741223671014, + "loss": 1.314, + "theoretical_loss": 3.441440960440225, + "tokens_seen": 2126708736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591173520561685, + "loss": 1.3919, + "theoretical_loss": 3.4414322970507234, + "tokens_seen": 2126774272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590972918756269, + "loss": 1.3416, + "theoretical_loss": 3.4414236340029243, + "tokens_seen": 2126839808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590772316950853, + "loss": 1.3243, + "theoretical_loss": 3.441414971296803, + "tokens_seen": 2126905344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035905717151454364, + "loss": 1.3324, + "theoretical_loss": 3.4414063089323363, + "tokens_seen": 2126970880 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359037111334002, + "loss": 1.187, + "theoretical_loss": 3.4413976469094996, + "tokens_seen": 2127036416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035901705115346037, + "loss": 1.3337, + "theoretical_loss": 3.4413889852282686, + "tokens_seen": 2127101952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589969909729188, + "loss": 1.4321, + "theoretical_loss": 3.4413803238886205, + "tokens_seen": 2127167488 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035897693079237715, + "loss": 1.3863, + "theoretical_loss": 3.44137166289053, + "tokens_seen": 2127233024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589568706118355, + "loss": 1.3384, + "theoretical_loss": 3.4413630022339743, + "tokens_seen": 2127298560 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035893681043129387, + "loss": 1.4916, + "theoretical_loss": 3.4413543419189283, + "tokens_seen": 2127364096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589167502507523, + "loss": 1.3072, + "theoretical_loss": 3.4413456819453687, + "tokens_seen": 2127429632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035889669007021065, + "loss": 1.3133, + "theoretical_loss": 3.441337022313272, + "tokens_seen": 2127495168 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358876629889669, + "loss": 1.2943, + "theoretical_loss": 3.441328363022613, + "tokens_seen": 2127560704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588565697091274, + "loss": 1.3519, + "theoretical_loss": 3.441319704073368, + "tokens_seen": 2127626240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035883650952858574, + "loss": 1.3547, + "theoretical_loss": 3.441311045465514, + "tokens_seen": 2127691776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035881644934804415, + "loss": 1.4391, + "theoretical_loss": 3.4413023871990256, + "tokens_seen": 2127757312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587963891675025, + "loss": 1.2724, + "theoretical_loss": 3.44129372927388, + "tokens_seen": 2127822848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587763289869609, + "loss": 1.205, + "theoretical_loss": 3.4412850716900527, + "tokens_seen": 2127888384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035875626880641924, + "loss": 1.3634, + "theoretical_loss": 3.4412764144475196, + "tokens_seen": 2127953920 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035873620862587766, + "loss": 1.2051, + "theoretical_loss": 3.4412677575462576, + "tokens_seen": 2128019456 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358716148445336, + "loss": 1.3038, + "theoretical_loss": 3.441259100986241, + "tokens_seen": 2128084992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586960882647944, + "loss": 1.2973, + "theoretical_loss": 3.441250444767448, + "tokens_seen": 2128150528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035867602808425274, + "loss": 1.4109, + "theoretical_loss": 3.441241788889853, + "tokens_seen": 2128216064 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035865596790371116, + "loss": 1.2411, + "theoretical_loss": 3.441233133353433, + "tokens_seen": 2128281600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586359077231695, + "loss": 1.2656, + "theoretical_loss": 3.441224478158164, + "tokens_seen": 2128347136 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 775219, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2397558689117432, + "objective/train/theoretical_loss": 3.441215823304021, + "objective/train/tokens_used": 498871776, + "theoretical_loss": 3.441215823304021, + "tokens_seen": 2128412672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586158475426279, + "loss": 1.3053, + "theoretical_loss": 3.441215823304021, + "tokens_seen": 2128412672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035859578736208625, + "loss": 1.4534, + "theoretical_loss": 3.441207168790981, + "tokens_seen": 2128478208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585757271815446, + "loss": 1.3226, + "theoretical_loss": 3.4411985146190203, + "tokens_seen": 2128543744 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358555667001003, + "loss": 1.2199, + "theoretical_loss": 3.4411898607881137, + "tokens_seen": 2128609280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585356068204614, + "loss": 1.4343, + "theoretical_loss": 3.4411812072982384, + "tokens_seen": 2128674816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035851554663991975, + "loss": 1.321, + "theoretical_loss": 3.4411725541493703, + "tokens_seen": 2128740352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584954864593781, + "loss": 1.3199, + "theoretical_loss": 3.441163901341485, + "tokens_seen": 2128805888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035847542627883653, + "loss": 1.324, + "theoretical_loss": 3.4411552488745594, + "tokens_seen": 2128871424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584553660982949, + "loss": 1.2769, + "theoretical_loss": 3.441146596748568, + "tokens_seen": 2128936960 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035843530591775325, + "loss": 1.2738, + "theoretical_loss": 3.4411379449634887, + "tokens_seen": 2129002496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584152457372116, + "loss": 1.1917, + "theoretical_loss": 3.4411292935192965, + "tokens_seen": 2129068032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035839518555667, + "loss": 1.2898, + "theoretical_loss": 3.4411206424159677, + "tokens_seen": 2129133568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583751253761284, + "loss": 1.2502, + "theoretical_loss": 3.4411119916534787, + "tokens_seen": 2129199104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035835506519558676, + "loss": 1.3959, + "theoretical_loss": 3.4411033412318046, + "tokens_seen": 2129264640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583350050150451, + "loss": 1.4017, + "theoretical_loss": 3.441094691150923, + "tokens_seen": 2129330176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583149448345035, + "loss": 1.3461, + "theoretical_loss": 3.441086041410809, + "tokens_seen": 2129395712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582948846539619, + "loss": 1.4017, + "theoretical_loss": 3.4410773920114384, + "tokens_seen": 2129461248 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035827482447342026, + "loss": 1.2654, + "theoretical_loss": 3.441068742952788, + "tokens_seen": 2129526784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582547642928786, + "loss": 1.4366, + "theoretical_loss": 3.4410600942348335, + "tokens_seen": 2129592320 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358234704112337, + "loss": 1.3301, + "theoretical_loss": 3.4410514458575516, + "tokens_seen": 2129657856 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035821464393179535, + "loss": 1.2691, + "theoretical_loss": 3.4410427978209173, + "tokens_seen": 2129723392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035819458375125376, + "loss": 1.3385, + "theoretical_loss": 3.4410341501249078, + "tokens_seen": 2129788928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581745235707121, + "loss": 1.3358, + "theoretical_loss": 3.4410255027694987, + "tokens_seen": 2129854464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581544633901705, + "loss": 1.3992, + "theoretical_loss": 3.441016855754666, + "tokens_seen": 2129920000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035813440320962885, + "loss": 1.4394, + "theoretical_loss": 3.441008209080386, + "tokens_seen": 2129985536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035811434302908727, + "loss": 1.4123, + "theoretical_loss": 3.4409995627466348, + "tokens_seen": 2130051072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580942828485457, + "loss": 1.4891, + "theoretical_loss": 3.440990916753388, + "tokens_seen": 2130116608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035807422266800404, + "loss": 1.3302, + "theoretical_loss": 3.440982271100623, + "tokens_seen": 2130182144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580541624874624, + "loss": 1.2888, + "theoretical_loss": 3.440973625788314, + "tokens_seen": 2130247680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035803410230692077, + "loss": 1.2651, + "theoretical_loss": 3.4409649808164393, + "tokens_seen": 2130313216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580140421263792, + "loss": 1.3167, + "theoretical_loss": 3.440956336184974, + "tokens_seen": 2130378752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035799398194583755, + "loss": 1.3723, + "theoretical_loss": 3.4409476918938937, + "tokens_seen": 2130444288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579739217652959, + "loss": 1.423, + "theoretical_loss": 3.440939047943175, + "tokens_seen": 2130509824 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035795386158475427, + "loss": 1.3125, + "theoretical_loss": 3.4409304043327937, + "tokens_seen": 2130575360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579338014042127, + "loss": 1.422, + "theoretical_loss": 3.4409217610627265, + "tokens_seen": 2130640896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035791374122367105, + "loss": 1.3242, + "theoretical_loss": 3.4409131181329498, + "tokens_seen": 2130706432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578936810431294, + "loss": 1.4027, + "theoretical_loss": 3.4409044755434386, + "tokens_seen": 2130771968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578736208625878, + "loss": 1.3327, + "theoretical_loss": 3.44089583329417, + "tokens_seen": 2130837504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035785356068204614, + "loss": 1.2811, + "theoretical_loss": 3.4408871913851193, + "tokens_seen": 2130903040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035783350050150455, + "loss": 1.3479, + "theoretical_loss": 3.4408785498162637, + "tokens_seen": 2130968576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578134403209629, + "loss": 1.197, + "theoretical_loss": 3.4408699085875787, + "tokens_seen": 2131034112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577933801404213, + "loss": 1.3662, + "theoretical_loss": 3.44086126769904, + "tokens_seen": 2131099648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035777331995987964, + "loss": 1.272, + "theoretical_loss": 3.4408526271506252, + "tokens_seen": 2131165184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035775325977933806, + "loss": 1.2617, + "theoretical_loss": 3.4408439869423093, + "tokens_seen": 2131230720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577331995987964, + "loss": 1.3017, + "theoretical_loss": 3.440835347074068, + "tokens_seen": 2131296256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577131394182548, + "loss": 1.2578, + "theoretical_loss": 3.440826707545879, + "tokens_seen": 2131361792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035769307923771314, + "loss": 1.2312, + "theoretical_loss": 3.4408180683577174, + "tokens_seen": 2131427328 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035767301905717156, + "loss": 1.3423, + "theoretical_loss": 3.4408094295095593, + "tokens_seen": 2131492864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576529588766299, + "loss": 1.2767, + "theoretical_loss": 3.4408007910013816, + "tokens_seen": 2131558400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576328986960883, + "loss": 1.3621, + "theoretical_loss": 3.4407921528331595, + "tokens_seen": 2131623936 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 776312, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.494091510772705, + "objective/train/theoretical_loss": 3.4407835150048705, + "objective/train/tokens_used": 502148576, + "theoretical_loss": 3.4407835150048705, + "tokens_seen": 2131689472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035761283851554665, + "loss": 1.4551, + "theoretical_loss": 3.4407835150048705, + "tokens_seen": 2131689472 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357592778335005, + "loss": 1.2621, + "theoretical_loss": 3.440774877516489, + "tokens_seen": 2131755008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575727181544634, + "loss": 1.4304, + "theoretical_loss": 3.440766240367993, + "tokens_seen": 2131820544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575526579739218, + "loss": 1.2241, + "theoretical_loss": 3.4407576035593572, + "tokens_seen": 2131886080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035753259779338015, + "loss": 1.3136, + "theoretical_loss": 3.440748967090559, + "tokens_seen": 2131951616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575125376128385, + "loss": 1.3389, + "theoretical_loss": 3.440740330961573, + "tokens_seen": 2132017152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035749247743229693, + "loss": 1.3099, + "theoretical_loss": 3.440731695172377, + "tokens_seen": 2132082688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574724172517553, + "loss": 1.3085, + "theoretical_loss": 3.440723059722947, + "tokens_seen": 2132148224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035745235707121365, + "loss": 1.3017, + "theoretical_loss": 3.440714424613258, + "tokens_seen": 2132213760 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357432296890672, + "loss": 1.3108, + "theoretical_loss": 3.440705789843287, + "tokens_seen": 2132279296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574122367101304, + "loss": 1.4492, + "theoretical_loss": 3.4406971554130106, + "tokens_seen": 2132344832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573921765295888, + "loss": 1.3548, + "theoretical_loss": 3.440688521322404, + "tokens_seen": 2132410368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035737211634904716, + "loss": 1.5501, + "theoretical_loss": 3.440679887571444, + "tokens_seen": 2132475904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573520561685055, + "loss": 1.451, + "theoretical_loss": 3.440671254160107, + "tokens_seen": 2132541440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573319959879639, + "loss": 1.3102, + "theoretical_loss": 3.440662621088369, + "tokens_seen": 2132606976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573119358074223, + "loss": 1.3613, + "theoretical_loss": 3.440653988356206, + "tokens_seen": 2132672512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035729187562688066, + "loss": 1.3527, + "theoretical_loss": 3.440645355963594, + "tokens_seen": 2132738048 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357271815446339, + "loss": 1.352, + "theoretical_loss": 3.44063672391051, + "tokens_seen": 2132803584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572517552657974, + "loss": 1.3763, + "theoretical_loss": 3.44062809219693, + "tokens_seen": 2132869120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035723169508525575, + "loss": 1.2878, + "theoretical_loss": 3.4406194608228295, + "tokens_seen": 2132934656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035721163490471416, + "loss": 1.324, + "theoretical_loss": 3.440610829788185, + "tokens_seen": 2133000192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571915747241725, + "loss": 1.1916, + "theoretical_loss": 3.4406021990929734, + "tokens_seen": 2133065728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571715145436309, + "loss": 1.3533, + "theoretical_loss": 3.44059356873717, + "tokens_seen": 2133131264 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035715145436308925, + "loss": 1.4111, + "theoretical_loss": 3.440584938720752, + "tokens_seen": 2133196800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035713139418254767, + "loss": 1.3214, + "theoretical_loss": 3.4405763090436947, + "tokens_seen": 2133262336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035711133400200603, + "loss": 1.294, + "theoretical_loss": 3.4405676797059748, + "tokens_seen": 2133327872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570912738214644, + "loss": 1.3682, + "theoretical_loss": 3.440559050707569, + "tokens_seen": 2133393408 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035707121364092275, + "loss": 1.3797, + "theoretical_loss": 3.440550422048452, + "tokens_seen": 2133458944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570511534603811, + "loss": 1.3337, + "theoretical_loss": 3.440541793728602, + "tokens_seen": 2133524480 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035703109327983953, + "loss": 1.3732, + "theoretical_loss": 3.440533165747994, + "tokens_seen": 2133590016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570110330992979, + "loss": 1.2371, + "theoretical_loss": 3.4405245381066045, + "tokens_seen": 2133655552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035699097291875626, + "loss": 1.3211, + "theoretical_loss": 3.440515910804409, + "tokens_seen": 2133721088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569709127382146, + "loss": 1.353, + "theoretical_loss": 3.4405072838413853, + "tokens_seen": 2133786624 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035695085255767303, + "loss": 1.351, + "theoretical_loss": 3.4404986572175087, + "tokens_seen": 2133852160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569307923771314, + "loss": 1.4066, + "theoretical_loss": 3.440490030932756, + "tokens_seen": 2133917696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035691073219658976, + "loss": 1.2886, + "theoretical_loss": 3.4404814049871026, + "tokens_seen": 2133983232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568906720160481, + "loss": 1.3413, + "theoretical_loss": 3.440472779380525, + "tokens_seen": 2134048768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568706118355065, + "loss": 1.4214, + "theoretical_loss": 3.440464154113, + "tokens_seen": 2134114304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568505516549649, + "loss": 1.3022, + "theoretical_loss": 3.440455529184504, + "tokens_seen": 2134179840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035683049147442326, + "loss": 1.3155, + "theoretical_loss": 3.440446904595012, + "tokens_seen": 2134245376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568104312938816, + "loss": 1.259, + "theoretical_loss": 3.4404382803445017, + "tokens_seen": 2134310912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035679037111334, + "loss": 1.2681, + "theoretical_loss": 3.4404296564329484, + "tokens_seen": 2134376448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567703109327984, + "loss": 1.2632, + "theoretical_loss": 3.440421032860329, + "tokens_seen": 2134441984 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035675025075225677, + "loss": 1.4561, + "theoretical_loss": 3.440412409626619, + "tokens_seen": 2134507520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035673019057171513, + "loss": 1.4086, + "theoretical_loss": 3.440403786731795, + "tokens_seen": 2134573056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567101303911735, + "loss": 1.369, + "theoretical_loss": 3.4403951641758344, + "tokens_seen": 2134638592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035669007021063185, + "loss": 1.2481, + "theoretical_loss": 3.440386541958712, + "tokens_seen": 2134704128 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035667001003009027, + "loss": 1.3134, + "theoretical_loss": 3.4403779200804046, + "tokens_seen": 2134769664 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035664994984954863, + "loss": 1.3466, + "theoretical_loss": 3.4403692985408885, + "tokens_seen": 2134835200 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356629889669007, + "loss": 1.3554, + "theoretical_loss": 3.4403606773401396, + "tokens_seen": 2134900736 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 777595, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3070225715637207, + "objective/train/theoretical_loss": 3.440352056478135, + "objective/train/tokens_used": 505425376, + "theoretical_loss": 3.440352056478135, + "tokens_seen": 2134966272 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035660982948846536, + "loss": 1.3461, + "theoretical_loss": 3.440352056478135, + "tokens_seen": 2134966272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565897693079238, + "loss": 1.3112, + "theoretical_loss": 3.4403434359548504, + "tokens_seen": 2135031808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565697091273822, + "loss": 1.2702, + "theoretical_loss": 3.4403348157702625, + "tokens_seen": 2135097344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035654964894684055, + "loss": 1.4314, + "theoretical_loss": 3.440326195924347, + "tokens_seen": 2135162880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565295887662989, + "loss": 1.3201, + "theoretical_loss": 3.4403175764170806, + "tokens_seen": 2135228416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035650952858575733, + "loss": 1.2803, + "theoretical_loss": 3.4403089572484395, + "tokens_seen": 2135293952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564894684052157, + "loss": 1.3105, + "theoretical_loss": 3.4403003384184005, + "tokens_seen": 2135359488 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035646940822467405, + "loss": 1.2803, + "theoretical_loss": 3.4402917199269396, + "tokens_seen": 2135425024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564493480441324, + "loss": 1.2174, + "theoretical_loss": 3.4402831017740323, + "tokens_seen": 2135490560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564292878635908, + "loss": 1.3404, + "theoretical_loss": 3.440274483959656, + "tokens_seen": 2135556096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564092276830492, + "loss": 1.3267, + "theoretical_loss": 3.440265866483786, + "tokens_seen": 2135621632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035638916750250756, + "loss": 1.4911, + "theoretical_loss": 3.4402572493464, + "tokens_seen": 2135687168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563691073219659, + "loss": 1.1375, + "theoretical_loss": 3.440248632547473, + "tokens_seen": 2135752704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563490471414243, + "loss": 1.2635, + "theoretical_loss": 3.440240016086982, + "tokens_seen": 2135818240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563289869608827, + "loss": 1.2706, + "theoretical_loss": 3.4402313999649037, + "tokens_seen": 2135883776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035630892678034106, + "loss": 1.2511, + "theoretical_loss": 3.440222784181213, + "tokens_seen": 2135949312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562888665997994, + "loss": 1.3204, + "theoretical_loss": 3.4402141687358876, + "tokens_seen": 2136014848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562688064192578, + "loss": 1.4885, + "theoretical_loss": 3.4402055536289033, + "tokens_seen": 2136080384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035624874623871615, + "loss": 1.3531, + "theoretical_loss": 3.4401969388602365, + "tokens_seen": 2136145920 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035622868605817456, + "loss": 1.1891, + "theoretical_loss": 3.4401883244298634, + "tokens_seen": 2136211456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562086258776329, + "loss": 1.3732, + "theoretical_loss": 3.4401797103377607, + "tokens_seen": 2136276992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561885656970913, + "loss": 1.2878, + "theoretical_loss": 3.4401710965839047, + "tokens_seen": 2136342528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035616850551654965, + "loss": 1.296, + "theoretical_loss": 3.440162483168271, + "tokens_seen": 2136408064 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035614844533600807, + "loss": 1.2962, + "theoretical_loss": 3.440153870090837, + "tokens_seen": 2136473600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035612838515546643, + "loss": 1.3415, + "theoretical_loss": 3.4401452573515785, + "tokens_seen": 2136539136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561083249749248, + "loss": 1.3779, + "theoretical_loss": 3.4401366449504716, + "tokens_seen": 2136604672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035608826479438315, + "loss": 1.2601, + "theoretical_loss": 3.440128032887493, + "tokens_seen": 2136670208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003560682046138415, + "loss": 1.2638, + "theoretical_loss": 3.4401194211626187, + "tokens_seen": 2136735744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035604814443329993, + "loss": 1.3308, + "theoretical_loss": 3.4401108097758257, + "tokens_seen": 2136801280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560280842527583, + "loss": 1.2922, + "theoretical_loss": 3.4401021987270903, + "tokens_seen": 2136866816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035600802407221666, + "loss": 1.3145, + "theoretical_loss": 3.440093588016388, + "tokens_seen": 2136932352 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355987963891675, + "loss": 1.4025, + "theoretical_loss": 3.4400849776436964, + "tokens_seen": 2136997888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035596790371113344, + "loss": 1.2907, + "theoretical_loss": 3.4400763676089907, + "tokens_seen": 2137063424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559478435305918, + "loss": 1.2305, + "theoretical_loss": 3.4400677579122476, + "tokens_seen": 2137128960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035592778335005016, + "loss": 1.3397, + "theoretical_loss": 3.440059148553444, + "tokens_seen": 2137194496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559077231695085, + "loss": 1.2745, + "theoretical_loss": 3.440050539532556, + "tokens_seen": 2137260032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558876629889669, + "loss": 1.3253, + "theoretical_loss": 3.44004193084956, + "tokens_seen": 2137325568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558676028084253, + "loss": 1.3965, + "theoretical_loss": 3.4400333225044317, + "tokens_seen": 2137391104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035584754262788366, + "loss": 1.4965, + "theoretical_loss": 3.4400247144971488, + "tokens_seen": 2137456640 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355827482447342, + "loss": 1.3132, + "theoretical_loss": 3.440016106827686, + "tokens_seen": 2137522176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558074222668004, + "loss": 1.3446, + "theoretical_loss": 3.440007499496021, + "tokens_seen": 2137587712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557873620862588, + "loss": 1.2975, + "theoretical_loss": 3.43999889250213, + "tokens_seen": 2137653248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035576730190571717, + "loss": 1.3259, + "theoretical_loss": 3.4399902858459894, + "tokens_seen": 2137718784 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035574724172517553, + "loss": 1.4234, + "theoretical_loss": 3.439981679527575, + "tokens_seen": 2137784320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557271815446339, + "loss": 1.2532, + "theoretical_loss": 3.439973073546864, + "tokens_seen": 2137849856 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035570712136409225, + "loss": 1.3639, + "theoretical_loss": 3.439964467903832, + "tokens_seen": 2137915392 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035568706118355067, + "loss": 1.3525, + "theoretical_loss": 3.439955862598456, + "tokens_seen": 2137980928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035566700100300903, + "loss": 1.291, + "theoretical_loss": 3.4399472576307124, + "tokens_seen": 2138046464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556469408224674, + "loss": 1.356, + "theoretical_loss": 3.439938653000577, + "tokens_seen": 2138112000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035562688064192576, + "loss": 1.4528, + "theoretical_loss": 3.4399300487080264, + "tokens_seen": 2138177536 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 778668, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3252495527267456, + "objective/train/theoretical_loss": 3.4399214447530375, + "objective/train/tokens_used": 508702176, + "theoretical_loss": 3.4399214447530375, + "tokens_seen": 2138243072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035560682046138417, + "loss": 1.3317, + "theoretical_loss": 3.4399214447530375, + "tokens_seen": 2138243072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035558676028084253, + "loss": 1.3404, + "theoretical_loss": 3.4399128411355866, + "tokens_seen": 2138308608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555667001003009, + "loss": 1.2442, + "theoretical_loss": 3.4399042378556497, + "tokens_seen": 2138374144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035554663991975926, + "loss": 1.3616, + "theoretical_loss": 3.4398956349132037, + "tokens_seen": 2138439680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555265797392176, + "loss": 1.2863, + "theoretical_loss": 3.4398870323082242, + "tokens_seen": 2138505216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035550651955867604, + "loss": 1.3091, + "theoretical_loss": 3.4398784300406886, + "tokens_seen": 2138570752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554864593781344, + "loss": 1.1826, + "theoretical_loss": 3.439869828110573, + "tokens_seen": 2138636288 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035546639919759276, + "loss": 1.3143, + "theoretical_loss": 3.4398612265178534, + "tokens_seen": 2138701824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554463390170511, + "loss": 1.3747, + "theoretical_loss": 3.439852625262507, + "tokens_seen": 2138767360 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035542627883650954, + "loss": 1.1323, + "theoretical_loss": 3.4398440243445094, + "tokens_seen": 2138832896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554062186559679, + "loss": 1.2308, + "theoretical_loss": 3.4398354237638378, + "tokens_seen": 2138898432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035538615847542627, + "loss": 1.2043, + "theoretical_loss": 3.439826823520468, + "tokens_seen": 2138963968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035536609829488463, + "loss": 1.3601, + "theoretical_loss": 3.4398182236143766, + "tokens_seen": 2139029504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035534603811434304, + "loss": 1.3758, + "theoretical_loss": 3.43980962404554, + "tokens_seen": 2139095040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553259779338014, + "loss": 1.4445, + "theoretical_loss": 3.439801024813935, + "tokens_seen": 2139160576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035530591775325977, + "loss": 1.2661, + "theoretical_loss": 3.439792425919538, + "tokens_seen": 2139226112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035528585757271813, + "loss": 1.3666, + "theoretical_loss": 3.439783827362325, + "tokens_seen": 2139291648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552657973921765, + "loss": 1.3466, + "theoretical_loss": 3.439775229142273, + "tokens_seen": 2139357184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552457372116349, + "loss": 1.3266, + "theoretical_loss": 3.439766631259358, + "tokens_seen": 2139422720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035522567703109327, + "loss": 1.3053, + "theoretical_loss": 3.4397580337135567, + "tokens_seen": 2139488256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035520561685055163, + "loss": 1.256, + "theoretical_loss": 3.4397494365048455, + "tokens_seen": 2139553792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035518555667001, + "loss": 1.3325, + "theoretical_loss": 3.4397408396332008, + "tokens_seen": 2139619328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551654964894684, + "loss": 1.322, + "theoretical_loss": 3.4397322430985993, + "tokens_seen": 2139684864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551454363089268, + "loss": 1.331, + "theoretical_loss": 3.439723646901017, + "tokens_seen": 2139750400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035512537612838514, + "loss": 1.2893, + "theoretical_loss": 3.43971505104043, + "tokens_seen": 2139815936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551053159478435, + "loss": 1.2141, + "theoretical_loss": 3.4397064555168164, + "tokens_seen": 2139881472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550852557673019, + "loss": 1.348, + "theoretical_loss": 3.439697860330152, + "tokens_seen": 2139947008 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035506519558676033, + "loss": 1.3245, + "theoretical_loss": 3.439689265480412, + "tokens_seen": 2140012544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550451354062187, + "loss": 1.3246, + "theoretical_loss": 3.439680670967574, + "tokens_seen": 2140078080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035502507522567706, + "loss": 1.3225, + "theoretical_loss": 3.4396720767916147, + "tokens_seen": 2140143616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550050150451354, + "loss": 1.3036, + "theoretical_loss": 3.4396634829525095, + "tokens_seen": 2140209152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035498495486459384, + "loss": 1.2314, + "theoretical_loss": 3.439654889450236, + "tokens_seen": 2140274688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549648946840522, + "loss": 1.3085, + "theoretical_loss": 3.4396462962847707, + "tokens_seen": 2140340224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035494483450351056, + "loss": 1.3708, + "theoretical_loss": 3.4396377034560888, + "tokens_seen": 2140405760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549247743229689, + "loss": 1.2907, + "theoretical_loss": 3.4396291109641677, + "tokens_seen": 2140471296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549047141424273, + "loss": 1.3083, + "theoretical_loss": 3.4396205188089843, + "tokens_seen": 2140536832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548846539618857, + "loss": 1.2757, + "theoretical_loss": 3.4396119269905143, + "tokens_seen": 2140602368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035486459378134406, + "loss": 1.2771, + "theoretical_loss": 3.4396033355087345, + "tokens_seen": 2140667904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548445336008024, + "loss": 1.3275, + "theoretical_loss": 3.439594744363622, + "tokens_seen": 2140733440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548244734202608, + "loss": 1.3031, + "theoretical_loss": 3.439586153555152, + "tokens_seen": 2140798976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548044132397192, + "loss": 1.3226, + "theoretical_loss": 3.4395775630833016, + "tokens_seen": 2140864512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035478435305917757, + "loss": 1.258, + "theoretical_loss": 3.439568972948048, + "tokens_seen": 2140930048 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035476429287863593, + "loss": 1.3212, + "theoretical_loss": 3.439560383149367, + "tokens_seen": 2140995584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547442326980943, + "loss": 1.2982, + "theoretical_loss": 3.439551793687235, + "tokens_seen": 2141061120 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035472417251755265, + "loss": 1.2696, + "theoretical_loss": 3.439543204561629, + "tokens_seen": 2141126656 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035470411233701107, + "loss": 1.3041, + "theoretical_loss": 3.439534615772525, + "tokens_seen": 2141192192 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035468405215646943, + "loss": 1.3282, + "theoretical_loss": 3.4395260273199, + "tokens_seen": 2141257728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546639919759278, + "loss": 1.3522, + "theoretical_loss": 3.4395174392037298, + "tokens_seen": 2141323264 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035464393179538616, + "loss": 1.3441, + "theoretical_loss": 3.439508851423992, + "tokens_seen": 2141388800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035462387161484457, + "loss": 1.3356, + "theoretical_loss": 3.4395002639806624, + "tokens_seen": 2141454336 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 779818, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.398399829864502, + "objective/train/theoretical_loss": 3.4394916768737174, + "objective/train/tokens_used": 511978976, + "theoretical_loss": 3.4394916768737174, + "tokens_seen": 2141519872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035460381143430293, + "loss": 1.3089, + "theoretical_loss": 3.4394916768737174, + "tokens_seen": 2141519872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545837512537613, + "loss": 1.287, + "theoretical_loss": 3.4394830901031344, + "tokens_seen": 2141585408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035456369107321966, + "loss": 1.3604, + "theoretical_loss": 3.4394745036688885, + "tokens_seen": 2141650944 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354543630892678, + "loss": 1.2881, + "theoretical_loss": 3.439465917570958, + "tokens_seen": 2141716480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035452357071213644, + "loss": 1.3275, + "theoretical_loss": 3.439457331809318, + "tokens_seen": 2141782016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545035105315948, + "loss": 1.3962, + "theoretical_loss": 3.4394487463839454, + "tokens_seen": 2141847552 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035448345035105316, + "loss": 1.2574, + "theoretical_loss": 3.4394401612948173, + "tokens_seen": 2141913088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544633901705115, + "loss": 1.4146, + "theoretical_loss": 3.4394315765419097, + "tokens_seen": 2141978624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035444332998996994, + "loss": 1.1949, + "theoretical_loss": 3.4394229921251993, + "tokens_seen": 2142044160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544232698094283, + "loss": 1.4292, + "theoretical_loss": 3.4394144080446623, + "tokens_seen": 2142109696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035440320962888667, + "loss": 1.2901, + "theoretical_loss": 3.439405824300276, + "tokens_seen": 2142175232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035438314944834503, + "loss": 1.318, + "theoretical_loss": 3.439397240892016, + "tokens_seen": 2142240768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035436308926780344, + "loss": 1.3866, + "theoretical_loss": 3.43938865781986, + "tokens_seen": 2142306304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543430290872618, + "loss": 1.4974, + "theoretical_loss": 3.4393800750837835, + "tokens_seen": 2142371840 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035432296890672017, + "loss": 1.2849, + "theoretical_loss": 3.4393714926837635, + "tokens_seen": 2142437376 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035430290872617853, + "loss": 1.2741, + "theoretical_loss": 3.439362910619777, + "tokens_seen": 2142502912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542828485456369, + "loss": 1.2831, + "theoretical_loss": 3.4393543288918, + "tokens_seen": 2142568448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542627883650953, + "loss": 1.2862, + "theoretical_loss": 3.439345747499809, + "tokens_seen": 2142633984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035424272818455367, + "loss": 1.3534, + "theoretical_loss": 3.4393371664437806, + "tokens_seen": 2142699520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035422266800401203, + "loss": 1.2924, + "theoretical_loss": 3.4393285857236915, + "tokens_seen": 2142765056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542026078234704, + "loss": 1.3513, + "theoretical_loss": 3.4393200053395185, + "tokens_seen": 2142830592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541825476429288, + "loss": 1.3135, + "theoretical_loss": 3.4393114252912387, + "tokens_seen": 2142896128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541624874623872, + "loss": 1.2822, + "theoretical_loss": 3.439302845578827, + "tokens_seen": 2142961664 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035414242728184554, + "loss": 1.3528, + "theoretical_loss": 3.4392942662022614, + "tokens_seen": 2143027200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541223671013039, + "loss": 1.2919, + "theoretical_loss": 3.439285687161518, + "tokens_seen": 2143092736 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035410230692076226, + "loss": 1.2921, + "theoretical_loss": 3.439277108456573, + "tokens_seen": 2143158272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540822467402207, + "loss": 1.3208, + "theoretical_loss": 3.439268530087404, + "tokens_seen": 2143223808 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035406218655967904, + "loss": 1.4625, + "theoretical_loss": 3.4392599520539866, + "tokens_seen": 2143289344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540421263791374, + "loss": 1.3268, + "theoretical_loss": 3.439251374356298, + "tokens_seen": 2143354880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035402206619859577, + "loss": 1.1718, + "theoretical_loss": 3.4392427969943147, + "tokens_seen": 2143420416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540020060180542, + "loss": 1.3166, + "theoretical_loss": 3.4392342199680126, + "tokens_seen": 2143485952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035398194583751254, + "loss": 1.2865, + "theoretical_loss": 3.4392256432773696, + "tokens_seen": 2143551488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539618856569709, + "loss": 1.3057, + "theoretical_loss": 3.439217066922361, + "tokens_seen": 2143617024 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035394182547642927, + "loss": 1.4133, + "theoretical_loss": 3.4392084909029643, + "tokens_seen": 2143682560 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035392176529588763, + "loss": 1.3409, + "theoretical_loss": 3.439199915219156, + "tokens_seen": 2143748096 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035390170511534605, + "loss": 1.3532, + "theoretical_loss": 3.4391913398709124, + "tokens_seen": 2143813632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538816449348044, + "loss": 1.2113, + "theoretical_loss": 3.4391827648582103, + "tokens_seen": 2143879168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035386158475426277, + "loss": 1.3304, + "theoretical_loss": 3.439174190181026, + "tokens_seen": 2143944704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035384152457372113, + "loss": 1.2251, + "theoretical_loss": 3.439165615839336, + "tokens_seen": 2144010240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035382146439317955, + "loss": 1.3451, + "theoretical_loss": 3.439157041833118, + "tokens_seen": 2144075776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538014042126379, + "loss": 1.3919, + "theoretical_loss": 3.4391484681623474, + "tokens_seen": 2144141312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537813440320963, + "loss": 1.3531, + "theoretical_loss": 3.4391398948270018, + "tokens_seen": 2144206848 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035376128385155464, + "loss": 1.3001, + "theoretical_loss": 3.439131321827057, + "tokens_seen": 2144272384 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353741223671013, + "loss": 1.2921, + "theoretical_loss": 3.4391227491624896, + "tokens_seen": 2144337920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537211634904714, + "loss": 1.1948, + "theoretical_loss": 3.4391141768332774, + "tokens_seen": 2144403456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537011033099298, + "loss": 1.3473, + "theoretical_loss": 3.439105604839396, + "tokens_seen": 2144468992 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035368104312938814, + "loss": 1.3074, + "theoretical_loss": 3.439097033180822, + "tokens_seen": 2144534528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536609829488465, + "loss": 1.2737, + "theoretical_loss": 3.4390884618575326, + "tokens_seen": 2144600064 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536409227683049, + "loss": 1.2875, + "theoretical_loss": 3.4390798908695035, + "tokens_seen": 2144665600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536208625877633, + "loss": 1.3816, + "theoretical_loss": 3.439071320216713, + "tokens_seen": 2144731136 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 780961, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3549442291259766, + "objective/train/theoretical_loss": 3.4390627498991355, + "objective/train/tokens_used": 515255776, + "theoretical_loss": 3.4390627498991355, + "tokens_seen": 2144796672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536008024072217, + "loss": 1.344, + "theoretical_loss": 3.4390627498991355, + "tokens_seen": 2144796672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035358074222668006, + "loss": 1.2742, + "theoretical_loss": 3.43905417991675, + "tokens_seen": 2144862208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535606820461384, + "loss": 1.3611, + "theoretical_loss": 3.4390456102695315, + "tokens_seen": 2144927744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035354062186559684, + "loss": 1.3172, + "theoretical_loss": 3.4390370409574573, + "tokens_seen": 2144993280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535205616850552, + "loss": 1.331, + "theoretical_loss": 3.439028471980504, + "tokens_seen": 2145058816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035350050150451356, + "loss": 1.3048, + "theoretical_loss": 3.439019903338648, + "tokens_seen": 2145124352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534804413239719, + "loss": 1.4079, + "theoretical_loss": 3.439011335031866, + "tokens_seen": 2145189888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035346038114343034, + "loss": 1.2809, + "theoretical_loss": 3.4390027670601353, + "tokens_seen": 2145255424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534403209628887, + "loss": 1.2763, + "theoretical_loss": 3.4389941994234317, + "tokens_seen": 2145320960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035342026078234707, + "loss": 1.2864, + "theoretical_loss": 3.4389856321217325, + "tokens_seen": 2145386496 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035340020060180543, + "loss": 1.2932, + "theoretical_loss": 3.4389770651550142, + "tokens_seen": 2145452032 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035338014042126384, + "loss": 1.3216, + "theoretical_loss": 3.438968498523253, + "tokens_seen": 2145517568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533600802407222, + "loss": 1.3375, + "theoretical_loss": 3.4389599322264264, + "tokens_seen": 2145583104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035334002006018057, + "loss": 1.3485, + "theoretical_loss": 3.43895136626451, + "tokens_seen": 2145648640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035331995987963893, + "loss": 1.3139, + "theoretical_loss": 3.4389428006374816, + "tokens_seen": 2145714176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532998996990973, + "loss": 1.4077, + "theoretical_loss": 3.438934235345317, + "tokens_seen": 2145779712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532798395185557, + "loss": 1.3511, + "theoretical_loss": 3.4389256703879942, + "tokens_seen": 2145845248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035325977933801407, + "loss": 1.4069, + "theoretical_loss": 3.438917105765488, + "tokens_seen": 2145910784 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035323971915747243, + "loss": 1.4001, + "theoretical_loss": 3.438908541477776, + "tokens_seen": 2145976320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532196589769308, + "loss": 1.2601, + "theoretical_loss": 3.4388999775248354, + "tokens_seen": 2146041856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531995987963892, + "loss": 1.2556, + "theoretical_loss": 3.438891413906642, + "tokens_seen": 2146107392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531795386158476, + "loss": 1.3329, + "theoretical_loss": 3.4388828506231737, + "tokens_seen": 2146172928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035315947843530594, + "loss": 1.3569, + "theoretical_loss": 3.438874287674406, + "tokens_seen": 2146238464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531394182547643, + "loss": 1.2781, + "theoretical_loss": 3.4388657250603156, + "tokens_seen": 2146304000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035311935807422266, + "loss": 1.3, + "theoretical_loss": 3.43885716278088, + "tokens_seen": 2146369536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530992978936811, + "loss": 1.339, + "theoretical_loss": 3.4388486008360752, + "tokens_seen": 2146435072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035307923771313944, + "loss": 1.3521, + "theoretical_loss": 3.4388400392258784, + "tokens_seen": 2146500608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530591775325978, + "loss": 1.282, + "theoretical_loss": 3.4388314779502664, + "tokens_seen": 2146566144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035303911735205617, + "loss": 1.3233, + "theoretical_loss": 3.4388229170092153, + "tokens_seen": 2146631680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530190571715146, + "loss": 1.3275, + "theoretical_loss": 3.4388143564027023, + "tokens_seen": 2146697216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035299899699097294, + "loss": 1.3534, + "theoretical_loss": 3.438805796130704, + "tokens_seen": 2146762752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529789368104313, + "loss": 1.3265, + "theoretical_loss": 3.4387972361931967, + "tokens_seen": 2146828288 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035295887662988967, + "loss": 1.2809, + "theoretical_loss": 3.438788676590158, + "tokens_seen": 2146893824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035293881644934803, + "loss": 1.3095, + "theoretical_loss": 3.4387801173215635, + "tokens_seen": 2146959360 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035291875626880645, + "loss": 1.3546, + "theoretical_loss": 3.4387715583873915, + "tokens_seen": 2147024896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528986960882648, + "loss": 1.3254, + "theoretical_loss": 3.438762999787617, + "tokens_seen": 2147090432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035287863590772317, + "loss": 1.3375, + "theoretical_loss": 3.4387544415222178, + "tokens_seen": 2147155968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035285857572718153, + "loss": 1.2386, + "theoretical_loss": 3.43874588359117, + "tokens_seen": 2147221504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035283851554663995, + "loss": 1.2849, + "theoretical_loss": 3.4387373259944507, + "tokens_seen": 2147287040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528184553660983, + "loss": 1.2986, + "theoretical_loss": 3.4387287687320374, + "tokens_seen": 2147352576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527983951855567, + "loss": 1.3696, + "theoretical_loss": 3.438720211803905, + "tokens_seen": 2147418112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035277833500501504, + "loss": 1.3382, + "theoretical_loss": 3.4387116552100316, + "tokens_seen": 2147483648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527582748244734, + "loss": 1.2614, + "theoretical_loss": 3.438703098950394, + "tokens_seen": 2147549184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527382146439318, + "loss": 1.3767, + "theoretical_loss": 3.4386945430249685, + "tokens_seen": 2147614720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527181544633902, + "loss": 1.3414, + "theoretical_loss": 3.4386859874337317, + "tokens_seen": 2147680256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035269809428284854, + "loss": 1.3447, + "theoretical_loss": 3.4386774321766604, + "tokens_seen": 2147745792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526780341023069, + "loss": 1.2385, + "theoretical_loss": 3.4386688772537317, + "tokens_seen": 2147811328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526579739217653, + "loss": 1.3352, + "theoretical_loss": 3.438660322664922, + "tokens_seen": 2147876864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526379137412237, + "loss": 1.3521, + "theoretical_loss": 3.4386517684102085, + "tokens_seen": 2147942400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035261785356068204, + "loss": 1.3489, + "theoretical_loss": 3.4386432144895673, + "tokens_seen": 2148007936 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 782164, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3615994453430176, + "objective/train/theoretical_loss": 3.4386346609029763, + "objective/train/tokens_used": 518532576, + "theoretical_loss": 3.4386346609029763, + "tokens_seen": 2148073472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525977933801404, + "loss": 1.3905, + "theoretical_loss": 3.4386346609029763, + "tokens_seen": 2148073472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035257773319959877, + "loss": 1.3062, + "theoretical_loss": 3.4386261076504105, + "tokens_seen": 2148139008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525576730190572, + "loss": 1.3979, + "theoretical_loss": 3.4386175547318487, + "tokens_seen": 2148204544 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035253761283851555, + "loss": 1.345, + "theoretical_loss": 3.438609002147266, + "tokens_seen": 2148270080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525175526579739, + "loss": 1.2995, + "theoretical_loss": 3.43860044989664, + "tokens_seen": 2148335616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035249749247743227, + "loss": 1.3768, + "theoretical_loss": 3.438591897979947, + "tokens_seen": 2148401152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524774322968907, + "loss": 1.3092, + "theoretical_loss": 3.4385833463971647, + "tokens_seen": 2148466688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035245737211634905, + "loss": 1.2958, + "theoretical_loss": 3.4385747951482686, + "tokens_seen": 2148532224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524373119358074, + "loss": 1.3365, + "theoretical_loss": 3.438566244233236, + "tokens_seen": 2148597760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524172517552658, + "loss": 1.373, + "theoretical_loss": 3.4385576936520446, + "tokens_seen": 2148663296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035239719157472414, + "loss": 1.2744, + "theoretical_loss": 3.4385491434046696, + "tokens_seen": 2148728832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035237713139418255, + "loss": 1.2524, + "theoretical_loss": 3.438540593491089, + "tokens_seen": 2148794368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523570712136409, + "loss": 1.3847, + "theoretical_loss": 3.4385320439112794, + "tokens_seen": 2148859904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523370110330993, + "loss": 1.294, + "theoretical_loss": 3.438523494665217, + "tokens_seen": 2148925440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035231695085255764, + "loss": 1.2763, + "theoretical_loss": 3.438514945752879, + "tokens_seen": 2148990976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035229689067201606, + "loss": 1.3273, + "theoretical_loss": 3.4385063971742422, + "tokens_seen": 2149056512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522768304914744, + "loss": 1.3004, + "theoretical_loss": 3.4384978489292832, + "tokens_seen": 2149122048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522567703109328, + "loss": 1.3441, + "theoretical_loss": 3.438489301017979, + "tokens_seen": 2149187584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035223671013039114, + "loss": 1.3159, + "theoretical_loss": 3.4384807534403063, + "tokens_seen": 2149253120 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035221664994984956, + "loss": 1.447, + "theoretical_loss": 3.4384722061962423, + "tokens_seen": 2149318656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521965897693079, + "loss": 1.1938, + "theoretical_loss": 3.438463659285763, + "tokens_seen": 2149384192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521765295887663, + "loss": 1.3454, + "theoretical_loss": 3.4384551127088456, + "tokens_seen": 2149449728 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035215646940822465, + "loss": 1.322, + "theoretical_loss": 3.4384465664654673, + "tokens_seen": 2149515264 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352136409227683, + "loss": 1.3285, + "theoretical_loss": 3.4384380205556044, + "tokens_seen": 2149580800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521163490471414, + "loss": 1.3322, + "theoretical_loss": 3.4384294749792343, + "tokens_seen": 2149646336 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035209628886659984, + "loss": 1.2728, + "theoretical_loss": 3.438420929736333, + "tokens_seen": 2149711872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520762286860582, + "loss": 1.296, + "theoretical_loss": 3.4384123848268784, + "tokens_seen": 2149777408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035205616850551657, + "loss": 1.346, + "theoretical_loss": 3.438403840250846, + "tokens_seen": 2149842944 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352036108324975, + "loss": 1.3266, + "theoretical_loss": 3.4383952960082134, + "tokens_seen": 2149908480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035201604814443334, + "loss": 1.4807, + "theoretical_loss": 3.4383867520989577, + "tokens_seen": 2149974016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519959879638917, + "loss": 1.2718, + "theoretical_loss": 3.438378208523055, + "tokens_seen": 2150039552 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035197592778335007, + "loss": 1.3319, + "theoretical_loss": 3.4383696652804825, + "tokens_seen": 2150105088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035195586760280843, + "loss": 1.2501, + "theoretical_loss": 3.438361122371217, + "tokens_seen": 2150170624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035193580742226685, + "loss": 1.361, + "theoretical_loss": 3.438352579795236, + "tokens_seen": 2150236160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519157472417252, + "loss": 1.2137, + "theoretical_loss": 3.4383440375525147, + "tokens_seen": 2150301696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035189568706118357, + "loss": 1.2792, + "theoretical_loss": 3.438335495643032, + "tokens_seen": 2150367232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035187562688064193, + "loss": 1.3469, + "theoretical_loss": 3.4383269540667625, + "tokens_seen": 2150432768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035185556670010035, + "loss": 1.3378, + "theoretical_loss": 3.438318412823685, + "tokens_seen": 2150498304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518355065195587, + "loss": 1.394, + "theoretical_loss": 3.4383098719137757, + "tokens_seen": 2150563840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518154463390171, + "loss": 1.3268, + "theoretical_loss": 3.438301331337011, + "tokens_seen": 2150629376 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035179538615847544, + "loss": 1.3358, + "theoretical_loss": 3.4382927910933683, + "tokens_seen": 2150694912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517753259779338, + "loss": 1.3563, + "theoretical_loss": 3.438284251182824, + "tokens_seen": 2150760448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517552657973922, + "loss": 1.3519, + "theoretical_loss": 3.438275711605356, + "tokens_seen": 2150825984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517352056168506, + "loss": 1.3287, + "theoretical_loss": 3.4382671723609395, + "tokens_seen": 2150891520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035171514543630894, + "loss": 1.4074, + "theoretical_loss": 3.4382586334495526, + "tokens_seen": 2150957056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516950852557673, + "loss": 1.4037, + "theoretical_loss": 3.4382500948711723, + "tokens_seen": 2151022592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516750250752257, + "loss": 1.2955, + "theoretical_loss": 3.4382415566257745, + "tokens_seen": 2151088128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516549648946841, + "loss": 1.4355, + "theoretical_loss": 3.438233018713336, + "tokens_seen": 2151153664 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035163490471414244, + "loss": 1.45, + "theoretical_loss": 3.438224481133835, + "tokens_seen": 2151219200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516148445336008, + "loss": 1.2596, + "theoretical_loss": 3.438215943887247, + "tokens_seen": 2151284736 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 783310, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.378399133682251, + "objective/train/theoretical_loss": 3.43820740697355, + "objective/train/tokens_used": 521809376, + "theoretical_loss": 3.43820740697355, + "tokens_seen": 2151350272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035159478435305917, + "loss": 1.3267, + "theoretical_loss": 3.43820740697355, + "tokens_seen": 2151350272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515747241725176, + "loss": 1.2686, + "theoretical_loss": 3.4381988703927204, + "tokens_seen": 2151415808 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035155466399197595, + "loss": 1.3023, + "theoretical_loss": 3.4381903341447346, + "tokens_seen": 2151481344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515346038114343, + "loss": 1.2003, + "theoretical_loss": 3.4381817982295697, + "tokens_seen": 2151546880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035151454363089267, + "loss": 1.301, + "theoretical_loss": 3.4381732626472035, + "tokens_seen": 2151612416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514944834503511, + "loss": 1.3024, + "theoretical_loss": 3.438164727397612, + "tokens_seen": 2151677952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035147442326980945, + "loss": 1.3875, + "theoretical_loss": 3.438156192480772, + "tokens_seen": 2151743488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514543630892678, + "loss": 1.4739, + "theoretical_loss": 3.438147657896661, + "tokens_seen": 2151809024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514343029087262, + "loss": 1.3442, + "theoretical_loss": 3.4381391236452554, + "tokens_seen": 2151874560 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035141424272818454, + "loss": 1.2211, + "theoretical_loss": 3.438130589726532, + "tokens_seen": 2151940096 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035139418254764295, + "loss": 1.3373, + "theoretical_loss": 3.4381220561404686, + "tokens_seen": 2152005632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513741223671013, + "loss": 1.3237, + "theoretical_loss": 3.4381135228870408, + "tokens_seen": 2152071168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513540621865597, + "loss": 1.4257, + "theoretical_loss": 3.438104989966227, + "tokens_seen": 2152136704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035133400200601804, + "loss": 1.3333, + "theoretical_loss": 3.4380964573780024, + "tokens_seen": 2152202240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035131394182547646, + "loss": 1.2998, + "theoretical_loss": 3.4380879251223453, + "tokens_seen": 2152267776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512938816449348, + "loss": 1.2912, + "theoretical_loss": 3.438079393199232, + "tokens_seen": 2152333312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512738214643932, + "loss": 1.2987, + "theoretical_loss": 3.4380708616086393, + "tokens_seen": 2152398848 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035125376128385154, + "loss": 1.2368, + "theoretical_loss": 3.4380623303505446, + "tokens_seen": 2152464384 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035123370110330996, + "loss": 1.3106, + "theoretical_loss": 3.4380537994249245, + "tokens_seen": 2152529920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512136409227683, + "loss": 1.3073, + "theoretical_loss": 3.4380452688317558, + "tokens_seen": 2152595456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511935807422267, + "loss": 1.3273, + "theoretical_loss": 3.438036738571016, + "tokens_seen": 2152660992 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035117352056168505, + "loss": 1.3947, + "theoretical_loss": 3.4380282086426814, + "tokens_seen": 2152726528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511534603811434, + "loss": 1.273, + "theoretical_loss": 3.4380196790467292, + "tokens_seen": 2152792064 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511334002006018, + "loss": 1.2551, + "theoretical_loss": 3.4380111497831365, + "tokens_seen": 2152857600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511133400200602, + "loss": 1.4666, + "theoretical_loss": 3.4380026208518792, + "tokens_seen": 2152923136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035109327983951855, + "loss": 1.2774, + "theoretical_loss": 3.4379940922529357, + "tokens_seen": 2152988672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510732196589769, + "loss": 1.3835, + "theoretical_loss": 3.4379855639862824, + "tokens_seen": 2153054208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035105315947843533, + "loss": 1.2825, + "theoretical_loss": 3.4379770360518958, + "tokens_seen": 2153119744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510330992978937, + "loss": 1.2703, + "theoretical_loss": 3.437968508449753, + "tokens_seen": 2153185280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035101303911735205, + "loss": 1.321, + "theoretical_loss": 3.437959981179832, + "tokens_seen": 2153250816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509929789368104, + "loss": 1.3321, + "theoretical_loss": 3.437951454242108, + "tokens_seen": 2153316352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509729187562688, + "loss": 1.3462, + "theoretical_loss": 3.437942927636559, + "tokens_seen": 2153381888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509528585757272, + "loss": 1.3044, + "theoretical_loss": 3.4379344013631616, + "tokens_seen": 2153447424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035093279839518556, + "loss": 1.267, + "theoretical_loss": 3.4379258754218935, + "tokens_seen": 2153512960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509127382146439, + "loss": 1.3815, + "theoretical_loss": 3.4379173498127304, + "tokens_seen": 2153578496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508926780341023, + "loss": 1.338, + "theoretical_loss": 3.4379088245356506, + "tokens_seen": 2153644032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508726178535607, + "loss": 1.2662, + "theoretical_loss": 3.43790029959063, + "tokens_seen": 2153709568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035085255767301906, + "loss": 1.3286, + "theoretical_loss": 3.437891774977646, + "tokens_seen": 2153775104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508324974924774, + "loss": 1.3535, + "theoretical_loss": 3.437883250696675, + "tokens_seen": 2153840640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508124373119358, + "loss": 1.3665, + "theoretical_loss": 3.4378747267476957, + "tokens_seen": 2153906176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035079237713139415, + "loss": 1.3857, + "theoretical_loss": 3.4378662031306826, + "tokens_seen": 2153971712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035077231695085256, + "loss": 1.2865, + "theoretical_loss": 3.4378576798456146, + "tokens_seen": 2154037248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507522567703109, + "loss": 1.3488, + "theoretical_loss": 3.437849156892468, + "tokens_seen": 2154102784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507321965897693, + "loss": 1.3197, + "theoretical_loss": 3.4378406342712196, + "tokens_seen": 2154168320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035071213640922765, + "loss": 1.2979, + "theoretical_loss": 3.4378321119818462, + "tokens_seen": 2154233856 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035069207622868607, + "loss": 1.3715, + "theoretical_loss": 3.4378235900243257, + "tokens_seen": 2154299392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035067201604814443, + "loss": 1.3941, + "theoretical_loss": 3.4378150683986344, + "tokens_seen": 2154364928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506519558676028, + "loss": 1.2239, + "theoretical_loss": 3.437806547104749, + "tokens_seen": 2154430464 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035063189568706115, + "loss": 1.4198, + "theoretical_loss": 3.4377980261426475, + "tokens_seen": 2154496000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506118355065195, + "loss": 1.2291, + "theoretical_loss": 3.437789505512306, + "tokens_seen": 2154561536 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 784476, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3676598072052002, + "objective/train/theoretical_loss": 3.437780985213702, + "objective/train/tokens_used": 525086176, + "theoretical_loss": 3.437780985213702, + "tokens_seen": 2154627072 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350591775325978, + "loss": 1.3837, + "theoretical_loss": 3.437780985213702, + "tokens_seen": 2154627072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035057171514543635, + "loss": 1.2263, + "theoretical_loss": 3.4377724652468116, + "tokens_seen": 2154692608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505516549648947, + "loss": 1.3726, + "theoretical_loss": 3.4377639456116134, + "tokens_seen": 2154758144 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035053159478435307, + "loss": 1.3508, + "theoretical_loss": 3.4377554263080827, + "tokens_seen": 2154823680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505115346038115, + "loss": 1.3855, + "theoretical_loss": 3.4377469073361975, + "tokens_seen": 2154889216 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035049147442326985, + "loss": 1.3991, + "theoretical_loss": 3.4377383886959345, + "tokens_seen": 2154954752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504714142427282, + "loss": 1.3087, + "theoretical_loss": 3.4377298703872707, + "tokens_seen": 2155020288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504513540621866, + "loss": 1.2403, + "theoretical_loss": 3.4377213524101835, + "tokens_seen": 2155085824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035043129388164494, + "loss": 1.2824, + "theoretical_loss": 3.4377128347646497, + "tokens_seen": 2155151360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035041123370110335, + "loss": 1.2955, + "theoretical_loss": 3.4377043174506454, + "tokens_seen": 2155216896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503911735205617, + "loss": 1.3288, + "theoretical_loss": 3.4376958004681493, + "tokens_seen": 2155282432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503711133400201, + "loss": 1.3505, + "theoretical_loss": 3.4376872838171373, + "tokens_seen": 2155347968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035035105315947844, + "loss": 1.2374, + "theoretical_loss": 3.4376787674975864, + "tokens_seen": 2155413504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035033099297893686, + "loss": 1.4408, + "theoretical_loss": 3.437670251509474, + "tokens_seen": 2155479040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503109327983952, + "loss": 1.3667, + "theoretical_loss": 3.4376617358527772, + "tokens_seen": 2155544576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502908726178536, + "loss": 1.3007, + "theoretical_loss": 3.4376532205274724, + "tokens_seen": 2155610112 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035027081243731194, + "loss": 1.3497, + "theoretical_loss": 3.437644705533537, + "tokens_seen": 2155675648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035025075225677036, + "loss": 1.2769, + "theoretical_loss": 3.4376361908709487, + "tokens_seen": 2155741184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502306920762287, + "loss": 1.3185, + "theoretical_loss": 3.437627676539684, + "tokens_seen": 2155806720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502106318956871, + "loss": 1.3205, + "theoretical_loss": 3.437619162539719, + "tokens_seen": 2155872256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035019057171514545, + "loss": 1.3376, + "theoretical_loss": 3.4376106488710327, + "tokens_seen": 2155937792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501705115346038, + "loss": 1.3628, + "theoretical_loss": 3.4376021355336004, + "tokens_seen": 2156003328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501504513540622, + "loss": 1.3153, + "theoretical_loss": 3.4375936225273995, + "tokens_seen": 2156068864 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501303911735206, + "loss": 1.2844, + "theoretical_loss": 3.4375851098524075, + "tokens_seen": 2156134400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035011033099297895, + "loss": 1.2758, + "theoretical_loss": 3.4375765975086017, + "tokens_seen": 2156199936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500902708124373, + "loss": 1.2258, + "theoretical_loss": 3.4375680854959585, + "tokens_seen": 2156265472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035007021063189573, + "loss": 1.2413, + "theoretical_loss": 3.437559573814455, + "tokens_seen": 2156331008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500501504513541, + "loss": 1.2511, + "theoretical_loss": 3.4375510624640686, + "tokens_seen": 2156396544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035003009027081245, + "loss": 1.3138, + "theoretical_loss": 3.437542551444776, + "tokens_seen": 2156462080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500100300902708, + "loss": 1.3907, + "theoretical_loss": 3.437534040756555, + "tokens_seen": 2156527616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499899699097292, + "loss": 1.272, + "theoretical_loss": 3.4375255303993812, + "tokens_seen": 2156593152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499699097291876, + "loss": 1.2678, + "theoretical_loss": 3.4375170203732335, + "tokens_seen": 2156658688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034994984954864596, + "loss": 1.3548, + "theoretical_loss": 3.4375085106780876, + "tokens_seen": 2156724224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499297893681043, + "loss": 1.4314, + "theoretical_loss": 3.437500001313921, + "tokens_seen": 2156789760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499097291875627, + "loss": 1.2993, + "theoretical_loss": 3.437491492280711, + "tokens_seen": 2156855296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498896690070211, + "loss": 1.2253, + "theoretical_loss": 3.437482983578434, + "tokens_seen": 2156920832 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034986960882647946, + "loss": 1.2981, + "theoretical_loss": 3.437474475207068, + "tokens_seen": 2156986368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498495486459378, + "loss": 1.337, + "theoretical_loss": 3.437465967166589, + "tokens_seen": 2157051904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498294884653962, + "loss": 1.2337, + "theoretical_loss": 3.4374574594569753, + "tokens_seen": 2157117440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034980942828485455, + "loss": 1.3061, + "theoretical_loss": 3.437448952078203, + "tokens_seen": 2157182976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034978936810431296, + "loss": 1.4652, + "theoretical_loss": 3.4374404450302496, + "tokens_seen": 2157248512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497693079237713, + "loss": 1.2987, + "theoretical_loss": 3.4374319383130922, + "tokens_seen": 2157314048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497492477432297, + "loss": 1.4275, + "theoretical_loss": 3.4374234319267076, + "tokens_seen": 2157379584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034972918756268805, + "loss": 1.4213, + "theoretical_loss": 3.4374149258710736, + "tokens_seen": 2157445120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034970912738214647, + "loss": 1.3919, + "theoretical_loss": 3.437406420146167, + "tokens_seen": 2157510656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034968906720160483, + "loss": 1.3731, + "theoretical_loss": 3.4373979147519638, + "tokens_seen": 2157576192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496690070210632, + "loss": 1.226, + "theoretical_loss": 3.4373894096884423, + "tokens_seen": 2157641728 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034964894684052155, + "loss": 1.3293, + "theoretical_loss": 3.4373809049555795, + "tokens_seen": 2157707264 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496288866599799, + "loss": 1.3446, + "theoretical_loss": 3.437372400553352, + "tokens_seen": 2157772800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034960882647943833, + "loss": 1.445, + "theoretical_loss": 3.4373638964817377, + "tokens_seen": 2157838336 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 785665, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1829450130462646, + "objective/train/theoretical_loss": 3.437355392740713, + "objective/train/tokens_used": 528362976, + "theoretical_loss": 3.437355392740713, + "tokens_seen": 2157903872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495887662988967, + "loss": 1.2758, + "theoretical_loss": 3.437355392740713, + "tokens_seen": 2157903872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034956870611835506, + "loss": 1.3502, + "theoretical_loss": 3.437346889330255, + "tokens_seen": 2157969408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495486459378134, + "loss": 1.2818, + "theoretical_loss": 3.4373383862503415, + "tokens_seen": 2158034944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034952858575727183, + "loss": 1.269, + "theoretical_loss": 3.4373298835009485, + "tokens_seen": 2158100480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495085255767302, + "loss": 1.3553, + "theoretical_loss": 3.437321381082054, + "tokens_seen": 2158166016 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034948846539618856, + "loss": 1.2537, + "theoretical_loss": 3.437312878993635, + "tokens_seen": 2158231552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494684052156469, + "loss": 1.3779, + "theoretical_loss": 3.4373043772356686, + "tokens_seen": 2158297088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494483450351053, + "loss": 1.4004, + "theoretical_loss": 3.4372958758081316, + "tokens_seen": 2158362624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494282848545637, + "loss": 1.3365, + "theoretical_loss": 3.4372873747110013, + "tokens_seen": 2158428160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034940822467402206, + "loss": 1.289, + "theoretical_loss": 3.4372788739442552, + "tokens_seen": 2158493696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493881644934804, + "loss": 1.4093, + "theoretical_loss": 3.43727037350787, + "tokens_seen": 2158559232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493681043129388, + "loss": 1.3102, + "theoretical_loss": 3.4372618734018228, + "tokens_seen": 2158624768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493480441323972, + "loss": 1.3522, + "theoretical_loss": 3.437253373626091, + "tokens_seen": 2158690304 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034932798395185557, + "loss": 1.2482, + "theoretical_loss": 3.437244874180651, + "tokens_seen": 2158755840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034930792377131393, + "loss": 1.2846, + "theoretical_loss": 3.4372363750654813, + "tokens_seen": 2158821376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492878635907723, + "loss": 1.2888, + "theoretical_loss": 3.4372278762805584, + "tokens_seen": 2158886912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034926780341023065, + "loss": 1.3336, + "theoretical_loss": 3.437219377825859, + "tokens_seen": 2158952448 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034924774322968907, + "loss": 1.3491, + "theoretical_loss": 3.43721087970136, + "tokens_seen": 2159017984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034922768304914743, + "loss": 1.3504, + "theoretical_loss": 3.43720238190704, + "tokens_seen": 2159083520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492076228686058, + "loss": 1.2928, + "theoretical_loss": 3.437193884442875, + "tokens_seen": 2159149056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034918756268806416, + "loss": 1.336, + "theoretical_loss": 3.4371853873088423, + "tokens_seen": 2159214592 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034916750250752257, + "loss": 1.2966, + "theoretical_loss": 3.4371768905049196, + "tokens_seen": 2159280128 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034914744232698093, + "loss": 1.3895, + "theoretical_loss": 3.437168394031083, + "tokens_seen": 2159345664 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491273821464393, + "loss": 1.4201, + "theoretical_loss": 3.4371598978873106, + "tokens_seen": 2159411200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034910732196589766, + "loss": 1.3285, + "theoretical_loss": 3.4371514020735794, + "tokens_seen": 2159476736 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034908726178535613, + "loss": 1.3024, + "theoretical_loss": 3.437142906589866, + "tokens_seen": 2159542272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490672016048145, + "loss": 1.3293, + "theoretical_loss": 3.437134411436148, + "tokens_seen": 2159607808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034904714142427285, + "loss": 1.3073, + "theoretical_loss": 3.437125916612403, + "tokens_seen": 2159673344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490270812437312, + "loss": 1.3164, + "theoretical_loss": 3.4371174221186074, + "tokens_seen": 2159738880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490070210631896, + "loss": 1.3424, + "theoretical_loss": 3.437108927954739, + "tokens_seen": 2159804416 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348986960882648, + "loss": 1.371, + "theoretical_loss": 3.4371004341207745, + "tokens_seen": 2159869952 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034896690070210636, + "loss": 1.3036, + "theoretical_loss": 3.4370919406166913, + "tokens_seen": 2159935488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489468405215647, + "loss": 1.3252, + "theoretical_loss": 3.4370834474424665, + "tokens_seen": 2160001024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489267803410231, + "loss": 1.3924, + "theoretical_loss": 3.437074954598077, + "tokens_seen": 2160066560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489067201604815, + "loss": 1.3545, + "theoretical_loss": 3.437066462083501, + "tokens_seen": 2160132096 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034888665997993986, + "loss": 1.4518, + "theoretical_loss": 3.4370579698987145, + "tokens_seen": 2160197632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488665997993982, + "loss": 1.3248, + "theoretical_loss": 3.4370494780436953, + "tokens_seen": 2160263168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488465396188566, + "loss": 1.3962, + "theoretical_loss": 3.4370409865184204, + "tokens_seen": 2160328704 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034882647943831495, + "loss": 1.2966, + "theoretical_loss": 3.437032495322867, + "tokens_seen": 2160394240 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034880641925777336, + "loss": 1.3209, + "theoretical_loss": 3.437024004457012, + "tokens_seen": 2160459776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487863590772317, + "loss": 1.3036, + "theoretical_loss": 3.4370155139208336, + "tokens_seen": 2160525312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487662988966901, + "loss": 1.4217, + "theoretical_loss": 3.437007023714308, + "tokens_seen": 2160590848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034874623871614845, + "loss": 1.4746, + "theoretical_loss": 3.436998533837413, + "tokens_seen": 2160656384 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034872617853560687, + "loss": 1.2652, + "theoretical_loss": 3.436990044290125, + "tokens_seen": 2160721920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034870611835506523, + "loss": 1.4176, + "theoretical_loss": 3.4369815550724225, + "tokens_seen": 2160787456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486860581745236, + "loss": 1.2972, + "theoretical_loss": 3.4369730661842817, + "tokens_seen": 2160852992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034866599799398195, + "loss": 1.4011, + "theoretical_loss": 3.43696457762568, + "tokens_seen": 2160918528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486459378134403, + "loss": 1.2829, + "theoretical_loss": 3.4369560893965945, + "tokens_seen": 2160984064 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034862587763289873, + "loss": 1.2168, + "theoretical_loss": 3.436947601497003, + "tokens_seen": 2161049600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486058174523571, + "loss": 1.3849, + "theoretical_loss": 3.436939113926882, + "tokens_seen": 2161115136 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 786923, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1785483360290527, + "objective/train/theoretical_loss": 3.436930626686209, + "objective/train/tokens_used": 531639776, + "theoretical_loss": 3.436930626686209, + "tokens_seen": 2161180672 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034858575727181546, + "loss": 1.1877, + "theoretical_loss": 3.436930626686209, + "tokens_seen": 2161180672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485656970912738, + "loss": 1.4342, + "theoretical_loss": 3.436922139774962, + "tokens_seen": 2161246208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034854563691073223, + "loss": 1.2839, + "theoretical_loss": 3.436913653193117, + "tokens_seen": 2161311744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485255767301906, + "loss": 1.229, + "theoretical_loss": 3.4369051669406514, + "tokens_seen": 2161377280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034850551654964896, + "loss": 1.2479, + "theoretical_loss": 3.436896681017543, + "tokens_seen": 2161442816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484854563691073, + "loss": 1.2655, + "theoretical_loss": 3.436888195423769, + "tokens_seen": 2161508352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484653961885657, + "loss": 1.2562, + "theoretical_loss": 3.4368797101593063, + "tokens_seen": 2161573888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484453360080241, + "loss": 1.4088, + "theoretical_loss": 3.436871225224132, + "tokens_seen": 2161639424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034842527582748246, + "loss": 1.2679, + "theoretical_loss": 3.4368627406182237, + "tokens_seen": 2161704960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484052156469408, + "loss": 1.2422, + "theoretical_loss": 3.436854256341559, + "tokens_seen": 2161770496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483851554663992, + "loss": 1.2552, + "theoretical_loss": 3.436845772394114, + "tokens_seen": 2161836032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483650952858576, + "loss": 1.3491, + "theoretical_loss": 3.4368372887758674, + "tokens_seen": 2161901568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034834503510531597, + "loss": 1.4595, + "theoretical_loss": 3.436828805486795, + "tokens_seen": 2161967104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034832497492477433, + "loss": 1.4305, + "theoretical_loss": 3.436820322526875, + "tokens_seen": 2162032640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483049147442327, + "loss": 1.2658, + "theoretical_loss": 3.4368118398960843, + "tokens_seen": 2162098176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034828485456369105, + "loss": 1.2528, + "theoretical_loss": 3.4368033575944, + "tokens_seen": 2162163712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034826479438314947, + "loss": 1.4206, + "theoretical_loss": 3.4367948756218, + "tokens_seen": 2162229248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034824473420260783, + "loss": 1.3028, + "theoretical_loss": 3.4367863939782612, + "tokens_seen": 2162294784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482246740220662, + "loss": 1.2927, + "theoretical_loss": 3.4367779126637603, + "tokens_seen": 2162360320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034820461384152456, + "loss": 1.3685, + "theoretical_loss": 3.4367694316782753, + "tokens_seen": 2162425856 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034818455366098297, + "loss": 1.3137, + "theoretical_loss": 3.4367609510217836, + "tokens_seen": 2162491392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034816449348044133, + "loss": 1.2694, + "theoretical_loss": 3.4367524706942616, + "tokens_seen": 2162556928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481444332998997, + "loss": 1.3111, + "theoretical_loss": 3.436743990695687, + "tokens_seen": 2162622464 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034812437311935806, + "loss": 1.2871, + "theoretical_loss": 3.4367355110260376, + "tokens_seen": 2162688000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481043129388164, + "loss": 1.3616, + "theoretical_loss": 3.4367270316852903, + "tokens_seen": 2162753536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034808425275827484, + "loss": 1.3638, + "theoretical_loss": 3.4367185526734216, + "tokens_seen": 2162819072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480641925777332, + "loss": 1.4093, + "theoretical_loss": 3.43671007399041, + "tokens_seen": 2162884608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034804413239719156, + "loss": 1.3259, + "theoretical_loss": 3.436701595636232, + "tokens_seen": 2162950144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480240722166499, + "loss": 1.3571, + "theoretical_loss": 3.4366931176108655, + "tokens_seen": 2163015680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034800401203610834, + "loss": 1.2948, + "theoretical_loss": 3.4366846399142874, + "tokens_seen": 2163081216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479839518555667, + "loss": 1.2727, + "theoretical_loss": 3.436676162546475, + "tokens_seen": 2163146752 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034796389167502506, + "loss": 1.3393, + "theoretical_loss": 3.4366676855074054, + "tokens_seen": 2163212288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034794383149448343, + "loss": 1.3181, + "theoretical_loss": 3.4366592087970558, + "tokens_seen": 2163277824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034792377131394184, + "loss": 1.3202, + "theoretical_loss": 3.436650732415404, + "tokens_seen": 2163343360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479037111334002, + "loss": 1.328, + "theoretical_loss": 3.4366422563624277, + "tokens_seen": 2163408896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034788365095285857, + "loss": 1.3365, + "theoretical_loss": 3.436633780638103, + "tokens_seen": 2163474432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034786359077231693, + "loss": 1.2109, + "theoretical_loss": 3.436625305242408, + "tokens_seen": 2163539968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478435305917753, + "loss": 1.3071, + "theoretical_loss": 3.4366168301753195, + "tokens_seen": 2163605504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478234704112337, + "loss": 1.3247, + "theoretical_loss": 3.4366083554368156, + "tokens_seen": 2163671040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034780341023069207, + "loss": 1.2939, + "theoretical_loss": 3.4365998810268725, + "tokens_seen": 2163736576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034778335005015043, + "loss": 1.337, + "theoretical_loss": 3.4365914069454684, + "tokens_seen": 2163802112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477632898696088, + "loss": 1.3165, + "theoretical_loss": 3.4365829331925806, + "tokens_seen": 2163867648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477432296890672, + "loss": 1.387, + "theoretical_loss": 3.436574459768186, + "tokens_seen": 2163933184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477231695085256, + "loss": 1.4402, + "theoretical_loss": 3.436565986672262, + "tokens_seen": 2163998720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034770310932798394, + "loss": 1.4596, + "theoretical_loss": 3.436557513904786, + "tokens_seen": 2164064256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476830491474423, + "loss": 1.336, + "theoretical_loss": 3.436549041465735, + "tokens_seen": 2164129792 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034766298896690066, + "loss": 1.336, + "theoretical_loss": 3.436540569355087, + "tokens_seen": 2164195328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476429287863591, + "loss": 1.2667, + "theoretical_loss": 3.436532097572819, + "tokens_seen": 2164260864 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034762286860581744, + "loss": 1.4381, + "theoretical_loss": 3.436523626118908, + "tokens_seen": 2164326400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476028084252758, + "loss": 1.3469, + "theoretical_loss": 3.436515154993332, + "tokens_seen": 2164391936 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 788153, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5332597494125366, + "objective/train/theoretical_loss": 3.436506684196068, + "objective/train/tokens_used": 534916576, + "theoretical_loss": 3.436506684196068, + "tokens_seen": 2164457472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475827482447342, + "loss": 1.4794, + "theoretical_loss": 3.436506684196068, + "tokens_seen": 2164457472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034756268806419263, + "loss": 1.3252, + "theoretical_loss": 3.4364982137270927, + "tokens_seen": 2164523008 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347542627883651, + "loss": 1.294, + "theoretical_loss": 3.4364897435863844, + "tokens_seen": 2164588544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034752256770310936, + "loss": 1.2614, + "theoretical_loss": 3.43648127377392, + "tokens_seen": 2164654080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475025075225677, + "loss": 1.3492, + "theoretical_loss": 3.436472804289677, + "tokens_seen": 2164719616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474824473420261, + "loss": 1.275, + "theoretical_loss": 3.436464335133633, + "tokens_seen": 2164785152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474623871614845, + "loss": 1.3473, + "theoretical_loss": 3.4364558663057645, + "tokens_seen": 2164850688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034744232698094286, + "loss": 1.3333, + "theoretical_loss": 3.4364473978060492, + "tokens_seen": 2164916224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474222668004012, + "loss": 1.235, + "theoretical_loss": 3.4364389296344653, + "tokens_seen": 2164981760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474022066198596, + "loss": 1.3302, + "theoretical_loss": 3.4364304617909887, + "tokens_seen": 2165047296 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347382146439318, + "loss": 1.3896, + "theoretical_loss": 3.436421994275598, + "tokens_seen": 2165112832 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034736208625877637, + "loss": 1.2921, + "theoretical_loss": 3.43641352708827, + "tokens_seen": 2165178368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034734202607823473, + "loss": 1.4465, + "theoretical_loss": 3.436405060228982, + "tokens_seen": 2165243904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473219658976931, + "loss": 1.3738, + "theoretical_loss": 3.4363965936977117, + "tokens_seen": 2165309440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034730190571715145, + "loss": 1.4406, + "theoretical_loss": 3.436388127494436, + "tokens_seen": 2165374976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034728184553660987, + "loss": 1.3367, + "theoretical_loss": 3.4363796616191324, + "tokens_seen": 2165440512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034726178535606823, + "loss": 1.3057, + "theoretical_loss": 3.4363711960717787, + "tokens_seen": 2165506048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472417251755266, + "loss": 1.2725, + "theoretical_loss": 3.436362730852352, + "tokens_seen": 2165571584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034722166499498496, + "loss": 1.2551, + "theoretical_loss": 3.4363542659608295, + "tokens_seen": 2165637120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034720160481444337, + "loss": 1.3699, + "theoretical_loss": 3.436345801397189, + "tokens_seen": 2165702656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034718154463390173, + "loss": 1.4416, + "theoretical_loss": 3.4363373371614068, + "tokens_seen": 2165768192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471614844533601, + "loss": 1.3418, + "theoretical_loss": 3.4363288732534616, + "tokens_seen": 2165833728 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034714142427281846, + "loss": 1.2469, + "theoretical_loss": 3.43632040967333, + "tokens_seen": 2165899264 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471213640922768, + "loss": 1.3655, + "theoretical_loss": 3.4363119464209904, + "tokens_seen": 2165964800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034710130391173524, + "loss": 1.3427, + "theoretical_loss": 3.4363034834964186, + "tokens_seen": 2166030336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470812437311936, + "loss": 1.3184, + "theoretical_loss": 3.4362950208995935, + "tokens_seen": 2166095872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034706118355065196, + "loss": 1.2642, + "theoretical_loss": 3.436286558630491, + "tokens_seen": 2166161408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470411233701103, + "loss": 1.3457, + "theoretical_loss": 3.4362780966890902, + "tokens_seen": 2166226944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034702106318956874, + "loss": 1.3559, + "theoretical_loss": 3.436269635075367, + "tokens_seen": 2166292480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470010030090271, + "loss": 1.3273, + "theoretical_loss": 3.4362611737893, + "tokens_seen": 2166358016 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034698094282848547, + "loss": 1.3022, + "theoretical_loss": 3.436252712830865, + "tokens_seen": 2166423552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034696088264794383, + "loss": 1.2717, + "theoretical_loss": 3.436244252200041, + "tokens_seen": 2166489088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034694082246740224, + "loss": 1.2509, + "theoretical_loss": 3.436235791896805, + "tokens_seen": 2166554624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003469207622868606, + "loss": 1.2517, + "theoretical_loss": 3.436227331921134, + "tokens_seen": 2166620160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034690070210631897, + "loss": 1.3783, + "theoretical_loss": 3.4362188722730056, + "tokens_seen": 2166685696 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034688064192577733, + "loss": 1.4356, + "theoretical_loss": 3.4362104129523976, + "tokens_seen": 2166751232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468605817452357, + "loss": 1.3083, + "theoretical_loss": 3.4362019539592863, + "tokens_seen": 2166816768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468405215646941, + "loss": 1.3557, + "theoretical_loss": 3.436193495293651, + "tokens_seen": 2166882304 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034682046138415247, + "loss": 1.3302, + "theoretical_loss": 3.4361850369554667, + "tokens_seen": 2166947840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034680040120361083, + "loss": 1.3662, + "theoretical_loss": 3.4361765789447127, + "tokens_seen": 2167013376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467803410230692, + "loss": 1.3596, + "theoretical_loss": 3.4361681212613657, + "tokens_seen": 2167078912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467602808425276, + "loss": 1.3058, + "theoretical_loss": 3.4361596639054035, + "tokens_seen": 2167144448 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346740220661986, + "loss": 1.3088, + "theoretical_loss": 3.4361512068768034, + "tokens_seen": 2167209984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034672016048144434, + "loss": 1.1991, + "theoretical_loss": 3.4361427501755424, + "tokens_seen": 2167275520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467001003009027, + "loss": 1.3397, + "theoretical_loss": 3.436134293801598, + "tokens_seen": 2167341056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034668004012036106, + "loss": 1.3777, + "theoretical_loss": 3.4361258377549486, + "tokens_seen": 2167406592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466599799398195, + "loss": 1.3044, + "theoretical_loss": 3.4361173820355706, + "tokens_seen": 2167472128 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034663991975927784, + "loss": 1.4952, + "theoretical_loss": 3.4361089266434415, + "tokens_seen": 2167537664 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466198595787362, + "loss": 1.2997, + "theoretical_loss": 3.4361004715785395, + "tokens_seen": 2167603200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034659979939819456, + "loss": 1.33, + "theoretical_loss": 3.436092016840841, + "tokens_seen": 2167668736 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 789244, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3871870040893555, + "objective/train/theoretical_loss": 3.436083562430324, + "objective/train/tokens_used": 538193376, + "theoretical_loss": 3.436083562430324, + "tokens_seen": 2167734272 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346579739217653, + "loss": 1.3178, + "theoretical_loss": 3.436083562430324, + "tokens_seen": 2167734272 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034655967903711134, + "loss": 1.2407, + "theoretical_loss": 3.4360751083469663, + "tokens_seen": 2167799808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465396188565697, + "loss": 1.287, + "theoretical_loss": 3.4360666545907446, + "tokens_seen": 2167865344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034651955867602807, + "loss": 1.3855, + "theoretical_loss": 3.436058201161637, + "tokens_seen": 2167930880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034649949849548643, + "loss": 1.435, + "theoretical_loss": 3.4360497480596206, + "tokens_seen": 2167996416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034647943831494485, + "loss": 1.4191, + "theoretical_loss": 3.4360412952846735, + "tokens_seen": 2168061952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464593781344032, + "loss": 1.3908, + "theoretical_loss": 3.436032842836772, + "tokens_seen": 2168127488 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034643931795386157, + "loss": 1.3212, + "theoretical_loss": 3.436024390715894, + "tokens_seen": 2168193024 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034641925777331993, + "loss": 1.3645, + "theoretical_loss": 3.4360159389220177, + "tokens_seen": 2168258560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034639919759277835, + "loss": 1.2613, + "theoretical_loss": 3.4360074874551194, + "tokens_seen": 2168324096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463791374122367, + "loss": 1.3637, + "theoretical_loss": 3.4359990363151773, + "tokens_seen": 2168389632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463590772316951, + "loss": 1.2942, + "theoretical_loss": 3.4359905855021693, + "tokens_seen": 2168455168 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034633901705115344, + "loss": 1.2805, + "theoretical_loss": 3.4359821350160717, + "tokens_seen": 2168520704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463189568706118, + "loss": 1.3219, + "theoretical_loss": 3.435973684856863, + "tokens_seen": 2168586240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462988966900702, + "loss": 1.4089, + "theoretical_loss": 3.4359652350245202, + "tokens_seen": 2168651776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462788365095286, + "loss": 1.2908, + "theoretical_loss": 3.4359567855190205, + "tokens_seen": 2168717312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034625877632898694, + "loss": 1.3103, + "theoretical_loss": 3.435948336340342, + "tokens_seen": 2168782848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462387161484453, + "loss": 1.3133, + "theoretical_loss": 3.4359398874884617, + "tokens_seen": 2168848384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462186559679037, + "loss": 1.3377, + "theoretical_loss": 3.4359314389633573, + "tokens_seen": 2168913920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461985957873621, + "loss": 1.3764, + "theoretical_loss": 3.4359229907650066, + "tokens_seen": 2168979456 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034617853560682044, + "loss": 1.3872, + "theoretical_loss": 3.4359145428933866, + "tokens_seen": 2169044992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461584754262788, + "loss": 1.2725, + "theoretical_loss": 3.435906095348475, + "tokens_seen": 2169110528 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034613841524573717, + "loss": 1.2686, + "theoretical_loss": 3.435897648130249, + "tokens_seen": 2169176064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461183550651956, + "loss": 1.414, + "theoretical_loss": 3.4358892012386866, + "tokens_seen": 2169241600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034609829488465395, + "loss": 1.2807, + "theoretical_loss": 3.435880754673765, + "tokens_seen": 2169307136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034607823470411236, + "loss": 1.2975, + "theoretical_loss": 3.435872308435462, + "tokens_seen": 2169372672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460581745235707, + "loss": 1.3427, + "theoretical_loss": 3.435863862523754, + "tokens_seen": 2169438208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034603811434302914, + "loss": 1.41, + "theoretical_loss": 3.43585541693862, + "tokens_seen": 2169503744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460180541624875, + "loss": 1.3423, + "theoretical_loss": 3.435846971680037, + "tokens_seen": 2169569280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034599799398194587, + "loss": 1.2122, + "theoretical_loss": 3.435838526747982, + "tokens_seen": 2169634816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034597793380140423, + "loss": 1.2868, + "theoretical_loss": 3.4358300821424335, + "tokens_seen": 2169700352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034595787362086264, + "loss": 1.2497, + "theoretical_loss": 3.4358216378633677, + "tokens_seen": 2169765888 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345937813440321, + "loss": 1.3116, + "theoretical_loss": 3.435813193910763, + "tokens_seen": 2169831424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034591775325977937, + "loss": 1.3055, + "theoretical_loss": 3.435804750284597, + "tokens_seen": 2169896960 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034589769307923773, + "loss": 1.2995, + "theoretical_loss": 3.4357963069848467, + "tokens_seen": 2169962496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458776328986961, + "loss": 1.3358, + "theoretical_loss": 3.4357878640114903, + "tokens_seen": 2170028032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458575727181545, + "loss": 1.2948, + "theoretical_loss": 3.4357794213645043, + "tokens_seen": 2170093568 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034583751253761287, + "loss": 1.3773, + "theoretical_loss": 3.435770979043867, + "tokens_seen": 2170159104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034581745235707123, + "loss": 1.3403, + "theoretical_loss": 3.4357625370495564, + "tokens_seen": 2170224640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457973921765296, + "loss": 1.3853, + "theoretical_loss": 3.4357540953815486, + "tokens_seen": 2170290176 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345777331995988, + "loss": 1.3735, + "theoretical_loss": 3.4357456540398226, + "tokens_seen": 2170355712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457572718154464, + "loss": 1.3183, + "theoretical_loss": 3.435737213024355, + "tokens_seen": 2170421248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034573721163490474, + "loss": 1.3896, + "theoretical_loss": 3.4357287723351235, + "tokens_seen": 2170486784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457171514543631, + "loss": 1.3508, + "theoretical_loss": 3.4357203319721057, + "tokens_seen": 2170552320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034569709127382146, + "loss": 1.3997, + "theoretical_loss": 3.4357118919352794, + "tokens_seen": 2170617856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456770310932799, + "loss": 1.4799, + "theoretical_loss": 3.435703452224622, + "tokens_seen": 2170683392 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034565697091273824, + "loss": 1.3092, + "theoretical_loss": 3.4356950128401107, + "tokens_seen": 2170748928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456369107321966, + "loss": 1.3922, + "theoretical_loss": 3.4356865737817235, + "tokens_seen": 2170814464 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034561685055165496, + "loss": 1.3961, + "theoretical_loss": 3.435678135049438, + "tokens_seen": 2170880000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455967903711134, + "loss": 1.2927, + "theoretical_loss": 3.4356696966432314, + "tokens_seen": 2170945536 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 790517, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.431609869003296, + "objective/train/theoretical_loss": 3.4356612585630812, + "objective/train/tokens_used": 541470176, + "theoretical_loss": 3.4356612585630812, + "tokens_seen": 2171011072 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034557673019057174, + "loss": 1.4045, + "theoretical_loss": 3.4356612585630812, + "tokens_seen": 2171011072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455566700100301, + "loss": 1.3023, + "theoretical_loss": 3.4356528208089654, + "tokens_seen": 2171076608 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034553660982948847, + "loss": 1.4038, + "theoretical_loss": 3.435644383380861, + "tokens_seen": 2171142144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034551654964894683, + "loss": 1.3352, + "theoretical_loss": 3.4356359462787465, + "tokens_seen": 2171207680 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034549648946840525, + "loss": 1.3481, + "theoretical_loss": 3.4356275095025985, + "tokens_seen": 2171273216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454764292878636, + "loss": 1.2947, + "theoretical_loss": 3.4356190730523952, + "tokens_seen": 2171338752 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034545636910732197, + "loss": 1.3783, + "theoretical_loss": 3.4356106369281134, + "tokens_seen": 2171404288 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034543630892678033, + "loss": 1.4503, + "theoretical_loss": 3.435602201129732, + "tokens_seen": 2171469824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034541624874623875, + "loss": 1.3653, + "theoretical_loss": 3.4355937656572264, + "tokens_seen": 2171535360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453961885656971, + "loss": 1.3756, + "theoretical_loss": 3.435585330510577, + "tokens_seen": 2171600896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453761283851555, + "loss": 1.3216, + "theoretical_loss": 3.435576895689759, + "tokens_seen": 2171666432 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034535606820461384, + "loss": 1.3816, + "theoretical_loss": 3.435568461194751, + "tokens_seen": 2171731968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453360080240722, + "loss": 1.3697, + "theoretical_loss": 3.4355600270255304, + "tokens_seen": 2171797504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453159478435306, + "loss": 1.4883, + "theoretical_loss": 3.4355515931820753, + "tokens_seen": 2171863040 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345295887662989, + "loss": 1.3121, + "theoretical_loss": 3.4355431596643626, + "tokens_seen": 2171928576 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034527582748244734, + "loss": 1.3712, + "theoretical_loss": 3.43553472647237, + "tokens_seen": 2171994112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452557673019057, + "loss": 1.3335, + "theoretical_loss": 3.4355262936060753, + "tokens_seen": 2172059648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452357071213641, + "loss": 1.3112, + "theoretical_loss": 3.435517861065456, + "tokens_seen": 2172125184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452156469408225, + "loss": 1.3316, + "theoretical_loss": 3.43550942885049, + "tokens_seen": 2172190720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034519558676028084, + "loss": 1.3152, + "theoretical_loss": 3.435500996961154, + "tokens_seen": 2172256256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451755265797392, + "loss": 1.3333, + "theoretical_loss": 3.4354925653974266, + "tokens_seen": 2172321792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034515546639919757, + "loss": 1.2781, + "theoretical_loss": 3.435484134159285, + "tokens_seen": 2172387328 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345135406218656, + "loss": 1.2835, + "theoretical_loss": 3.435475703246707, + "tokens_seen": 2172452864 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034511534603811435, + "loss": 1.4316, + "theoretical_loss": 3.4354672726596696, + "tokens_seen": 2172518400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450952858575727, + "loss": 1.2724, + "theoretical_loss": 3.4354588423981514, + "tokens_seen": 2172583936 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034507522567703107, + "loss": 1.3588, + "theoretical_loss": 3.4354504124621292, + "tokens_seen": 2172649472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450551654964895, + "loss": 1.1856, + "theoretical_loss": 3.4354419828515805, + "tokens_seen": 2172715008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034503510531594785, + "loss": 1.3569, + "theoretical_loss": 3.435433553566484, + "tokens_seen": 2172780544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450150451354062, + "loss": 1.3674, + "theoretical_loss": 3.435425124606816, + "tokens_seen": 2172846080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449949849548646, + "loss": 1.2325, + "theoretical_loss": 3.435416695972555, + "tokens_seen": 2172911616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034497492477432294, + "loss": 1.3763, + "theoretical_loss": 3.4354082676636786, + "tokens_seen": 2172977152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034495486459378135, + "loss": 1.2315, + "theoretical_loss": 3.4353998396801635, + "tokens_seen": 2173042688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449348044132397, + "loss": 1.1306, + "theoretical_loss": 3.4353914120219886, + "tokens_seen": 2173108224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449147442326981, + "loss": 1.3038, + "theoretical_loss": 3.435382984689131, + "tokens_seen": 2173173760 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034489468405215644, + "loss": 1.194, + "theoretical_loss": 3.4353745576815675, + "tokens_seen": 2173239296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034487462387161486, + "loss": 1.3488, + "theoretical_loss": 3.4353661309992773, + "tokens_seen": 2173304832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448545636910732, + "loss": 1.4117, + "theoretical_loss": 3.4353577046422368, + "tokens_seen": 2173370368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448345035105316, + "loss": 1.4061, + "theoretical_loss": 3.435349278610424, + "tokens_seen": 2173435904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034481444332998994, + "loss": 1.1974, + "theoretical_loss": 3.435340852903817, + "tokens_seen": 2173501440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034479438314944836, + "loss": 1.35, + "theoretical_loss": 3.4353324275223933, + "tokens_seen": 2173566976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447743229689067, + "loss": 1.2944, + "theoretical_loss": 3.43532400246613, + "tokens_seen": 2173632512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447542627883651, + "loss": 1.3432, + "theoretical_loss": 3.435315577735005, + "tokens_seen": 2173698048 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034473420260782345, + "loss": 1.3156, + "theoretical_loss": 3.4353071533289956, + "tokens_seen": 2173763584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447141424272818, + "loss": 1.3341, + "theoretical_loss": 3.43529872924808, + "tokens_seen": 2173829120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446940822467402, + "loss": 1.244, + "theoretical_loss": 3.435290305492236, + "tokens_seen": 2173894656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446740220661986, + "loss": 1.3551, + "theoretical_loss": 3.4352818820614415, + "tokens_seen": 2173960192 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034465396188565695, + "loss": 1.2405, + "theoretical_loss": 3.435273458955673, + "tokens_seen": 2174025728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446339017051153, + "loss": 1.2419, + "theoretical_loss": 3.4352650361749086, + "tokens_seen": 2174091264 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034461384152457373, + "loss": 1.381, + "theoretical_loss": 3.4352566137191265, + "tokens_seen": 2174156800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034459378134403214, + "loss": 1.3899, + "theoretical_loss": 3.435248191588304, + "tokens_seen": 2174222336 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 791682, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.361458659172058, + "objective/train/theoretical_loss": 3.435239769782419, + "objective/train/tokens_used": 544746976, + "theoretical_loss": 3.435239769782419, + "tokens_seen": 2174287872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445737211634905, + "loss": 1.356, + "theoretical_loss": 3.435239769782419, + "tokens_seen": 2174287872 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034455366098294887, + "loss": 1.378, + "theoretical_loss": 3.4352313483014485, + "tokens_seen": 2174353408 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034453360080240723, + "loss": 1.2636, + "theoretical_loss": 3.4352229271453707, + "tokens_seen": 2174418944 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034451354062186565, + "loss": 1.3618, + "theoretical_loss": 3.4352145063141633, + "tokens_seen": 2174484480 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344493480441324, + "loss": 1.3451, + "theoretical_loss": 3.435206085807804, + "tokens_seen": 2174550016 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034447342026078237, + "loss": 1.4435, + "theoretical_loss": 3.43519766562627, + "tokens_seen": 2174615552 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034445336008024073, + "loss": 1.3167, + "theoretical_loss": 3.43518924576954, + "tokens_seen": 2174681088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034443329989969915, + "loss": 1.3294, + "theoretical_loss": 3.4351808262375907, + "tokens_seen": 2174746624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444132397191575, + "loss": 1.2333, + "theoretical_loss": 3.4351724070304, + "tokens_seen": 2174812160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443931795386159, + "loss": 1.3757, + "theoretical_loss": 3.435163988147946, + "tokens_seen": 2174877696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034437311935807424, + "loss": 1.3469, + "theoretical_loss": 3.4351555695902056, + "tokens_seen": 2174943232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443530591775326, + "loss": 1.3429, + "theoretical_loss": 3.435147151357157, + "tokens_seen": 2175008768 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344332998996991, + "loss": 1.3447, + "theoretical_loss": 3.4351387334487784, + "tokens_seen": 2175074304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443129388164494, + "loss": 1.3634, + "theoretical_loss": 3.4351303158650466, + "tokens_seen": 2175139840 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034429287863590774, + "loss": 1.2966, + "theoretical_loss": 3.4351218986059395, + "tokens_seen": 2175205376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442728184553661, + "loss": 1.2259, + "theoretical_loss": 3.4351134816714355, + "tokens_seen": 2175270912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442527582748245, + "loss": 1.2976, + "theoretical_loss": 3.4351050650615114, + "tokens_seen": 2175336448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442326980942829, + "loss": 1.2581, + "theoretical_loss": 3.4350966487761454, + "tokens_seen": 2175401984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034421263791374124, + "loss": 1.3174, + "theoretical_loss": 3.435088232815315, + "tokens_seen": 2175467520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441925777331996, + "loss": 1.168, + "theoretical_loss": 3.4350798171789982, + "tokens_seen": 2175533056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034417251755265797, + "loss": 1.2699, + "theoretical_loss": 3.435071401867172, + "tokens_seen": 2175598592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441524573721164, + "loss": 1.3749, + "theoretical_loss": 3.435062986879815, + "tokens_seen": 2175664128 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034413239719157475, + "loss": 1.3102, + "theoretical_loss": 3.4350545722169046, + "tokens_seen": 2175729664 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441123370110331, + "loss": 1.2661, + "theoretical_loss": 3.4350461578784186, + "tokens_seen": 2175795200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034409227683049147, + "loss": 1.299, + "theoretical_loss": 3.435037743864334, + "tokens_seen": 2175860736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440722166499499, + "loss": 1.287, + "theoretical_loss": 3.4350293301746295, + "tokens_seen": 2175926272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034405215646940825, + "loss": 1.3887, + "theoretical_loss": 3.4350209168092825, + "tokens_seen": 2175991808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440320962888666, + "loss": 1.1957, + "theoretical_loss": 3.43501250376827, + "tokens_seen": 2176057344 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344012036108325, + "loss": 1.3083, + "theoretical_loss": 3.4350040910515713, + "tokens_seen": 2176122880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034399197592778334, + "loss": 1.2747, + "theoretical_loss": 3.434995678659163, + "tokens_seen": 2176188416 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034397191574724175, + "loss": 1.1947, + "theoretical_loss": 3.4349872665910226, + "tokens_seen": 2176253952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439518555667001, + "loss": 1.2829, + "theoretical_loss": 3.4349788548471287, + "tokens_seen": 2176319488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439317953861585, + "loss": 1.3004, + "theoretical_loss": 3.4349704434274586, + "tokens_seen": 2176385024 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034391173520561684, + "loss": 1.403, + "theoretical_loss": 3.43496203233199, + "tokens_seen": 2176450560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034389167502507526, + "loss": 1.3556, + "theoretical_loss": 3.4349536215607004, + "tokens_seen": 2176516096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438716148445336, + "loss": 1.2716, + "theoretical_loss": 3.4349452111135683, + "tokens_seen": 2176581632 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343851554663992, + "loss": 1.3452, + "theoretical_loss": 3.4349368009905707, + "tokens_seen": 2176647168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034383149448345034, + "loss": 1.2354, + "theoretical_loss": 3.4349283911916855, + "tokens_seen": 2176712704 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034381143430290876, + "loss": 1.2828, + "theoretical_loss": 3.4349199817168907, + "tokens_seen": 2176778240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437913741223671, + "loss": 1.3771, + "theoretical_loss": 3.4349115725661643, + "tokens_seen": 2176843776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437713139418255, + "loss": 1.2748, + "theoretical_loss": 3.4349031637394836, + "tokens_seen": 2176909312 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034375125376128385, + "loss": 1.4442, + "theoretical_loss": 3.4348947552368263, + "tokens_seen": 2176974848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437311935807422, + "loss": 1.3574, + "theoretical_loss": 3.4348863470581703, + "tokens_seen": 2177040384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437111334002006, + "loss": 1.364, + "theoretical_loss": 3.434877939203494, + "tokens_seen": 2177105920 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343691073219659, + "loss": 1.3288, + "theoretical_loss": 3.4348695316727733, + "tokens_seen": 2177171456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034367101303911735, + "loss": 1.3275, + "theoretical_loss": 3.4348611244659883, + "tokens_seen": 2177236992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436509528585757, + "loss": 1.433, + "theoretical_loss": 3.4348527175831154, + "tokens_seen": 2177302528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034363089267803413, + "loss": 1.3813, + "theoretical_loss": 3.4348443110241327, + "tokens_seen": 2177368064 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436108324974925, + "loss": 1.395, + "theoretical_loss": 3.4348359047890176, + "tokens_seen": 2177433600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034359077231695085, + "loss": 1.2783, + "theoretical_loss": 3.4348274988777483, + "tokens_seen": 2177499136 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 792987, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.399841070175171, + "objective/train/theoretical_loss": 3.4348190932903027, + "objective/train/tokens_used": 548023776, + "theoretical_loss": 3.4348190932903027, + "tokens_seen": 2177564672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435707121364092, + "loss": 1.3861, + "theoretical_loss": 3.4348190932903027, + "tokens_seen": 2177564672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435506519558676, + "loss": 1.3846, + "theoretical_loss": 3.4348106880266585, + "tokens_seen": 2177630208 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343530591775326, + "loss": 1.3781, + "theoretical_loss": 3.434802283086793, + "tokens_seen": 2177695744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034351053159478436, + "loss": 1.3168, + "theoretical_loss": 3.4347938784706846, + "tokens_seen": 2177761280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434904714142427, + "loss": 1.2561, + "theoretical_loss": 3.4347854741783106, + "tokens_seen": 2177826816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434704112337011, + "loss": 1.3239, + "theoretical_loss": 3.434777070209649, + "tokens_seen": 2177892352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434503510531595, + "loss": 1.3563, + "theoretical_loss": 3.4347686665646777, + "tokens_seen": 2177957888 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034343029087261786, + "loss": 1.2932, + "theoretical_loss": 3.4347602632433745, + "tokens_seen": 2178023424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434102306920762, + "loss": 1.3324, + "theoretical_loss": 3.4347518602457168, + "tokens_seen": 2178088960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433901705115346, + "loss": 1.2846, + "theoretical_loss": 3.434743457571683, + "tokens_seen": 2178154496 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034337011033099295, + "loss": 1.3624, + "theoretical_loss": 3.4347350552212506, + "tokens_seen": 2178220032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034335005015045136, + "loss": 1.2685, + "theoretical_loss": 3.434726653194397, + "tokens_seen": 2178285568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433299899699097, + "loss": 1.3453, + "theoretical_loss": 3.4347182514911005, + "tokens_seen": 2178351104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433099297893681, + "loss": 1.2145, + "theoretical_loss": 3.434709850111339, + "tokens_seen": 2178416640 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034328986960882645, + "loss": 1.2773, + "theoretical_loss": 3.43470144905509, + "tokens_seen": 2178482176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034326980942828487, + "loss": 1.3635, + "theoretical_loss": 3.4346930483223312, + "tokens_seen": 2178547712 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034324974924774323, + "loss": 1.3572, + "theoretical_loss": 3.4346846479130413, + "tokens_seen": 2178613248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432296890672016, + "loss": 1.3026, + "theoretical_loss": 3.434676247827197, + "tokens_seen": 2178678784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034320962888665995, + "loss": 1.3476, + "theoretical_loss": 3.4346678480647768, + "tokens_seen": 2178744320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431895687061183, + "loss": 1.2264, + "theoretical_loss": 3.434659448625758, + "tokens_seen": 2178809856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034316950852557673, + "loss": 1.2787, + "theoretical_loss": 3.4346510495101183, + "tokens_seen": 2178875392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431494483450351, + "loss": 1.2984, + "theoretical_loss": 3.4346426507178367, + "tokens_seen": 2178940928 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034312938816449345, + "loss": 1.3013, + "theoretical_loss": 3.4346342522488897, + "tokens_seen": 2179006464 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431093279839518, + "loss": 1.3285, + "theoretical_loss": 3.434625854103256, + "tokens_seen": 2179072000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430892678034103, + "loss": 1.3527, + "theoretical_loss": 3.4346174562809133, + "tokens_seen": 2179137536 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034306920762286865, + "loss": 1.3568, + "theoretical_loss": 3.434609058781839, + "tokens_seen": 2179203072 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343049147442327, + "loss": 1.1975, + "theoretical_loss": 3.434600661606011, + "tokens_seen": 2179268608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430290872617854, + "loss": 1.323, + "theoretical_loss": 3.4345922647534075, + "tokens_seen": 2179334144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034300902708124374, + "loss": 1.3763, + "theoretical_loss": 3.4345838682240064, + "tokens_seen": 2179399680 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034298896690070215, + "loss": 1.2692, + "theoretical_loss": 3.434575472017785, + "tokens_seen": 2179465216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429689067201605, + "loss": 1.3164, + "theoretical_loss": 3.4345670761347216, + "tokens_seen": 2179530752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429488465396189, + "loss": 1.3078, + "theoretical_loss": 3.434558680574794, + "tokens_seen": 2179596288 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034292878635907724, + "loss": 1.3579, + "theoretical_loss": 3.4345502853379792, + "tokens_seen": 2179661824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034290872617853566, + "loss": 1.28, + "theoretical_loss": 3.4345418904242564, + "tokens_seen": 2179727360 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342888665997994, + "loss": 1.3472, + "theoretical_loss": 3.4345334958336027, + "tokens_seen": 2179792896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428686058174524, + "loss": 1.2463, + "theoretical_loss": 3.4345251015659963, + "tokens_seen": 2179858432 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034284854563691074, + "loss": 1.3217, + "theoretical_loss": 3.4345167076214143, + "tokens_seen": 2179923968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428284854563691, + "loss": 1.3139, + "theoretical_loss": 3.4345083139998356, + "tokens_seen": 2179989504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428084252758275, + "loss": 1.3851, + "theoretical_loss": 3.4344999207012377, + "tokens_seen": 2180055040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427883650952859, + "loss": 1.427, + "theoretical_loss": 3.434491527725598, + "tokens_seen": 2180120576 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034276830491474425, + "loss": 1.4454, + "theoretical_loss": 3.4344831350728944, + "tokens_seen": 2180186112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427482447342026, + "loss": 1.4002, + "theoretical_loss": 3.434474742743106, + "tokens_seen": 2180251648 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342728184553661, + "loss": 1.2849, + "theoretical_loss": 3.4344663507362085, + "tokens_seen": 2180317184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427081243731194, + "loss": 1.3261, + "theoretical_loss": 3.4344579590521818, + "tokens_seen": 2180382720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034268806419257775, + "loss": 1.4017, + "theoretical_loss": 3.434449567691003, + "tokens_seen": 2180448256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426680040120361, + "loss": 1.3601, + "theoretical_loss": 3.43444117665265, + "tokens_seen": 2180513792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034264794383149453, + "loss": 1.3857, + "theoretical_loss": 3.4344327859371004, + "tokens_seen": 2180579328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426278836509529, + "loss": 1.282, + "theoretical_loss": 3.4344243955443323, + "tokens_seen": 2180644864 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034260782347041125, + "loss": 1.4175, + "theoretical_loss": 3.4344160054743234, + "tokens_seen": 2180710400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425877632898696, + "loss": 1.3413, + "theoretical_loss": 3.434407615727052, + "tokens_seen": 2180775936 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 794081, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3060002326965332, + "objective/train/theoretical_loss": 3.434399226302496, + "objective/train/tokens_used": 551300576, + "theoretical_loss": 3.434399226302496, + "tokens_seen": 2180841472 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342567703109328, + "loss": 1.3029, + "theoretical_loss": 3.434399226302496, + "tokens_seen": 2180841472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425476429287864, + "loss": 1.3117, + "theoretical_loss": 3.434390837200633, + "tokens_seen": 2180907008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034252758274824476, + "loss": 1.3131, + "theoretical_loss": 3.4343824484214407, + "tokens_seen": 2180972544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425075225677031, + "loss": 1.329, + "theoretical_loss": 3.434374059964897, + "tokens_seen": 2181038080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424874623871615, + "loss": 1.351, + "theoretical_loss": 3.4343656718309807, + "tokens_seen": 2181103616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424674022066199, + "loss": 1.2812, + "theoretical_loss": 3.4343572840196686, + "tokens_seen": 2181169152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034244734202607826, + "loss": 1.3166, + "theoretical_loss": 3.434348896530939, + "tokens_seen": 2181234688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424272818455366, + "loss": 1.265, + "theoretical_loss": 3.4343405093647705, + "tokens_seen": 2181300224 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342407221664995, + "loss": 1.2505, + "theoretical_loss": 3.4343321225211394, + "tokens_seen": 2181365760 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034238716148445335, + "loss": 1.389, + "theoretical_loss": 3.4343237360000254, + "tokens_seen": 2181431296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034236710130391176, + "loss": 1.3569, + "theoretical_loss": 3.434315349801405, + "tokens_seen": 2181496832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423470411233701, + "loss": 1.3176, + "theoretical_loss": 3.434306963925257, + "tokens_seen": 2181562368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423269809428285, + "loss": 1.2503, + "theoretical_loss": 3.4342985783715587, + "tokens_seen": 2181627904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034230692076228685, + "loss": 1.4076, + "theoretical_loss": 3.4342901931402885, + "tokens_seen": 2181693440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034228686058174527, + "loss": 1.3911, + "theoretical_loss": 3.434281808231424, + "tokens_seen": 2181758976 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034226680040120363, + "loss": 1.3134, + "theoretical_loss": 3.4342734236449433, + "tokens_seen": 2181824512 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342246740220662, + "loss": 1.247, + "theoretical_loss": 3.434265039380824, + "tokens_seen": 2181890048 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034222668004012035, + "loss": 1.3527, + "theoretical_loss": 3.434256655439045, + "tokens_seen": 2181955584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422066198595787, + "loss": 1.3526, + "theoretical_loss": 3.4342482718195826, + "tokens_seen": 2182021120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034218655967903713, + "loss": 1.27, + "theoretical_loss": 3.4342398885224164, + "tokens_seen": 2182086656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421664994984955, + "loss": 1.255, + "theoretical_loss": 3.434231505547523, + "tokens_seen": 2182152192 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034214643931795386, + "loss": 1.2233, + "theoretical_loss": 3.434223122894881, + "tokens_seen": 2182217728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421263791374122, + "loss": 1.3433, + "theoretical_loss": 3.4342147405644683, + "tokens_seen": 2182283264 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034210631895687063, + "loss": 1.3015, + "theoretical_loss": 3.434206358556263, + "tokens_seen": 2182348800 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342086258776329, + "loss": 1.3572, + "theoretical_loss": 3.4341979768702426, + "tokens_seen": 2182414336 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034206619859578736, + "loss": 1.3685, + "theoretical_loss": 3.4341895955063855, + "tokens_seen": 2182479872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420461384152457, + "loss": 1.34, + "theoretical_loss": 3.434181214464669, + "tokens_seen": 2182545408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420260782347041, + "loss": 1.3865, + "theoretical_loss": 3.434172833745072, + "tokens_seen": 2182610944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420060180541625, + "loss": 1.3439, + "theoretical_loss": 3.4341644533475715, + "tokens_seen": 2182676480 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034198595787362086, + "loss": 1.3034, + "theoretical_loss": 3.434156073272146, + "tokens_seen": 2182742016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419658976930792, + "loss": 1.2918, + "theoretical_loss": 3.434147693518773, + "tokens_seen": 2182807552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419458375125376, + "loss": 1.3659, + "theoretical_loss": 3.434139314087431, + "tokens_seen": 2182873088 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341925777331996, + "loss": 1.3824, + "theoretical_loss": 3.4341309349780977, + "tokens_seen": 2182938624 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034190571715145436, + "loss": 1.344, + "theoretical_loss": 3.434122556190751, + "tokens_seen": 2183004160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418856569709127, + "loss": 1.2866, + "theoretical_loss": 3.4341141777253688, + "tokens_seen": 2183069696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418655967903711, + "loss": 1.3307, + "theoretical_loss": 3.4341057995819293, + "tokens_seen": 2183135232 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034184553660982945, + "loss": 1.2127, + "theoretical_loss": 3.4340974217604106, + "tokens_seen": 2183200768 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034182547642928787, + "loss": 1.3412, + "theoretical_loss": 3.4340890442607903, + "tokens_seen": 2183266304 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034180541624874623, + "loss": 1.2727, + "theoretical_loss": 3.4340806670830464, + "tokens_seen": 2183331840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417853560682046, + "loss": 1.3589, + "theoretical_loss": 3.434072290227157, + "tokens_seen": 2183397376 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034176529588766295, + "loss": 1.3892, + "theoretical_loss": 3.4340639136931, + "tokens_seen": 2183462912 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034174523570712137, + "loss": 1.2902, + "theoretical_loss": 3.4340555374808535, + "tokens_seen": 2183528448 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034172517552657973, + "loss": 1.3418, + "theoretical_loss": 3.4340471615903954, + "tokens_seen": 2183593984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417051153460381, + "loss": 1.3557, + "theoretical_loss": 3.4340387860217034, + "tokens_seen": 2183659520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034168505516549646, + "loss": 1.3078, + "theoretical_loss": 3.434030410774756, + "tokens_seen": 2183725056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416649949849548, + "loss": 1.3491, + "theoretical_loss": 3.434022035849531, + "tokens_seen": 2183790592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034164493480441324, + "loss": 1.2932, + "theoretical_loss": 3.434013661246006, + "tokens_seen": 2183856128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416248746238716, + "loss": 1.359, + "theoretical_loss": 3.43400528696416, + "tokens_seen": 2183921664 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034160481444332996, + "loss": 1.2571, + "theoretical_loss": 3.43399691300397, + "tokens_seen": 2183987200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415847542627884, + "loss": 1.4629, + "theoretical_loss": 3.433988539365414, + "tokens_seen": 2184052736 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 795264, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.33791184425354, + "objective/train/theoretical_loss": 3.43398016604847, + "objective/train/tokens_used": 554577376, + "theoretical_loss": 3.43398016604847, + "tokens_seen": 2184118272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415646940822468, + "loss": 1.3418, + "theoretical_loss": 3.43398016604847, + "tokens_seen": 2184118272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034154463390170516, + "loss": 1.3694, + "theoretical_loss": 3.433971793053117, + "tokens_seen": 2184183808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415245737211635, + "loss": 1.3126, + "theoretical_loss": 3.433963420379332, + "tokens_seen": 2184249344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415045135406219, + "loss": 1.4299, + "theoretical_loss": 3.4339550480270935, + "tokens_seen": 2184314880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414844533600803, + "loss": 1.3505, + "theoretical_loss": 3.433946675996379, + "tokens_seen": 2184380416 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034146439317953866, + "loss": 1.4095, + "theoretical_loss": 3.4339383042871674, + "tokens_seen": 2184445952 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341444332998997, + "loss": 1.295, + "theoretical_loss": 3.4339299328994355, + "tokens_seen": 2184511488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414242728184554, + "loss": 1.3499, + "theoretical_loss": 3.433921561833162, + "tokens_seen": 2184577024 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034140421263791375, + "loss": 1.3926, + "theoretical_loss": 3.433913191088325, + "tokens_seen": 2184642560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034138415245737216, + "loss": 1.3065, + "theoretical_loss": 3.4339048206649023, + "tokens_seen": 2184708096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413640922768305, + "loss": 1.3218, + "theoretical_loss": 3.433896450562872, + "tokens_seen": 2184773632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413440320962889, + "loss": 1.2751, + "theoretical_loss": 3.4338880807822125, + "tokens_seen": 2184839168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034132397191574725, + "loss": 1.3726, + "theoretical_loss": 3.4338797113229007, + "tokens_seen": 2184904704 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034130391173520567, + "loss": 1.2598, + "theoretical_loss": 3.433871342184916, + "tokens_seen": 2184970240 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034128385155466403, + "loss": 1.3376, + "theoretical_loss": 3.4338629733682353, + "tokens_seen": 2185035776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412637913741224, + "loss": 1.284, + "theoretical_loss": 3.4338546048728373, + "tokens_seen": 2185101312 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034124373119358075, + "loss": 1.3145, + "theoretical_loss": 3.4338462366986997, + "tokens_seen": 2185166848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412236710130391, + "loss": 1.3778, + "theoretical_loss": 3.433837868845801, + "tokens_seen": 2185232384 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034120361083249753, + "loss": 1.3896, + "theoretical_loss": 3.4338295013141185, + "tokens_seen": 2185297920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411835506519559, + "loss": 1.333, + "theoretical_loss": 3.433821134103631, + "tokens_seen": 2185363456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034116349047141426, + "loss": 1.3688, + "theoretical_loss": 3.4338127672143157, + "tokens_seen": 2185428992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411434302908726, + "loss": 1.279, + "theoretical_loss": 3.4338044006461512, + "tokens_seen": 2185494528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034112337011033103, + "loss": 1.3386, + "theoretical_loss": 3.4337960343991156, + "tokens_seen": 2185560064 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411033099297894, + "loss": 1.4406, + "theoretical_loss": 3.4337876684731867, + "tokens_seen": 2185625600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034108324974924776, + "loss": 1.3079, + "theoretical_loss": 3.4337793028683428, + "tokens_seen": 2185691136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410631895687061, + "loss": 1.3677, + "theoretical_loss": 3.433770937584562, + "tokens_seen": 2185756672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410431293881645, + "loss": 1.3359, + "theoretical_loss": 3.4337625726218217, + "tokens_seen": 2185822208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410230692076229, + "loss": 1.3223, + "theoretical_loss": 3.4337542079801007, + "tokens_seen": 2185887744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034100300902708126, + "loss": 1.4324, + "theoretical_loss": 3.4337458436593766, + "tokens_seen": 2185953280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409829488465396, + "loss": 1.4117, + "theoretical_loss": 3.433737479659628, + "tokens_seen": 2186018816 + }, + { + "epoch": 0.32, + "learning_rate": 0.000340962888665998, + "loss": 1.286, + "theoretical_loss": 3.433729115980832, + "tokens_seen": 2186084352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409428284854564, + "loss": 1.4227, + "theoretical_loss": 3.4337207526229676, + "tokens_seen": 2186149888 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034092276830491477, + "loss": 1.3973, + "theoretical_loss": 3.433712389586012, + "tokens_seen": 2186215424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034090270812437313, + "loss": 1.267, + "theoretical_loss": 3.4337040268699446, + "tokens_seen": 2186280960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408826479438315, + "loss": 1.4434, + "theoretical_loss": 3.433695664474742, + "tokens_seen": 2186346496 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034086258776328985, + "loss": 1.3399, + "theoretical_loss": 3.4336873024003833, + "tokens_seen": 2186412032 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034084252758274827, + "loss": 1.4267, + "theoretical_loss": 3.433678940646846, + "tokens_seen": 2186477568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034082246740220663, + "loss": 1.2892, + "theoretical_loss": 3.4336705792141085, + "tokens_seen": 2186543104 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340802407221665, + "loss": 1.2968, + "theoretical_loss": 3.4336622181021488, + "tokens_seen": 2186608640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034078234704112335, + "loss": 1.292, + "theoretical_loss": 3.4336538573109445, + "tokens_seen": 2186674176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034076228686058177, + "loss": 1.3133, + "theoretical_loss": 3.4336454968404744, + "tokens_seen": 2186739712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034074222668004013, + "loss": 1.3695, + "theoretical_loss": 3.4336371366907166, + "tokens_seen": 2186805248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407221664994985, + "loss": 1.4023, + "theoretical_loss": 3.4336287768616485, + "tokens_seen": 2186870784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034070210631895686, + "loss": 1.3003, + "theoretical_loss": 3.4336204173532483, + "tokens_seen": 2186936320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406820461384152, + "loss": 1.3085, + "theoretical_loss": 3.4336120581654948, + "tokens_seen": 2187001856 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034066198595787364, + "loss": 1.376, + "theoretical_loss": 3.4336036992983656, + "tokens_seen": 2187067392 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340641925777332, + "loss": 1.4446, + "theoretical_loss": 3.4335953407518387, + "tokens_seen": 2187132928 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034062186559679036, + "loss": 1.3059, + "theoretical_loss": 3.4335869825258922, + "tokens_seen": 2187198464 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406018054162487, + "loss": 1.3361, + "theoretical_loss": 3.4335786246205044, + "tokens_seen": 2187264000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034058174523570714, + "loss": 1.3258, + "theoretical_loss": 3.4335702670356536, + "tokens_seen": 2187329536 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 796380, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4696393013000488, + "objective/train/theoretical_loss": 3.4335619097713175, + "objective/train/tokens_used": 557854176, + "theoretical_loss": 3.4335619097713175, + "tokens_seen": 2187395072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405616850551655, + "loss": 1.4422, + "theoretical_loss": 3.4335619097713175, + "tokens_seen": 2187395072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034054162487462386, + "loss": 1.3469, + "theoretical_loss": 3.4335535528274743, + "tokens_seen": 2187460608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405215646940822, + "loss": 1.2971, + "theoretical_loss": 3.4335451962041024, + "tokens_seen": 2187526144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034050150451354064, + "loss": 1.3619, + "theoretical_loss": 3.4335368399011794, + "tokens_seen": 2187591680 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340481444332999, + "loss": 1.3903, + "theoretical_loss": 3.4335284839186837, + "tokens_seen": 2187657216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034046138415245737, + "loss": 1.3446, + "theoretical_loss": 3.4335201282565935, + "tokens_seen": 2187722752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034044132397191573, + "loss": 1.2827, + "theoretical_loss": 3.433511772914887, + "tokens_seen": 2187788288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404212637913741, + "loss": 1.3151, + "theoretical_loss": 3.433503417893542, + "tokens_seen": 2187853824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404012036108325, + "loss": 1.3519, + "theoretical_loss": 3.4334950631925363, + "tokens_seen": 2187919360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034038114343029087, + "loss": 1.3695, + "theoretical_loss": 3.433486708811849, + "tokens_seen": 2187984896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034036108324974923, + "loss": 1.3254, + "theoretical_loss": 3.4334783547514576, + "tokens_seen": 2188050432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403410230692076, + "loss": 1.3506, + "theoretical_loss": 3.43347000101134, + "tokens_seen": 2188115968 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340320962888666, + "loss": 1.3177, + "theoretical_loss": 3.4334616475914754, + "tokens_seen": 2188181504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403009027081244, + "loss": 1.3301, + "theoretical_loss": 3.433453294491841, + "tokens_seen": 2188247040 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034028084252758274, + "loss": 1.2995, + "theoretical_loss": 3.4334449417124144, + "tokens_seen": 2188312576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402607823470411, + "loss": 1.4326, + "theoretical_loss": 3.433436589253175, + "tokens_seen": 2188378112 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034024072216649946, + "loss": 1.3466, + "theoretical_loss": 3.4334282371141, + "tokens_seen": 2188443648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402206619859579, + "loss": 1.292, + "theoretical_loss": 3.4334198852951685, + "tokens_seen": 2188509184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034020060180541624, + "loss": 1.4296, + "theoretical_loss": 3.4334115337963578, + "tokens_seen": 2188574720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401805416248746, + "loss": 1.3808, + "theoretical_loss": 3.4334031826176465, + "tokens_seen": 2188640256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034016048144433296, + "loss": 1.3303, + "theoretical_loss": 3.433394831759012, + "tokens_seen": 2188705792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401404212637914, + "loss": 1.3042, + "theoretical_loss": 3.4333864812204338, + "tokens_seen": 2188771328 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034012036108324974, + "loss": 1.3298, + "theoretical_loss": 3.4333781310018887, + "tokens_seen": 2188836864 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401003009027081, + "loss": 1.3884, + "theoretical_loss": 3.4333697811033557, + "tokens_seen": 2188902400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400802407221665, + "loss": 1.3479, + "theoretical_loss": 3.433361431524813, + "tokens_seen": 2188967936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400601805416249, + "loss": 1.3489, + "theoretical_loss": 3.4333530822662377, + "tokens_seen": 2189033472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400401203610833, + "loss": 1.3704, + "theoretical_loss": 3.4333447333276093, + "tokens_seen": 2189099008 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034002006018054166, + "loss": 1.3014, + "theoretical_loss": 3.433336384708905, + "tokens_seen": 2189164544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034, + "loss": 1.3024, + "theoretical_loss": 3.4333280364101038, + "tokens_seen": 2189230080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399799398194584, + "loss": 1.3214, + "theoretical_loss": 3.4333196884311827, + "tokens_seen": 2189295616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399598796389168, + "loss": 1.186, + "theoretical_loss": 3.433311340772121, + "tokens_seen": 2189361152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033993981945837517, + "loss": 1.3232, + "theoretical_loss": 3.4333029934328962, + "tokens_seen": 2189426688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033991975927783353, + "loss": 1.2205, + "theoretical_loss": 3.433294646413487, + "tokens_seen": 2189492224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398996990972919, + "loss": 1.3335, + "theoretical_loss": 3.433286299713871, + "tokens_seen": 2189557760 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033987963891675025, + "loss": 1.3321, + "theoretical_loss": 3.4332779533340263, + "tokens_seen": 2189623296 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033985957873620867, + "loss": 1.4256, + "theoretical_loss": 3.433269607273932, + "tokens_seen": 2189688832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033983951855566703, + "loss": 1.2873, + "theoretical_loss": 3.433261261533566, + "tokens_seen": 2189754368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398194583751254, + "loss": 1.3906, + "theoretical_loss": 3.4332529161129055, + "tokens_seen": 2189819904 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033979939819458376, + "loss": 1.311, + "theoretical_loss": 3.4332445710119295, + "tokens_seen": 2189885440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033977933801404217, + "loss": 1.3967, + "theoretical_loss": 3.4332362262306164, + "tokens_seen": 2189950976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033975927783350053, + "loss": 1.2852, + "theoretical_loss": 3.4332278817689437, + "tokens_seen": 2190016512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397392176529589, + "loss": 1.3093, + "theoretical_loss": 3.43321953762689, + "tokens_seen": 2190082048 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033971915747241726, + "loss": 1.4804, + "theoretical_loss": 3.433211193804434, + "tokens_seen": 2190147584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396990972918756, + "loss": 1.3167, + "theoretical_loss": 3.4332028503015524, + "tokens_seen": 2190213120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033967903711133404, + "loss": 1.3219, + "theoretical_loss": 3.433194507118225, + "tokens_seen": 2190278656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396589769307924, + "loss": 1.3668, + "theoretical_loss": 3.4331861642544292, + "tokens_seen": 2190344192 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033963891675025076, + "loss": 1.3353, + "theoretical_loss": 3.433177821710143, + "tokens_seen": 2190409728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396188565697091, + "loss": 1.2263, + "theoretical_loss": 3.4331694794853456, + "tokens_seen": 2190475264 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033959879638916754, + "loss": 1.3763, + "theoretical_loss": 3.433161137580014, + "tokens_seen": 2190540800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395787362086259, + "loss": 1.2612, + "theoretical_loss": 3.4331527959941273, + "tokens_seen": 2190606336 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 797544, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4318948984146118, + "objective/train/theoretical_loss": 3.4331444547276635, + "objective/train/tokens_used": 561130976, + "theoretical_loss": 3.4331444547276635, + "tokens_seen": 2190671872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033955867602808426, + "loss": 1.3687, + "theoretical_loss": 3.4331444547276635, + "tokens_seen": 2190671872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395386158475426, + "loss": 1.2755, + "theoretical_loss": 3.4331361137806002, + "tokens_seen": 2190737408 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033951855566700104, + "loss": 1.3143, + "theoretical_loss": 3.433127773152916, + "tokens_seen": 2190802944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394984954864594, + "loss": 1.3477, + "theoretical_loss": 3.43311943284459, + "tokens_seen": 2190868480 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033947843530591777, + "loss": 1.3112, + "theoretical_loss": 3.433111092855599, + "tokens_seen": 2190934016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033945837512537613, + "loss": 1.3379, + "theoretical_loss": 3.433102753185922, + "tokens_seen": 2190999552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394383149448345, + "loss": 1.3878, + "theoretical_loss": 3.433094413835537, + "tokens_seen": 2191065088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394182547642929, + "loss": 1.1772, + "theoretical_loss": 3.4330860748044225, + "tokens_seen": 2191130624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033939819458375127, + "loss": 1.4139, + "theoretical_loss": 3.4330777360925566, + "tokens_seen": 2191196160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033937813440320963, + "loss": 1.3543, + "theoretical_loss": 3.433069397699917, + "tokens_seen": 2191261696 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339358074222668, + "loss": 1.2785, + "theoretical_loss": 3.433061059626483, + "tokens_seen": 2191327232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393380140421264, + "loss": 1.2888, + "theoretical_loss": 3.433052721872232, + "tokens_seen": 2191392768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393179538615848, + "loss": 1.351, + "theoretical_loss": 3.4330443844371423, + "tokens_seen": 2191458304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033929789368104314, + "loss": 1.2236, + "theoretical_loss": 3.4330360473211923, + "tokens_seen": 2191523840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392778335005015, + "loss": 1.3972, + "theoretical_loss": 3.4330277105243603, + "tokens_seen": 2191589376 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033925777331995986, + "loss": 1.2715, + "theoretical_loss": 3.4330193740466246, + "tokens_seen": 2191654912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392377131394183, + "loss": 1.3, + "theoretical_loss": 3.4330110378879635, + "tokens_seen": 2191720448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033921765295887664, + "loss": 1.4087, + "theoretical_loss": 3.4330027020483547, + "tokens_seen": 2191785984 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339197592778335, + "loss": 1.3882, + "theoretical_loss": 3.432994366527777, + "tokens_seen": 2191851520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033917753259779336, + "loss": 1.3216, + "theoretical_loss": 3.432986031326209, + "tokens_seen": 2191917056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391574724172518, + "loss": 1.3639, + "theoretical_loss": 3.4329776964436274, + "tokens_seen": 2191982592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033913741223671014, + "loss": 1.2528, + "theoretical_loss": 3.432969361880012, + "tokens_seen": 2192048128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391173520561685, + "loss": 1.3301, + "theoretical_loss": 3.432961027635341, + "tokens_seen": 2192113664 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033909729187562687, + "loss": 1.3384, + "theoretical_loss": 3.4329526937095913, + "tokens_seen": 2192179200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033907723169508523, + "loss": 1.4283, + "theoretical_loss": 3.4329443601027427, + "tokens_seen": 2192244736 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033905717151454365, + "loss": 1.2136, + "theoretical_loss": 3.432936026814773, + "tokens_seen": 2192310272 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339037111334002, + "loss": 1.2605, + "theoretical_loss": 3.43292769384566, + "tokens_seen": 2192375808 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033901705115346037, + "loss": 1.3571, + "theoretical_loss": 3.432919361195382, + "tokens_seen": 2192441344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033899699097291873, + "loss": 1.2635, + "theoretical_loss": 3.4329110288639177, + "tokens_seen": 2192506880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033897693079237715, + "loss": 1.3853, + "theoretical_loss": 3.4329026968512455, + "tokens_seen": 2192572416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389568706118355, + "loss": 1.2294, + "theoretical_loss": 3.432894365157343, + "tokens_seen": 2192637952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389368104312939, + "loss": 1.2674, + "theoretical_loss": 3.4328860337821894, + "tokens_seen": 2192703488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033891675025075224, + "loss": 1.357, + "theoretical_loss": 3.432877702725762, + "tokens_seen": 2192769024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388966900702106, + "loss": 1.3789, + "theoretical_loss": 3.4328693719880397, + "tokens_seen": 2192834560 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338876629889669, + "loss": 1.3345, + "theoretical_loss": 3.4328610415690006, + "tokens_seen": 2192900096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388565697091274, + "loss": 1.2225, + "theoretical_loss": 3.432852711468623, + "tokens_seen": 2192965632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033883650952858574, + "loss": 1.319, + "theoretical_loss": 3.432844381686885, + "tokens_seen": 2193031168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388164493480441, + "loss": 1.2773, + "theoretical_loss": 3.4328360522237653, + "tokens_seen": 2193096704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387963891675025, + "loss": 1.3942, + "theoretical_loss": 3.4328277230792423, + "tokens_seen": 2193162240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387763289869609, + "loss": 1.2898, + "theoretical_loss": 3.4328193942532934, + "tokens_seen": 2193227776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033875626880641924, + "loss": 1.3954, + "theoretical_loss": 3.4328110657458977, + "tokens_seen": 2193293312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387362086258776, + "loss": 1.2491, + "theoretical_loss": 3.432802737557033, + "tokens_seen": 2193358848 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033871614844533597, + "loss": 1.3824, + "theoretical_loss": 3.4327944096866783, + "tokens_seen": 2193424384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386960882647944, + "loss": 1.336, + "theoretical_loss": 3.432786082134811, + "tokens_seen": 2193489920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033867602808425275, + "loss": 1.3324, + "theoretical_loss": 3.43277775490141, + "tokens_seen": 2193555456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386559679037111, + "loss": 1.3167, + "theoretical_loss": 3.432769427986454, + "tokens_seen": 2193620992 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033863590772316947, + "loss": 1.3703, + "theoretical_loss": 3.43276110138992, + "tokens_seen": 2193686528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386158475426279, + "loss": 1.4705, + "theoretical_loss": 3.4327527751117874, + "tokens_seen": 2193752064 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033859578736208625, + "loss": 1.2581, + "theoretical_loss": 3.4327444491520342, + "tokens_seen": 2193817600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033857572718154467, + "loss": 1.3588, + "theoretical_loss": 3.432736123510639, + "tokens_seen": 2193883136 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 798764, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3836020231246948, + "objective/train/theoretical_loss": 3.4327277981875794, + "objective/train/tokens_used": 564407776, + "theoretical_loss": 3.4327277981875794, + "tokens_seen": 2193948672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033855566700100303, + "loss": 1.3783, + "theoretical_loss": 3.4327277981875794, + "tokens_seen": 2193948672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033853560682046144, + "loss": 1.304, + "theoretical_loss": 3.4327194731828348, + "tokens_seen": 2194014208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385155466399198, + "loss": 1.4275, + "theoretical_loss": 3.432711148496382, + "tokens_seen": 2194079744 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033849548645937817, + "loss": 1.328, + "theoretical_loss": 3.4327028241282007, + "tokens_seen": 2194145280 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033847542627883653, + "loss": 1.3396, + "theoretical_loss": 3.4326945000782687, + "tokens_seen": 2194210816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384553660982949, + "loss": 1.2473, + "theoretical_loss": 3.4326861763465644, + "tokens_seen": 2194276352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384353059177533, + "loss": 1.4009, + "theoretical_loss": 3.432677852933066, + "tokens_seen": 2194341888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033841524573721167, + "loss": 1.2956, + "theoretical_loss": 3.432669529837752, + "tokens_seen": 2194407424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033839518555667003, + "loss": 1.377, + "theoretical_loss": 3.4326612070606006, + "tokens_seen": 2194472960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383751253761284, + "loss": 1.2978, + "theoretical_loss": 3.4326528846015902, + "tokens_seen": 2194538496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383550651955868, + "loss": 1.3589, + "theoretical_loss": 3.432644562460699, + "tokens_seen": 2194604032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383350050150452, + "loss": 1.2523, + "theoretical_loss": 3.4326362406379056, + "tokens_seen": 2194669568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033831494483450354, + "loss": 1.3193, + "theoretical_loss": 3.4326279191331883, + "tokens_seen": 2194735104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382948846539619, + "loss": 1.3393, + "theoretical_loss": 3.4326195979465255, + "tokens_seen": 2194800640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033827482447342026, + "loss": 1.447, + "theoretical_loss": 3.432611277077895, + "tokens_seen": 2194866176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382547642928787, + "loss": 1.3865, + "theoretical_loss": 3.4326029565272758, + "tokens_seen": 2194931712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033823470411233704, + "loss": 1.347, + "theoretical_loss": 3.432594636294646, + "tokens_seen": 2194997248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382146439317954, + "loss": 1.3352, + "theoretical_loss": 3.4325863163799837, + "tokens_seen": 2195062784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033819458375125376, + "loss": 1.2813, + "theoretical_loss": 3.4325779967832677, + "tokens_seen": 2195128320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381745235707122, + "loss": 1.3839, + "theoretical_loss": 3.4325696775044765, + "tokens_seen": 2195193856 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033815446339017054, + "loss": 1.2652, + "theoretical_loss": 3.4325613585435875, + "tokens_seen": 2195259392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381344032096289, + "loss": 1.2375, + "theoretical_loss": 3.43255303990058, + "tokens_seen": 2195324928 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033811434302908727, + "loss": 1.3222, + "theoretical_loss": 3.4325447215754323, + "tokens_seen": 2195390464 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033809428284854563, + "loss": 1.3751, + "theoretical_loss": 3.4325364035681223, + "tokens_seen": 2195456000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033807422266800405, + "loss": 1.2995, + "theoretical_loss": 3.4325280858786287, + "tokens_seen": 2195521536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380541624874624, + "loss": 1.2695, + "theoretical_loss": 3.4325197685069297, + "tokens_seen": 2195587072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033803410230692077, + "loss": 1.2899, + "theoretical_loss": 3.4325114514530037, + "tokens_seen": 2195652608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033801404212637913, + "loss": 1.3608, + "theoretical_loss": 3.4325031347168293, + "tokens_seen": 2195718144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033799398194583755, + "loss": 1.4839, + "theoretical_loss": 3.4324948182983843, + "tokens_seen": 2195783680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379739217652959, + "loss": 1.2855, + "theoretical_loss": 3.432486502197648, + "tokens_seen": 2195849216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379538615847543, + "loss": 1.3818, + "theoretical_loss": 3.4324781864145977, + "tokens_seen": 2195914752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033793380140421264, + "loss": 1.2523, + "theoretical_loss": 3.4324698709492125, + "tokens_seen": 2195980288 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337913741223671, + "loss": 1.3566, + "theoretical_loss": 3.432461555801471, + "tokens_seen": 2196045824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378936810431294, + "loss": 1.3287, + "theoretical_loss": 3.432453240971351, + "tokens_seen": 2196111360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378736208625878, + "loss": 1.2999, + "theoretical_loss": 3.432444926458831, + "tokens_seen": 2196176896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033785356068204614, + "loss": 1.2603, + "theoretical_loss": 3.4324366122638894, + "tokens_seen": 2196242432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378335005015045, + "loss": 1.3015, + "theoretical_loss": 3.4324282983865047, + "tokens_seen": 2196307968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378134403209629, + "loss": 1.3631, + "theoretical_loss": 3.4324199848266557, + "tokens_seen": 2196373504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377933801404213, + "loss": 1.2407, + "theoretical_loss": 3.43241167158432, + "tokens_seen": 2196439040 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033777331995987964, + "loss": 1.305, + "theoretical_loss": 3.432403358659476, + "tokens_seen": 2196504576 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337753259779338, + "loss": 1.4281, + "theoretical_loss": 3.432395046052103, + "tokens_seen": 2196570112 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033773319959879637, + "loss": 1.3704, + "theoretical_loss": 3.432386733762179, + "tokens_seen": 2196635648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377131394182548, + "loss": 1.3947, + "theoretical_loss": 3.432378421789682, + "tokens_seen": 2196701184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033769307923771315, + "loss": 1.3973, + "theoretical_loss": 3.432370110134591, + "tokens_seen": 2196766720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376730190571715, + "loss": 1.2445, + "theoretical_loss": 3.432361798796884, + "tokens_seen": 2196832256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033765295887662987, + "loss": 1.2544, + "theoretical_loss": 3.4323534877765396, + "tokens_seen": 2196897792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376328986960883, + "loss": 1.3833, + "theoretical_loss": 3.4323451770735356, + "tokens_seen": 2196963328 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033761283851554665, + "loss": 1.3316, + "theoretical_loss": 3.4323368666878515, + "tokens_seen": 2197028864 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337592778335005, + "loss": 1.3844, + "theoretical_loss": 3.4323285566194652, + "tokens_seen": 2197094400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375727181544634, + "loss": 1.345, + "theoretical_loss": 3.432320246868355, + "tokens_seen": 2197159936 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 799965, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.29378080368042, + "objective/train/theoretical_loss": 3.4323119374344992, + "objective/train/tokens_used": 567684576, + "theoretical_loss": 3.4323119374344992, + "tokens_seen": 2197225472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033755265797392174, + "loss": 1.3068, + "theoretical_loss": 3.4323119374344992, + "tokens_seen": 2197225472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033753259779338015, + "loss": 1.2938, + "theoretical_loss": 3.4323036283178765, + "tokens_seen": 2197291008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375125376128385, + "loss": 1.3309, + "theoretical_loss": 3.4322953195184653, + "tokens_seen": 2197356544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374924774322969, + "loss": 1.3439, + "theoretical_loss": 3.432287011036244, + "tokens_seen": 2197422080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033747241725175524, + "loss": 1.3396, + "theoretical_loss": 3.432278702871191, + "tokens_seen": 2197487616 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033745235707121366, + "loss": 1.3109, + "theoretical_loss": 3.432270395023285, + "tokens_seen": 2197553152 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337432296890672, + "loss": 1.3985, + "theoretical_loss": 3.432262087492504, + "tokens_seen": 2197618688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374122367101304, + "loss": 1.277, + "theoretical_loss": 3.4322537802788267, + "tokens_seen": 2197684224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033739217652958874, + "loss": 1.3417, + "theoretical_loss": 3.4322454733822316, + "tokens_seen": 2197749760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373721163490471, + "loss": 1.294, + "theoretical_loss": 3.4322371668026967, + "tokens_seen": 2197815296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373520561685055, + "loss": 1.461, + "theoretical_loss": 3.432228860540201, + "tokens_seen": 2197880832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373319959879639, + "loss": 1.3607, + "theoretical_loss": 3.4322205545947226, + "tokens_seen": 2197946368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033731193580742225, + "loss": 1.3439, + "theoretical_loss": 3.4322122489662403, + "tokens_seen": 2198011904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372918756268806, + "loss": 1.3775, + "theoretical_loss": 3.432203943654732, + "tokens_seen": 2198077440 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337271815446339, + "loss": 1.3674, + "theoretical_loss": 3.432195638660177, + "tokens_seen": 2198142976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372517552657974, + "loss": 1.2804, + "theoretical_loss": 3.4321873339825526, + "tokens_seen": 2198208512 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033723169508525575, + "loss": 1.3526, + "theoretical_loss": 3.432179029621838, + "tokens_seen": 2198274048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372116349047141, + "loss": 1.3247, + "theoretical_loss": 3.432170725578012, + "tokens_seen": 2198339584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371915747241725, + "loss": 1.383, + "theoretical_loss": 3.4321624218510522, + "tokens_seen": 2198405120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371715145436309, + "loss": 1.2943, + "theoretical_loss": 3.4321541184409377, + "tokens_seen": 2198470656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033715145436308925, + "loss": 1.2926, + "theoretical_loss": 3.4321458153476465, + "tokens_seen": 2198536192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371313941825476, + "loss": 1.3219, + "theoretical_loss": 3.432137512571157, + "tokens_seen": 2198601728 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337111334002006, + "loss": 1.3274, + "theoretical_loss": 3.432129210111449, + "tokens_seen": 2198667264 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370912738214644, + "loss": 1.3279, + "theoretical_loss": 3.432120907968499, + "tokens_seen": 2198732800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370712136409228, + "loss": 1.3657, + "theoretical_loss": 3.4321126061422866, + "tokens_seen": 2198798336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033705115346038117, + "loss": 1.4111, + "theoretical_loss": 3.43210430463279, + "tokens_seen": 2198863872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033703109327983953, + "loss": 1.1815, + "theoretical_loss": 3.4320960034399883, + "tokens_seen": 2198929408 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033701103309929795, + "loss": 1.1981, + "theoretical_loss": 3.4320877025638588, + "tokens_seen": 2198994944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369909729187563, + "loss": 1.2987, + "theoretical_loss": 3.432079402004381, + "tokens_seen": 2199060480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369709127382147, + "loss": 1.3709, + "theoretical_loss": 3.432071101761533, + "tokens_seen": 2199126016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033695085255767304, + "loss": 1.3555, + "theoretical_loss": 3.4320628018352934, + "tokens_seen": 2199191552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369307923771314, + "loss": 1.3236, + "theoretical_loss": 3.4320545022256406, + "tokens_seen": 2199257088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369107321965898, + "loss": 1.2404, + "theoretical_loss": 3.4320462029325527, + "tokens_seen": 2199322624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368906720160482, + "loss": 1.2502, + "theoretical_loss": 3.432037903956009, + "tokens_seen": 2199388160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033687061183550654, + "loss": 1.3871, + "theoretical_loss": 3.432029605295987, + "tokens_seen": 2199453696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368505516549649, + "loss": 1.3112, + "theoretical_loss": 3.4320213069524663, + "tokens_seen": 2199519232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368304914744233, + "loss": 1.3443, + "theoretical_loss": 3.432013008925425, + "tokens_seen": 2199584768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368104312938817, + "loss": 1.3577, + "theoretical_loss": 3.432004711214841, + "tokens_seen": 2199650304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033679037111334004, + "loss": 1.3898, + "theoretical_loss": 3.4319964138206935, + "tokens_seen": 2199715840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367703109327984, + "loss": 1.345, + "theoretical_loss": 3.4319881167429607, + "tokens_seen": 2199781376 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033675025075225677, + "loss": 1.3213, + "theoretical_loss": 3.4319798199816214, + "tokens_seen": 2199846912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367301905717152, + "loss": 1.314, + "theoretical_loss": 3.431971523536654, + "tokens_seen": 2199912448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033671013039117355, + "loss": 1.2828, + "theoretical_loss": 3.431963227408036, + "tokens_seen": 2199977984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366900702106319, + "loss": 1.3471, + "theoretical_loss": 3.431954931595748, + "tokens_seen": 2200043520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033667001003009027, + "loss": 1.3046, + "theoretical_loss": 3.4319466360997666, + "tokens_seen": 2200109056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366499498495487, + "loss": 1.3941, + "theoretical_loss": 3.431938340920071, + "tokens_seen": 2200174592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033662988966900705, + "loss": 1.3828, + "theoretical_loss": 3.43193004605664, + "tokens_seen": 2200240128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366098294884654, + "loss": 1.308, + "theoretical_loss": 3.431921751509452, + "tokens_seen": 2200305664 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365897693079238, + "loss": 1.3843, + "theoretical_loss": 3.4319134572784855, + "tokens_seen": 2200371200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033656970912738214, + "loss": 1.3077, + "theoretical_loss": 3.4319051633637185, + "tokens_seen": 2200436736 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 801168, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3015751838684082, + "objective/train/theoretical_loss": 3.4318968697651306, + "objective/train/tokens_used": 570961376, + "theoretical_loss": 3.4318968697651306, + "tokens_seen": 2200502272 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033654964894684055, + "loss": 1.2766, + "theoretical_loss": 3.4318968697651306, + "tokens_seen": 2200502272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365295887662989, + "loss": 1.32, + "theoretical_loss": 3.431888576482699, + "tokens_seen": 2200567808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365095285857573, + "loss": 1.3601, + "theoretical_loss": 3.4318802835164037, + "tokens_seen": 2200633344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033648946840521564, + "loss": 1.1883, + "theoretical_loss": 3.431871990866222, + "tokens_seen": 2200698880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033646940822467406, + "loss": 1.333, + "theoretical_loss": 3.431863698532133, + "tokens_seen": 2200764416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364493480441324, + "loss": 1.2618, + "theoretical_loss": 3.4318554065141154, + "tokens_seen": 2200829952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364292878635908, + "loss": 1.3489, + "theoretical_loss": 3.4318471148121468, + "tokens_seen": 2200895488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033640922768304914, + "loss": 1.3973, + "theoretical_loss": 3.431838823426207, + "tokens_seen": 2200961024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363891675025075, + "loss": 1.2477, + "theoretical_loss": 3.431830532356274, + "tokens_seen": 2201026560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363691073219659, + "loss": 1.2898, + "theoretical_loss": 3.431822241602326, + "tokens_seen": 2201092096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363490471414243, + "loss": 1.3257, + "theoretical_loss": 3.431813951164342, + "tokens_seen": 2201157632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033632898696088265, + "loss": 1.4136, + "theoretical_loss": 3.4318056610423007, + "tokens_seen": 2201223168 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336308926780341, + "loss": 1.2406, + "theoretical_loss": 3.43179737123618, + "tokens_seen": 2201288704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362888665997994, + "loss": 1.3525, + "theoretical_loss": 3.431789081745959, + "tokens_seen": 2201354240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362688064192578, + "loss": 1.2073, + "theoretical_loss": 3.431780792571616, + "tokens_seen": 2201419776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033624874623871615, + "loss": 1.3003, + "theoretical_loss": 3.43177250371313, + "tokens_seen": 2201485312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362286860581745, + "loss": 1.3523, + "theoretical_loss": 3.431764215170479, + "tokens_seen": 2201550848 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033620862587763293, + "loss": 1.3401, + "theoretical_loss": 3.4317559269436417, + "tokens_seen": 2201616384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361885656970913, + "loss": 1.2454, + "theoretical_loss": 3.431747639032597, + "tokens_seen": 2201681920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033616850551654965, + "loss": 1.2756, + "theoretical_loss": 3.431739351437323, + "tokens_seen": 2201747456 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336148445336008, + "loss": 1.3579, + "theoretical_loss": 3.4317310641577987, + "tokens_seen": 2201812992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361283851554664, + "loss": 1.2667, + "theoretical_loss": 3.431722777194002, + "tokens_seen": 2201878528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361083249749248, + "loss": 1.2843, + "theoretical_loss": 3.4317144905459127, + "tokens_seen": 2201944064 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033608826479438316, + "loss": 1.3839, + "theoretical_loss": 3.4317062042135076, + "tokens_seen": 2202009600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360682046138415, + "loss": 1.337, + "theoretical_loss": 3.4316979181967673, + "tokens_seen": 2202075136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360481444332999, + "loss": 1.3916, + "theoretical_loss": 3.4316896324956687, + "tokens_seen": 2202140672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360280842527583, + "loss": 1.2635, + "theoretical_loss": 3.4316813471101915, + "tokens_seen": 2202206208 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033600802407221666, + "loss": 1.3362, + "theoretical_loss": 3.4316730620403133, + "tokens_seen": 2202271744 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335987963891675, + "loss": 1.3782, + "theoretical_loss": 3.431664777286014, + "tokens_seen": 2202337280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359679037111334, + "loss": 1.3283, + "theoretical_loss": 3.431656492847271, + "tokens_seen": 2202402816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033594784353059174, + "loss": 1.3423, + "theoretical_loss": 3.431648208724063, + "tokens_seen": 2202468352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033592778335005016, + "loss": 1.296, + "theoretical_loss": 3.4316399249163694, + "tokens_seen": 2202533888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359077231695085, + "loss": 1.2728, + "theoretical_loss": 3.431631641424168, + "tokens_seen": 2202599424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003358876629889669, + "loss": 1.3436, + "theoretical_loss": 3.4316233582474376, + "tokens_seen": 2202664960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033586760280842525, + "loss": 1.2932, + "theoretical_loss": 3.431615075386157, + "tokens_seen": 2202730496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033584754262788366, + "loss": 1.4493, + "theoretical_loss": 3.431606792840305, + "tokens_seen": 2202796032 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335827482447342, + "loss": 1.5376, + "theoretical_loss": 3.4315985106098594, + "tokens_seen": 2202861568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358074222668004, + "loss": 1.35, + "theoretical_loss": 3.4315902286948, + "tokens_seen": 2202927104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033578736208625875, + "loss": 1.3767, + "theoretical_loss": 3.431581947095104, + "tokens_seen": 2202992640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357673019057171, + "loss": 1.3946, + "theoretical_loss": 3.431573665810751, + "tokens_seen": 2203058176 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033574724172517553, + "loss": 1.2454, + "theoretical_loss": 3.43156538484172, + "tokens_seen": 2203123712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357271815446339, + "loss": 1.3536, + "theoretical_loss": 3.4315571041879878, + "tokens_seen": 2203189248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033570712136409225, + "loss": 1.4043, + "theoretical_loss": 3.431548823849535, + "tokens_seen": 2203254784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356870611835506, + "loss": 1.3067, + "theoretical_loss": 3.431540543826339, + "tokens_seen": 2203320320 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033566700100300903, + "loss": 1.3333, + "theoretical_loss": 3.431532264118379, + "tokens_seen": 2203385856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356469408224674, + "loss": 1.3819, + "theoretical_loss": 3.4315239847256334, + "tokens_seen": 2203451392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033562688064192576, + "loss": 1.2752, + "theoretical_loss": 3.4315157056480805, + "tokens_seen": 2203516928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356068204613841, + "loss": 1.3737, + "theoretical_loss": 3.4315074268856995, + "tokens_seen": 2203582464 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355867602808425, + "loss": 1.2647, + "theoretical_loss": 3.431499148438469, + "tokens_seen": 2203648000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033556670010030095, + "loss": 1.3371, + "theoretical_loss": 3.4314908703063676, + "tokens_seen": 2203713536 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 802328, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3683021068572998, + "objective/train/theoretical_loss": 3.4314825924893735, + "objective/train/tokens_used": 574238176, + "theoretical_loss": 3.4314825924893735, + "tokens_seen": 2203779072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355466399197593, + "loss": 1.3298, + "theoretical_loss": 3.4314825924893735, + "tokens_seen": 2203779072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355265797392177, + "loss": 1.3622, + "theoretical_loss": 3.4314743149874656, + "tokens_seen": 2203844608 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033550651955867604, + "loss": 1.3813, + "theoretical_loss": 3.4314660378006225, + "tokens_seen": 2203910144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033548645937813446, + "loss": 1.3239, + "theoretical_loss": 3.431457760928823, + "tokens_seen": 2203975680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354663991975928, + "loss": 1.2864, + "theoretical_loss": 3.4314494843720458, + "tokens_seen": 2204041216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354463390170512, + "loss": 1.352, + "theoretical_loss": 3.431441208130269, + "tokens_seen": 2204106752 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033542627883650954, + "loss": 1.1718, + "theoretical_loss": 3.431432932203472, + "tokens_seen": 2204172288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354062186559679, + "loss": 1.3385, + "theoretical_loss": 3.4314246565916324, + "tokens_seen": 2204237824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353861584754263, + "loss": 1.294, + "theoretical_loss": 3.4314163812947305, + "tokens_seen": 2204303360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353660982948847, + "loss": 1.4892, + "theoretical_loss": 3.4314081063127433, + "tokens_seen": 2204368896 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033534603811434305, + "loss": 1.3164, + "theoretical_loss": 3.4313998316456504, + "tokens_seen": 2204434432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353259779338014, + "loss": 1.3477, + "theoretical_loss": 3.4313915572934297, + "tokens_seen": 2204499968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353059177532598, + "loss": 1.398, + "theoretical_loss": 3.431383283256061, + "tokens_seen": 2204565504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352858575727182, + "loss": 1.2768, + "theoretical_loss": 3.431375009533522, + "tokens_seen": 2204631040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033526579739217655, + "loss": 1.3906, + "theoretical_loss": 3.4313667361257916, + "tokens_seen": 2204696576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352457372116349, + "loss": 1.3745, + "theoretical_loss": 3.4313584630328484, + "tokens_seen": 2204762112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033522567703109333, + "loss": 1.2807, + "theoretical_loss": 3.4313501902546717, + "tokens_seen": 2204827648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352056168505517, + "loss": 1.3377, + "theoretical_loss": 3.431341917791239, + "tokens_seen": 2204893184 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033518555667001005, + "loss": 1.3502, + "theoretical_loss": 3.43133364564253, + "tokens_seen": 2204958720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351654964894684, + "loss": 1.238, + "theoretical_loss": 3.431325373808523, + "tokens_seen": 2205024256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351454363089268, + "loss": 1.3744, + "theoretical_loss": 3.4313171022891966, + "tokens_seen": 2205089792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351253761283852, + "loss": 1.3221, + "theoretical_loss": 3.4313088310845297, + "tokens_seen": 2205155328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033510531594784356, + "loss": 1.3182, + "theoretical_loss": 3.431300560194501, + "tokens_seen": 2205220864 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350852557673019, + "loss": 1.3753, + "theoretical_loss": 3.4312922896190887, + "tokens_seen": 2205286400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350651955867603, + "loss": 1.5038, + "theoretical_loss": 3.4312840193582717, + "tokens_seen": 2205351936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350451354062187, + "loss": 1.3746, + "theoretical_loss": 3.431275749412029, + "tokens_seen": 2205417472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033502507522567706, + "loss": 1.3086, + "theoretical_loss": 3.431267479780339, + "tokens_seen": 2205483008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350050150451354, + "loss": 1.4448, + "theoretical_loss": 3.4312592104631805, + "tokens_seen": 2205548544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349849548645938, + "loss": 1.2769, + "theoretical_loss": 3.431250941460532, + "tokens_seen": 2205614080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033496489468405215, + "loss": 1.3541, + "theoretical_loss": 3.431242672772373, + "tokens_seen": 2205679616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033494483450351056, + "loss": 1.3862, + "theoretical_loss": 3.4312344043986807, + "tokens_seen": 2205745152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349247743229689, + "loss": 1.365, + "theoretical_loss": 3.431226136339435, + "tokens_seen": 2205810688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349047141424273, + "loss": 1.3345, + "theoretical_loss": 3.431217868594614, + "tokens_seen": 2205876224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033488465396188565, + "loss": 1.391, + "theoretical_loss": 3.431209601164197, + "tokens_seen": 2205941760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033486459378134406, + "loss": 1.2849, + "theoretical_loss": 3.431201334048162, + "tokens_seen": 2206007296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348445336008024, + "loss": 1.352, + "theoretical_loss": 3.4311930672464888, + "tokens_seen": 2206072832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348244734202608, + "loss": 1.2794, + "theoretical_loss": 3.4311848007591546, + "tokens_seen": 2206138368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033480441323971915, + "loss": 1.2393, + "theoretical_loss": 3.431176534586139, + "tokens_seen": 2206203904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347843530591775, + "loss": 1.4392, + "theoretical_loss": 3.4311682687274208, + "tokens_seen": 2206269440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033476429287863593, + "loss": 1.2982, + "theoretical_loss": 3.431160003182978, + "tokens_seen": 2206334976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347442326980943, + "loss": 1.2828, + "theoretical_loss": 3.4311517379527903, + "tokens_seen": 2206400512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033472417251755265, + "loss": 1.3521, + "theoretical_loss": 3.431143473036836, + "tokens_seen": 2206466048 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334704112337011, + "loss": 1.2801, + "theoretical_loss": 3.4311352084350935, + "tokens_seen": 2206531584 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033468405215646943, + "loss": 1.2954, + "theoretical_loss": 3.431126944147542, + "tokens_seen": 2206597120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346639919759278, + "loss": 1.3527, + "theoretical_loss": 3.4311186801741598, + "tokens_seen": 2206662656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033464393179538616, + "loss": 1.3429, + "theoretical_loss": 3.4311104165149255, + "tokens_seen": 2206728192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346238716148445, + "loss": 1.3928, + "theoretical_loss": 3.4311021531698187, + "tokens_seen": 2206793728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346038114343029, + "loss": 1.2796, + "theoretical_loss": 3.4310938901388175, + "tokens_seen": 2206859264 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345837512537613, + "loss": 1.3227, + "theoretical_loss": 3.4310856274219006, + "tokens_seen": 2206924800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033456369107321966, + "loss": 1.4448, + "theoretical_loss": 3.4310773650190467, + "tokens_seen": 2206990336 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 803600, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3918547630310059, + "objective/train/theoretical_loss": 3.431069102930235, + "objective/train/tokens_used": 577514976, + "theoretical_loss": 3.431069102930235, + "tokens_seen": 2207055872 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334543630892678, + "loss": 1.3267, + "theoretical_loss": 3.431069102930235, + "tokens_seen": 2207055872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345235707121364, + "loss": 1.308, + "theoretical_loss": 3.4310608411554435, + "tokens_seen": 2207121408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345035105315948, + "loss": 1.341, + "theoretical_loss": 3.4310525796946516, + "tokens_seen": 2207186944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033448345035105316, + "loss": 1.3016, + "theoretical_loss": 3.431044318547838, + "tokens_seen": 2207252480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344633901705115, + "loss": 1.2061, + "theoretical_loss": 3.431036057714981, + "tokens_seen": 2207318016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344433299899699, + "loss": 1.3517, + "theoretical_loss": 3.4310277971960597, + "tokens_seen": 2207383552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033442326980942825, + "loss": 1.3882, + "theoretical_loss": 3.4310195369910526, + "tokens_seen": 2207449088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033440320962888667, + "loss": 1.3396, + "theoretical_loss": 3.431011277099939, + "tokens_seen": 2207514624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033438314944834503, + "loss": 1.3227, + "theoretical_loss": 3.431003017522697, + "tokens_seen": 2207580160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343630892678034, + "loss": 1.3462, + "theoretical_loss": 3.4309947582593057, + "tokens_seen": 2207645696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033434302908726175, + "loss": 1.3299, + "theoretical_loss": 3.4309864993097436, + "tokens_seen": 2207711232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033432296890672017, + "loss": 1.3047, + "theoretical_loss": 3.4309782406739897, + "tokens_seen": 2207776768 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033430290872617853, + "loss": 1.3421, + "theoretical_loss": 3.430969982352023, + "tokens_seen": 2207842304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342828485456369, + "loss": 1.3673, + "theoretical_loss": 3.4309617243438217, + "tokens_seen": 2207907840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033426278836509526, + "loss": 1.2267, + "theoretical_loss": 3.4309534666493646, + "tokens_seen": 2207973376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342427281845536, + "loss": 1.3883, + "theoretical_loss": 3.4309452092686312, + "tokens_seen": 2208038912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033422266800401204, + "loss": 1.4103, + "theoretical_loss": 3.4309369522015993, + "tokens_seen": 2208104448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342026078234704, + "loss": 1.3459, + "theoretical_loss": 3.4309286954482485, + "tokens_seen": 2208169984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033418254764292876, + "loss": 1.2135, + "theoretical_loss": 3.430920439008557, + "tokens_seen": 2208235520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341624874623871, + "loss": 1.3554, + "theoretical_loss": 3.430912182882504, + "tokens_seen": 2208301056 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033414242728184554, + "loss": 1.3319, + "theoretical_loss": 3.4309039270700676, + "tokens_seen": 2208366592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341223671013039, + "loss": 1.2252, + "theoretical_loss": 3.4308956715712275, + "tokens_seen": 2208432128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033410230692076226, + "loss": 1.3462, + "theoretical_loss": 3.430887416385962, + "tokens_seen": 2208497664 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340822467402207, + "loss": 1.3496, + "theoretical_loss": 3.4308791615142495, + "tokens_seen": 2208563200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340621865596791, + "loss": 1.3064, + "theoretical_loss": 3.43087090695607, + "tokens_seen": 2208628736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033404212637913746, + "loss": 1.3807, + "theoretical_loss": 3.430862652711401, + "tokens_seen": 2208694272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340220661985958, + "loss": 1.3175, + "theoretical_loss": 3.4308543987802214, + "tokens_seen": 2208759808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340020060180542, + "loss": 1.3441, + "theoretical_loss": 3.4308461451625107, + "tokens_seen": 2208825344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033398194583751255, + "loss": 1.3901, + "theoretical_loss": 3.430837891858247, + "tokens_seen": 2208890880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033396188565697096, + "loss": 1.4168, + "theoretical_loss": 3.43082963886741, + "tokens_seen": 2208956416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339418254764293, + "loss": 1.3062, + "theoretical_loss": 3.430821386189978, + "tokens_seen": 2209021952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339217652958877, + "loss": 1.3759, + "theoretical_loss": 3.4308131338259296, + "tokens_seen": 2209087488 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033390170511534605, + "loss": 1.2549, + "theoretical_loss": 3.4308048817752437, + "tokens_seen": 2209153024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033388164493480447, + "loss": 1.4478, + "theoretical_loss": 3.430796630037899, + "tokens_seen": 2209218560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033386158475426283, + "loss": 1.4023, + "theoretical_loss": 3.430788378613875, + "tokens_seen": 2209284096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338415245737212, + "loss": 1.4027, + "theoretical_loss": 3.4307801275031498, + "tokens_seen": 2209349632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033382146439317955, + "loss": 1.3824, + "theoretical_loss": 3.4307718767057023, + "tokens_seen": 2209415168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338014042126379, + "loss": 1.4902, + "theoretical_loss": 3.4307636262215113, + "tokens_seen": 2209480704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033378134403209633, + "loss": 1.2937, + "theoretical_loss": 3.4307553760505556, + "tokens_seen": 2209546240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337612838515547, + "loss": 1.3211, + "theoretical_loss": 3.4307471261928146, + "tokens_seen": 2209611776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033374122367101306, + "loss": 1.2264, + "theoretical_loss": 3.4307388766482667, + "tokens_seen": 2209677312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337211634904714, + "loss": 1.3344, + "theoretical_loss": 3.43073062741689, + "tokens_seen": 2209742848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033370110330992983, + "loss": 1.3912, + "theoretical_loss": 3.4307223784986647, + "tokens_seen": 2209808384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336810431293882, + "loss": 1.2656, + "theoretical_loss": 3.430714129893569, + "tokens_seen": 2209873920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033366098294884656, + "loss": 1.3834, + "theoretical_loss": 3.430705881601581, + "tokens_seen": 2209939456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336409227683049, + "loss": 1.2637, + "theoretical_loss": 3.4306976336226804, + "tokens_seen": 2210004992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336208625877633, + "loss": 1.3339, + "theoretical_loss": 3.430689385956846, + "tokens_seen": 2210070528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336008024072217, + "loss": 1.2492, + "theoretical_loss": 3.4306811386040565, + "tokens_seen": 2210136064 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033358074222668006, + "loss": 1.3958, + "theoretical_loss": 3.430672891564291, + "tokens_seen": 2210201600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335606820461384, + "loss": 1.3039, + "theoretical_loss": 3.4306646448375275, + "tokens_seen": 2210267136 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 804834, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.344569444656372, + "objective/train/theoretical_loss": 3.4306563984237455, + "objective/train/tokens_used": 580791776, + "theoretical_loss": 3.4306563984237455, + "tokens_seen": 2210332672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335406218655968, + "loss": 1.2756, + "theoretical_loss": 3.4306563984237455, + "tokens_seen": 2210332672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335205616850552, + "loss": 1.3968, + "theoretical_loss": 3.4306481523229237, + "tokens_seen": 2210398208 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033350050150451356, + "loss": 1.2591, + "theoretical_loss": 3.430639906535041, + "tokens_seen": 2210463744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334804413239719, + "loss": 1.3303, + "theoretical_loss": 3.4306316610600764, + "tokens_seen": 2210529280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334603811434303, + "loss": 1.3544, + "theoretical_loss": 3.4306234158980082, + "tokens_seen": 2210594816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033344032096288865, + "loss": 1.3693, + "theoretical_loss": 3.430615171048816, + "tokens_seen": 2210660352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033342026078234707, + "loss": 1.2581, + "theoretical_loss": 3.4306069265124783, + "tokens_seen": 2210725888 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033340020060180543, + "loss": 1.2585, + "theoretical_loss": 3.4305986822889736, + "tokens_seen": 2210791424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333801404212638, + "loss": 1.3018, + "theoretical_loss": 3.4305904383782813, + "tokens_seen": 2210856960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033336008024072215, + "loss": 1.3816, + "theoretical_loss": 3.4305821947803796, + "tokens_seen": 2210922496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033334002006018057, + "loss": 1.2539, + "theoretical_loss": 3.4305739514952482, + "tokens_seen": 2210988032 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033331995987963893, + "loss": 1.3827, + "theoretical_loss": 3.430565708522866, + "tokens_seen": 2211053568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332998996990973, + "loss": 1.4157, + "theoretical_loss": 3.4305574658632105, + "tokens_seen": 2211119104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033327983951855566, + "loss": 1.2953, + "theoretical_loss": 3.4305492235162616, + "tokens_seen": 2211184640 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333259779338014, + "loss": 1.4728, + "theoretical_loss": 3.4305409814819985, + "tokens_seen": 2211250176 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033323971915747244, + "loss": 1.2951, + "theoretical_loss": 3.4305327397603995, + "tokens_seen": 2211315712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332196589769308, + "loss": 1.2814, + "theoretical_loss": 3.430524498351444, + "tokens_seen": 2211381248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033319959879638916, + "loss": 1.4162, + "theoretical_loss": 3.4305162572551096, + "tokens_seen": 2211446784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331795386158475, + "loss": 1.3947, + "theoretical_loss": 3.430508016471377, + "tokens_seen": 2211512320 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033315947843530594, + "loss": 1.2438, + "theoretical_loss": 3.4304997760002234, + "tokens_seen": 2211577856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331394182547643, + "loss": 1.343, + "theoretical_loss": 3.430491535841629, + "tokens_seen": 2211643392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033311935807422266, + "loss": 1.3006, + "theoretical_loss": 3.4304832959955713, + "tokens_seen": 2211708928 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333099297893681, + "loss": 1.4471, + "theoretical_loss": 3.4304750564620305, + "tokens_seen": 2211774464 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033307923771313944, + "loss": 1.4741, + "theoretical_loss": 3.4304668172409847, + "tokens_seen": 2211840000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330591775325978, + "loss": 1.4021, + "theoretical_loss": 3.4304585783324133, + "tokens_seen": 2211905536 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033303911735205617, + "loss": 1.3122, + "theoretical_loss": 3.430450339736295, + "tokens_seen": 2211971072 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033301905717151453, + "loss": 1.3565, + "theoretical_loss": 3.4304421014526083, + "tokens_seen": 2212036608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329989969909729, + "loss": 1.3251, + "theoretical_loss": 3.430433863481333, + "tokens_seen": 2212102144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329789368104313, + "loss": 1.2868, + "theoretical_loss": 3.430425625822447, + "tokens_seen": 2212167680 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033295887662988967, + "loss": 1.2844, + "theoretical_loss": 3.43041738847593, + "tokens_seen": 2212233216 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033293881644934803, + "loss": 1.3552, + "theoretical_loss": 3.4304091514417605, + "tokens_seen": 2212298752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329187562688064, + "loss": 1.2922, + "theoretical_loss": 3.4304009147199173, + "tokens_seen": 2212364288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328986960882648, + "loss": 1.3541, + "theoretical_loss": 3.430392678310379, + "tokens_seen": 2212429824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328786359077232, + "loss": 1.3362, + "theoretical_loss": 3.4303844422131258, + "tokens_seen": 2212495360 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033285857572718154, + "loss": 1.3668, + "theoretical_loss": 3.430376206428135, + "tokens_seen": 2212560896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328385155466399, + "loss": 1.3328, + "theoretical_loss": 3.4303679709553867, + "tokens_seen": 2212626432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033281845536609826, + "loss": 1.4272, + "theoretical_loss": 3.4303597357948594, + "tokens_seen": 2212691968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327983951855567, + "loss": 1.333, + "theoretical_loss": 3.430351500946532, + "tokens_seen": 2212757504 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033277833500501504, + "loss": 1.3385, + "theoretical_loss": 3.4303432664103832, + "tokens_seen": 2212823040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327582748244734, + "loss": 1.4265, + "theoretical_loss": 3.4303350321863926, + "tokens_seen": 2212888576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033273821464393176, + "loss": 1.3897, + "theoretical_loss": 3.4303267982745385, + "tokens_seen": 2212954112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327181544633902, + "loss": 1.2506, + "theoretical_loss": 3.4303185646747996, + "tokens_seen": 2213019648 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033269809428284854, + "loss": 1.3113, + "theoretical_loss": 3.4303103313871555, + "tokens_seen": 2213085184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326780341023069, + "loss": 1.3464, + "theoretical_loss": 3.430302098411585, + "tokens_seen": 2213150720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033265797392176527, + "loss": 1.3402, + "theoretical_loss": 3.430293865748067, + "tokens_seen": 2213216256 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033263791374122363, + "loss": 1.351, + "theoretical_loss": 3.43028563339658, + "tokens_seen": 2213281792 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033261785356068205, + "loss": 1.271, + "theoretical_loss": 3.4302774013571034, + "tokens_seen": 2213347328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325977933801404, + "loss": 1.3101, + "theoretical_loss": 3.430269169629616, + "tokens_seen": 2213412864 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325777331995988, + "loss": 1.2616, + "theoretical_loss": 3.4302609382140967, + "tokens_seen": 2213478400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325576730190572, + "loss": 1.3884, + "theoretical_loss": 3.430252707110524, + "tokens_seen": 2213543936 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 805966, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3321585655212402, + "objective/train/theoretical_loss": 3.430244476318878, + "objective/train/tokens_used": 584068576, + "theoretical_loss": 3.430244476318878, + "tokens_seen": 2213609472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325376128385156, + "loss": 1.3494, + "theoretical_loss": 3.430244476318878, + "tokens_seen": 2213609472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033251755265797396, + "loss": 1.3688, + "theoretical_loss": 3.4302362458391364, + "tokens_seen": 2213675008 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033249749247743233, + "loss": 1.3037, + "theoretical_loss": 3.4302280156712794, + "tokens_seen": 2213740544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324774322968907, + "loss": 1.4126, + "theoretical_loss": 3.430219785815285, + "tokens_seen": 2213806080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033245737211634905, + "loss": 1.3166, + "theoretical_loss": 3.430211556271132, + "tokens_seen": 2213871616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033243731193580747, + "loss": 1.3076, + "theoretical_loss": 3.4302033270388, + "tokens_seen": 2213937152 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033241725175526583, + "loss": 1.3585, + "theoretical_loss": 3.4301950981182676, + "tokens_seen": 2214002688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323971915747242, + "loss": 1.2564, + "theoretical_loss": 3.430186869509514, + "tokens_seen": 2214068224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033237713139418255, + "loss": 1.3511, + "theoretical_loss": 3.4301786412125184, + "tokens_seen": 2214133760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033235707121364097, + "loss": 1.3087, + "theoretical_loss": 3.4301704132272586, + "tokens_seen": 2214199296 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033233701103309933, + "loss": 1.3705, + "theoretical_loss": 3.4301621855537148, + "tokens_seen": 2214264832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323169508525577, + "loss": 1.351, + "theoretical_loss": 3.4301539581918656, + "tokens_seen": 2214330368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033229689067201606, + "loss": 1.288, + "theoretical_loss": 3.43014573114169, + "tokens_seen": 2214395904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322768304914744, + "loss": 1.3084, + "theoretical_loss": 3.4301375044031666, + "tokens_seen": 2214461440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033225677031093284, + "loss": 1.193, + "theoretical_loss": 3.4301292779762744, + "tokens_seen": 2214526976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322367101303912, + "loss": 1.3568, + "theoretical_loss": 3.430121051860993, + "tokens_seen": 2214592512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033221664994984956, + "loss": 1.3756, + "theoretical_loss": 3.4301128260573006, + "tokens_seen": 2214658048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321965897693079, + "loss": 1.4211, + "theoretical_loss": 3.4301046005651767, + "tokens_seen": 2214723584 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033217652958876634, + "loss": 1.3621, + "theoretical_loss": 3.4300963753846005, + "tokens_seen": 2214789120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321564694082247, + "loss": 1.2664, + "theoretical_loss": 3.4300881505155503, + "tokens_seen": 2214854656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033213640922768306, + "loss": 1.2217, + "theoretical_loss": 3.4300799259580055, + "tokens_seen": 2214920192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321163490471414, + "loss": 1.3188, + "theoretical_loss": 3.4300717017119444, + "tokens_seen": 2214985728 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033209628886659984, + "loss": 1.2905, + "theoretical_loss": 3.430063477777347, + "tokens_seen": 2215051264 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320762286860582, + "loss": 1.3232, + "theoretical_loss": 3.430055254154192, + "tokens_seen": 2215116800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033205616850551657, + "loss": 1.2838, + "theoretical_loss": 3.4300470308424584, + "tokens_seen": 2215182336 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033203610832497493, + "loss": 1.1825, + "theoretical_loss": 3.430038807842125, + "tokens_seen": 2215247872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320160481444333, + "loss": 1.3243, + "theoretical_loss": 3.4300305851531703, + "tokens_seen": 2215313408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319959879638917, + "loss": 1.3269, + "theoretical_loss": 3.4300223627755746, + "tokens_seen": 2215378944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033197592778335007, + "loss": 1.2704, + "theoretical_loss": 3.4300141407093157, + "tokens_seen": 2215444480 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033195586760280843, + "loss": 1.3595, + "theoretical_loss": 3.430005918954373, + "tokens_seen": 2215510016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319358074222668, + "loss": 1.3242, + "theoretical_loss": 3.429997697510726, + "tokens_seen": 2215575552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319157472417252, + "loss": 1.2508, + "theoretical_loss": 3.4299894763783527, + "tokens_seen": 2215641088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318956870611836, + "loss": 1.3513, + "theoretical_loss": 3.429981255557233, + "tokens_seen": 2215706624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033187562688064194, + "loss": 1.4151, + "theoretical_loss": 3.429973035047345, + "tokens_seen": 2215772160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318555667001003, + "loss": 1.4775, + "theoretical_loss": 3.4299648148486686, + "tokens_seen": 2215837696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033183550651955866, + "loss": 1.3503, + "theoretical_loss": 3.4299565949611828, + "tokens_seen": 2215903232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318154463390171, + "loss": 1.4167, + "theoretical_loss": 3.429948375384866, + "tokens_seen": 2215968768 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033179538615847544, + "loss": 1.1877, + "theoretical_loss": 3.429940156119698, + "tokens_seen": 2216034304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317753259779338, + "loss": 1.3089, + "theoretical_loss": 3.4299319371656565, + "tokens_seen": 2216099840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033175526579739216, + "loss": 1.3692, + "theoretical_loss": 3.429923718522722, + "tokens_seen": 2216165376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317352056168506, + "loss": 1.3783, + "theoretical_loss": 3.429915500190873, + "tokens_seen": 2216230912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033171514543630894, + "loss": 1.3537, + "theoretical_loss": 3.429907282170088, + "tokens_seen": 2216296448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316950852557673, + "loss": 1.3731, + "theoretical_loss": 3.4298990644603466, + "tokens_seen": 2216361984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033167502507522567, + "loss": 1.2365, + "theoretical_loss": 3.429890847061628, + "tokens_seen": 2216427520 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033165496489468403, + "loss": 1.3151, + "theoretical_loss": 3.4298826299739105, + "tokens_seen": 2216493056 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033163490471414245, + "loss": 1.2895, + "theoretical_loss": 3.4298744131971737, + "tokens_seen": 2216558592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316148445336008, + "loss": 1.3609, + "theoretical_loss": 3.429866196731396, + "tokens_seen": 2216624128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033159478435305917, + "loss": 1.2262, + "theoretical_loss": 3.429857980576558, + "tokens_seen": 2216689664 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033157472417251753, + "loss": 1.3608, + "theoretical_loss": 3.429849764732637, + "tokens_seen": 2216755200 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033155466399197595, + "loss": 1.2636, + "theoretical_loss": 3.4298415491996126, + "tokens_seen": 2216820736 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 807026, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.281714677810669, + "objective/train/theoretical_loss": 3.4298333339774643, + "objective/train/tokens_used": 587345376, + "theoretical_loss": 3.4298333339774643, + "tokens_seen": 2216886272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315346038114343, + "loss": 1.3427, + "theoretical_loss": 3.4298333339774643, + "tokens_seen": 2216886272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315145436308927, + "loss": 1.2907, + "theoretical_loss": 3.4298251190661704, + "tokens_seen": 2216951808 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033149448345035104, + "loss": 1.2677, + "theoretical_loss": 3.429816904465711, + "tokens_seen": 2217017344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314744232698094, + "loss": 1.4184, + "theoretical_loss": 3.429808690176064, + "tokens_seen": 2217082880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314543630892678, + "loss": 1.2704, + "theoretical_loss": 3.429800476197209, + "tokens_seen": 2217148416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314343029087262, + "loss": 1.2691, + "theoretical_loss": 3.429792262529125, + "tokens_seen": 2217213952 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033141424272818454, + "loss": 1.1882, + "theoretical_loss": 3.4297840491717912, + "tokens_seen": 2217279488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313941825476429, + "loss": 1.3455, + "theoretical_loss": 3.429775836125186, + "tokens_seen": 2217345024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313741223671013, + "loss": 1.4167, + "theoretical_loss": 3.42976762338929, + "tokens_seen": 2217410560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313540621865597, + "loss": 1.355, + "theoretical_loss": 3.4297594109640803, + "tokens_seen": 2217476096 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033133400200601804, + "loss": 1.3368, + "theoretical_loss": 3.4297511988495377, + "tokens_seen": 2217541632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313139418254764, + "loss": 1.2772, + "theoretical_loss": 3.42974298704564, + "tokens_seen": 2217607168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033129388164493477, + "loss": 1.2859, + "theoretical_loss": 3.429734775552367, + "tokens_seen": 2217672704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312738214643932, + "loss": 1.3015, + "theoretical_loss": 3.4297265643696973, + "tokens_seen": 2217738240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033125376128385155, + "loss": 1.3132, + "theoretical_loss": 3.42971835349761, + "tokens_seen": 2217803776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312337011033099, + "loss": 1.1991, + "theoretical_loss": 3.4297101429360852, + "tokens_seen": 2217869312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033121364092276827, + "loss": 1.3119, + "theoretical_loss": 3.4297019326851004, + "tokens_seen": 2217934848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311935807422267, + "loss": 1.3279, + "theoretical_loss": 3.4296937227446356, + "tokens_seen": 2218000384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033117352056168505, + "loss": 1.409, + "theoretical_loss": 3.4296855131146695, + "tokens_seen": 2218065920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311534603811434, + "loss": 1.3572, + "theoretical_loss": 3.4296773037951818, + "tokens_seen": 2218131456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033113340020060177, + "loss": 1.2476, + "theoretical_loss": 3.429669094786151, + "tokens_seen": 2218196992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033111334002006013, + "loss": 1.3752, + "theoretical_loss": 3.4296608860875564, + "tokens_seen": 2218262528 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033109327983951855, + "loss": 1.3066, + "theoretical_loss": 3.429652677699377, + "tokens_seen": 2218328064 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033107321965897697, + "loss": 1.4104, + "theoretical_loss": 3.429644469621592, + "tokens_seen": 2218393600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033105315947843533, + "loss": 1.2898, + "theoretical_loss": 3.4296362618541805, + "tokens_seen": 2218459136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310330992978937, + "loss": 1.3612, + "theoretical_loss": 3.429628054397121, + "tokens_seen": 2218524672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310130391173521, + "loss": 1.2916, + "theoretical_loss": 3.429619847250394, + "tokens_seen": 2218590208 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033099297893681047, + "loss": 1.3593, + "theoretical_loss": 3.4296116404139774, + "tokens_seen": 2218655744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033097291875626883, + "loss": 1.2953, + "theoretical_loss": 3.4296034338878503, + "tokens_seen": 2218721280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309528585757272, + "loss": 1.2146, + "theoretical_loss": 3.4295952276719923, + "tokens_seen": 2218786816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309327983951856, + "loss": 1.3254, + "theoretical_loss": 3.4295870217663826, + "tokens_seen": 2218852352 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330912738214644, + "loss": 1.3326, + "theoretical_loss": 3.4295788161709995, + "tokens_seen": 2218917888 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033089267803410234, + "loss": 1.3057, + "theoretical_loss": 3.4295706108858233, + "tokens_seen": 2218983424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308726178535607, + "loss": 1.2973, + "theoretical_loss": 3.4295624059108323, + "tokens_seen": 2219048960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033085255767301906, + "loss": 1.3042, + "theoretical_loss": 3.4295542012460056, + "tokens_seen": 2219114496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308324974924775, + "loss": 1.2186, + "theoretical_loss": 3.4295459968913224, + "tokens_seen": 2219180032 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033081243731193584, + "loss": 1.2339, + "theoretical_loss": 3.4295377928467623, + "tokens_seen": 2219245568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307923771313942, + "loss": 1.3124, + "theoretical_loss": 3.429529589112304, + "tokens_seen": 2219311104 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033077231695085256, + "loss": 1.4591, + "theoretical_loss": 3.4295213856879263, + "tokens_seen": 2219376640 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330752256770311, + "loss": 1.2751, + "theoretical_loss": 3.4295131825736087, + "tokens_seen": 2219442176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033073219658976934, + "loss": 1.3162, + "theoretical_loss": 3.4295049797693307, + "tokens_seen": 2219507712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307121364092277, + "loss": 1.2756, + "theoretical_loss": 3.429496777275071, + "tokens_seen": 2219573248 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033069207622868607, + "loss": 1.249, + "theoretical_loss": 3.4294885750908084, + "tokens_seen": 2219638784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033067201604814443, + "loss": 1.409, + "theoretical_loss": 3.4294803732165224, + "tokens_seen": 2219704320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033065195586760285, + "loss": 1.2564, + "theoretical_loss": 3.4294721716521925, + "tokens_seen": 2219769856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306318956870612, + "loss": 1.2923, + "theoretical_loss": 3.4294639703977974, + "tokens_seen": 2219835392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033061183550651957, + "loss": 1.2263, + "theoretical_loss": 3.429455769453316, + "tokens_seen": 2219900928 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033059177532597793, + "loss": 1.2222, + "theoretical_loss": 3.4294475688187283, + "tokens_seen": 2219966464 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033057171514543635, + "loss": 1.3128, + "theoretical_loss": 3.4294393684940125, + "tokens_seen": 2220032000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305516549648947, + "loss": 1.2149, + "theoretical_loss": 3.429431168479148, + "tokens_seen": 2220097536 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 808099, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.131688117980957, + "objective/train/theoretical_loss": 3.4294229687741145, + "objective/train/tokens_used": 590622176, + "theoretical_loss": 3.4294229687741145, + "tokens_seen": 2220163072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305315947843531, + "loss": 1.3087, + "theoretical_loss": 3.4294229687741145, + "tokens_seen": 2220163072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033051153460381144, + "loss": 1.3767, + "theoretical_loss": 3.4294147693788903, + "tokens_seen": 2220228608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304914744232698, + "loss": 1.3219, + "theoretical_loss": 3.4294065702934553, + "tokens_seen": 2220294144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304714142427282, + "loss": 1.2563, + "theoretical_loss": 3.429398371517788, + "tokens_seen": 2220359680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304513540621866, + "loss": 1.3356, + "theoretical_loss": 3.4293901730518686, + "tokens_seen": 2220425216 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033043129388164494, + "loss": 1.329, + "theoretical_loss": 3.4293819748956746, + "tokens_seen": 2220490752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304112337011033, + "loss": 1.3432, + "theoretical_loss": 3.4293737770491868, + "tokens_seen": 2220556288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303911735205617, + "loss": 1.2847, + "theoretical_loss": 3.429365579512383, + "tokens_seen": 2220621824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303711133400201, + "loss": 1.3271, + "theoretical_loss": 3.429357382285244, + "tokens_seen": 2220687360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033035105315947844, + "loss": 1.359, + "theoretical_loss": 3.429349185367747, + "tokens_seen": 2220752896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303309929789368, + "loss": 1.4198, + "theoretical_loss": 3.429340988759873, + "tokens_seen": 2220818432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033031093279839517, + "loss": 1.3725, + "theoretical_loss": 3.4293327924615995, + "tokens_seen": 2220883968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302908726178536, + "loss": 1.3385, + "theoretical_loss": 3.429324596472907, + "tokens_seen": 2220949504 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033027081243731195, + "loss": 1.347, + "theoretical_loss": 3.429316400793774, + "tokens_seen": 2221015040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302507522567703, + "loss": 1.4334, + "theoretical_loss": 3.4293082054241797, + "tokens_seen": 2221080576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033023069207622867, + "loss": 1.4178, + "theoretical_loss": 3.429300010364104, + "tokens_seen": 2221146112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302106318956871, + "loss": 1.2217, + "theoretical_loss": 3.429291815613525, + "tokens_seen": 2221211648 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033019057171514545, + "loss": 1.3204, + "theoretical_loss": 3.4292836211724222, + "tokens_seen": 2221277184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301705115346038, + "loss": 1.2518, + "theoretical_loss": 3.4292754270407753, + "tokens_seen": 2221342720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301504513540622, + "loss": 1.3269, + "theoretical_loss": 3.429267233218563, + "tokens_seen": 2221408256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033013039117352054, + "loss": 1.3638, + "theoretical_loss": 3.4292590397057645, + "tokens_seen": 2221473792 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033011033099297895, + "loss": 1.256, + "theoretical_loss": 3.4292508465023595, + "tokens_seen": 2221539328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300902708124373, + "loss": 1.3411, + "theoretical_loss": 3.4292426536083265, + "tokens_seen": 2221604864 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300702106318957, + "loss": 1.2805, + "theoretical_loss": 3.429234461023645, + "tokens_seen": 2221670400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033005015045135404, + "loss": 1.3341, + "theoretical_loss": 3.429226268748294, + "tokens_seen": 2221735936 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033003009027081245, + "loss": 1.2051, + "theoretical_loss": 3.4292180767822535, + "tokens_seen": 2221801472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300100300902708, + "loss": 1.373, + "theoretical_loss": 3.429209885125502, + "tokens_seen": 2221867008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299899699097292, + "loss": 1.2722, + "theoretical_loss": 3.429201693778018, + "tokens_seen": 2221932544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032996990972918754, + "loss": 1.2885, + "theoretical_loss": 3.429193502739782, + "tokens_seen": 2221998080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299498495486459, + "loss": 1.3181, + "theoretical_loss": 3.4291853120107727, + "tokens_seen": 2222063616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299297893681043, + "loss": 1.2666, + "theoretical_loss": 3.4291771215909694, + "tokens_seen": 2222129152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299097291875627, + "loss": 1.4222, + "theoretical_loss": 3.4291689314803513, + "tokens_seen": 2222194688 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032988966900702104, + "loss": 1.3218, + "theoretical_loss": 3.4291607416788974, + "tokens_seen": 2222260224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298696088264794, + "loss": 1.3142, + "theoretical_loss": 3.429152552186587, + "tokens_seen": 2222325760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298495486459378, + "loss": 1.3737, + "theoretical_loss": 3.4291443630033993, + "tokens_seen": 2222391296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298294884653962, + "loss": 1.3868, + "theoretical_loss": 3.4291361741293134, + "tokens_seen": 2222456832 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032980942828485455, + "loss": 1.3815, + "theoretical_loss": 3.4291279855643086, + "tokens_seen": 2222522368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297893681043129, + "loss": 1.2976, + "theoretical_loss": 3.4291197973083647, + "tokens_seen": 2222587904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297693079237713, + "loss": 1.3086, + "theoretical_loss": 3.42911160936146, + "tokens_seen": 2222653440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297492477432297, + "loss": 1.3181, + "theoretical_loss": 3.4291034217235747, + "tokens_seen": 2222718976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032972918756268805, + "loss": 1.3175, + "theoretical_loss": 3.429095234394687, + "tokens_seen": 2222784512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297091273821464, + "loss": 1.3317, + "theoretical_loss": 3.4290870473747765, + "tokens_seen": 2222850048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296890672016048, + "loss": 1.28, + "theoretical_loss": 3.429078860663823, + "tokens_seen": 2222915584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296690070210632, + "loss": 1.3931, + "theoretical_loss": 3.429070674261805, + "tokens_seen": 2222981120 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032964894684052155, + "loss": 1.3234, + "theoretical_loss": 3.429062488168702, + "tokens_seen": 2223046656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296288866599799, + "loss": 1.3378, + "theoretical_loss": 3.4290543023844933, + "tokens_seen": 2223112192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296088264794383, + "loss": 1.2408, + "theoretical_loss": 3.429046116909158, + "tokens_seen": 2223177728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295887662988967, + "loss": 1.2164, + "theoretical_loss": 3.429037931742675, + "tokens_seen": 2223243264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295687061183551, + "loss": 1.3058, + "theoretical_loss": 3.429029746885025, + "tokens_seen": 2223308800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295486459378135, + "loss": 1.4393, + "theoretical_loss": 3.4290215623361853, + "tokens_seen": 2223374336 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 809438, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2450859546661377, + "objective/train/theoretical_loss": 3.4290133780961365, + "objective/train/tokens_used": 593898976, + "theoretical_loss": 3.4290133780961365, + "tokens_seen": 2223439872 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032952858575727184, + "loss": 1.3189, + "theoretical_loss": 3.4290133780961365, + "tokens_seen": 2223439872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295085255767302, + "loss": 1.296, + "theoretical_loss": 3.429005194164857, + "tokens_seen": 2223505408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294884653961886, + "loss": 1.422, + "theoretical_loss": 3.428997010542327, + "tokens_seen": 2223570944 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329468405215647, + "loss": 1.3577, + "theoretical_loss": 3.4289888272285247, + "tokens_seen": 2223636480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032944834503510534, + "loss": 1.316, + "theoretical_loss": 3.4289806442234303, + "tokens_seen": 2223702016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294282848545637, + "loss": 1.4083, + "theoretical_loss": 3.4289724615270223, + "tokens_seen": 2223767552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294082246740221, + "loss": 1.3205, + "theoretical_loss": 3.4289642791392803, + "tokens_seen": 2223833088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293881644934805, + "loss": 1.4501, + "theoretical_loss": 3.428956097060184, + "tokens_seen": 2223898624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032936810431293884, + "loss": 1.4362, + "theoretical_loss": 3.4289479152897115, + "tokens_seen": 2223964160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293480441323972, + "loss": 1.2713, + "theoretical_loss": 3.428939733827843, + "tokens_seen": 2224029696 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032932798395185557, + "loss": 1.2722, + "theoretical_loss": 3.4289315526745576, + "tokens_seen": 2224095232 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329307923771314, + "loss": 1.3474, + "theoretical_loss": 3.4289233718298346, + "tokens_seen": 2224160768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032928786359077235, + "loss": 1.353, + "theoretical_loss": 3.428915191293653, + "tokens_seen": 2224226304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292678034102307, + "loss": 1.4524, + "theoretical_loss": 3.4289070110659923, + "tokens_seen": 2224291840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032924774322968907, + "loss": 1.4401, + "theoretical_loss": 3.428898831146832, + "tokens_seen": 2224357376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292276830491475, + "loss": 1.3023, + "theoretical_loss": 3.428890651536151, + "tokens_seen": 2224422912 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032920762286860585, + "loss": 1.1897, + "theoretical_loss": 3.428882472233928, + "tokens_seen": 2224488448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291875626880642, + "loss": 1.3022, + "theoretical_loss": 3.4288742932401437, + "tokens_seen": 2224553984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291675025075226, + "loss": 1.3435, + "theoretical_loss": 3.4288661145547765, + "tokens_seen": 2224619520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032914744232698094, + "loss": 1.2625, + "theoretical_loss": 3.4288579361778053, + "tokens_seen": 2224685056 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032912738214643935, + "loss": 1.32, + "theoretical_loss": 3.4288497581092106, + "tokens_seen": 2224750592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291073219658977, + "loss": 1.2711, + "theoretical_loss": 3.428841580348971, + "tokens_seen": 2224816128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290872617853561, + "loss": 1.387, + "theoretical_loss": 3.428833402897065, + "tokens_seen": 2224881664 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032906720160481444, + "loss": 1.4263, + "theoretical_loss": 3.4288252257534735, + "tokens_seen": 2224947200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032904714142427286, + "loss": 1.4015, + "theoretical_loss": 3.4288170489181744, + "tokens_seen": 2225012736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290270812437312, + "loss": 1.3984, + "theoretical_loss": 3.428808872391148, + "tokens_seen": 2225078272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290070210631896, + "loss": 1.3592, + "theoretical_loss": 3.428800696172373, + "tokens_seen": 2225143808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032898696088264794, + "loss": 1.4372, + "theoretical_loss": 3.428792520261829, + "tokens_seen": 2225209344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289669007021063, + "loss": 1.2257, + "theoretical_loss": 3.428784344659495, + "tokens_seen": 2225274880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289468405215647, + "loss": 1.3395, + "theoretical_loss": 3.4287761693653502, + "tokens_seen": 2225340416 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289267803410231, + "loss": 1.3004, + "theoretical_loss": 3.4287679943793745, + "tokens_seen": 2225405952 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032890672016048145, + "loss": 1.2406, + "theoretical_loss": 3.428759819701547, + "tokens_seen": 2225471488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288866599799398, + "loss": 1.352, + "theoretical_loss": 3.4287516453318467, + "tokens_seen": 2225537024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288665997993982, + "loss": 1.2338, + "theoretical_loss": 3.4287434712702534, + "tokens_seen": 2225602560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288465396188566, + "loss": 1.3808, + "theoretical_loss": 3.428735297516746, + "tokens_seen": 2225668096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032882647943831495, + "loss": 1.2405, + "theoretical_loss": 3.4287271240713038, + "tokens_seen": 2225733632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288064192577733, + "loss": 1.4105, + "theoretical_loss": 3.4287189509339067, + "tokens_seen": 2225799168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287863590772317, + "loss": 1.2982, + "theoretical_loss": 3.428710778104533, + "tokens_seen": 2225864704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287662988966901, + "loss": 1.3408, + "theoretical_loss": 3.4287026055831626, + "tokens_seen": 2225930240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032874623871614845, + "loss": 1.194, + "theoretical_loss": 3.4286944333697753, + "tokens_seen": 2225995776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287261785356068, + "loss": 1.3261, + "theoretical_loss": 3.4286862614643496, + "tokens_seen": 2226061312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287061183550652, + "loss": 1.2783, + "theoretical_loss": 3.4286780898668656, + "tokens_seen": 2226126848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286860581745236, + "loss": 1.39, + "theoretical_loss": 3.428669918577302, + "tokens_seen": 2226192384 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032866599799398195, + "loss": 1.2794, + "theoretical_loss": 3.4286617475956382, + "tokens_seen": 2226257920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286459378134403, + "loss": 1.3567, + "theoretical_loss": 3.428653576921854, + "tokens_seen": 2226323456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286258776328987, + "loss": 1.4562, + "theoretical_loss": 3.428645406555928, + "tokens_seen": 2226388992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286058174523571, + "loss": 1.3584, + "theoretical_loss": 3.42863723649784, + "tokens_seen": 2226454528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032858575727181546, + "loss": 1.3346, + "theoretical_loss": 3.42862906674757, + "tokens_seen": 2226520064 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285656970912738, + "loss": 1.1057, + "theoretical_loss": 3.428620897305096, + "tokens_seen": 2226585600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285456369107322, + "loss": 1.3695, + "theoretical_loss": 3.4286127281703984, + "tokens_seen": 2226651136 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 810670, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.169150471687317, + "objective/train/theoretical_loss": 3.428604559343456, + "objective/train/tokens_used": 597175776, + "theoretical_loss": 3.428604559343456, + "tokens_seen": 2226716672 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032852557673019054, + "loss": 1.2969, + "theoretical_loss": 3.428604559343456, + "tokens_seen": 2226716672 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032850551654964896, + "loss": 1.3127, + "theoretical_loss": 3.4285963908242483, + "tokens_seen": 2226782208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284854563691073, + "loss": 1.3027, + "theoretical_loss": 3.4285882226127544, + "tokens_seen": 2226847744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284653961885657, + "loss": 1.321, + "theoretical_loss": 3.428580054708954, + "tokens_seen": 2226913280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032844533600802405, + "loss": 1.3251, + "theoretical_loss": 3.428571887112826, + "tokens_seen": 2226978816 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032842527582748246, + "loss": 1.4209, + "theoretical_loss": 3.428563719824351, + "tokens_seen": 2227044352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284052156469408, + "loss": 1.3158, + "theoretical_loss": 3.4285555528435068, + "tokens_seen": 2227109888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283851554663992, + "loss": 1.291, + "theoretical_loss": 3.4285473861702735, + "tokens_seen": 2227175424 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032836509528585755, + "loss": 1.2967, + "theoretical_loss": 3.4285392198046303, + "tokens_seen": 2227240960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283450351053159, + "loss": 1.3661, + "theoretical_loss": 3.428531053746557, + "tokens_seen": 2227306496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032832497492477433, + "loss": 1.2764, + "theoretical_loss": 3.4285228879960323, + "tokens_seen": 2227372032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283049147442327, + "loss": 1.3949, + "theoretical_loss": 3.428514722553036, + "tokens_seen": 2227437568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032828485456369105, + "loss": 1.2696, + "theoretical_loss": 3.4285065574175473, + "tokens_seen": 2227503104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282647943831494, + "loss": 1.2822, + "theoretical_loss": 3.428498392589546, + "tokens_seen": 2227568640 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032824473420260783, + "loss": 1.3233, + "theoretical_loss": 3.428490228069011, + "tokens_seen": 2227634176 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282246740220662, + "loss": 1.2422, + "theoretical_loss": 3.4284820638559212, + "tokens_seen": 2227699712 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032820461384152456, + "loss": 1.4608, + "theoretical_loss": 3.428473899950257, + "tokens_seen": 2227765248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281845536609829, + "loss": 1.2156, + "theoretical_loss": 3.4284657363519973, + "tokens_seen": 2227830784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281644934804413, + "loss": 1.3026, + "theoretical_loss": 3.4284575730611215, + "tokens_seen": 2227896320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281444332998997, + "loss": 1.4539, + "theoretical_loss": 3.428449410077609, + "tokens_seen": 2227961856 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032812437311935806, + "loss": 1.2724, + "theoretical_loss": 3.428441247401439, + "tokens_seen": 2228027392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281043129388164, + "loss": 1.3554, + "theoretical_loss": 3.4284330850325917, + "tokens_seen": 2228092928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280842527582748, + "loss": 1.2716, + "theoretical_loss": 3.428424922971045, + "tokens_seen": 2228158464 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032806419257773326, + "loss": 1.2671, + "theoretical_loss": 3.4284167612167797, + "tokens_seen": 2228224000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280441323971916, + "loss": 1.209, + "theoretical_loss": 3.428408599769775, + "tokens_seen": 2228289536 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032802407221665, + "loss": 1.3737, + "theoretical_loss": 3.428400438630009, + "tokens_seen": 2228355072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032800401203610834, + "loss": 1.3199, + "theoretical_loss": 3.428392277797463, + "tokens_seen": 2228420608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279839518555667, + "loss": 1.418, + "theoretical_loss": 3.428384117272115, + "tokens_seen": 2228486144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279638916750251, + "loss": 1.2577, + "theoretical_loss": 3.428375957053945, + "tokens_seen": 2228551680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279438314944835, + "loss": 1.3428, + "theoretical_loss": 3.428367797142932, + "tokens_seen": 2228617216 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032792377131394185, + "loss": 1.3765, + "theoretical_loss": 3.4283596375390557, + "tokens_seen": 2228682752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279037111334002, + "loss": 1.3115, + "theoretical_loss": 3.4283514782422957, + "tokens_seen": 2228748288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278836509528586, + "loss": 1.2102, + "theoretical_loss": 3.4283433192526314, + "tokens_seen": 2228813824 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327863590772317, + "loss": 1.3156, + "theoretical_loss": 3.4283351605700414, + "tokens_seen": 2228879360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032784353059177535, + "loss": 1.3174, + "theoretical_loss": 3.4283270021945063, + "tokens_seen": 2228944896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278234704112337, + "loss": 1.307, + "theoretical_loss": 3.4283188441260046, + "tokens_seen": 2229010432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032780341023069213, + "loss": 1.3305, + "theoretical_loss": 3.428310686364516, + "tokens_seen": 2229075968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277833500501505, + "loss": 1.4379, + "theoretical_loss": 3.4283025289100197, + "tokens_seen": 2229141504 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032776328986960885, + "loss": 1.3166, + "theoretical_loss": 3.428294371762496, + "tokens_seen": 2229207040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277432296890672, + "loss": 1.35, + "theoretical_loss": 3.4282862149219233, + "tokens_seen": 2229272576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277231695085256, + "loss": 1.3105, + "theoretical_loss": 3.4282780583882815, + "tokens_seen": 2229338112 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327703109327984, + "loss": 1.3376, + "theoretical_loss": 3.42826990216155, + "tokens_seen": 2229403648 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032768304914744235, + "loss": 1.428, + "theoretical_loss": 3.4282617462417084, + "tokens_seen": 2229469184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276629889669007, + "loss": 1.3062, + "theoretical_loss": 3.428253590628736, + "tokens_seen": 2229534720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276429287863591, + "loss": 1.3647, + "theoretical_loss": 3.428245435322612, + "tokens_seen": 2229600256 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276228686058175, + "loss": 1.3005, + "theoretical_loss": 3.428237280323316, + "tokens_seen": 2229665792 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032760280842527586, + "loss": 1.2641, + "theoretical_loss": 3.4282291256308275, + "tokens_seen": 2229731328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275827482447342, + "loss": 1.4044, + "theoretical_loss": 3.4282209712451257, + "tokens_seen": 2229796864 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275626880641926, + "loss": 1.2942, + "theoretical_loss": 3.42821281716619, + "tokens_seen": 2229862400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032754262788365094, + "loss": 1.3273, + "theoretical_loss": 3.4282046633940007, + "tokens_seen": 2229927936 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 811859, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4204270839691162, + "objective/train/theoretical_loss": 3.428196509928536, + "objective/train/tokens_used": 600452576, + "theoretical_loss": 3.428196509928536, + "tokens_seen": 2229993472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032752256770310936, + "loss": 1.3451, + "theoretical_loss": 3.428196509928536, + "tokens_seen": 2229993472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275025075225677, + "loss": 1.2626, + "theoretical_loss": 3.4281883567697764, + "tokens_seen": 2230059008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274824473420261, + "loss": 1.4101, + "theoretical_loss": 3.4281802039177007, + "tokens_seen": 2230124544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032746238716148445, + "loss": 1.406, + "theoretical_loss": 3.4281720513722886, + "tokens_seen": 2230190080 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032744232698094286, + "loss": 1.2918, + "theoretical_loss": 3.4281638991335197, + "tokens_seen": 2230255616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274222668004012, + "loss": 1.2664, + "theoretical_loss": 3.428155747201373, + "tokens_seen": 2230321152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274022066198596, + "loss": 1.3028, + "theoretical_loss": 3.4281475955758287, + "tokens_seen": 2230386688 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032738214643931795, + "loss": 1.3383, + "theoretical_loss": 3.4281394442568653, + "tokens_seen": 2230452224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273620862587763, + "loss": 1.3342, + "theoretical_loss": 3.428131293244463, + "tokens_seen": 2230517760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032734202607823473, + "loss": 1.3421, + "theoretical_loss": 3.4281231425386007, + "tokens_seen": 2230583296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273219658976931, + "loss": 1.3, + "theoretical_loss": 3.4281149921392586, + "tokens_seen": 2230648832 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032730190571715145, + "loss": 1.2941, + "theoretical_loss": 3.4281068420464154, + "tokens_seen": 2230714368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272818455366098, + "loss": 1.3592, + "theoretical_loss": 3.428098692260051, + "tokens_seen": 2230779904 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032726178535606823, + "loss": 1.3582, + "theoretical_loss": 3.428090542780145, + "tokens_seen": 2230845440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272417251755266, + "loss": 1.2604, + "theoretical_loss": 3.4280823936066764, + "tokens_seen": 2230910976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032722166499498496, + "loss": 1.4166, + "theoretical_loss": 3.428074244739625, + "tokens_seen": 2230976512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272016048144433, + "loss": 1.2577, + "theoretical_loss": 3.4280660961789704, + "tokens_seen": 2231042048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271815446339017, + "loss": 1.2681, + "theoretical_loss": 3.4280579479246915, + "tokens_seen": 2231107584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271614844533601, + "loss": 1.2704, + "theoretical_loss": 3.428049799976769, + "tokens_seen": 2231173120 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032714142427281846, + "loss": 1.2891, + "theoretical_loss": 3.428041652335181, + "tokens_seen": 2231238656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271213640922768, + "loss": 1.2308, + "theoretical_loss": 3.4280335049999073, + "tokens_seen": 2231304192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271013039117352, + "loss": 1.2789, + "theoretical_loss": 3.428025357970928, + "tokens_seen": 2231369728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270812437311936, + "loss": 1.2353, + "theoretical_loss": 3.428017211248222, + "tokens_seen": 2231435264 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032706118355065196, + "loss": 1.3457, + "theoretical_loss": 3.4280090648317696, + "tokens_seen": 2231500800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270411233701103, + "loss": 1.3621, + "theoretical_loss": 3.428000918721549, + "tokens_seen": 2231566336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270210631895687, + "loss": 1.351, + "theoretical_loss": 3.427992772917541, + "tokens_seen": 2231631872 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032700100300902705, + "loss": 1.426, + "theoretical_loss": 3.4279846274197237, + "tokens_seen": 2231697408 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032698094282848547, + "loss": 1.3148, + "theoretical_loss": 3.427976482228078, + "tokens_seen": 2231762944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032696088264794383, + "loss": 1.4199, + "theoretical_loss": 3.427968337342583, + "tokens_seen": 2231828480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269408224674022, + "loss": 1.2463, + "theoretical_loss": 3.4279601927632175, + "tokens_seen": 2231894016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032692076228686055, + "loss": 1.379, + "theoretical_loss": 3.4279520484899617, + "tokens_seen": 2231959552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032690070210631897, + "loss": 1.3451, + "theoretical_loss": 3.4279439045227953, + "tokens_seen": 2232025088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032688064192577733, + "loss": 1.3589, + "theoretical_loss": 3.427935760861697, + "tokens_seen": 2232090624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268605817452357, + "loss": 1.3003, + "theoretical_loss": 3.4279276175066467, + "tokens_seen": 2232156160 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032684052156469406, + "loss": 1.4179, + "theoretical_loss": 3.427919474457624, + "tokens_seen": 2232221696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268204613841524, + "loss": 1.2903, + "theoretical_loss": 3.4279113317146086, + "tokens_seen": 2232287232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032680040120361084, + "loss": 1.3979, + "theoretical_loss": 3.4279031892775795, + "tokens_seen": 2232352768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267803410230692, + "loss": 1.3002, + "theoretical_loss": 3.427895047146517, + "tokens_seen": 2232418304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032676028084252756, + "loss": 1.3035, + "theoretical_loss": 3.4278869053213996, + "tokens_seen": 2232483840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267402206619859, + "loss": 1.3255, + "theoretical_loss": 3.4278787638022075, + "tokens_seen": 2232549376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032672016048144434, + "loss": 1.3246, + "theoretical_loss": 3.4278706225889204, + "tokens_seen": 2232614912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267001003009027, + "loss": 1.282, + "theoretical_loss": 3.427862481681517, + "tokens_seen": 2232680448 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032668004012036106, + "loss": 1.323, + "theoretical_loss": 3.4278543410799776, + "tokens_seen": 2232745984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266599799398194, + "loss": 1.3203, + "theoretical_loss": 3.4278462007842814, + "tokens_seen": 2232811520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032663991975927784, + "loss": 1.318, + "theoretical_loss": 3.427838060794408, + "tokens_seen": 2232877056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266198595787362, + "loss": 1.3438, + "theoretical_loss": 3.427829921110337, + "tokens_seen": 2232942592 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032659979939819457, + "loss": 1.282, + "theoretical_loss": 3.4278217817320478, + "tokens_seen": 2233008128 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032657973921765293, + "loss": 1.3253, + "theoretical_loss": 3.42781364265952, + "tokens_seen": 2233073664 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032655967903711135, + "loss": 1.3459, + "theoretical_loss": 3.4278055038927335, + "tokens_seen": 2233139200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032653961885656976, + "loss": 1.4018, + "theoretical_loss": 3.427797365431667, + "tokens_seen": 2233204736 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 813220, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3615901470184326, + "objective/train/theoretical_loss": 3.4277892272763006, + "objective/train/tokens_used": 603729376, + "theoretical_loss": 3.4277892272763006, + "tokens_seen": 2233270272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265195586760281, + "loss": 1.2716, + "theoretical_loss": 3.4277892272763006, + "tokens_seen": 2233270272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264994984954865, + "loss": 1.3807, + "theoretical_loss": 3.427781089426614, + "tokens_seen": 2233335808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032647943831494485, + "loss": 1.3369, + "theoretical_loss": 3.4277729518825866, + "tokens_seen": 2233401344 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032645937813440326, + "loss": 1.2356, + "theoretical_loss": 3.4277648146441977, + "tokens_seen": 2233466880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264393179538616, + "loss": 1.3094, + "theoretical_loss": 3.427756677711427, + "tokens_seen": 2233532416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032641925777332, + "loss": 1.3786, + "theoretical_loss": 3.427748541084254, + "tokens_seen": 2233597952 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032639919759277835, + "loss": 1.308, + "theoretical_loss": 3.427740404762659, + "tokens_seen": 2233663488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263791374122367, + "loss": 1.2941, + "theoretical_loss": 3.4277322687466203, + "tokens_seen": 2233729024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032635907723169513, + "loss": 1.2268, + "theoretical_loss": 3.427724133036118, + "tokens_seen": 2233794560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263390170511535, + "loss": 1.2973, + "theoretical_loss": 3.427715997631132, + "tokens_seen": 2233860096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032631895687061185, + "loss": 1.2539, + "theoretical_loss": 3.4277078625316415, + "tokens_seen": 2233925632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262988966900702, + "loss": 1.3549, + "theoretical_loss": 3.427699727737626, + "tokens_seen": 2233991168 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032627883650952863, + "loss": 1.2435, + "theoretical_loss": 3.4276915932490657, + "tokens_seen": 2234056704 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326258776328987, + "loss": 1.2663, + "theoretical_loss": 3.4276834590659395, + "tokens_seen": 2234122240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032623871614844536, + "loss": 1.4961, + "theoretical_loss": 3.4276753251882273, + "tokens_seen": 2234187776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262186559679037, + "loss": 1.3321, + "theoretical_loss": 3.427667191615908, + "tokens_seen": 2234253312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261985957873621, + "loss": 1.4481, + "theoretical_loss": 3.4276590583489623, + "tokens_seen": 2234318848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261785356068205, + "loss": 1.3234, + "theoretical_loss": 3.427650925387369, + "tokens_seen": 2234384384 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032615847542627886, + "loss": 1.4162, + "theoretical_loss": 3.4276427927311075, + "tokens_seen": 2234449920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261384152457372, + "loss": 1.2544, + "theoretical_loss": 3.427634660380158, + "tokens_seen": 2234515456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261183550651956, + "loss": 1.3533, + "theoretical_loss": 3.4276265283345, + "tokens_seen": 2234580992 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326098294884654, + "loss": 1.3068, + "theoretical_loss": 3.427618396594113, + "tokens_seen": 2234646528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032607823470411236, + "loss": 1.3381, + "theoretical_loss": 3.4276102651589766, + "tokens_seen": 2234712064 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260581745235707, + "loss": 1.2924, + "theoretical_loss": 3.42760213402907, + "tokens_seen": 2234777600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260381143430291, + "loss": 1.3524, + "theoretical_loss": 3.4275940032043732, + "tokens_seen": 2234843136 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032601805416248745, + "loss": 1.3387, + "theoretical_loss": 3.427585872684866, + "tokens_seen": 2234908672 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032599799398194587, + "loss": 1.4948, + "theoretical_loss": 3.427577742470527, + "tokens_seen": 2234974208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032597793380140423, + "loss": 1.3096, + "theoretical_loss": 3.4275696125613373, + "tokens_seen": 2235039744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259578736208626, + "loss": 1.2825, + "theoretical_loss": 3.427561482957275, + "tokens_seen": 2235105280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032593781344032095, + "loss": 1.3896, + "theoretical_loss": 3.427553353658321, + "tokens_seen": 2235170816 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032591775325977937, + "loss": 1.2537, + "theoretical_loss": 3.427545224664454, + "tokens_seen": 2235236352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032589769307923773, + "loss": 1.3955, + "theoretical_loss": 3.4275370959756537, + "tokens_seen": 2235301888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258776328986961, + "loss": 1.3798, + "theoretical_loss": 3.4275289675919, + "tokens_seen": 2235367424 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032585757271815446, + "loss": 1.242, + "theoretical_loss": 3.4275208395131727, + "tokens_seen": 2235432960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258375125376128, + "loss": 1.3125, + "theoretical_loss": 3.427512711739451, + "tokens_seen": 2235498496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032581745235707124, + "loss": 1.3612, + "theoretical_loss": 3.427504584270715, + "tokens_seen": 2235564032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003257973921765296, + "loss": 1.3237, + "theoretical_loss": 3.427496457106943, + "tokens_seen": 2235629568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032577733199598796, + "loss": 1.3474, + "theoretical_loss": 3.4274883302481163, + "tokens_seen": 2235695104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257572718154463, + "loss": 1.4031, + "theoretical_loss": 3.4274802036942136, + "tokens_seen": 2235760640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032573721163490474, + "loss": 1.314, + "theoretical_loss": 3.427472077445215, + "tokens_seen": 2235826176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257171514543631, + "loss": 1.3287, + "theoretical_loss": 3.4274639515010996, + "tokens_seen": 2235891712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032569709127382146, + "loss": 1.2919, + "theoretical_loss": 3.4274558258618475, + "tokens_seen": 2235957248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256770310932798, + "loss": 1.3112, + "theoretical_loss": 3.4274477005274377, + "tokens_seen": 2236022784 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032565697091273824, + "loss": 1.2952, + "theoretical_loss": 3.4274395754978504, + "tokens_seen": 2236088320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256369107321966, + "loss": 1.2735, + "theoretical_loss": 3.427431450773065, + "tokens_seen": 2236153856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032561685055165497, + "loss": 1.2751, + "theoretical_loss": 3.4274233263530616, + "tokens_seen": 2236219392 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032559679037111333, + "loss": 1.3584, + "theoretical_loss": 3.427415202237819, + "tokens_seen": 2236284928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255767301905717, + "loss": 1.3446, + "theoretical_loss": 3.427407078427317, + "tokens_seen": 2236350464 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255566700100301, + "loss": 1.3523, + "theoretical_loss": 3.4273989549215362, + "tokens_seen": 2236416000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032553660982948847, + "loss": 1.2586, + "theoretical_loss": 3.427390831720455, + "tokens_seen": 2236481536 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 814455, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4246420860290527, + "objective/train/theoretical_loss": 3.4273827088240543, + "objective/train/tokens_used": 607006176, + "theoretical_loss": 3.4273827088240543, + "tokens_seen": 2236547072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032551654964894683, + "loss": 1.3625, + "theoretical_loss": 3.4273827088240543, + "tokens_seen": 2236547072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254964894684052, + "loss": 1.4448, + "theoretical_loss": 3.4273745862323124, + "tokens_seen": 2236612608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254764292878636, + "loss": 1.381, + "theoretical_loss": 3.4273664639452095, + "tokens_seen": 2236678144 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325456369107322, + "loss": 1.3451, + "theoretical_loss": 3.4273583419627256, + "tokens_seen": 2236743680 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032543630892678034, + "loss": 1.3672, + "theoretical_loss": 3.42735022028484, + "tokens_seen": 2236809216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254162487462387, + "loss": 1.2712, + "theoretical_loss": 3.4273420989115326, + "tokens_seen": 2236874752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032539618856569706, + "loss": 1.4407, + "theoretical_loss": 3.4273339778427827, + "tokens_seen": 2236940288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253761283851555, + "loss": 1.339, + "theoretical_loss": 3.42732585707857, + "tokens_seen": 2237005824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032535606820461384, + "loss": 1.3412, + "theoretical_loss": 3.4273177366188747, + "tokens_seen": 2237071360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253360080240722, + "loss": 1.2962, + "theoretical_loss": 3.427309616463676, + "tokens_seen": 2237136896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032531594784353056, + "loss": 1.3918, + "theoretical_loss": 3.4273014966129534, + "tokens_seen": 2237202432 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325295887662989, + "loss": 1.4419, + "theoretical_loss": 3.4272933770666865, + "tokens_seen": 2237267968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032527582748244734, + "loss": 1.3683, + "theoretical_loss": 3.4272852578248556, + "tokens_seen": 2237333504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252557673019057, + "loss": 1.3307, + "theoretical_loss": 3.4272771388874403, + "tokens_seen": 2237399040 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032523570712136407, + "loss": 1.3817, + "theoretical_loss": 3.4272690202544194, + "tokens_seen": 2237464576 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032521564694082243, + "loss": 1.2603, + "theoretical_loss": 3.4272609019257736, + "tokens_seen": 2237530112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032519558676028084, + "loss": 1.3102, + "theoretical_loss": 3.427252783901482, + "tokens_seen": 2237595648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251755265797392, + "loss": 1.3353, + "theoretical_loss": 3.4272446661815246, + "tokens_seen": 2237661184 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032515546639919757, + "loss": 1.3055, + "theoretical_loss": 3.4272365487658805, + "tokens_seen": 2237726720 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032513540621865593, + "loss": 1.251, + "theoretical_loss": 3.4272284316545303, + "tokens_seen": 2237792256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032511534603811435, + "loss": 1.3998, + "theoretical_loss": 3.4272203148474527, + "tokens_seen": 2237857792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250952858575727, + "loss": 1.444, + "theoretical_loss": 3.4272121983446278, + "tokens_seen": 2237923328 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032507522567703107, + "loss": 1.391, + "theoretical_loss": 3.427204082146036, + "tokens_seen": 2237988864 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250551654964895, + "loss": 1.3426, + "theoretical_loss": 3.427195966251656, + "tokens_seen": 2238054400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032503510531594785, + "loss": 1.2968, + "theoretical_loss": 3.4271878506614675, + "tokens_seen": 2238119936 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032501504513540627, + "loss": 1.2998, + "theoretical_loss": 3.4271797353754505, + "tokens_seen": 2238185472 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032499498495486463, + "loss": 1.43, + "theoretical_loss": 3.427171620393585, + "tokens_seen": 2238251008 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324974924774323, + "loss": 1.3812, + "theoretical_loss": 3.42716350571585, + "tokens_seen": 2238316544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032495486459378135, + "loss": 1.4087, + "theoretical_loss": 3.427155391342226, + "tokens_seen": 2238382080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032493480441323977, + "loss": 1.2584, + "theoretical_loss": 3.427147277272692, + "tokens_seen": 2238447616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032491474423269813, + "loss": 1.385, + "theoretical_loss": 3.427139163507228, + "tokens_seen": 2238513152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248946840521565, + "loss": 1.381, + "theoretical_loss": 3.427131050045814, + "tokens_seen": 2238578688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032487462387161486, + "loss": 1.337, + "theoretical_loss": 3.427122936888429, + "tokens_seen": 2238644224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248545636910732, + "loss": 1.2905, + "theoretical_loss": 3.427114824035053, + "tokens_seen": 2238709760 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032483450351053164, + "loss": 1.3592, + "theoretical_loss": 3.427106711485666, + "tokens_seen": 2238775296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032481444332999, + "loss": 1.4158, + "theoretical_loss": 3.4270985992402476, + "tokens_seen": 2238840832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032479438314944836, + "loss": 1.3595, + "theoretical_loss": 3.427090487298778, + "tokens_seen": 2238906368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247743229689067, + "loss": 1.3745, + "theoretical_loss": 3.4270823756612354, + "tokens_seen": 2238971904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032475426278836514, + "loss": 1.3984, + "theoretical_loss": 3.427074264327601, + "tokens_seen": 2239037440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247342026078235, + "loss": 1.2992, + "theoretical_loss": 3.4270661532978535, + "tokens_seen": 2239102976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032471414242728186, + "loss": 1.3502, + "theoretical_loss": 3.4270580425719737, + "tokens_seen": 2239168512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246940822467402, + "loss": 1.2706, + "theoretical_loss": 3.4270499321499406, + "tokens_seen": 2239234048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032467402206619864, + "loss": 1.273, + "theoretical_loss": 3.427041822031734, + "tokens_seen": 2239299584 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324653961885657, + "loss": 1.3548, + "theoretical_loss": 3.4270337122173333, + "tokens_seen": 2239365120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032463390170511537, + "loss": 1.3315, + "theoretical_loss": 3.427025602706719, + "tokens_seen": 2239430656 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032461384152457373, + "loss": 1.2724, + "theoretical_loss": 3.4270174934998705, + "tokens_seen": 2239496192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245937813440321, + "loss": 1.3239, + "theoretical_loss": 3.4270093845967673, + "tokens_seen": 2239561728 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245737211634905, + "loss": 1.2367, + "theoretical_loss": 3.427001275997389, + "tokens_seen": 2239627264 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032455366098294887, + "loss": 1.355, + "theoretical_loss": 3.4269931677017165, + "tokens_seen": 2239692800 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032453360080240723, + "loss": 1.2892, + "theoretical_loss": 3.426985059709728, + "tokens_seen": 2239758336 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 815683, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.283194899559021, + "objective/train/theoretical_loss": 3.426976952021404, + "objective/train/tokens_used": 610282976, + "theoretical_loss": 3.426976952021404, + "tokens_seen": 2239823872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245135406218656, + "loss": 1.3528, + "theoretical_loss": 3.426976952021404, + "tokens_seen": 2239823872 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324493480441324, + "loss": 1.2866, + "theoretical_loss": 3.4269688446367246, + "tokens_seen": 2239889408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244734202607824, + "loss": 1.314, + "theoretical_loss": 3.4269607375556683, + "tokens_seen": 2239954944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032445336008024074, + "loss": 1.4064, + "theoretical_loss": 3.4269526307782163, + "tokens_seen": 2240020480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244332998996991, + "loss": 1.3326, + "theoretical_loss": 3.4269445243043477, + "tokens_seen": 2240086016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032441323971915746, + "loss": 1.2591, + "theoretical_loss": 3.426936418134042, + "tokens_seen": 2240151552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243931795386159, + "loss": 1.372, + "theoretical_loss": 3.4269283122672793, + "tokens_seen": 2240217088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032437311935807424, + "loss": 1.3037, + "theoretical_loss": 3.426920206704039, + "tokens_seen": 2240282624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243530591775326, + "loss": 1.4723, + "theoretical_loss": 3.4269121014443016, + "tokens_seen": 2240348160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032433299899699096, + "loss": 1.437, + "theoretical_loss": 3.426903996488046, + "tokens_seen": 2240413696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243129388164494, + "loss": 1.3004, + "theoretical_loss": 3.426895891835253, + "tokens_seen": 2240479232 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032429287863590774, + "loss": 1.311, + "theoretical_loss": 3.426887787485901, + "tokens_seen": 2240544768 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242728184553661, + "loss": 1.3843, + "theoretical_loss": 3.426879683439971, + "tokens_seen": 2240610304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032425275827482447, + "loss": 1.3201, + "theoretical_loss": 3.4268715796974414, + "tokens_seen": 2240675840 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032423269809428283, + "loss": 1.3486, + "theoretical_loss": 3.4268634762582937, + "tokens_seen": 2240741376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032421263791374125, + "loss": 1.3286, + "theoretical_loss": 3.426855373122506, + "tokens_seen": 2240806912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241925777331996, + "loss": 1.3047, + "theoretical_loss": 3.426847270290059, + "tokens_seen": 2240872448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032417251755265797, + "loss": 1.3084, + "theoretical_loss": 3.426839167760933, + "tokens_seen": 2240937984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032415245737211633, + "loss": 1.39, + "theoretical_loss": 3.4268310655351066, + "tokens_seen": 2241003520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032413239719157475, + "loss": 1.3173, + "theoretical_loss": 3.42682296361256, + "tokens_seen": 2241069056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241123370110331, + "loss": 1.3257, + "theoretical_loss": 3.426814861993273, + "tokens_seen": 2241134592 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032409227683049147, + "loss": 1.2635, + "theoretical_loss": 3.4268067606772257, + "tokens_seen": 2241200128 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032407221664994984, + "loss": 1.3251, + "theoretical_loss": 3.4267986596643976, + "tokens_seen": 2241265664 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240521564694082, + "loss": 1.3612, + "theoretical_loss": 3.4267905589547683, + "tokens_seen": 2241331200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240320962888666, + "loss": 1.2525, + "theoretical_loss": 3.4267824585483178, + "tokens_seen": 2241396736 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324012036108325, + "loss": 1.3227, + "theoretical_loss": 3.4267743584450256, + "tokens_seen": 2241462272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032399197592778334, + "loss": 1.2418, + "theoretical_loss": 3.4267662586448724, + "tokens_seen": 2241527808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239719157472417, + "loss": 1.3262, + "theoretical_loss": 3.4267581591478367, + "tokens_seen": 2241593344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239518555667001, + "loss": 1.3126, + "theoretical_loss": 3.4267500599538994, + "tokens_seen": 2241658880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239317953861585, + "loss": 1.3415, + "theoretical_loss": 3.4267419610630396, + "tokens_seen": 2241724416 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032391173520561684, + "loss": 1.3013, + "theoretical_loss": 3.4267338624752375, + "tokens_seen": 2241789952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238916750250752, + "loss": 1.3569, + "theoretical_loss": 3.426725764190473, + "tokens_seen": 2241855488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032387161484453357, + "loss": 1.3318, + "theoretical_loss": 3.426717666208725, + "tokens_seen": 2241921024 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323851554663992, + "loss": 1.3828, + "theoretical_loss": 3.426709568529974, + "tokens_seen": 2241986560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032383149448345034, + "loss": 1.3944, + "theoretical_loss": 3.4267014711542005, + "tokens_seen": 2242052096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238114343029087, + "loss": 1.2655, + "theoretical_loss": 3.426693374081383, + "tokens_seen": 2242117632 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032379137412236707, + "loss": 1.2596, + "theoretical_loss": 3.4266852773115017, + "tokens_seen": 2242183168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237713139418255, + "loss": 1.3989, + "theoretical_loss": 3.4266771808445373, + "tokens_seen": 2242248704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032375125376128385, + "loss": 1.2744, + "theoretical_loss": 3.4266690846804684, + "tokens_seen": 2242314240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237311935807422, + "loss": 1.2889, + "theoretical_loss": 3.4266609888192754, + "tokens_seen": 2242379776 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032371113340020057, + "loss": 1.3091, + "theoretical_loss": 3.426652893260938, + "tokens_seen": 2242445312 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032369107321965893, + "loss": 1.3262, + "theoretical_loss": 3.426644798005436, + "tokens_seen": 2242510848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032367101303911735, + "loss": 1.3671, + "theoretical_loss": 3.4266367030527496, + "tokens_seen": 2242576384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236509528585757, + "loss": 1.2783, + "theoretical_loss": 3.426628608402858, + "tokens_seen": 2242641920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236308926780341, + "loss": 1.3074, + "theoretical_loss": 3.4266205140557413, + "tokens_seen": 2242707456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032361083249749244, + "loss": 1.4128, + "theoretical_loss": 3.4266124200113794, + "tokens_seen": 2242772992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032359077231695085, + "loss": 1.3533, + "theoretical_loss": 3.4266043262697523, + "tokens_seen": 2242838528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032357071213640927, + "loss": 1.2707, + "theoretical_loss": 3.4265962328308395, + "tokens_seen": 2242904064 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032355065195586763, + "loss": 1.3119, + "theoretical_loss": 3.426588139694621, + "tokens_seen": 2242969600 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323530591775326, + "loss": 1.2203, + "theoretical_loss": 3.4265800468610763, + "tokens_seen": 2243035136 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 816816, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2776150703430176, + "objective/train/theoretical_loss": 3.4265719543301856, + "objective/train/tokens_used": 613559776, + "theoretical_loss": 3.4265719543301856, + "tokens_seen": 2243100672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235105315947844, + "loss": 1.3051, + "theoretical_loss": 3.4265719543301856, + "tokens_seen": 2243100672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234904714142428, + "loss": 1.1838, + "theoretical_loss": 3.4265638621019288, + "tokens_seen": 2243166208 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032347041123370114, + "loss": 1.2919, + "theoretical_loss": 3.426555770176286, + "tokens_seen": 2243231744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234503510531595, + "loss": 1.3196, + "theoretical_loss": 3.426547678553236, + "tokens_seen": 2243297280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032343029087261786, + "loss": 1.3738, + "theoretical_loss": 3.4265395872327593, + "tokens_seen": 2243362816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234102306920763, + "loss": 1.3357, + "theoretical_loss": 3.426531496214836, + "tokens_seen": 2243428352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032339017051153464, + "loss": 1.3721, + "theoretical_loss": 3.426523405499446, + "tokens_seen": 2243493888 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323370110330993, + "loss": 1.3602, + "theoretical_loss": 3.4265153150865686, + "tokens_seen": 2243559424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032335005015045136, + "loss": 1.4463, + "theoretical_loss": 3.4265072249761834, + "tokens_seen": 2243624960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233299899699098, + "loss": 1.3772, + "theoretical_loss": 3.4264991351682714, + "tokens_seen": 2243690496 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032330992978936814, + "loss": 1.4138, + "theoretical_loss": 3.4264910456628117, + "tokens_seen": 2243756032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232898696088265, + "loss": 1.2472, + "theoretical_loss": 3.4264829564597843, + "tokens_seen": 2243821568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032326980942828487, + "loss": 1.3262, + "theoretical_loss": 3.4264748675591687, + "tokens_seen": 2243887104 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032324974924774323, + "loss": 1.3571, + "theoretical_loss": 3.426466778960945, + "tokens_seen": 2243952640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032322968906720165, + "loss": 1.4162, + "theoretical_loss": 3.4264586906650933, + "tokens_seen": 2244018176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032320962888666, + "loss": 1.345, + "theoretical_loss": 3.4264506026715935, + "tokens_seen": 2244083712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032318956870611837, + "loss": 1.3167, + "theoretical_loss": 3.426442514980425, + "tokens_seen": 2244149248 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032316950852557673, + "loss": 1.4397, + "theoretical_loss": 3.4264344275915684, + "tokens_seen": 2244214784 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032314944834503515, + "loss": 1.4028, + "theoretical_loss": 3.426426340505003, + "tokens_seen": 2244280320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231293881644935, + "loss": 1.3731, + "theoretical_loss": 3.4264182537207084, + "tokens_seen": 2244345856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231093279839519, + "loss": 1.3621, + "theoretical_loss": 3.4264101672386653, + "tokens_seen": 2244411392 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032308926780341024, + "loss": 1.2781, + "theoretical_loss": 3.4264020810588534, + "tokens_seen": 2244476928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230692076228686, + "loss": 1.2786, + "theoretical_loss": 3.4263939951812517, + "tokens_seen": 2244542464 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323049147442327, + "loss": 1.3825, + "theoretical_loss": 3.426385909605841, + "tokens_seen": 2244608000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230290872617854, + "loss": 1.2917, + "theoretical_loss": 3.426377824332601, + "tokens_seen": 2244673536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032300902708124374, + "loss": 1.4216, + "theoretical_loss": 3.4263697393615113, + "tokens_seen": 2244739072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229889669007021, + "loss": 1.2464, + "theoretical_loss": 3.426361654692552, + "tokens_seen": 2244804608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229689067201605, + "loss": 1.2479, + "theoretical_loss": 3.426353570325703, + "tokens_seen": 2244870144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229488465396189, + "loss": 1.4348, + "theoretical_loss": 3.426345486260944, + "tokens_seen": 2244935680 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032292878635907724, + "loss": 1.3056, + "theoretical_loss": 3.4263374024982554, + "tokens_seen": 2245001216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229087261785356, + "loss": 1.4138, + "theoretical_loss": 3.4263293190376167, + "tokens_seen": 2245066752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032288866599799397, + "loss": 1.335, + "theoretical_loss": 3.4263212358790076, + "tokens_seen": 2245132288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228686058174524, + "loss": 1.4341, + "theoretical_loss": 3.4263131530224085, + "tokens_seen": 2245197824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032284854563691074, + "loss": 1.3424, + "theoretical_loss": 3.426305070467799, + "tokens_seen": 2245263360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228284854563691, + "loss": 1.3051, + "theoretical_loss": 3.426296988215159, + "tokens_seen": 2245328896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032280842527582747, + "loss": 1.293, + "theoretical_loss": 3.4262889062644684, + "tokens_seen": 2245394432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227883650952859, + "loss": 1.3867, + "theoretical_loss": 3.4262808246157066, + "tokens_seen": 2245459968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032276830491474425, + "loss": 1.3451, + "theoretical_loss": 3.426272743268855, + "tokens_seen": 2245525504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227482447342026, + "loss": 1.323, + "theoretical_loss": 3.426264662223892, + "tokens_seen": 2245591040 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032272818455366097, + "loss": 1.3401, + "theoretical_loss": 3.426256581480798, + "tokens_seen": 2245656576 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032270812437311933, + "loss": 1.3086, + "theoretical_loss": 3.4262485010395536, + "tokens_seen": 2245722112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032268806419257775, + "loss": 1.3879, + "theoretical_loss": 3.426240420900138, + "tokens_seen": 2245787648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226680040120361, + "loss": 1.4148, + "theoretical_loss": 3.426232341062531, + "tokens_seen": 2245853184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226479438314945, + "loss": 1.3904, + "theoretical_loss": 3.4262242615267127, + "tokens_seen": 2245918720 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032262788365095284, + "loss": 1.364, + "theoretical_loss": 3.4262161822926633, + "tokens_seen": 2245984256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032260782347041125, + "loss": 1.2714, + "theoretical_loss": 3.4262081033603624, + "tokens_seen": 2246049792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225877632898696, + "loss": 1.2822, + "theoretical_loss": 3.42620002472979, + "tokens_seen": 2246115328 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322567703109328, + "loss": 1.3282, + "theoretical_loss": 3.426191946400926, + "tokens_seen": 2246180864 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032254764292878634, + "loss": 1.3302, + "theoretical_loss": 3.4261838683737507, + "tokens_seen": 2246246400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225275827482447, + "loss": 1.3235, + "theoretical_loss": 3.4261757906482435, + "tokens_seen": 2246311936 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 817879, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2966686487197876, + "objective/train/theoretical_loss": 3.4261677132243844, + "objective/train/tokens_used": 616836576, + "theoretical_loss": 3.4261677132243844, + "tokens_seen": 2246377472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225075225677031, + "loss": 1.3322, + "theoretical_loss": 3.4261677132243844, + "tokens_seen": 2246377472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224874623871615, + "loss": 1.295, + "theoretical_loss": 3.4261596361021533, + "tokens_seen": 2246443008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032246740220661984, + "loss": 1.4355, + "theoretical_loss": 3.426151559281531, + "tokens_seen": 2246508544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224473420260782, + "loss": 1.2629, + "theoretical_loss": 3.4261434827624964, + "tokens_seen": 2246574080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224272818455366, + "loss": 1.2509, + "theoretical_loss": 3.42613540654503, + "tokens_seen": 2246639616 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322407221664995, + "loss": 1.4527, + "theoretical_loss": 3.4261273306291113, + "tokens_seen": 2246705152 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032238716148445335, + "loss": 1.4129, + "theoretical_loss": 3.4261192550147204, + "tokens_seen": 2246770688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223671013039117, + "loss": 1.4208, + "theoretical_loss": 3.4261111797018375, + "tokens_seen": 2246836224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223470411233701, + "loss": 1.3964, + "theoretical_loss": 3.4261031046904424, + "tokens_seen": 2246901760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223269809428285, + "loss": 1.2715, + "theoretical_loss": 3.426095029980515, + "tokens_seen": 2246967296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032230692076228685, + "loss": 1.3555, + "theoretical_loss": 3.4260869555720355, + "tokens_seen": 2247032832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222868605817452, + "loss": 1.3502, + "theoretical_loss": 3.4260788814649836, + "tokens_seen": 2247098368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222668004012036, + "loss": 1.2704, + "theoretical_loss": 3.4260708076593396, + "tokens_seen": 2247163904 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322246740220662, + "loss": 1.2961, + "theoretical_loss": 3.4260627341550824, + "tokens_seen": 2247229440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032222668004012035, + "loss": 1.3102, + "theoretical_loss": 3.426054660952193, + "tokens_seen": 2247294976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222066198595787, + "loss": 1.3569, + "theoretical_loss": 3.4260465880506517, + "tokens_seen": 2247360512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221865596790371, + "loss": 1.3768, + "theoretical_loss": 3.426038515450437, + "tokens_seen": 2247426048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221664994984955, + "loss": 1.3061, + "theoretical_loss": 3.4260304431515305, + "tokens_seen": 2247491584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032214643931795386, + "loss": 1.3181, + "theoretical_loss": 3.426022371153911, + "tokens_seen": 2247557120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221263791374122, + "loss": 1.3183, + "theoretical_loss": 3.4260142994575586, + "tokens_seen": 2247622656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221063189568706, + "loss": 1.1854, + "theoretical_loss": 3.426006228062454, + "tokens_seen": 2247688192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032208625877632894, + "loss": 1.2646, + "theoretical_loss": 3.4259981569685767, + "tokens_seen": 2247753728 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220661985957874, + "loss": 1.3579, + "theoretical_loss": 3.425990086175906, + "tokens_seen": 2247819264 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220461384152458, + "loss": 1.3166, + "theoretical_loss": 3.4259820156844234, + "tokens_seen": 2247884800 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032202607823470414, + "loss": 1.4076, + "theoretical_loss": 3.425973945494108, + "tokens_seen": 2247950336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220060180541625, + "loss": 1.261, + "theoretical_loss": 3.425965875604939, + "tokens_seen": 2248015872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219859578736209, + "loss": 1.4944, + "theoretical_loss": 3.425957806016898, + "tokens_seen": 2248081408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219658976930793, + "loss": 1.2509, + "theoretical_loss": 3.425949736729964, + "tokens_seen": 2248146944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032194583751253764, + "loss": 1.4037, + "theoretical_loss": 3.425941667744117, + "tokens_seen": 2248212480 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321925777331996, + "loss": 1.3695, + "theoretical_loss": 3.425933599059337, + "tokens_seen": 2248278016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032190571715145437, + "loss": 1.3075, + "theoretical_loss": 3.4259255306756042, + "tokens_seen": 2248343552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218856569709128, + "loss": 1.4285, + "theoretical_loss": 3.425917462592899, + "tokens_seen": 2248409088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032186559679037115, + "loss": 1.2572, + "theoretical_loss": 3.4259093948112005, + "tokens_seen": 2248474624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218455366098295, + "loss": 1.3999, + "theoretical_loss": 3.4259013273304895, + "tokens_seen": 2248540160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032182547642928787, + "loss": 1.3358, + "theoretical_loss": 3.4258932601507457, + "tokens_seen": 2248605696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218054162487463, + "loss": 1.5248, + "theoretical_loss": 3.4258851932719487, + "tokens_seen": 2248671232 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032178535606820465, + "loss": 1.2936, + "theoretical_loss": 3.425877126694079, + "tokens_seen": 2248736768 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321765295887663, + "loss": 1.2266, + "theoretical_loss": 3.4258690604171163, + "tokens_seen": 2248802304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032174523570712137, + "loss": 1.309, + "theoretical_loss": 3.425860994441041, + "tokens_seen": 2248867840 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032172517552657974, + "loss": 1.3537, + "theoretical_loss": 3.425852928765833, + "tokens_seen": 2248933376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032170511534603815, + "loss": 1.4119, + "theoretical_loss": 3.4258448633914718, + "tokens_seen": 2248998912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216850551654965, + "loss": 1.3467, + "theoretical_loss": 3.425836798317938, + "tokens_seen": 2249064448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216649949849549, + "loss": 1.204, + "theoretical_loss": 3.4258287335452113, + "tokens_seen": 2249129984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032164493480441324, + "loss": 1.3691, + "theoretical_loss": 3.4258206690732718, + "tokens_seen": 2249195520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032162487462387165, + "loss": 1.2847, + "theoretical_loss": 3.4258126049020996, + "tokens_seen": 2249261056 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032160481444333, + "loss": 1.2781, + "theoretical_loss": 3.425804541031675, + "tokens_seen": 2249326592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215847542627884, + "loss": 1.2514, + "theoretical_loss": 3.425796477461977, + "tokens_seen": 2249392128 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032156469408224674, + "loss": 1.3658, + "theoretical_loss": 3.425788414192987, + "tokens_seen": 2249457664 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215446339017051, + "loss": 1.2879, + "theoretical_loss": 3.4257803512246836, + "tokens_seen": 2249523200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215245737211635, + "loss": 1.2631, + "theoretical_loss": 3.425772288557048, + "tokens_seen": 2249588736 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 819025, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2979280948638916, + "objective/train/theoretical_loss": 3.4257642261900596, + "objective/train/tokens_used": 620113376, + "theoretical_loss": 3.4257642261900596, + "tokens_seen": 2249654272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215045135406219, + "loss": 1.3743, + "theoretical_loss": 3.4257642261900596, + "tokens_seen": 2249654272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032148445336008024, + "loss": 1.2688, + "theoretical_loss": 3.425756164123699, + "tokens_seen": 2249719808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214643931795386, + "loss": 1.2962, + "theoretical_loss": 3.4257481023579457, + "tokens_seen": 2249785344 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321444332998997, + "loss": 1.3528, + "theoretical_loss": 3.425740040892779, + "tokens_seen": 2249850880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214242728184554, + "loss": 1.3516, + "theoretical_loss": 3.425731979728181, + "tokens_seen": 2249916416 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032140421263791375, + "loss": 1.3329, + "theoretical_loss": 3.42572391886413, + "tokens_seen": 2249981952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213841524573721, + "loss": 1.3181, + "theoretical_loss": 3.425715858300607, + "tokens_seen": 2250047488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213640922768305, + "loss": 1.377, + "theoretical_loss": 3.425707798037591, + "tokens_seen": 2250113024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213440320962889, + "loss": 1.3133, + "theoretical_loss": 3.425699738075063, + "tokens_seen": 2250178560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032132397191574725, + "loss": 1.3571, + "theoretical_loss": 3.425691678413003, + "tokens_seen": 2250244096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213039117352056, + "loss": 1.4143, + "theoretical_loss": 3.425683619051391, + "tokens_seen": 2250309632 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321283851554664, + "loss": 1.2556, + "theoretical_loss": 3.4256755599902062, + "tokens_seen": 2250375168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212637913741224, + "loss": 1.2576, + "theoretical_loss": 3.4256675012294293, + "tokens_seen": 2250440704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032124373119358075, + "loss": 1.2748, + "theoretical_loss": 3.425659442769041, + "tokens_seen": 2250506240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212236710130391, + "loss": 1.2606, + "theoretical_loss": 3.42565138460902, + "tokens_seen": 2250571776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212036108324975, + "loss": 1.4338, + "theoretical_loss": 3.4256433267493476, + "tokens_seen": 2250637312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211835506519559, + "loss": 1.2458, + "theoretical_loss": 3.425635269190003, + "tokens_seen": 2250702848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032116349047141426, + "loss": 1.2956, + "theoretical_loss": 3.4256272119309665, + "tokens_seen": 2250768384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211434302908726, + "loss": 1.2375, + "theoretical_loss": 3.4256191549722184, + "tokens_seen": 2250833920 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321123370110331, + "loss": 1.3663, + "theoretical_loss": 3.4256110983137384, + "tokens_seen": 2250899456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032110330992978934, + "loss": 1.3749, + "theoretical_loss": 3.4256030419555072, + "tokens_seen": 2250964992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032108324974924776, + "loss": 1.4324, + "theoretical_loss": 3.4255949858975043, + "tokens_seen": 2251030528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210631895687061, + "loss": 1.3478, + "theoretical_loss": 3.4255869301397097, + "tokens_seen": 2251096064 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210431293881645, + "loss": 1.3242, + "theoretical_loss": 3.4255788746821034, + "tokens_seen": 2251161600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032102306920762285, + "loss": 1.3252, + "theoretical_loss": 3.4255708195246664, + "tokens_seen": 2251227136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032100300902708126, + "loss": 1.3484, + "theoretical_loss": 3.4255627646673776, + "tokens_seen": 2251292672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209829488465396, + "loss": 1.3683, + "theoretical_loss": 3.425554710110218, + "tokens_seen": 2251358208 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320962888665998, + "loss": 1.3374, + "theoretical_loss": 3.4255466558531675, + "tokens_seen": 2251423744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032094282848545635, + "loss": 1.336, + "theoretical_loss": 3.4255386018962057, + "tokens_seen": 2251489280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209227683049147, + "loss": 1.4272, + "theoretical_loss": 3.425530548239313, + "tokens_seen": 2251554816 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032090270812437313, + "loss": 1.2396, + "theoretical_loss": 3.425522494882469, + "tokens_seen": 2251620352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208826479438315, + "loss": 1.3155, + "theoretical_loss": 3.425514441825655, + "tokens_seen": 2251685888 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032086258776328985, + "loss": 1.3442, + "theoretical_loss": 3.42550638906885, + "tokens_seen": 2251751424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208425275827482, + "loss": 1.2906, + "theoretical_loss": 3.4254983366120344, + "tokens_seen": 2251816960 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032082246740220663, + "loss": 1.3499, + "theoretical_loss": 3.425490284455188, + "tokens_seen": 2251882496 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320802407221665, + "loss": 1.2708, + "theoretical_loss": 3.425482232598292, + "tokens_seen": 2251948032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032078234704112336, + "loss": 1.3997, + "theoretical_loss": 3.4254741810413254, + "tokens_seen": 2252013568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207622868605817, + "loss": 1.3667, + "theoretical_loss": 3.4254661297842683, + "tokens_seen": 2252079104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207422266800401, + "loss": 1.2637, + "theoretical_loss": 3.4254580788271016, + "tokens_seen": 2252144640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207221664994985, + "loss": 1.3521, + "theoretical_loss": 3.425450028169805, + "tokens_seen": 2252210176 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032070210631895686, + "loss": 1.3381, + "theoretical_loss": 3.4254419778123575, + "tokens_seen": 2252275712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206820461384152, + "loss": 1.2596, + "theoretical_loss": 3.425433927754741, + "tokens_seen": 2252341248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206619859578736, + "loss": 1.3667, + "theoretical_loss": 3.425425877996935, + "tokens_seen": 2252406784 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320641925777332, + "loss": 1.3873, + "theoretical_loss": 3.4254178285389187, + "tokens_seen": 2252472320 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032062186559679036, + "loss": 1.2714, + "theoretical_loss": 3.425409779380674, + "tokens_seen": 2252537856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206018054162487, + "loss": 1.3055, + "theoretical_loss": 3.425401730522179, + "tokens_seen": 2252603392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205817452357071, + "loss": 1.2931, + "theoretical_loss": 3.4253936819634156, + "tokens_seen": 2252668928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205616850551655, + "loss": 1.3049, + "theoretical_loss": 3.4253856337043627, + "tokens_seen": 2252734464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205416248746239, + "loss": 1.3062, + "theoretical_loss": 3.425377585745001, + "tokens_seen": 2252800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205215646940823, + "loss": 1.3105, + "theoretical_loss": 3.4253695380853104, + "tokens_seen": 2252865536 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 820362, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4370191097259521, + "objective/train/theoretical_loss": 3.4253614907252707, + "objective/train/tokens_used": 623390176, + "theoretical_loss": 3.4253614907252707, + "tokens_seen": 2252931072 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032050150451354064, + "loss": 1.394, + "theoretical_loss": 3.4253614907252707, + "tokens_seen": 2252931072 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320481444332999, + "loss": 1.3374, + "theoretical_loss": 3.425353443664863, + "tokens_seen": 2252996608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204613841524574, + "loss": 1.2829, + "theoretical_loss": 3.4253453969040666, + "tokens_seen": 2253062144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204413239719158, + "loss": 1.2971, + "theoretical_loss": 3.425337350442862, + "tokens_seen": 2253127680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032042126379137415, + "loss": 1.3441, + "theoretical_loss": 3.425329304281229, + "tokens_seen": 2253193216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204012036108325, + "loss": 1.4001, + "theoretical_loss": 3.4253212584191477, + "tokens_seen": 2253258752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203811434302909, + "loss": 1.3022, + "theoretical_loss": 3.425313212856599, + "tokens_seen": 2253324288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203610832497493, + "loss": 1.3169, + "theoretical_loss": 3.4253051675935624, + "tokens_seen": 2253389824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032034102306920765, + "loss": 1.2853, + "theoretical_loss": 3.425297122630018, + "tokens_seen": 2253455360 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320320962888666, + "loss": 1.3437, + "theoretical_loss": 3.4252890779659464, + "tokens_seen": 2253520896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203009027081244, + "loss": 1.2632, + "theoretical_loss": 3.425281033601327, + "tokens_seen": 2253586432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202808425275828, + "loss": 1.3172, + "theoretical_loss": 3.4252729895361407, + "tokens_seen": 2253651968 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032026078234704115, + "loss": 1.4267, + "theoretical_loss": 3.425264945770367, + "tokens_seen": 2253717504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202407221664995, + "loss": 1.3465, + "theoretical_loss": 3.4252569023039867, + "tokens_seen": 2253783040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202206619859579, + "loss": 1.3125, + "theoretical_loss": 3.4252488591369796, + "tokens_seen": 2253848576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202006018054163, + "loss": 1.3946, + "theoretical_loss": 3.4252408162693255, + "tokens_seen": 2253914112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032018054162487466, + "loss": 1.421, + "theoretical_loss": 3.4252327737010053, + "tokens_seen": 2253979648 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320160481444333, + "loss": 1.3479, + "theoretical_loss": 3.425224731431999, + "tokens_seen": 2254045184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201404212637914, + "loss": 1.3725, + "theoretical_loss": 3.425216689462286, + "tokens_seen": 2254110720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032012036108324974, + "loss": 1.3045, + "theoretical_loss": 3.4252086477918473, + "tokens_seen": 2254176256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032010030090270816, + "loss": 1.3119, + "theoretical_loss": 3.425200606420663, + "tokens_seen": 2254241792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200802407221665, + "loss": 1.2955, + "theoretical_loss": 3.4251925653487127, + "tokens_seen": 2254307328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200601805416249, + "loss": 1.3212, + "theoretical_loss": 3.425184524575977, + "tokens_seen": 2254372864 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032004012036108325, + "loss": 1.4203, + "theoretical_loss": 3.425176484102436, + "tokens_seen": 2254438400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032002006018054166, + "loss": 1.3079, + "theoretical_loss": 3.4251684439280696, + "tokens_seen": 2254503936 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032, + "loss": 1.3803, + "theoretical_loss": 3.4251604040528587, + "tokens_seen": 2254569472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199799398194584, + "loss": 1.2478, + "theoretical_loss": 3.4251523644767827, + "tokens_seen": 2254635008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031995987963891675, + "loss": 1.3882, + "theoretical_loss": 3.425144325199822, + "tokens_seen": 2254700544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199398194583751, + "loss": 1.1777, + "theoretical_loss": 3.425136286221957, + "tokens_seen": 2254766080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031991975927783353, + "loss": 1.3226, + "theoretical_loss": 3.4251282475431677, + "tokens_seen": 2254831616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198996990972919, + "loss": 1.3984, + "theoretical_loss": 3.4251202091634343, + "tokens_seen": 2254897152 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031987963891675025, + "loss": 1.3665, + "theoretical_loss": 3.4251121710827372, + "tokens_seen": 2254962688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198595787362086, + "loss": 1.3718, + "theoretical_loss": 3.425104133301056, + "tokens_seen": 2255028224 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031983951855566703, + "loss": 1.3711, + "theoretical_loss": 3.4250960958183714, + "tokens_seen": 2255093760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198194583751254, + "loss": 1.3844, + "theoretical_loss": 3.4250880586346635, + "tokens_seen": 2255159296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031979939819458376, + "loss": 1.3275, + "theoretical_loss": 3.4250800217499124, + "tokens_seen": 2255224832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197793380140421, + "loss": 1.2946, + "theoretical_loss": 3.4250719851640983, + "tokens_seen": 2255290368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197592778335005, + "loss": 1.3391, + "theoretical_loss": 3.4250639488772014, + "tokens_seen": 2255355904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197392176529589, + "loss": 1.285, + "theoretical_loss": 3.425055912889202, + "tokens_seen": 2255421440 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031971915747241726, + "loss": 1.3366, + "theoretical_loss": 3.4250478772000803, + "tokens_seen": 2255486976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196990972918756, + "loss": 1.3796, + "theoretical_loss": 3.4250398418098165, + "tokens_seen": 2255552512 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319679037111334, + "loss": 1.3664, + "theoretical_loss": 3.4250318067183905, + "tokens_seen": 2255618048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196589769307924, + "loss": 1.2685, + "theoretical_loss": 3.4250237719257823, + "tokens_seen": 2255683584 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031963891675025076, + "loss": 1.2698, + "theoretical_loss": 3.4250157374319734, + "tokens_seen": 2255749120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031963891675025076, + "loss": 1.4421, + "theoretical_loss": 3.4250077032369424, + "tokens_seen": 2255814656 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196188565697091, + "loss": 1.337, + "theoretical_loss": 3.4249996693406706, + "tokens_seen": 2255880192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195987963891675, + "loss": 1.234, + "theoretical_loss": 3.424991635743138, + "tokens_seen": 2255945728 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031957873620862585, + "loss": 1.3528, + "theoretical_loss": 3.4249836024443248, + "tokens_seen": 2256011264 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031955867602808427, + "loss": 1.35, + "theoretical_loss": 3.4249755694442108, + "tokens_seen": 2256076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031953861584754263, + "loss": 1.2912, + "theoretical_loss": 3.4249675367427765, + "tokens_seen": 2256142336 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 821597, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2336771488189697, + "objective/train/theoretical_loss": 3.424959504340002, + "objective/train/tokens_used": 626666976, + "theoretical_loss": 3.424959504340002, + "tokens_seen": 2256207872 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319518555667001, + "loss": 1.2972, + "theoretical_loss": 3.424959504340002, + "tokens_seen": 2256207872 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031949849548645935, + "loss": 1.2471, + "theoretical_loss": 3.424951472235868, + "tokens_seen": 2256273408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031947843530591777, + "loss": 1.3624, + "theoretical_loss": 3.4249434404303543, + "tokens_seen": 2256338944 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031945837512537613, + "loss": 1.3168, + "theoretical_loss": 3.424935408923441, + "tokens_seen": 2256404480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194383149448345, + "loss": 1.3819, + "theoretical_loss": 3.4249273777151084, + "tokens_seen": 2256470016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031941825476429286, + "loss": 1.3789, + "theoretical_loss": 3.4249193468053374, + "tokens_seen": 2256535552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193981945837512, + "loss": 1.3234, + "theoretical_loss": 3.424911316194107, + "tokens_seen": 2256601088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031937813440320964, + "loss": 1.2714, + "theoretical_loss": 3.4249032858813986, + "tokens_seen": 2256666624 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319358074222668, + "loss": 1.3801, + "theoretical_loss": 3.4248952558671917, + "tokens_seen": 2256732160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031933801404212636, + "loss": 1.3342, + "theoretical_loss": 3.424887226151467, + "tokens_seen": 2256797696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193179538615847, + "loss": 1.4121, + "theoretical_loss": 3.4248791967342043, + "tokens_seen": 2256863232 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031929789368104314, + "loss": 1.3731, + "theoretical_loss": 3.4248711676153842, + "tokens_seen": 2256928768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192778335005015, + "loss": 1.3239, + "theoretical_loss": 3.424863138794987, + "tokens_seen": 2256994304 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031925777331995986, + "loss": 1.286, + "theoretical_loss": 3.4248551102729925, + "tokens_seen": 2257059840 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192377131394182, + "loss": 1.3224, + "theoretical_loss": 3.4248470820493813, + "tokens_seen": 2257125376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192176529588766, + "loss": 1.2979, + "theoretical_loss": 3.424839054124133, + "tokens_seen": 2257190912 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319197592778335, + "loss": 1.4244, + "theoretical_loss": 3.424831026497229, + "tokens_seen": 2257256448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031917753259779337, + "loss": 1.3088, + "theoretical_loss": 3.424822999168649, + "tokens_seen": 2257321984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031915747241725173, + "loss": 1.3579, + "theoretical_loss": 3.424814972138373, + "tokens_seen": 2257387520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191374122367101, + "loss": 1.3548, + "theoretical_loss": 3.4248069454063814, + "tokens_seen": 2257453056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191173520561685, + "loss": 1.2612, + "theoretical_loss": 3.4247989189726544, + "tokens_seen": 2257518592 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031909729187562687, + "loss": 1.3465, + "theoretical_loss": 3.4247908928371724, + "tokens_seen": 2257584128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031907723169508523, + "loss": 1.2989, + "theoretical_loss": 3.424782866999916, + "tokens_seen": 2257649664 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031905717151454365, + "loss": 1.3458, + "theoretical_loss": 3.4247748414608647, + "tokens_seen": 2257715200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031903711133400206, + "loss": 1.3281, + "theoretical_loss": 3.424766816219999, + "tokens_seen": 2257780736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190170511534604, + "loss": 1.2821, + "theoretical_loss": 3.4247587912773, + "tokens_seen": 2257846272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189969909729188, + "loss": 1.309, + "theoretical_loss": 3.424750766632747, + "tokens_seen": 2257911808 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031897693079237715, + "loss": 1.2786, + "theoretical_loss": 3.42474274228632, + "tokens_seen": 2257977344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189568706118355, + "loss": 1.3446, + "theoretical_loss": 3.4247347182380006, + "tokens_seen": 2258042880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031893681043129393, + "loss": 1.4057, + "theoretical_loss": 3.424726694487768, + "tokens_seen": 2258108416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189167502507523, + "loss": 1.3338, + "theoretical_loss": 3.4247186710356026, + "tokens_seen": 2258173952 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031889669007021065, + "loss": 1.4403, + "theoretical_loss": 3.424710647881485, + "tokens_seen": 2258239488 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318876629889669, + "loss": 1.3422, + "theoretical_loss": 3.4247026250253954, + "tokens_seen": 2258305024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031885656970912743, + "loss": 1.3196, + "theoretical_loss": 3.424694602467314, + "tokens_seen": 2258370560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188365095285858, + "loss": 1.3789, + "theoretical_loss": 3.424686580207221, + "tokens_seen": 2258436096 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031881644934804416, + "loss": 1.3325, + "theoretical_loss": 3.4246785582450965, + "tokens_seen": 2258501632 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187963891675025, + "loss": 1.381, + "theoretical_loss": 3.424670536580922, + "tokens_seen": 2258567168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187763289869609, + "loss": 1.2465, + "theoretical_loss": 3.424662515214676, + "tokens_seen": 2258632704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187562688064193, + "loss": 1.3982, + "theoretical_loss": 3.42465449414634, + "tokens_seen": 2258698240 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031873620862587766, + "loss": 1.3873, + "theoretical_loss": 3.4246464733758937, + "tokens_seen": 2258763776 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318716148445336, + "loss": 1.2336, + "theoretical_loss": 3.424638452903318, + "tokens_seen": 2258829312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186960882647944, + "loss": 1.2054, + "theoretical_loss": 3.4246304327285926, + "tokens_seen": 2258894848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186760280842528, + "loss": 1.3343, + "theoretical_loss": 3.424622412851698, + "tokens_seen": 2258960384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031865596790371116, + "loss": 1.3511, + "theoretical_loss": 3.4246143932726145, + "tokens_seen": 2259025920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186359077231695, + "loss": 1.3493, + "theoretical_loss": 3.424606373991322, + "tokens_seen": 2259091456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186158475426279, + "loss": 1.3013, + "theoretical_loss": 3.4245983550078023, + "tokens_seen": 2259156992 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031859578736208625, + "loss": 1.2058, + "theoretical_loss": 3.424590336322034, + "tokens_seen": 2259222528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031857572718154467, + "loss": 1.3551, + "theoretical_loss": 3.4245823179339983, + "tokens_seen": 2259288064 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031855566700100303, + "loss": 1.3394, + "theoretical_loss": 3.424574299843675, + "tokens_seen": 2259353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185356068204614, + "loss": 1.3188, + "theoretical_loss": 3.4245662820510443, + "tokens_seen": 2259419136 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 822867, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5087814331054688, + "objective/train/theoretical_loss": 3.4245582645560875, + "objective/train/tokens_used": 629943776, + "theoretical_loss": 3.4245582645560875, + "tokens_seen": 2259484672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031851554663991975, + "loss": 1.3787, + "theoretical_loss": 3.4245582645560875, + "tokens_seen": 2259484672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031849548645937817, + "loss": 1.2638, + "theoretical_loss": 3.424550247358784, + "tokens_seen": 2259550208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031847542627883653, + "loss": 1.2854, + "theoretical_loss": 3.4245422304591147, + "tokens_seen": 2259615744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184553660982949, + "loss": 1.2466, + "theoretical_loss": 3.4245342138570596, + "tokens_seen": 2259681280 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031843530591775326, + "loss": 1.3687, + "theoretical_loss": 3.424526197552599, + "tokens_seen": 2259746816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184152457372116, + "loss": 1.3883, + "theoretical_loss": 3.424518181545713, + "tokens_seen": 2259812352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031839518555667004, + "loss": 1.2954, + "theoretical_loss": 3.4245101658363826, + "tokens_seen": 2259877888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183751253761284, + "loss": 1.2757, + "theoretical_loss": 3.4245021504245874, + "tokens_seen": 2259943424 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031835506519558676, + "loss": 1.2809, + "theoretical_loss": 3.424494135310308, + "tokens_seen": 2260008960 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183350050150451, + "loss": 1.2842, + "theoretical_loss": 3.4244861204935253, + "tokens_seen": 2260074496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031831494483450354, + "loss": 1.3217, + "theoretical_loss": 3.4244781059742184, + "tokens_seen": 2260140032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182948846539619, + "loss": 1.3503, + "theoretical_loss": 3.424470091752369, + "tokens_seen": 2260205568 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031827482447342026, + "loss": 1.1934, + "theoretical_loss": 3.4244620778279566, + "tokens_seen": 2260271104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182547642928786, + "loss": 1.2986, + "theoretical_loss": 3.4244540642009613, + "tokens_seen": 2260336640 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318234704112337, + "loss": 1.3644, + "theoretical_loss": 3.4244460508713646, + "tokens_seen": 2260402176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182146439317954, + "loss": 1.3134, + "theoretical_loss": 3.4244380378391455, + "tokens_seen": 2260467712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031819458375125377, + "loss": 1.266, + "theoretical_loss": 3.424430025104285, + "tokens_seen": 2260533248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031817452357071213, + "loss": 1.3428, + "theoretical_loss": 3.4244220126667635, + "tokens_seen": 2260598784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181544633901705, + "loss": 1.299, + "theoretical_loss": 3.4244140005265615, + "tokens_seen": 2260664320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181344032096289, + "loss": 1.379, + "theoretical_loss": 3.4244059886836586, + "tokens_seen": 2260729856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031811434302908727, + "loss": 1.4492, + "theoretical_loss": 3.424397977138036, + "tokens_seen": 2260795392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031809428284854563, + "loss": 1.446, + "theoretical_loss": 3.4243899658896737, + "tokens_seen": 2260860928 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318074222668004, + "loss": 1.3207, + "theoretical_loss": 3.4243819549385517, + "tokens_seen": 2260926464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180541624874624, + "loss": 1.3248, + "theoretical_loss": 3.424373944284651, + "tokens_seen": 2260992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031803410230692077, + "loss": 1.2993, + "theoretical_loss": 3.4243659339279517, + "tokens_seen": 2261057536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031801404212637913, + "loss": 1.3959, + "theoretical_loss": 3.424357923868434, + "tokens_seen": 2261123072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179939819458375, + "loss": 1.332, + "theoretical_loss": 3.4243499141060783, + "tokens_seen": 2261188608 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031797392176529586, + "loss": 1.2128, + "theoretical_loss": 3.4243419046408654, + "tokens_seen": 2261254144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179538615847543, + "loss": 1.3677, + "theoretical_loss": 3.424333895472775, + "tokens_seen": 2261319680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031793380140421264, + "loss": 1.3694, + "theoretical_loss": 3.4243258866017876, + "tokens_seen": 2261385216 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317913741223671, + "loss": 1.2748, + "theoretical_loss": 3.4243178780278836, + "tokens_seen": 2261450752 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031789368104312936, + "loss": 1.3419, + "theoretical_loss": 3.424309869751044, + "tokens_seen": 2261516288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178736208625878, + "loss": 1.3376, + "theoretical_loss": 3.424301861771249, + "tokens_seen": 2261581824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031785356068204614, + "loss": 1.3809, + "theoretical_loss": 3.4242938540884778, + "tokens_seen": 2261647360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178335005015045, + "loss": 1.3321, + "theoretical_loss": 3.424285846702712, + "tokens_seen": 2261712896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031781344032096287, + "loss": 1.4432, + "theoretical_loss": 3.4242778396139317, + "tokens_seen": 2261778432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031779338014042123, + "loss": 1.3648, + "theoretical_loss": 3.4242698328221173, + "tokens_seen": 2261843968 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031777331995987964, + "loss": 1.404, + "theoretical_loss": 3.424261826327249, + "tokens_seen": 2261909504 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317753259779338, + "loss": 1.2429, + "theoretical_loss": 3.424253820129307, + "tokens_seen": 2261975040 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031773319959879637, + "loss": 1.3101, + "theoretical_loss": 3.4242458142282723, + "tokens_seen": 2262040576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031771313941825473, + "loss": 1.1376, + "theoretical_loss": 3.4242378086241247, + "tokens_seen": 2262106112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031769307923771315, + "loss": 1.4223, + "theoretical_loss": 3.424229803316845, + "tokens_seen": 2262171648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176730190571715, + "loss": 1.2826, + "theoretical_loss": 3.424221798306413, + "tokens_seen": 2262237184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031765295887662987, + "loss": 1.3168, + "theoretical_loss": 3.42421379359281, + "tokens_seen": 2262302720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031763289869608823, + "loss": 1.3673, + "theoretical_loss": 3.424205789176016, + "tokens_seen": 2262368256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176128385155466, + "loss": 1.4524, + "theoretical_loss": 3.424197785056011, + "tokens_seen": 2262433792 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317592778335005, + "loss": 1.3922, + "theoretical_loss": 3.4241897812327755, + "tokens_seen": 2262499328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175727181544634, + "loss": 1.3567, + "theoretical_loss": 3.4241817777062904, + "tokens_seen": 2262564864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175526579739218, + "loss": 1.3194, + "theoretical_loss": 3.424173774476536, + "tokens_seen": 2262630400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031753259779338015, + "loss": 1.3001, + "theoretical_loss": 3.4241657715434917, + "tokens_seen": 2262695936 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 824193, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4341137409210205, + "objective/train/theoretical_loss": 3.4241577689071394, + "objective/train/tokens_used": 633220576, + "theoretical_loss": 3.4241577689071394, + "tokens_seen": 2262761472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031751253761283857, + "loss": 1.3582, + "theoretical_loss": 3.4241577689071394, + "tokens_seen": 2262761472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031749247743229693, + "loss": 1.3299, + "theoretical_loss": 3.4241497665674587, + "tokens_seen": 2262827008 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174724172517553, + "loss": 1.351, + "theoretical_loss": 3.42414176452443, + "tokens_seen": 2262892544 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031745235707121366, + "loss": 1.2903, + "theoretical_loss": 3.424133762778034, + "tokens_seen": 2262958080 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317432296890672, + "loss": 1.3457, + "theoretical_loss": 3.424125761328251, + "tokens_seen": 2263023616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031741223671013044, + "loss": 1.2391, + "theoretical_loss": 3.4241177601750614, + "tokens_seen": 2263089152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173921765295888, + "loss": 1.2918, + "theoretical_loss": 3.4241097593184455, + "tokens_seen": 2263154688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031737211634904716, + "loss": 1.3542, + "theoretical_loss": 3.424101758758384, + "tokens_seen": 2263220224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173520561685055, + "loss": 1.3067, + "theoretical_loss": 3.424093758494857, + "tokens_seen": 2263285760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031733199598796394, + "loss": 1.345, + "theoretical_loss": 3.424085758527845, + "tokens_seen": 2263351296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173119358074223, + "loss": 1.3262, + "theoretical_loss": 3.4240777588573286, + "tokens_seen": 2263416832 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031729187562688066, + "loss": 1.3295, + "theoretical_loss": 3.424069759483288, + "tokens_seen": 2263482368 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317271815446339, + "loss": 1.3212, + "theoretical_loss": 3.4240617604057038, + "tokens_seen": 2263547904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172517552657974, + "loss": 1.4052, + "theoretical_loss": 3.424053761624556, + "tokens_seen": 2263613440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172316950852558, + "loss": 1.3852, + "theoretical_loss": 3.424045763139826, + "tokens_seen": 2263678976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031721163490471417, + "loss": 1.3656, + "theoretical_loss": 3.4240377649514935, + "tokens_seen": 2263744512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031719157472417253, + "loss": 1.3539, + "theoretical_loss": 3.4240297670595385, + "tokens_seen": 2263810048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171715145436309, + "loss": 1.3526, + "theoretical_loss": 3.4240217694639425, + "tokens_seen": 2263875584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171514543630893, + "loss": 1.3976, + "theoretical_loss": 3.4240137721646855, + "tokens_seen": 2263941120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031713139418254767, + "loss": 1.2782, + "theoretical_loss": 3.424005775161748, + "tokens_seen": 2264006656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031711133400200603, + "loss": 1.4166, + "theoretical_loss": 3.42399777845511, + "tokens_seen": 2264072192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170912738214644, + "loss": 1.384, + "theoretical_loss": 3.4239897820447522, + "tokens_seen": 2264137728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170712136409228, + "loss": 1.2788, + "theoretical_loss": 3.4239817859306556, + "tokens_seen": 2264203264 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170511534603812, + "loss": 1.3884, + "theoretical_loss": 3.4239737901128, + "tokens_seen": 2264268800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031703109327983954, + "loss": 1.3159, + "theoretical_loss": 3.423965794591166, + "tokens_seen": 2264334336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170110330992979, + "loss": 1.2609, + "theoretical_loss": 3.423957799365734, + "tokens_seen": 2264399872 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031699097291875626, + "loss": 1.2845, + "theoretical_loss": 3.4239498044364844, + "tokens_seen": 2264465408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169709127382147, + "loss": 1.3472, + "theoretical_loss": 3.423941809803398, + "tokens_seen": 2264530944 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031695085255767304, + "loss": 1.2235, + "theoretical_loss": 3.423933815466455, + "tokens_seen": 2264596480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169307923771314, + "loss": 1.3438, + "theoretical_loss": 3.423925821425636, + "tokens_seen": 2264662016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031691073219658976, + "loss": 1.3512, + "theoretical_loss": 3.4239178276809215, + "tokens_seen": 2264727552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168906720160482, + "loss": 1.3445, + "theoretical_loss": 3.4239098342322913, + "tokens_seen": 2264793088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031687061183550654, + "loss": 1.3948, + "theoretical_loss": 3.4239018410797266, + "tokens_seen": 2264858624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168505516549649, + "loss": 1.3745, + "theoretical_loss": 3.4238938482232077, + "tokens_seen": 2264924160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031683049147442327, + "loss": 1.367, + "theoretical_loss": 3.423885855662715, + "tokens_seen": 2264989696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031681043129388163, + "loss": 1.3853, + "theoretical_loss": 3.4238778633982294, + "tokens_seen": 2265055232 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031679037111334004, + "loss": 1.3939, + "theoretical_loss": 3.423869871429731, + "tokens_seen": 2265120768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167703109327984, + "loss": 1.3539, + "theoretical_loss": 3.4238618797572, + "tokens_seen": 2265186304 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031675025075225677, + "loss": 1.3538, + "theoretical_loss": 3.4238538883806173, + "tokens_seen": 2265251840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031673019057171513, + "loss": 1.3526, + "theoretical_loss": 3.423845897299963, + "tokens_seen": 2265317376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031671013039117355, + "loss": 1.3548, + "theoretical_loss": 3.423837906515218, + "tokens_seen": 2265382912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166900702106319, + "loss": 1.2499, + "theoretical_loss": 3.423829916026362, + "tokens_seen": 2265448448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031667001003009027, + "loss": 1.3064, + "theoretical_loss": 3.423821925833377, + "tokens_seen": 2265513984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031664994984954863, + "loss": 1.2685, + "theoretical_loss": 3.423813935936242, + "tokens_seen": 2265579520 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316629889669007, + "loss": 1.3814, + "theoretical_loss": 3.4238059463349377, + "tokens_seen": 2265645056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166098294884654, + "loss": 1.2939, + "theoretical_loss": 3.4237979570294454, + "tokens_seen": 2265710592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165897693079238, + "loss": 1.2421, + "theoretical_loss": 3.423789968019745, + "tokens_seen": 2265776128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031656970912738214, + "loss": 1.3809, + "theoretical_loss": 3.4237819793058173, + "tokens_seen": 2265841664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165496489468405, + "loss": 1.4027, + "theoretical_loss": 3.4237739908876423, + "tokens_seen": 2265907200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165295887662989, + "loss": 1.2975, + "theoretical_loss": 3.423766002765201, + "tokens_seen": 2265972736 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 825438, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3243449926376343, + "objective/train/theoretical_loss": 3.423758014938474, + "objective/train/tokens_used": 636497376, + "theoretical_loss": 3.423758014938474, + "tokens_seen": 2266038272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165095285857573, + "loss": 1.2886, + "theoretical_loss": 3.423758014938474, + "tokens_seen": 2266038272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031648946840521564, + "loss": 1.3231, + "theoretical_loss": 3.423750027407441, + "tokens_seen": 2266103808 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316469408224674, + "loss": 1.2709, + "theoretical_loss": 3.4237420401720833, + "tokens_seen": 2266169344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031644934804413237, + "loss": 1.2743, + "theoretical_loss": 3.423734053232381, + "tokens_seen": 2266234880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164292878635908, + "loss": 1.2997, + "theoretical_loss": 3.4237260665883147, + "tokens_seen": 2266300416 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031640922768304914, + "loss": 1.5299, + "theoretical_loss": 3.423718080239865, + "tokens_seen": 2266365952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163891675025075, + "loss": 1.349, + "theoretical_loss": 3.423710094187012, + "tokens_seen": 2266431488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031636910732196587, + "loss": 1.2911, + "theoretical_loss": 3.423702108429737, + "tokens_seen": 2266497024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163490471414243, + "loss": 1.3395, + "theoretical_loss": 3.42369412296802, + "tokens_seen": 2266562560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031632898696088265, + "loss": 1.3541, + "theoretical_loss": 3.4236861378018415, + "tokens_seen": 2266628096 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316308926780341, + "loss": 1.4256, + "theoretical_loss": 3.423678152931182, + "tokens_seen": 2266693632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031628886659979937, + "loss": 1.4446, + "theoretical_loss": 3.4236701683560224, + "tokens_seen": 2266759168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031626880641925773, + "loss": 1.309, + "theoretical_loss": 3.423662184076343, + "tokens_seen": 2266824704 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031624874623871615, + "loss": 1.3731, + "theoretical_loss": 3.4236542000921237, + "tokens_seen": 2266890240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162286860581745, + "loss": 1.2647, + "theoretical_loss": 3.4236462164033457, + "tokens_seen": 2266955776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162086258776329, + "loss": 1.3859, + "theoretical_loss": 3.42363823300999, + "tokens_seen": 2267021312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031618856569709124, + "loss": 1.3212, + "theoretical_loss": 3.423630249912036, + "tokens_seen": 2267086848 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031616850551654965, + "loss": 1.3167, + "theoretical_loss": 3.423622267109465, + "tokens_seen": 2267152384 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316148445336008, + "loss": 1.3949, + "theoretical_loss": 3.423614284602257, + "tokens_seen": 2267217920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161283851554664, + "loss": 1.293, + "theoretical_loss": 3.423606302390393, + "tokens_seen": 2267283456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031610832497492474, + "loss": 1.3901, + "theoretical_loss": 3.423598320473854, + "tokens_seen": 2267348992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160882647943831, + "loss": 1.3159, + "theoretical_loss": 3.423590338852619, + "tokens_seen": 2267414528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160682046138415, + "loss": 1.2792, + "theoretical_loss": 3.42358235752667, + "tokens_seen": 2267480064 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031604814443329994, + "loss": 1.2882, + "theoretical_loss": 3.4235743764959867, + "tokens_seen": 2267545600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160280842527583, + "loss": 1.3013, + "theoretical_loss": 3.42356639576055, + "tokens_seen": 2267611136 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031600802407221666, + "loss": 1.3033, + "theoretical_loss": 3.4235584153203407, + "tokens_seen": 2267676672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159879638916751, + "loss": 1.2979, + "theoretical_loss": 3.423550435175339, + "tokens_seen": 2267742208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031596790371113344, + "loss": 1.2566, + "theoretical_loss": 3.423542455325525, + "tokens_seen": 2267807744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159478435305918, + "loss": 1.3325, + "theoretical_loss": 3.42353447577088, + "tokens_seen": 2267873280 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031592778335005016, + "loss": 1.2413, + "theoretical_loss": 3.4235264965113843, + "tokens_seen": 2267938816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159077231695086, + "loss": 1.4014, + "theoretical_loss": 3.4235185175470186, + "tokens_seen": 2268004352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031588766298896694, + "loss": 1.4239, + "theoretical_loss": 3.423510538877763, + "tokens_seen": 2268069888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158676028084253, + "loss": 1.416, + "theoretical_loss": 3.4235025605035987, + "tokens_seen": 2268135424 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031584754262788367, + "loss": 1.4103, + "theoretical_loss": 3.4234945824245058, + "tokens_seen": 2268200960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031582748244734203, + "loss": 1.2994, + "theoretical_loss": 3.423486604640465, + "tokens_seen": 2268266496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031580742226680044, + "loss": 1.255, + "theoretical_loss": 3.4234786271514563, + "tokens_seen": 2268332032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157873620862588, + "loss": 1.362, + "theoretical_loss": 3.423470649957461, + "tokens_seen": 2268397568 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031576730190571717, + "loss": 1.3785, + "theoretical_loss": 3.4234626730584603, + "tokens_seen": 2268463104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031574724172517553, + "loss": 1.2947, + "theoretical_loss": 3.4234546964544332, + "tokens_seen": 2268528640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031572718154463395, + "loss": 1.2728, + "theoretical_loss": 3.4234467201453613, + "tokens_seen": 2268594176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157071213640923, + "loss": 1.2468, + "theoretical_loss": 3.4234387441312246, + "tokens_seen": 2268659712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031568706118355067, + "loss": 1.4021, + "theoretical_loss": 3.423430768412004, + "tokens_seen": 2268725248 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031566700100300903, + "loss": 1.273, + "theoretical_loss": 3.4234227929876804, + "tokens_seen": 2268790784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156469408224674, + "loss": 1.3642, + "theoretical_loss": 3.4234148178582338, + "tokens_seen": 2268856320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156268806419258, + "loss": 1.3198, + "theoretical_loss": 3.4234068430236446, + "tokens_seen": 2268921856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156068204613842, + "loss": 1.3041, + "theoretical_loss": 3.4233988684838943, + "tokens_seen": 2268987392 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031558676028084254, + "loss": 1.3536, + "theoretical_loss": 3.423390894238963, + "tokens_seen": 2269052928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155667001003009, + "loss": 1.2933, + "theoretical_loss": 3.423382920288831, + "tokens_seen": 2269118464 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155466399197593, + "loss": 1.1906, + "theoretical_loss": 3.423374946633479, + "tokens_seen": 2269184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155265797392177, + "loss": 1.2789, + "theoretical_loss": 3.423366973272888, + "tokens_seen": 2269249536 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 826474, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2791855335235596, + "objective/train/theoretical_loss": 3.4233590002070384, + "objective/train/tokens_used": 639774176, + "theoretical_loss": 3.4233590002070384, + "tokens_seen": 2269315072 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031550651955867604, + "loss": 1.3293, + "theoretical_loss": 3.4233590002070384, + "tokens_seen": 2269315072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154864593781344, + "loss": 1.3329, + "theoretical_loss": 3.4233510274359107, + "tokens_seen": 2269380608 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031546639919759277, + "loss": 1.3198, + "theoretical_loss": 3.4233430549594854, + "tokens_seen": 2269446144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154463390170512, + "loss": 1.2804, + "theoretical_loss": 3.423335082777743, + "tokens_seen": 2269511680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031542627883650954, + "loss": 1.3649, + "theoretical_loss": 3.4233271108906647, + "tokens_seen": 2269577216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154062186559679, + "loss": 1.2806, + "theoretical_loss": 3.4233191392982305, + "tokens_seen": 2269642752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031538615847542627, + "loss": 1.2899, + "theoretical_loss": 3.423311168000421, + "tokens_seen": 2269708288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153660982948847, + "loss": 1.3163, + "theoretical_loss": 3.4233031969972174, + "tokens_seen": 2269773824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031534603811434305, + "loss": 1.2895, + "theoretical_loss": 3.4232952262886, + "tokens_seen": 2269839360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153259779338014, + "loss": 1.3795, + "theoretical_loss": 3.423287255874549, + "tokens_seen": 2269904896 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031530591775325977, + "loss": 1.3685, + "theoretical_loss": 3.423279285755045, + "tokens_seen": 2269970432 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031528585757271813, + "loss": 1.3008, + "theoretical_loss": 3.4232713159300694, + "tokens_seen": 2270035968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031526579739217655, + "loss": 1.1915, + "theoretical_loss": 3.4232633463996023, + "tokens_seen": 2270101504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152457372116349, + "loss": 1.2787, + "theoretical_loss": 3.4232553771636245, + "tokens_seen": 2270167040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152256770310933, + "loss": 1.2514, + "theoretical_loss": 3.4232474082221165, + "tokens_seen": 2270232576 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031520561685055164, + "loss": 1.2126, + "theoretical_loss": 3.4232394395750587, + "tokens_seen": 2270298112 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031518555667001005, + "loss": 1.3392, + "theoretical_loss": 3.4232314712224317, + "tokens_seen": 2270363648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151654964894684, + "loss": 1.2786, + "theoretical_loss": 3.423223503164217, + "tokens_seen": 2270429184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151454363089268, + "loss": 1.2944, + "theoretical_loss": 3.423215535400394, + "tokens_seen": 2270494720 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031512537612838514, + "loss": 1.3857, + "theoretical_loss": 3.423207567930944, + "tokens_seen": 2270560256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151053159478435, + "loss": 1.2976, + "theoretical_loss": 3.423199600755848, + "tokens_seen": 2270625792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150852557673019, + "loss": 1.3315, + "theoretical_loss": 3.423191633875086, + "tokens_seen": 2270691328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150651955867603, + "loss": 1.3245, + "theoretical_loss": 3.4231836672886384, + "tokens_seen": 2270756864 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031504513540621864, + "loss": 1.3049, + "theoretical_loss": 3.4231757009964863, + "tokens_seen": 2270822400 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315025075225677, + "loss": 1.3287, + "theoretical_loss": 3.4231677349986107, + "tokens_seen": 2270887936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150050150451354, + "loss": 1.1428, + "theoretical_loss": 3.4231597692949913, + "tokens_seen": 2270953472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149849548645938, + "loss": 1.2547, + "theoretical_loss": 3.4231518038856095, + "tokens_seen": 2271019008 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031496489468405215, + "loss": 1.3618, + "theoretical_loss": 3.4231438387704456, + "tokens_seen": 2271084544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149448345035105, + "loss": 1.3367, + "theoretical_loss": 3.42313587394948, + "tokens_seen": 2271150080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149247743229689, + "loss": 1.451, + "theoretical_loss": 3.4231279094226945, + "tokens_seen": 2271215616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149047141424273, + "loss": 1.3541, + "theoretical_loss": 3.4231199451900682, + "tokens_seen": 2271281152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031488465396188565, + "loss": 1.2798, + "theoretical_loss": 3.4231119812515827, + "tokens_seen": 2271346688 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314864593781344, + "loss": 1.2459, + "theoretical_loss": 3.4231040176072187, + "tokens_seen": 2271412224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148445336008024, + "loss": 1.4042, + "theoretical_loss": 3.4230960542569564, + "tokens_seen": 2271477760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148244734202608, + "loss": 1.2704, + "theoretical_loss": 3.4230880912007766, + "tokens_seen": 2271543296 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031480441323971915, + "loss": 1.2606, + "theoretical_loss": 3.4230801284386594, + "tokens_seen": 2271608832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147843530591775, + "loss": 1.2651, + "theoretical_loss": 3.423072165970587, + "tokens_seen": 2271674368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147642928786359, + "loss": 1.3158, + "theoretical_loss": 3.4230642037965384, + "tokens_seen": 2271739904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147442326980943, + "loss": 1.3353, + "theoretical_loss": 3.423056241916495, + "tokens_seen": 2271805440 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031472417251755266, + "loss": 1.2985, + "theoretical_loss": 3.4230482803304376, + "tokens_seen": 2271870976 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314704112337011, + "loss": 1.3788, + "theoretical_loss": 3.423040319038347, + "tokens_seen": 2271936512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146840521564694, + "loss": 1.2801, + "theoretical_loss": 3.423032358040203, + "tokens_seen": 2272002048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031466399197592774, + "loss": 1.2869, + "theoretical_loss": 3.423024397335987, + "tokens_seen": 2272067584 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031464393179538616, + "loss": 1.3256, + "theoretical_loss": 3.4230164369256797, + "tokens_seen": 2272133120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146238716148445, + "loss": 1.385, + "theoretical_loss": 3.4230084768092612, + "tokens_seen": 2272198656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146038114343029, + "loss": 1.2548, + "theoretical_loss": 3.423000516986713, + "tokens_seen": 2272264192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031458375125376125, + "loss": 1.3888, + "theoretical_loss": 3.4229925574580147, + "tokens_seen": 2272329728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145636910732197, + "loss": 1.3687, + "theoretical_loss": 3.422984598223148, + "tokens_seen": 2272395264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145436308926781, + "loss": 1.2808, + "theoretical_loss": 3.4229766392820933, + "tokens_seen": 2272460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031452357071213644, + "loss": 1.228, + "theoretical_loss": 3.4229686806348307, + "tokens_seen": 2272526336 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 827716, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3444836139678955, + "objective/train/theoretical_loss": 3.4229607222813416, + "objective/train/tokens_used": 643050976, + "theoretical_loss": 3.4229607222813416, + "tokens_seen": 2272591872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145035105315948, + "loss": 1.3083, + "theoretical_loss": 3.4229607222813416, + "tokens_seen": 2272591872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031448345035105317, + "loss": 1.3633, + "theoretical_loss": 3.4229527642216064, + "tokens_seen": 2272657408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144633901705116, + "loss": 1.3269, + "theoretical_loss": 3.4229448064556056, + "tokens_seen": 2272722944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031444332998996994, + "loss": 1.408, + "theoretical_loss": 3.42293684898332, + "tokens_seen": 2272788480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144232698094283, + "loss": 1.2728, + "theoretical_loss": 3.422928891804731, + "tokens_seen": 2272854016 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031440320962888667, + "loss": 1.2904, + "theoretical_loss": 3.4229209349198184, + "tokens_seen": 2272919552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143831494483451, + "loss": 1.3569, + "theoretical_loss": 3.422912978328563, + "tokens_seen": 2272985088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031436308926780345, + "loss": 1.2309, + "theoretical_loss": 3.4229050220309456, + "tokens_seen": 2273050624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143430290872618, + "loss": 1.3575, + "theoretical_loss": 3.4228970660269473, + "tokens_seen": 2273116160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031432296890672017, + "loss": 1.3659, + "theoretical_loss": 3.422889110316548, + "tokens_seen": 2273181696 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031430290872617853, + "loss": 1.2539, + "theoretical_loss": 3.422881154899729, + "tokens_seen": 2273247232 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031428284854563695, + "loss": 1.3712, + "theoretical_loss": 3.422873199776471, + "tokens_seen": 2273312768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142627883650953, + "loss": 1.2843, + "theoretical_loss": 3.4228652449467547, + "tokens_seen": 2273378304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142427281845537, + "loss": 1.2933, + "theoretical_loss": 3.4228572904105605, + "tokens_seen": 2273443840 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031422266800401204, + "loss": 1.3838, + "theoretical_loss": 3.4228493361678693, + "tokens_seen": 2273509376 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031420260782347045, + "loss": 1.3896, + "theoretical_loss": 3.4228413822186616, + "tokens_seen": 2273574912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141825476429288, + "loss": 1.3701, + "theoretical_loss": 3.4228334285629187, + "tokens_seen": 2273640448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141624874623872, + "loss": 1.3951, + "theoretical_loss": 3.4228254752006206, + "tokens_seen": 2273705984 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031414242728184554, + "loss": 1.4413, + "theoretical_loss": 3.4228175221317483, + "tokens_seen": 2273771520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141223671013039, + "loss": 1.3033, + "theoretical_loss": 3.422809569356283, + "tokens_seen": 2273837056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141023069207623, + "loss": 1.3422, + "theoretical_loss": 3.4228016168742044, + "tokens_seen": 2273902592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140822467402207, + "loss": 1.3284, + "theoretical_loss": 3.422793664685494, + "tokens_seen": 2273968128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031406218655967904, + "loss": 1.3892, + "theoretical_loss": 3.4227857127901324, + "tokens_seen": 2274033664 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140421263791374, + "loss": 1.2324, + "theoretical_loss": 3.4227777611881, + "tokens_seen": 2274099200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140220661985958, + "loss": 1.3167, + "theoretical_loss": 3.4227698098793784, + "tokens_seen": 2274164736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140020060180542, + "loss": 1.4027, + "theoretical_loss": 3.422761858863947, + "tokens_seen": 2274230272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031398194583751255, + "loss": 1.3724, + "theoretical_loss": 3.4227539081417877, + "tokens_seen": 2274295808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139618856569709, + "loss": 1.376, + "theoretical_loss": 3.4227459577128805, + "tokens_seen": 2274361344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139418254764293, + "loss": 1.3556, + "theoretical_loss": 3.4227380075772067, + "tokens_seen": 2274426880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139217652958877, + "loss": 1.4243, + "theoretical_loss": 3.4227300577347464, + "tokens_seen": 2274492416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031390170511534605, + "loss": 1.4111, + "theoretical_loss": 3.422722108185481, + "tokens_seen": 2274557952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138816449348044, + "loss": 1.3285, + "theoretical_loss": 3.4227141589293906, + "tokens_seen": 2274623488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138615847542628, + "loss": 1.2785, + "theoretical_loss": 3.4227062099664565, + "tokens_seen": 2274689024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138415245737212, + "loss": 1.3156, + "theoretical_loss": 3.4226982612966586, + "tokens_seen": 2274754560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031382146439317955, + "loss": 1.1823, + "theoretical_loss": 3.4226903129199786, + "tokens_seen": 2274820096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138014042126379, + "loss": 1.275, + "theoretical_loss": 3.422682364836397, + "tokens_seen": 2274885632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137813440320963, + "loss": 1.2978, + "theoretical_loss": 3.4226744170458945, + "tokens_seen": 2274951168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137612838515547, + "loss": 1.3652, + "theoretical_loss": 3.4226664695484517, + "tokens_seen": 2275016704 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031374122367101306, + "loss": 1.3871, + "theoretical_loss": 3.4226585223440495, + "tokens_seen": 2275082240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137211634904714, + "loss": 1.347, + "theoretical_loss": 3.4226505754326686, + "tokens_seen": 2275147776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137011033099298, + "loss": 1.3375, + "theoretical_loss": 3.4226426288142893, + "tokens_seen": 2275213312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031368104312938814, + "loss": 1.3626, + "theoretical_loss": 3.4226346824888934, + "tokens_seen": 2275278848 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031366098294884656, + "loss": 1.2423, + "theoretical_loss": 3.422626736456461, + "tokens_seen": 2275344384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136409227683049, + "loss": 1.312, + "theoretical_loss": 3.4226187907169723, + "tokens_seen": 2275409920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136208625877633, + "loss": 1.3829, + "theoretical_loss": 3.4226108452704094, + "tokens_seen": 2275475456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031360080240722165, + "loss": 1.2626, + "theoretical_loss": 3.422602900116752, + "tokens_seen": 2275540992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031358074222668006, + "loss": 1.3898, + "theoretical_loss": 3.422594955255981, + "tokens_seen": 2275606528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135606820461384, + "loss": 1.3207, + "theoretical_loss": 3.422587010688078, + "tokens_seen": 2275672064 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135406218655968, + "loss": 1.4256, + "theoretical_loss": 3.422579066413023, + "tokens_seen": 2275737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031352056168505515, + "loss": 1.3792, + "theoretical_loss": 3.4225711224307966, + "tokens_seen": 2275803136 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 828910, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2416036128997803, + "objective/train/theoretical_loss": 3.42256317874138, + "objective/train/tokens_used": 646327776, + "theoretical_loss": 3.42256317874138, + "tokens_seen": 2275868672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135005015045135, + "loss": 1.3088, + "theoretical_loss": 3.42256317874138, + "tokens_seen": 2275868672 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031348044132397193, + "loss": 1.4286, + "theoretical_loss": 3.422555235344754, + "tokens_seen": 2275934208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134603811434303, + "loss": 1.1684, + "theoretical_loss": 3.4225472922408993, + "tokens_seen": 2275999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031344032096288865, + "loss": 1.3829, + "theoretical_loss": 3.4225393494297967, + "tokens_seen": 2276065280 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313420260782347, + "loss": 1.3313, + "theoretical_loss": 3.4225314069114265, + "tokens_seen": 2276130816 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031340020060180543, + "loss": 1.2778, + "theoretical_loss": 3.4225234646857703, + "tokens_seen": 2276196352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133801404212638, + "loss": 1.378, + "theoretical_loss": 3.4225155227528083, + "tokens_seen": 2276261888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031336008024072216, + "loss": 1.3645, + "theoretical_loss": 3.4225075811125216, + "tokens_seen": 2276327424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133400200601805, + "loss": 1.3042, + "theoretical_loss": 3.422499639764891, + "tokens_seen": 2276392960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133199598796389, + "loss": 1.3678, + "theoretical_loss": 3.422491698709897, + "tokens_seen": 2276458496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132998996990973, + "loss": 1.2056, + "theoretical_loss": 3.4224837579475205, + "tokens_seen": 2276524032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031327983951855566, + "loss": 1.2938, + "theoretical_loss": 3.422475817477742, + "tokens_seen": 2276589568 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313259779338014, + "loss": 1.4211, + "theoretical_loss": 3.422467877300543, + "tokens_seen": 2276655104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132397191574724, + "loss": 1.2548, + "theoretical_loss": 3.422459937415904, + "tokens_seen": 2276720640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132196589769308, + "loss": 1.3966, + "theoretical_loss": 3.4224519978238055, + "tokens_seen": 2276786176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031319959879638916, + "loss": 1.3433, + "theoretical_loss": 3.422444058524229, + "tokens_seen": 2276851712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131795386158475, + "loss": 1.2742, + "theoretical_loss": 3.4224361195171547, + "tokens_seen": 2276917248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131594784353059, + "loss": 1.3375, + "theoretical_loss": 3.422428180802563, + "tokens_seen": 2276982784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031313941825476425, + "loss": 1.2373, + "theoretical_loss": 3.422420242380436, + "tokens_seen": 2277048320 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031311935807422267, + "loss": 1.252, + "theoretical_loss": 3.422412304250753, + "tokens_seen": 2277113856 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031309929789368103, + "loss": 1.3981, + "theoretical_loss": 3.422404366413496, + "tokens_seen": 2277179392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130792377131394, + "loss": 1.3659, + "theoretical_loss": 3.4223964288686455, + "tokens_seen": 2277244928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130591775325978, + "loss": 1.3125, + "theoretical_loss": 3.422388491616182, + "tokens_seen": 2277310464 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130391173520562, + "loss": 1.249, + "theoretical_loss": 3.422380554656087, + "tokens_seen": 2277376000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130190571715146, + "loss": 1.413, + "theoretical_loss": 3.42237261798834, + "tokens_seen": 2277441536 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031299899699097295, + "loss": 1.3406, + "theoretical_loss": 3.422364681612923, + "tokens_seen": 2277507072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129789368104313, + "loss": 1.3259, + "theoretical_loss": 3.4223567455298167, + "tokens_seen": 2277572608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129588766298897, + "loss": 1.3627, + "theoretical_loss": 3.4223488097390016, + "tokens_seen": 2277638144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129388164493481, + "loss": 1.3205, + "theoretical_loss": 3.422340874240458, + "tokens_seen": 2277703680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031291875626880645, + "loss": 1.4216, + "theoretical_loss": 3.422332939034168, + "tokens_seen": 2277769216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128986960882648, + "loss": 1.3796, + "theoretical_loss": 3.4223250041201116, + "tokens_seen": 2277834752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128786359077232, + "loss": 1.2808, + "theoretical_loss": 3.42231706949827, + "tokens_seen": 2277900288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128585757271816, + "loss": 1.3852, + "theoretical_loss": 3.4223091351686237, + "tokens_seen": 2277965824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031283851554663995, + "loss": 1.3091, + "theoretical_loss": 3.4223012011311535, + "tokens_seen": 2278031360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128184553660983, + "loss": 1.3835, + "theoretical_loss": 3.4222932673858404, + "tokens_seen": 2278096896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127983951855567, + "loss": 1.2637, + "theoretical_loss": 3.4222853339326655, + "tokens_seen": 2278162432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127783350050151, + "loss": 1.3812, + "theoretical_loss": 3.4222774007716095, + "tokens_seen": 2278227968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031275827482447346, + "loss": 1.238, + "theoretical_loss": 3.4222694679026526, + "tokens_seen": 2278293504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127382146439318, + "loss": 1.2695, + "theoretical_loss": 3.4222615353257764, + "tokens_seen": 2278359040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127181544633902, + "loss": 1.3677, + "theoretical_loss": 3.422253603040962, + "tokens_seen": 2278424576 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031269809428284854, + "loss": 1.3223, + "theoretical_loss": 3.422245671048189, + "tokens_seen": 2278490112 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031267803410230696, + "loss": 1.4013, + "theoretical_loss": 3.4222377393474392, + "tokens_seen": 2278555648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126579739217653, + "loss": 1.4126, + "theoretical_loss": 3.422229807938694, + "tokens_seen": 2278621184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126379137412237, + "loss": 1.4429, + "theoretical_loss": 3.422221876821933, + "tokens_seen": 2278686720 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031261785356068205, + "loss": 1.2284, + "theoretical_loss": 3.422213945997137, + "tokens_seen": 2278752256 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031259779338014046, + "loss": 1.2272, + "theoretical_loss": 3.422206015464288, + "tokens_seen": 2278817792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125777331995988, + "loss": 1.3482, + "theoretical_loss": 3.422198085223366, + "tokens_seen": 2278883328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125576730190572, + "loss": 1.2693, + "theoretical_loss": 3.422190155274352, + "tokens_seen": 2278948864 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031253761283851555, + "loss": 1.2568, + "theoretical_loss": 3.422182225617228, + "tokens_seen": 2279014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125175526579739, + "loss": 1.3014, + "theoretical_loss": 3.422174296251973, + "tokens_seen": 2279079936 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 830070, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4052246809005737, + "objective/train/theoretical_loss": 3.4221663671785687, + "objective/train/tokens_used": 649604576, + "theoretical_loss": 3.4221663671785687, + "tokens_seen": 2279145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031249749247743233, + "loss": 1.348, + "theoretical_loss": 3.4221663671785687, + "tokens_seen": 2279145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124774322968907, + "loss": 1.399, + "theoretical_loss": 3.4221584383969965, + "tokens_seen": 2279211008 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031245737211634905, + "loss": 1.2513, + "theoretical_loss": 3.422150509907236, + "tokens_seen": 2279276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124373119358074, + "loss": 1.2671, + "theoretical_loss": 3.422142581709269, + "tokens_seen": 2279342080 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031241725175526583, + "loss": 1.2308, + "theoretical_loss": 3.422134653803077, + "tokens_seen": 2279407616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123971915747242, + "loss": 1.2963, + "theoretical_loss": 3.4221267261886394, + "tokens_seen": 2279473152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031237713139418256, + "loss": 1.4848, + "theoretical_loss": 3.422118798865938, + "tokens_seen": 2279538688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123570712136409, + "loss": 1.3358, + "theoretical_loss": 3.422110871834953, + "tokens_seen": 2279604224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123370110330993, + "loss": 1.3508, + "theoretical_loss": 3.422102945095666, + "tokens_seen": 2279669760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123169508525577, + "loss": 1.2728, + "theoretical_loss": 3.422095018648058, + "tokens_seen": 2279735296 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031229689067201606, + "loss": 1.3979, + "theoretical_loss": 3.4220870924921085, + "tokens_seen": 2279800832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122768304914744, + "loss": 1.4036, + "theoretical_loss": 3.4220791666278, + "tokens_seen": 2279866368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122567703109328, + "loss": 1.3083, + "theoretical_loss": 3.4220712410551126, + "tokens_seen": 2279931904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122367101303912, + "loss": 1.3162, + "theoretical_loss": 3.4220633157740274, + "tokens_seen": 2279997440 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031221664994984956, + "loss": 1.2242, + "theoretical_loss": 3.422055390784525, + "tokens_seen": 2280062976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121965897693079, + "loss": 1.3682, + "theoretical_loss": 3.422047466086587, + "tokens_seen": 2280128512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121765295887663, + "loss": 1.2556, + "theoretical_loss": 3.4220395416801934, + "tokens_seen": 2280194048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031215646940822465, + "loss": 1.2739, + "theoretical_loss": 3.4220316175653256, + "tokens_seen": 2280259584 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031213640922768307, + "loss": 1.3798, + "theoretical_loss": 3.4220236937419646, + "tokens_seen": 2280325120 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031211634904714143, + "loss": 1.2226, + "theoretical_loss": 3.422015770210091, + "tokens_seen": 2280390656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120962888665998, + "loss": 1.3171, + "theoretical_loss": 3.4220078469696853, + "tokens_seen": 2280456192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031207622868605815, + "loss": 1.4636, + "theoretical_loss": 3.4219999240207297, + "tokens_seen": 2280521728 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031205616850551657, + "loss": 1.3077, + "theoretical_loss": 3.4219920013632037, + "tokens_seen": 2280587264 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031203610832497493, + "loss": 1.3283, + "theoretical_loss": 3.421984078997089, + "tokens_seen": 2280652800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120160481444333, + "loss": 1.262, + "theoretical_loss": 3.421976156922366, + "tokens_seen": 2280718336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031199598796389166, + "loss": 1.349, + "theoretical_loss": 3.4219682351390164, + "tokens_seen": 2280783872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031197592778335, + "loss": 1.3937, + "theoretical_loss": 3.4219603136470207, + "tokens_seen": 2280849408 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031195586760280843, + "loss": 1.3059, + "theoretical_loss": 3.4219523924463595, + "tokens_seen": 2280914944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119358074222668, + "loss": 1.2466, + "theoretical_loss": 3.4219444715370138, + "tokens_seen": 2280980480 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031191574724172516, + "loss": 1.3576, + "theoretical_loss": 3.4219365509189648, + "tokens_seen": 2281046016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118956870611835, + "loss": 1.3172, + "theoretical_loss": 3.4219286305921934, + "tokens_seen": 2281111552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031187562688064194, + "loss": 1.3591, + "theoretical_loss": 3.4219207105566802, + "tokens_seen": 2281177088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118555667001003, + "loss": 1.4031, + "theoretical_loss": 3.421912790812407, + "tokens_seen": 2281242624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031183550651955866, + "loss": 1.2355, + "theoretical_loss": 3.421904871359353, + "tokens_seen": 2281308160 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311815446339017, + "loss": 1.3552, + "theoretical_loss": 3.421896952197501, + "tokens_seen": 2281373696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117953861584754, + "loss": 1.2175, + "theoretical_loss": 3.4218890333268313, + "tokens_seen": 2281439232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117753259779338, + "loss": 1.2584, + "theoretical_loss": 3.421881114747324, + "tokens_seen": 2281504768 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031175526579739217, + "loss": 1.2597, + "theoretical_loss": 3.421873196458961, + "tokens_seen": 2281570304 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031173520561685053, + "loss": 1.3005, + "theoretical_loss": 3.421865278461723, + "tokens_seen": 2281635840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117151454363089, + "loss": 1.3096, + "theoretical_loss": 3.4218573607555904, + "tokens_seen": 2281701376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116950852557673, + "loss": 1.4209, + "theoretical_loss": 3.421849443340545, + "tokens_seen": 2281766912 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031167502507522567, + "loss": 1.3921, + "theoretical_loss": 3.4218415262165673, + "tokens_seen": 2281832448 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031165496489468403, + "loss": 1.3128, + "theoretical_loss": 3.421833609383638, + "tokens_seen": 2281897984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116349047141424, + "loss": 1.3118, + "theoretical_loss": 3.4218256928417388, + "tokens_seen": 2281963520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116148445336008, + "loss": 1.3656, + "theoretical_loss": 3.4218177765908497, + "tokens_seen": 2282029056 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031159478435305917, + "loss": 1.3162, + "theoretical_loss": 3.421809860630952, + "tokens_seen": 2282094592 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031157472417251753, + "loss": 1.3124, + "theoretical_loss": 3.421801944962027, + "tokens_seen": 2282160128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031155466399197595, + "loss": 1.1769, + "theoretical_loss": 3.4217940295840554, + "tokens_seen": 2282225664 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115346038114343, + "loss": 1.2758, + "theoretical_loss": 3.421786114497018, + "tokens_seen": 2282291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031151454363089273, + "loss": 1.3223, + "theoretical_loss": 3.4217781997008965, + "tokens_seen": 2282356736 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 831223, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.272815465927124, + "objective/train/theoretical_loss": 3.4217702851956706, + "objective/train/tokens_used": 652881376, + "theoretical_loss": 3.4217702851956706, + "tokens_seen": 2282422272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114944834503511, + "loss": 1.3403, + "theoretical_loss": 3.4217702851956706, + "tokens_seen": 2282422272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031147442326980945, + "loss": 1.2478, + "theoretical_loss": 3.421762370981322, + "tokens_seen": 2282487808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114543630892678, + "loss": 1.3738, + "theoretical_loss": 3.4217544570578315, + "tokens_seen": 2282553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031143430290872623, + "loss": 1.2698, + "theoretical_loss": 3.4217465434251806, + "tokens_seen": 2282618880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114142427281846, + "loss": 1.3156, + "theoretical_loss": 3.4217386300833494, + "tokens_seen": 2282684416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031139418254764296, + "loss": 1.2899, + "theoretical_loss": 3.421730717032319, + "tokens_seen": 2282749952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113741223671013, + "loss": 1.4426, + "theoretical_loss": 3.421722804272071, + "tokens_seen": 2282815488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113540621865597, + "loss": 1.4258, + "theoretical_loss": 3.421714891802586, + "tokens_seen": 2282881024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113340020060181, + "loss": 1.2838, + "theoretical_loss": 3.421706979623845, + "tokens_seen": 2282946560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031131394182547646, + "loss": 1.3136, + "theoretical_loss": 3.4216990677358288, + "tokens_seen": 2283012096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112938816449348, + "loss": 1.2838, + "theoretical_loss": 3.421691156138518, + "tokens_seen": 2283077632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112738214643932, + "loss": 1.3447, + "theoretical_loss": 3.421683244831895, + "tokens_seen": 2283143168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112537612838516, + "loss": 1.2075, + "theoretical_loss": 3.4216753338159394, + "tokens_seen": 2283208704 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031123370110330996, + "loss": 1.2258, + "theoretical_loss": 3.4216674230906325, + "tokens_seen": 2283274240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112136409227683, + "loss": 1.2774, + "theoretical_loss": 3.4216595126559555, + "tokens_seen": 2283339776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111935807422267, + "loss": 1.2819, + "theoretical_loss": 3.4216516025118895, + "tokens_seen": 2283405312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031117352056168505, + "loss": 1.4573, + "theoretical_loss": 3.421643692658415, + "tokens_seen": 2283470848 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031115346038114347, + "loss": 1.3288, + "theoretical_loss": 3.421635783095513, + "tokens_seen": 2283536384 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031113340020060183, + "loss": 1.4551, + "theoretical_loss": 3.4216278738231654, + "tokens_seen": 2283601920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111133400200602, + "loss": 1.2476, + "theoretical_loss": 3.4216199648413523, + "tokens_seen": 2283667456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031109327983951855, + "loss": 1.3921, + "theoretical_loss": 3.421612056150055, + "tokens_seen": 2283732992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031107321965897697, + "loss": 1.2785, + "theoretical_loss": 3.421604147749254, + "tokens_seen": 2283798528 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031105315947843533, + "loss": 1.3602, + "theoretical_loss": 3.421596239638931, + "tokens_seen": 2283864064 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110330992978937, + "loss": 1.4148, + "theoretical_loss": 3.4215883318190667, + "tokens_seen": 2283929600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031101303911735206, + "loss": 1.316, + "theoretical_loss": 3.421580424289642, + "tokens_seen": 2283995136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109929789368104, + "loss": 1.3387, + "theoretical_loss": 3.421572517050638, + "tokens_seen": 2284060672 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031097291875626883, + "loss": 1.2477, + "theoretical_loss": 3.4215646101020356, + "tokens_seen": 2284126208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109528585757272, + "loss": 1.2883, + "theoretical_loss": 3.4215567034438163, + "tokens_seen": 2284191744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031093279839518556, + "loss": 1.257, + "theoretical_loss": 3.4215487970759604, + "tokens_seen": 2284257280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109127382146439, + "loss": 1.3497, + "theoretical_loss": 3.4215408909984495, + "tokens_seen": 2284322816 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031089267803410234, + "loss": 1.2615, + "theoretical_loss": 3.4215329852112637, + "tokens_seen": 2284388352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108726178535607, + "loss": 1.2576, + "theoretical_loss": 3.4215250797143852, + "tokens_seen": 2284453888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031085255767301906, + "loss": 1.3441, + "theoretical_loss": 3.4215171745077946, + "tokens_seen": 2284519424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108324974924774, + "loss": 1.3487, + "theoretical_loss": 3.421509269591472, + "tokens_seen": 2284584960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108124373119358, + "loss": 1.2536, + "theoretical_loss": 3.4215013649654, + "tokens_seen": 2284650496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107923771313942, + "loss": 1.281, + "theoretical_loss": 3.4214934606295584, + "tokens_seen": 2284716032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031077231695085257, + "loss": 1.3197, + "theoretical_loss": 3.4214855565839284, + "tokens_seen": 2284781568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031075225677031093, + "loss": 1.3433, + "theoretical_loss": 3.421477652828491, + "tokens_seen": 2284847104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107321965897693, + "loss": 1.3833, + "theoretical_loss": 3.4214697493632285, + "tokens_seen": 2284912640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107121364092277, + "loss": 1.3009, + "theoretical_loss": 3.4214618461881203, + "tokens_seen": 2284978176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031069207622868607, + "loss": 1.2542, + "theoretical_loss": 3.4214539433031477, + "tokens_seen": 2285043712 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031067201604814443, + "loss": 1.3658, + "theoretical_loss": 3.4214460407082923, + "tokens_seen": 2285109248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003106519558676028, + "loss": 1.3465, + "theoretical_loss": 3.421438138403535, + "tokens_seen": 2285174784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003106318956870612, + "loss": 1.3107, + "theoretical_loss": 3.4214302363888565, + "tokens_seen": 2285240320 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031061183550651957, + "loss": 1.4092, + "theoretical_loss": 3.4214223346642383, + "tokens_seen": 2285305856 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031059177532597793, + "loss": 1.3314, + "theoretical_loss": 3.421414433229661, + "tokens_seen": 2285371392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105717151454363, + "loss": 1.3858, + "theoretical_loss": 3.4214065320851055, + "tokens_seen": 2285436928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031055165496489466, + "loss": 1.258, + "theoretical_loss": 3.4213986312305535, + "tokens_seen": 2285502464 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105315947843531, + "loss": 1.2349, + "theoretical_loss": 3.4213907306659856, + "tokens_seen": 2285568000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031051153460381144, + "loss": 1.2747, + "theoretical_loss": 3.4213828303913827, + "tokens_seen": 2285633536 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 832464, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2004680633544922, + "objective/train/theoretical_loss": 3.4213749304067265, + "objective/train/tokens_used": 656158176, + "theoretical_loss": 3.4213749304067265, + "tokens_seen": 2285699072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104914744232698, + "loss": 1.2988, + "theoretical_loss": 3.4213749304067265, + "tokens_seen": 2285699072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031047141424272816, + "loss": 1.3727, + "theoretical_loss": 3.4213670307119974, + "tokens_seen": 2285764608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104513540621866, + "loss": 1.2657, + "theoretical_loss": 3.4213591313071765, + "tokens_seen": 2285830144 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031043129388164494, + "loss": 1.3149, + "theoretical_loss": 3.421351232192245, + "tokens_seen": 2285895680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104112337011033, + "loss": 1.2488, + "theoretical_loss": 3.4213433333671843, + "tokens_seen": 2285961216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031039117352056167, + "loss": 1.3831, + "theoretical_loss": 3.421335434831975, + "tokens_seen": 2286026752 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031037111334002003, + "loss": 1.3024, + "theoretical_loss": 3.421327536586598, + "tokens_seen": 2286092288 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031035105315947844, + "loss": 1.2364, + "theoretical_loss": 3.4213196386310347, + "tokens_seen": 2286157824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103309929789368, + "loss": 1.2459, + "theoretical_loss": 3.4213117409652662, + "tokens_seen": 2286223360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031031093279839517, + "loss": 1.4962, + "theoretical_loss": 3.4213038435892735, + "tokens_seen": 2286288896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031029087261785353, + "loss": 1.2534, + "theoretical_loss": 3.4212959465030375, + "tokens_seen": 2286354432 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031027081243731195, + "loss": 1.3141, + "theoretical_loss": 3.4212880497065394, + "tokens_seen": 2286419968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102507522567703, + "loss": 1.3077, + "theoretical_loss": 3.4212801531997603, + "tokens_seen": 2286485504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031023069207622867, + "loss": 1.3553, + "theoretical_loss": 3.4212722569826814, + "tokens_seen": 2286551040 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031021063189568703, + "loss": 1.2816, + "theoretical_loss": 3.421264361055283, + "tokens_seen": 2286616576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101905717151454, + "loss": 1.3052, + "theoretical_loss": 3.4212564654175472, + "tokens_seen": 2286682112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101705115346038, + "loss": 1.312, + "theoretical_loss": 3.421248570069454, + "tokens_seen": 2286747648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101504513540622, + "loss": 1.4145, + "theoretical_loss": 3.421240675010986, + "tokens_seen": 2286813184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031013039117352054, + "loss": 1.3362, + "theoretical_loss": 3.4212327802421227, + "tokens_seen": 2286878720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101103309929789, + "loss": 1.4111, + "theoretical_loss": 3.421224885762846, + "tokens_seen": 2286944256 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100902708124373, + "loss": 1.3658, + "theoretical_loss": 3.4212169915731367, + "tokens_seen": 2287009792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100702106318957, + "loss": 1.3521, + "theoretical_loss": 3.4212090976729765, + "tokens_seen": 2287075328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100501504513541, + "loss": 1.3366, + "theoretical_loss": 3.4212012040623456, + "tokens_seen": 2287140864 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031003009027081246, + "loss": 1.2581, + "theoretical_loss": 3.4211933107412253, + "tokens_seen": 2287206400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100100300902708, + "loss": 1.3943, + "theoretical_loss": 3.421185417709597, + "tokens_seen": 2287271936 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030998996990972924, + "loss": 1.4666, + "theoretical_loss": 3.421177524967442, + "tokens_seen": 2287337472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099699097291876, + "loss": 1.275, + "theoretical_loss": 3.4211696325147405, + "tokens_seen": 2287403008 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030994984954864596, + "loss": 1.2751, + "theoretical_loss": 3.4211617403514745, + "tokens_seen": 2287468544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099297893681043, + "loss": 1.2505, + "theoretical_loss": 3.4211538484776245, + "tokens_seen": 2287534080 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030990972918756274, + "loss": 1.3728, + "theoretical_loss": 3.421145956893172, + "tokens_seen": 2287599616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098896690070211, + "loss": 1.2774, + "theoretical_loss": 3.4211380655980976, + "tokens_seen": 2287665152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030986960882647946, + "loss": 1.3289, + "theoretical_loss": 3.421130174592383, + "tokens_seen": 2287730688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098495486459378, + "loss": 1.4023, + "theoretical_loss": 3.421122283876009, + "tokens_seen": 2287796224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098294884653962, + "loss": 1.4101, + "theoretical_loss": 3.4211143934489567, + "tokens_seen": 2287861760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098094282848546, + "loss": 1.3196, + "theoretical_loss": 3.421106503311207, + "tokens_seen": 2287927296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030978936810431297, + "loss": 1.4507, + "theoretical_loss": 3.4210986134627417, + "tokens_seen": 2287992832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030976930792377133, + "loss": 1.3148, + "theoretical_loss": 3.421090723903541, + "tokens_seen": 2288058368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097492477432297, + "loss": 1.3067, + "theoretical_loss": 3.4210828346335864, + "tokens_seen": 2288123904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097291875626881, + "loss": 1.436, + "theoretical_loss": 3.4210749456528595, + "tokens_seen": 2288189440 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030970912738214647, + "loss": 1.1947, + "theoretical_loss": 3.421067056961341, + "tokens_seen": 2288254976 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030968906720160483, + "loss": 1.3105, + "theoretical_loss": 3.4210591685590113, + "tokens_seen": 2288320512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096690070210632, + "loss": 1.3058, + "theoretical_loss": 3.4210512804458526, + "tokens_seen": 2288386048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096489468405216, + "loss": 1.4547, + "theoretical_loss": 3.4210433926218458, + "tokens_seen": 2288451584 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030962888665997997, + "loss": 1.2596, + "theoretical_loss": 3.4210355050869716, + "tokens_seen": 2288517120 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030960882647943833, + "loss": 1.3046, + "theoretical_loss": 3.4210276178412116, + "tokens_seen": 2288582656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095887662988967, + "loss": 1.3757, + "theoretical_loss": 3.421019730884546, + "tokens_seen": 2288648192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030956870611835506, + "loss": 1.295, + "theoretical_loss": 3.4210118442169573, + "tokens_seen": 2288713728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095486459378135, + "loss": 1.2446, + "theoretical_loss": 3.4210039578384257, + "tokens_seen": 2288779264 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030952858575727184, + "loss": 1.2992, + "theoretical_loss": 3.4209960717489327, + "tokens_seen": 2288844800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095085255767302, + "loss": 1.2972, + "theoretical_loss": 3.420988185948459, + "tokens_seen": 2288910336 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 833545, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3949838876724243, + "objective/train/theoretical_loss": 3.4209803004369865, + "objective/train/tokens_used": 659434976, + "theoretical_loss": 3.4209803004369865, + "tokens_seen": 2288975872 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030948846539618856, + "loss": 1.3159, + "theoretical_loss": 3.4209803004369865, + "tokens_seen": 2288975872 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309468405215647, + "loss": 1.3452, + "theoretical_loss": 3.4209724152144956, + "tokens_seen": 2289041408 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030944834503510534, + "loss": 1.3737, + "theoretical_loss": 3.420964530280968, + "tokens_seen": 2289106944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094282848545637, + "loss": 1.2262, + "theoretical_loss": 3.4209566456363842, + "tokens_seen": 2289172480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030940822467402207, + "loss": 1.3255, + "theoretical_loss": 3.4209487612807257, + "tokens_seen": 2289238016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030938816449348043, + "loss": 1.2949, + "theoretical_loss": 3.4209408772139738, + "tokens_seen": 2289303552 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030936810431293884, + "loss": 1.3541, + "theoretical_loss": 3.4209329934361095, + "tokens_seen": 2289369088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093480441323972, + "loss": 1.3476, + "theoretical_loss": 3.4209251099471136, + "tokens_seen": 2289434624 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030932798395185557, + "loss": 1.4121, + "theoretical_loss": 3.4209172267469676, + "tokens_seen": 2289500160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030930792377131393, + "loss": 1.347, + "theoretical_loss": 3.420909343835653, + "tokens_seen": 2289565696 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030928786359077235, + "loss": 1.2828, + "theoretical_loss": 3.42090146121315, + "tokens_seen": 2289631232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092678034102307, + "loss": 1.3231, + "theoretical_loss": 3.420893578879441, + "tokens_seen": 2289696768 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030924774322968907, + "loss": 1.361, + "theoretical_loss": 3.420885696834506, + "tokens_seen": 2289762304 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030922768304914743, + "loss": 1.2923, + "theoretical_loss": 3.4208778150783266, + "tokens_seen": 2289827840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092076228686058, + "loss": 1.2441, + "theoretical_loss": 3.4208699336108843, + "tokens_seen": 2289893376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091875626880642, + "loss": 1.3607, + "theoretical_loss": 3.4208620524321596, + "tokens_seen": 2289958912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091675025075226, + "loss": 1.4097, + "theoretical_loss": 3.420854171542134, + "tokens_seen": 2290024448 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030914744232698094, + "loss": 1.2348, + "theoretical_loss": 3.420846290940789, + "tokens_seen": 2290089984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091273821464393, + "loss": 1.34, + "theoretical_loss": 3.420838410628105, + "tokens_seen": 2290155520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091073219658977, + "loss": 1.3663, + "theoretical_loss": 3.420830530604064, + "tokens_seen": 2290221056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090872617853561, + "loss": 1.2582, + "theoretical_loss": 3.4208226508686463, + "tokens_seen": 2290286592 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030906720160481444, + "loss": 1.251, + "theoretical_loss": 3.420814771421834, + "tokens_seen": 2290352128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090471414242728, + "loss": 1.2802, + "theoretical_loss": 3.4208068922636077, + "tokens_seen": 2290417664 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030902708124373116, + "loss": 1.2286, + "theoretical_loss": 3.4207990133939488, + "tokens_seen": 2290483200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090070210631896, + "loss": 1.3673, + "theoretical_loss": 3.4207911348128377, + "tokens_seen": 2290548736 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030898696088264794, + "loss": 1.3027, + "theoretical_loss": 3.4207832565202567, + "tokens_seen": 2290614272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089669007021063, + "loss": 1.3198, + "theoretical_loss": 3.4207753785161867, + "tokens_seen": 2290679808 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030894684052156467, + "loss": 1.386, + "theoretical_loss": 3.4207675008006087, + "tokens_seen": 2290745344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089267803410231, + "loss": 1.2098, + "theoretical_loss": 3.4207596233735034, + "tokens_seen": 2290810880 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030890672016048145, + "loss": 1.3366, + "theoretical_loss": 3.4207517462348527, + "tokens_seen": 2290876416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088866599799398, + "loss": 1.3594, + "theoretical_loss": 3.420743869384638, + "tokens_seen": 2290941952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030886659979939817, + "loss": 1.389, + "theoretical_loss": 3.4207359928228396, + "tokens_seen": 2291007488 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030884653961885653, + "loss": 1.2543, + "theoretical_loss": 3.420728116549439, + "tokens_seen": 2291073024 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030882647943831495, + "loss": 1.2422, + "theoretical_loss": 3.4207202405644175, + "tokens_seen": 2291138560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088064192577733, + "loss": 1.3826, + "theoretical_loss": 3.4207123648677564, + "tokens_seen": 2291204096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087863590772317, + "loss": 1.3423, + "theoretical_loss": 3.420704489459437, + "tokens_seen": 2291269632 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030876629889669004, + "loss": 1.2938, + "theoretical_loss": 3.42069661433944, + "tokens_seen": 2291335168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030874623871614845, + "loss": 1.3459, + "theoretical_loss": 3.420688739507747, + "tokens_seen": 2291400704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087261785356068, + "loss": 1.3615, + "theoretical_loss": 3.420680864964339, + "tokens_seen": 2291466240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087061183550652, + "loss": 1.2909, + "theoretical_loss": 3.4206729907091975, + "tokens_seen": 2291531776 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030868605817452354, + "loss": 1.294, + "theoretical_loss": 3.4206651167423034, + "tokens_seen": 2291597312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086659979939819, + "loss": 1.3171, + "theoretical_loss": 3.420657243063638, + "tokens_seen": 2291662848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086459378134403, + "loss": 1.3536, + "theoretical_loss": 3.420649369673183, + "tokens_seen": 2291728384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086258776328987, + "loss": 1.3219, + "theoretical_loss": 3.4206414965709184, + "tokens_seen": 2291793920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030860581745235704, + "loss": 1.3707, + "theoretical_loss": 3.4206336237568267, + "tokens_seen": 2291859456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085857572718154, + "loss": 1.3637, + "theoretical_loss": 3.420625751230888, + "tokens_seen": 2291924992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085656970912738, + "loss": 1.3018, + "theoretical_loss": 3.4206178789930846, + "tokens_seen": 2291990528 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030854563691073224, + "loss": 1.356, + "theoretical_loss": 3.4206100070433965, + "tokens_seen": 2292056064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085255767301906, + "loss": 1.2556, + "theoretical_loss": 3.420602135381806, + "tokens_seen": 2292121600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030850551654964896, + "loss": 1.2167, + "theoretical_loss": 3.420594264008294, + "tokens_seen": 2292187136 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 834793, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2405096292495728, + "objective/train/theoretical_loss": 3.4205863929228415, + "objective/train/tokens_used": 662711776, + "theoretical_loss": 3.4205863929228415, + "tokens_seen": 2292252672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084854563691074, + "loss": 1.2682, + "theoretical_loss": 3.4205863929228415, + "tokens_seen": 2292252672 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030846539618856574, + "loss": 1.3506, + "theoretical_loss": 3.42057852212543, + "tokens_seen": 2292318208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084453360080241, + "loss": 1.1343, + "theoretical_loss": 3.4205706516160403, + "tokens_seen": 2292383744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030842527582748247, + "loss": 1.2668, + "theoretical_loss": 3.420562781394654, + "tokens_seen": 2292449280 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030840521564694083, + "loss": 1.3169, + "theoretical_loss": 3.4205549114612523, + "tokens_seen": 2292514816 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030838515546639924, + "loss": 1.3265, + "theoretical_loss": 3.4205470418158166, + "tokens_seen": 2292580352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083650952858576, + "loss": 1.3332, + "theoretical_loss": 3.4205391724583274, + "tokens_seen": 2292645888 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030834503510531597, + "loss": 1.382, + "theoretical_loss": 3.420531303388767, + "tokens_seen": 2292711424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030832497492477433, + "loss": 1.3819, + "theoretical_loss": 3.420523434607116, + "tokens_seen": 2292776960 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030830491474423275, + "loss": 1.3875, + "theoretical_loss": 3.4205155661133553, + "tokens_seen": 2292842496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082848545636911, + "loss": 1.3447, + "theoretical_loss": 3.4205076979074667, + "tokens_seen": 2292908032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030826479438314947, + "loss": 1.2513, + "theoretical_loss": 3.420499829989431, + "tokens_seen": 2292973568 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030824473420260783, + "loss": 1.256, + "theoretical_loss": 3.42049196235923, + "tokens_seen": 2293039104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082246740220662, + "loss": 1.2473, + "theoretical_loss": 3.4204840950168447, + "tokens_seen": 2293104640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082046138415246, + "loss": 1.2046, + "theoretical_loss": 3.420476227962256, + "tokens_seen": 2293170176 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308184553660983, + "loss": 1.3442, + "theoretical_loss": 3.420468361195446, + "tokens_seen": 2293235712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030816449348044134, + "loss": 1.3664, + "theoretical_loss": 3.4204604947163952, + "tokens_seen": 2293301248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081444332998997, + "loss": 1.2128, + "theoretical_loss": 3.4204526285250854, + "tokens_seen": 2293366784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081243731193581, + "loss": 1.3575, + "theoretical_loss": 3.420444762621497, + "tokens_seen": 2293432320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081043129388165, + "loss": 1.3835, + "theoretical_loss": 3.4204368970056116, + "tokens_seen": 2293497856 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030808425275827484, + "loss": 1.4044, + "theoretical_loss": 3.4204290316774113, + "tokens_seen": 2293563392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080641925777332, + "loss": 1.2343, + "theoretical_loss": 3.420421166636876, + "tokens_seen": 2293628928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030804413239719157, + "loss": 1.325, + "theoretical_loss": 3.420413301883988, + "tokens_seen": 2293694464 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030802407221665, + "loss": 1.1399, + "theoretical_loss": 3.4204054374187285, + "tokens_seen": 2293760000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030800401203610834, + "loss": 1.3272, + "theoretical_loss": 3.420397573241078, + "tokens_seen": 2293825536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079839518555667, + "loss": 1.3813, + "theoretical_loss": 3.4203897093510185, + "tokens_seen": 2293891072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030796389167502507, + "loss": 1.3791, + "theoretical_loss": 3.4203818457485307, + "tokens_seen": 2293956608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079438314944835, + "loss": 1.3351, + "theoretical_loss": 3.4203739824335964, + "tokens_seen": 2294022144 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030792377131394185, + "loss": 1.3899, + "theoretical_loss": 3.4203661194061965, + "tokens_seen": 2294087680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079037111334002, + "loss": 1.3845, + "theoretical_loss": 3.420358256666313, + "tokens_seen": 2294153216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030788365095285857, + "loss": 1.3103, + "theoretical_loss": 3.420350394213926, + "tokens_seen": 2294218752 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030786359077231693, + "loss": 1.2906, + "theoretical_loss": 3.4203425320490175, + "tokens_seen": 2294284288 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030784353059177535, + "loss": 1.2971, + "theoretical_loss": 3.4203346701715684, + "tokens_seen": 2294349824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078234704112337, + "loss": 1.3238, + "theoretical_loss": 3.4203268085815606, + "tokens_seen": 2294415360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078034102306921, + "loss": 1.3012, + "theoretical_loss": 3.420318947278975, + "tokens_seen": 2294480896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030778335005015044, + "loss": 1.3194, + "theoretical_loss": 3.4203110862637924, + "tokens_seen": 2294546432 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030776328986960885, + "loss": 1.4411, + "theoretical_loss": 3.420303225535995, + "tokens_seen": 2294611968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077432296890672, + "loss": 1.1563, + "theoretical_loss": 3.4202953650955634, + "tokens_seen": 2294677504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077231695085256, + "loss": 1.3424, + "theoretical_loss": 3.4202875049424795, + "tokens_seen": 2294743040 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030770310932798394, + "loss": 1.3091, + "theoretical_loss": 3.420279645076724, + "tokens_seen": 2294808576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076830491474423, + "loss": 1.4602, + "theoretical_loss": 3.4202717854982785, + "tokens_seen": 2294874112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076629889669007, + "loss": 1.3731, + "theoretical_loss": 3.420263926207124, + "tokens_seen": 2294939648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076429287863591, + "loss": 1.3165, + "theoretical_loss": 3.4202560672032423, + "tokens_seen": 2295005184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030762286860581744, + "loss": 1.285, + "theoretical_loss": 3.420248208486614, + "tokens_seen": 2295070720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076028084252758, + "loss": 1.3232, + "theoretical_loss": 3.420240350057221, + "tokens_seen": 2295136256 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075827482447342, + "loss": 1.1694, + "theoretical_loss": 3.4202324919150446, + "tokens_seen": 2295201792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075626880641926, + "loss": 1.3174, + "theoretical_loss": 3.420224634060066, + "tokens_seen": 2295267328 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030754262788365095, + "loss": 1.2465, + "theoretical_loss": 3.420216776492266, + "tokens_seen": 2295332864 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075225677031093, + "loss": 1.3986, + "theoretical_loss": 3.420208919211627, + "tokens_seen": 2295398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075025075225677, + "loss": 1.3003, + "theoretical_loss": 3.420201062218129, + "tokens_seen": 2295463936 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 836058, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1093416213989258, + "objective/train/theoretical_loss": 3.420193205511754, + "objective/train/tokens_used": 665988576, + "theoretical_loss": 3.420193205511754, + "tokens_seen": 2295529472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074824473420261, + "loss": 1.1661, + "theoretical_loss": 3.420193205511754, + "tokens_seen": 2295529472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030746238716148445, + "loss": 1.3943, + "theoretical_loss": 3.4201853490924834, + "tokens_seen": 2295595008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074423269809428, + "loss": 1.3037, + "theoretical_loss": 3.420177492960298, + "tokens_seen": 2295660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074222668004012, + "loss": 1.2376, + "theoretical_loss": 3.42016963711518, + "tokens_seen": 2295726080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074022066198596, + "loss": 1.3062, + "theoretical_loss": 3.42016178155711, + "tokens_seen": 2295791616 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030738214643931795, + "loss": 1.3919, + "theoretical_loss": 3.4201539262860696, + "tokens_seen": 2295857152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073620862587763, + "loss": 1.3921, + "theoretical_loss": 3.4201460713020397, + "tokens_seen": 2295922688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073420260782347, + "loss": 1.371, + "theoretical_loss": 3.420138216605002, + "tokens_seen": 2295988224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073219658976931, + "loss": 1.3823, + "theoretical_loss": 3.4201303621949384, + "tokens_seen": 2296053760 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030730190571715146, + "loss": 1.3091, + "theoretical_loss": 3.420122508071829, + "tokens_seen": 2296119296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072818455366098, + "loss": 1.3259, + "theoretical_loss": 3.4201146542356557, + "tokens_seen": 2296184832 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072617853560682, + "loss": 1.3361, + "theoretical_loss": 3.4201068006864, + "tokens_seen": 2296250368 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030724172517552654, + "loss": 1.4059, + "theoretical_loss": 3.420098947424043, + "tokens_seen": 2296315904 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030722166499498496, + "loss": 1.2967, + "theoretical_loss": 3.4200910944485665, + "tokens_seen": 2296381440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072016048144433, + "loss": 1.324, + "theoretical_loss": 3.4200832417599507, + "tokens_seen": 2296446976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071815446339017, + "loss": 1.4418, + "theoretical_loss": 3.4200753893581783, + "tokens_seen": 2296512512 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030716148445336005, + "loss": 1.4553, + "theoretical_loss": 3.42006753724323, + "tokens_seen": 2296578048 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030714142427281846, + "loss": 1.3193, + "theoretical_loss": 3.4200596854150866, + "tokens_seen": 2296643584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071213640922768, + "loss": 1.3432, + "theoretical_loss": 3.4200518338737305, + "tokens_seen": 2296709120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071013039117352, + "loss": 1.3294, + "theoretical_loss": 3.420043982619142, + "tokens_seen": 2296774656 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030708124373119355, + "loss": 1.2788, + "theoretical_loss": 3.4200361316513037, + "tokens_seen": 2296840192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070611835506519, + "loss": 1.3076, + "theoretical_loss": 3.420028280970196, + "tokens_seen": 2296905728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070411233701104, + "loss": 1.3058, + "theoretical_loss": 3.4200204305758004, + "tokens_seen": 2296971264 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030702106318956874, + "loss": 1.3349, + "theoretical_loss": 3.4200125804680983, + "tokens_seen": 2297036800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070010030090271, + "loss": 1.3601, + "theoretical_loss": 3.420004730647071, + "tokens_seen": 2297102336 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030698094282848547, + "loss": 1.3333, + "theoretical_loss": 3.4199968811127004, + "tokens_seen": 2297167872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069608826479439, + "loss": 1.2505, + "theoretical_loss": 3.419989031864967, + "tokens_seen": 2297233408 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030694082246740225, + "loss": 1.3008, + "theoretical_loss": 3.419981182903852, + "tokens_seen": 2297298944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069207622868606, + "loss": 1.2611, + "theoretical_loss": 3.419973334229338, + "tokens_seen": 2297364480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030690070210631897, + "loss": 1.3647, + "theoretical_loss": 3.4199654858414057, + "tokens_seen": 2297430016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030688064192577733, + "loss": 1.2623, + "theoretical_loss": 3.419957637740036, + "tokens_seen": 2297495552 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030686058174523575, + "loss": 1.3993, + "theoretical_loss": 3.419949789925211, + "tokens_seen": 2297561088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068405215646941, + "loss": 1.266, + "theoretical_loss": 3.4199419423969117, + "tokens_seen": 2297626624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068204613841525, + "loss": 1.2297, + "theoretical_loss": 3.4199340951551194, + "tokens_seen": 2297692160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030680040120361084, + "loss": 1.3165, + "theoretical_loss": 3.4199262481998156, + "tokens_seen": 2297757696 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030678034102306925, + "loss": 1.3014, + "theoretical_loss": 3.419918401530982, + "tokens_seen": 2297823232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067602808425276, + "loss": 1.2619, + "theoretical_loss": 3.419910555148599, + "tokens_seen": 2297888768 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306740220661986, + "loss": 1.3285, + "theoretical_loss": 3.419902709052649, + "tokens_seen": 2297954304 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030672016048144434, + "loss": 1.3551, + "theoretical_loss": 3.4198948632431128, + "tokens_seen": 2298019840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067001003009027, + "loss": 1.4281, + "theoretical_loss": 3.419887017719972, + "tokens_seen": 2298085376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066800401203611, + "loss": 1.3251, + "theoretical_loss": 3.419879172483208, + "tokens_seen": 2298150912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066599799398195, + "loss": 1.3003, + "theoretical_loss": 3.419871327532802, + "tokens_seen": 2298216448 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030663991975927784, + "loss": 1.3669, + "theoretical_loss": 3.4198634828687355, + "tokens_seen": 2298281984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066198595787362, + "loss": 1.3332, + "theoretical_loss": 3.41985563849099, + "tokens_seen": 2298347520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065997993981946, + "loss": 1.2826, + "theoretical_loss": 3.4198477943995464, + "tokens_seen": 2298413056 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306579739217653, + "loss": 1.3073, + "theoretical_loss": 3.419839950594387, + "tokens_seen": 2298478592 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030655967903711135, + "loss": 1.2733, + "theoretical_loss": 3.419832107075492, + "tokens_seen": 2298544128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065396188565697, + "loss": 1.4615, + "theoretical_loss": 3.419824263842844, + "tokens_seen": 2298609664 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065195586760281, + "loss": 1.3059, + "theoretical_loss": 3.4198164208964235, + "tokens_seen": 2298675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064994984954865, + "loss": 1.2271, + "theoretical_loss": 3.4198085782362124, + "tokens_seen": 2298740736 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 837138, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1962976455688477, + "objective/train/theoretical_loss": 3.419800735862192, + "objective/train/tokens_used": 669265376, + "theoretical_loss": 3.419800735862192, + "tokens_seen": 2298806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030647943831494485, + "loss": 1.1987, + "theoretical_loss": 3.419800735862192, + "tokens_seen": 2298806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064593781344032, + "loss": 1.4421, + "theoretical_loss": 3.4197928937743436, + "tokens_seen": 2298871808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064393179538616, + "loss": 1.4064, + "theoretical_loss": 3.4197850519726485, + "tokens_seen": 2298937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030641925777332, + "loss": 1.3172, + "theoretical_loss": 3.419777210457088, + "tokens_seen": 2299002880 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030639919759277835, + "loss": 1.1757, + "theoretical_loss": 3.4197693692276436, + "tokens_seen": 2299068416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063791374122367, + "loss": 1.2895, + "theoretical_loss": 3.4197615282842975, + "tokens_seen": 2299133952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063590772316951, + "loss": 1.3535, + "theoretical_loss": 3.41975368762703, + "tokens_seen": 2299199488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063390170511535, + "loss": 1.2948, + "theoretical_loss": 3.4197458472558226, + "tokens_seen": 2299265024 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030631895687061186, + "loss": 1.3044, + "theoretical_loss": 3.4197380071706576, + "tokens_seen": 2299330560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062988966900702, + "loss": 1.3765, + "theoretical_loss": 3.4197301673715157, + "tokens_seen": 2299396096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062788365095286, + "loss": 1.2523, + "theoretical_loss": 3.419722327858379, + "tokens_seen": 2299461632 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030625877632898694, + "loss": 1.2807, + "theoretical_loss": 3.4197144886312274, + "tokens_seen": 2299527168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030623871614844536, + "loss": 1.3466, + "theoretical_loss": 3.4197066496900437, + "tokens_seen": 2299592704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062186559679037, + "loss": 1.2634, + "theoretical_loss": 3.419698811034809, + "tokens_seen": 2299658240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061985957873621, + "loss": 1.2867, + "theoretical_loss": 3.4196909726655047, + "tokens_seen": 2299723776 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030617853560682045, + "loss": 1.2816, + "theoretical_loss": 3.4196831345821117, + "tokens_seen": 2299789312 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030615847542627886, + "loss": 1.293, + "theoretical_loss": 3.4196752967846127, + "tokens_seen": 2299854848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061384152457372, + "loss": 1.3621, + "theoretical_loss": 3.4196674592729877, + "tokens_seen": 2299920384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061183550651956, + "loss": 1.3215, + "theoretical_loss": 3.419659622047219, + "tokens_seen": 2299985920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030609829488465395, + "loss": 1.4327, + "theoretical_loss": 3.4196517851072876, + "tokens_seen": 2300051456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060782347041123, + "loss": 1.3545, + "theoretical_loss": 3.4196439484531753, + "tokens_seen": 2300116992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030605817452357073, + "loss": 1.4895, + "theoretical_loss": 3.419636112084863, + "tokens_seen": 2300182528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060381143430291, + "loss": 1.2612, + "theoretical_loss": 3.4196282760023324, + "tokens_seen": 2300248064 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030601805416248745, + "loss": 1.2977, + "theoretical_loss": 3.4196204402055654, + "tokens_seen": 2300313600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059979939819458, + "loss": 1.3587, + "theoretical_loss": 3.419612604694543, + "tokens_seen": 2300379136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030597793380140423, + "loss": 1.3322, + "theoretical_loss": 3.4196047694692466, + "tokens_seen": 2300444672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059578736208626, + "loss": 1.3294, + "theoretical_loss": 3.419596934529658, + "tokens_seen": 2300510208 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030593781344032096, + "loss": 1.3496, + "theoretical_loss": 3.419589099875758, + "tokens_seen": 2300575744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059177532597793, + "loss": 1.2959, + "theoretical_loss": 3.4195812655075284, + "tokens_seen": 2300641280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058976930792377, + "loss": 1.3593, + "theoretical_loss": 3.4195734314249506, + "tokens_seen": 2300706816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058776328986961, + "loss": 1.273, + "theoretical_loss": 3.4195655976280066, + "tokens_seen": 2300772352 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030585757271815446, + "loss": 1.3414, + "theoretical_loss": 3.419557764116677, + "tokens_seen": 2300837888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058375125376128, + "loss": 1.4565, + "theoretical_loss": 3.419549930890944, + "tokens_seen": 2300903424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058174523570712, + "loss": 1.2996, + "theoretical_loss": 3.419542097950788, + "tokens_seen": 2300968960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057973921765296, + "loss": 1.3554, + "theoretical_loss": 3.4195342652961918, + "tokens_seen": 2301034496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030577733199598796, + "loss": 1.4252, + "theoretical_loss": 3.4195264329271353, + "tokens_seen": 2301100032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057572718154463, + "loss": 1.3005, + "theoretical_loss": 3.4195186008436016, + "tokens_seen": 2301165568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057372116349047, + "loss": 1.278, + "theoretical_loss": 3.419510769045571, + "tokens_seen": 2301231104 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030571715145436305, + "loss": 1.3614, + "theoretical_loss": 3.4195029375330255, + "tokens_seen": 2301296640 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030569709127382147, + "loss": 1.4513, + "theoretical_loss": 3.4194951063059467, + "tokens_seen": 2301362176 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030567703109327983, + "loss": 1.3966, + "theoretical_loss": 3.419487275364315, + "tokens_seen": 2301427712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056569709127382, + "loss": 1.3468, + "theoretical_loss": 3.4194794447081134, + "tokens_seen": 2301493248 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030563691073219655, + "loss": 1.2386, + "theoretical_loss": 3.4194716143373225, + "tokens_seen": 2301558784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030561685055165497, + "loss": 1.2769, + "theoretical_loss": 3.4194637842519238, + "tokens_seen": 2301624320 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030559679037111333, + "loss": 1.3143, + "theoretical_loss": 3.4194559544518985, + "tokens_seen": 2301689856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055767301905717, + "loss": 1.3054, + "theoretical_loss": 3.419448124937229, + "tokens_seen": 2301755392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030555667001003006, + "loss": 1.1853, + "theoretical_loss": 3.419440295707896, + "tokens_seen": 2301820928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055366098294885, + "loss": 1.2656, + "theoretical_loss": 3.4194324667638814, + "tokens_seen": 2301886464 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055165496489469, + "loss": 1.4154, + "theoretical_loss": 3.4194246381051663, + "tokens_seen": 2301952000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030549648946840525, + "loss": 1.3523, + "theoretical_loss": 3.419416809731732, + "tokens_seen": 2302017536 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 838318, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2945561408996582, + "objective/train/theoretical_loss": 3.419408981643561, + "objective/train/tokens_used": 672542176, + "theoretical_loss": 3.419408981643561, + "tokens_seen": 2302083072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054764292878636, + "loss": 1.3049, + "theoretical_loss": 3.419408981643561, + "tokens_seen": 2302083072 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305456369107322, + "loss": 1.3339, + "theoretical_loss": 3.4194011538406333, + "tokens_seen": 2302148608 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054363089267804, + "loss": 1.3502, + "theoretical_loss": 3.4193933263229317, + "tokens_seen": 2302214144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030541624874623875, + "loss": 1.3428, + "theoretical_loss": 3.4193854990904375, + "tokens_seen": 2302279680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053961885656971, + "loss": 1.3427, + "theoretical_loss": 3.4193776721431313, + "tokens_seen": 2302345216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053761283851555, + "loss": 1.281, + "theoretical_loss": 3.4193698454809955, + "tokens_seen": 2302410752 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053560682046139, + "loss": 1.3703, + "theoretical_loss": 3.4193620191040113, + "tokens_seen": 2302476288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030533600802407226, + "loss": 1.3498, + "theoretical_loss": 3.4193541930121603, + "tokens_seen": 2302541824 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053159478435306, + "loss": 1.2243, + "theoretical_loss": 3.4193463672054234, + "tokens_seen": 2302607360 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305295887662989, + "loss": 1.1954, + "theoretical_loss": 3.419338541683783, + "tokens_seen": 2302672896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030527582748244734, + "loss": 1.3808, + "theoretical_loss": 3.41933071644722, + "tokens_seen": 2302738432 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030525576730190576, + "loss": 1.3674, + "theoretical_loss": 3.4193228914957157, + "tokens_seen": 2302803968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052357071213641, + "loss": 1.3431, + "theoretical_loss": 3.4193150668292525, + "tokens_seen": 2302869504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052156469408225, + "loss": 1.3908, + "theoretical_loss": 3.419307242447811, + "tokens_seen": 2302935040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030519558676028085, + "loss": 1.3625, + "theoretical_loss": 3.4192994183513736, + "tokens_seen": 2303000576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030517552657973926, + "loss": 1.375, + "theoretical_loss": 3.4192915945399207, + "tokens_seen": 2303066112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051554663991976, + "loss": 1.2921, + "theoretical_loss": 3.4192837710134345, + "tokens_seen": 2303131648 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305135406218656, + "loss": 1.3388, + "theoretical_loss": 3.419275947771897, + "tokens_seen": 2303197184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030511534603811435, + "loss": 1.3836, + "theoretical_loss": 3.4192681248152885, + "tokens_seen": 2303262720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050952858575727, + "loss": 1.2521, + "theoretical_loss": 3.4192603021435914, + "tokens_seen": 2303328256 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030507522567703113, + "loss": 1.2955, + "theoretical_loss": 3.419252479756787, + "tokens_seen": 2303393792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050551654964895, + "loss": 1.4169, + "theoretical_loss": 3.4192446576548567, + "tokens_seen": 2303459328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030503510531594785, + "loss": 1.3302, + "theoretical_loss": 3.419236835837782, + "tokens_seen": 2303524864 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050150451354062, + "loss": 1.2711, + "theoretical_loss": 3.419229014305545, + "tokens_seen": 2303590400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030499498495486463, + "loss": 1.2155, + "theoretical_loss": 3.419221193058126, + "tokens_seen": 2303655936 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304974924774323, + "loss": 1.3041, + "theoretical_loss": 3.4192133720955074, + "tokens_seen": 2303721472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030495486459378136, + "loss": 1.3629, + "theoretical_loss": 3.4192055514176714, + "tokens_seen": 2303787008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049348044132397, + "loss": 1.2605, + "theoretical_loss": 3.419197731024598, + "tokens_seen": 2303852544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049147442326981, + "loss": 1.3385, + "theoretical_loss": 3.4191899109162693, + "tokens_seen": 2303918080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048946840521565, + "loss": 1.4245, + "theoretical_loss": 3.4191820910926674, + "tokens_seen": 2303983616 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030487462387161486, + "loss": 1.3832, + "theoretical_loss": 3.419174271553773, + "tokens_seen": 2304049152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048545636910732, + "loss": 1.3049, + "theoretical_loss": 3.419166452299569, + "tokens_seen": 2304114688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048345035105316, + "loss": 1.313, + "theoretical_loss": 3.4191586333300354, + "tokens_seen": 2304180224 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030481444332999, + "loss": 1.2704, + "theoretical_loss": 3.419150814645154, + "tokens_seen": 2304245760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030479438314944836, + "loss": 1.3347, + "theoretical_loss": 3.4191429962449074, + "tokens_seen": 2304311296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047743229689067, + "loss": 1.3249, + "theoretical_loss": 3.419135178129276, + "tokens_seen": 2304376832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047542627883651, + "loss": 1.3527, + "theoretical_loss": 3.419127360298242, + "tokens_seen": 2304442368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030473420260782345, + "loss": 1.4751, + "theoretical_loss": 3.4191195427517864, + "tokens_seen": 2304507904 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030471414242728187, + "loss": 1.1912, + "theoretical_loss": 3.4191117254898913, + "tokens_seen": 2304573440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030469408224674023, + "loss": 1.3793, + "theoretical_loss": 3.419103908512538, + "tokens_seen": 2304638976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046740220661986, + "loss": 1.4098, + "theoretical_loss": 3.419096091819708, + "tokens_seen": 2304704512 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030465396188565695, + "loss": 1.306, + "theoretical_loss": 3.419088275411383, + "tokens_seen": 2304770048 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030463390170511537, + "loss": 1.3689, + "theoretical_loss": 3.4190804592875446, + "tokens_seen": 2304835584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030461384152457373, + "loss": 1.2591, + "theoretical_loss": 3.4190726434481737, + "tokens_seen": 2304901120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045937813440321, + "loss": 1.2996, + "theoretical_loss": 3.419064827893253, + "tokens_seen": 2304966656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030457372116349046, + "loss": 1.3937, + "theoretical_loss": 3.4190570126227633, + "tokens_seen": 2305032192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045536609829488, + "loss": 1.2631, + "theoretical_loss": 3.4190491976366864, + "tokens_seen": 2305097728 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030453360080240723, + "loss": 1.2879, + "theoretical_loss": 3.4190413829350037, + "tokens_seen": 2305163264 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045135406218656, + "loss": 1.2513, + "theoretical_loss": 3.4190335685176967, + "tokens_seen": 2305228800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030449348044132396, + "loss": 1.2981, + "theoretical_loss": 3.419025754384747, + "tokens_seen": 2305294336 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.753016582273028, + "debugging/entropy-1-grams": 5.229850717926142, + "debugging/length": 439.2307692307692, + "debugging/num_segments": 13, + "debugging/score": 0.013377460944710597, + "debugging/score_std": 0.02051348399645926, + "epoch": 0.4, + "objective/train/docs_used": 839437, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3634417057037354, + "objective/train/theoretical_loss": 3.4190179405361367, + "objective/train/tokens_used": 675818976, + "theoretical_loss": 3.4190179405361367, + "tokens_seen": 2305359872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044734202607823, + "loss": 1.4161, + "theoretical_loss": 3.4190179405361367, + "tokens_seen": 2305359872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030445336008024074, + "loss": 1.3611, + "theoretical_loss": 3.4190101269718465, + "tokens_seen": 2305425408 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044332998996991, + "loss": 1.3643, + "theoretical_loss": 3.419002313691859, + "tokens_seen": 2305490944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030441323971915746, + "loss": 1.4038, + "theoretical_loss": 3.418994500696155, + "tokens_seen": 2305556480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043931795386158, + "loss": 1.2971, + "theoretical_loss": 3.4189866879847157, + "tokens_seen": 2305622016 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043731193580742, + "loss": 1.3797, + "theoretical_loss": 3.4189788755575243, + "tokens_seen": 2305687552 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043530591775326, + "loss": 1.4518, + "theoretical_loss": 3.4189710634145607, + "tokens_seen": 2305753088 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030433299899699097, + "loss": 1.375, + "theoretical_loss": 3.418963251555807, + "tokens_seen": 2305818624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030431293881644933, + "loss": 1.352, + "theoretical_loss": 3.418955439981245, + "tokens_seen": 2305884160 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042928786359077, + "loss": 1.2615, + "theoretical_loss": 3.4189476286908564, + "tokens_seen": 2305949696 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042728184553661, + "loss": 1.2764, + "theoretical_loss": 3.418939817684622, + "tokens_seen": 2306015232 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030425275827482447, + "loss": 1.3208, + "theoretical_loss": 3.4189320069625246, + "tokens_seen": 2306080768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030423269809428283, + "loss": 1.403, + "theoretical_loss": 3.418924196524545, + "tokens_seen": 2306146304 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042126379137412, + "loss": 1.2701, + "theoretical_loss": 3.418916386370665, + "tokens_seen": 2306211840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041925777331996, + "loss": 1.2588, + "theoretical_loss": 3.418908576500866, + "tokens_seen": 2306277376 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030417251755265797, + "loss": 1.3732, + "theoretical_loss": 3.41890076691513, + "tokens_seen": 2306342912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030415245737211633, + "loss": 1.3369, + "theoretical_loss": 3.418892957613438, + "tokens_seen": 2306408448 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041323971915747, + "loss": 1.3356, + "theoretical_loss": 3.418885148595772, + "tokens_seen": 2306473984 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030411233701103306, + "loss": 1.2508, + "theoretical_loss": 3.4188773398621137, + "tokens_seen": 2306539520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040922768304915, + "loss": 1.2693, + "theoretical_loss": 3.4188695314124447, + "tokens_seen": 2306605056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030407221664994984, + "loss": 1.2421, + "theoretical_loss": 3.4188617232467458, + "tokens_seen": 2306670592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030405215646940825, + "loss": 1.2955, + "theoretical_loss": 3.4188539153649993, + "tokens_seen": 2306736128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040320962888666, + "loss": 1.3536, + "theoretical_loss": 3.4188461077671874, + "tokens_seen": 2306801664 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030401203610832503, + "loss": 1.3138, + "theoretical_loss": 3.4188383004532903, + "tokens_seen": 2306867200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039919759277834, + "loss": 1.3446, + "theoretical_loss": 3.418830493423291, + "tokens_seen": 2306932736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030397191574724176, + "loss": 1.379, + "theoretical_loss": 3.41882268667717, + "tokens_seen": 2306998272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039518555667001, + "loss": 1.4427, + "theoretical_loss": 3.4188148802149096, + "tokens_seen": 2307063808 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039317953861585, + "loss": 1.3429, + "theoretical_loss": 3.4188070740364913, + "tokens_seen": 2307129344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039117352056169, + "loss": 1.5064, + "theoretical_loss": 3.418799268141896, + "tokens_seen": 2307194880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030389167502507526, + "loss": 1.373, + "theoretical_loss": 3.418791462531107, + "tokens_seen": 2307260416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038716148445336, + "loss": 1.3274, + "theoretical_loss": 3.418783657204104, + "tokens_seen": 2307325952 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303851554663992, + "loss": 1.307, + "theoretical_loss": 3.41877585216087, + "tokens_seen": 2307391488 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038314944834504, + "loss": 1.2884, + "theoretical_loss": 3.418768047401386, + "tokens_seen": 2307457024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030381143430290876, + "loss": 1.3686, + "theoretical_loss": 3.418760242925633, + "tokens_seen": 2307522560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037913741223671, + "loss": 1.2705, + "theoretical_loss": 3.4187524387335944, + "tokens_seen": 2307588096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037713139418255, + "loss": 1.4601, + "theoretical_loss": 3.4187446348252504, + "tokens_seen": 2307653632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030375125376128385, + "loss": 1.2987, + "theoretical_loss": 3.418736831200583, + "tokens_seen": 2307719168 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030373119358074227, + "loss": 1.3119, + "theoretical_loss": 3.4187290278595737, + "tokens_seen": 2307784704 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030371113340020063, + "loss": 1.3028, + "theoretical_loss": 3.4187212248022045, + "tokens_seen": 2307850240 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303691073219659, + "loss": 1.3852, + "theoretical_loss": 3.4187134220284565, + "tokens_seen": 2307915776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030367101303911735, + "loss": 1.2782, + "theoretical_loss": 3.418705619538312, + "tokens_seen": 2307981312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030365095285857577, + "loss": 1.3356, + "theoretical_loss": 3.4186978173317524, + "tokens_seen": 2308046848 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030363089267803413, + "loss": 1.3757, + "theoretical_loss": 3.418690015408759, + "tokens_seen": 2308112384 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036108324974925, + "loss": 1.3665, + "theoretical_loss": 3.418682213769314, + "tokens_seen": 2308177920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030359077231695086, + "loss": 1.3203, + "theoretical_loss": 3.418674412413398, + "tokens_seen": 2308243456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035707121364092, + "loss": 1.2473, + "theoretical_loss": 3.418666611340994, + "tokens_seen": 2308308992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030355065195586763, + "loss": 1.4159, + "theoretical_loss": 3.418658810552083, + "tokens_seen": 2308374528 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303530591775326, + "loss": 1.4028, + "theoretical_loss": 3.4186510100466463, + "tokens_seen": 2308440064 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030351053159478436, + "loss": 1.4188, + "theoretical_loss": 3.418643209824666, + "tokens_seen": 2308505600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034904714142427, + "loss": 1.3897, + "theoretical_loss": 3.418635409886124, + "tokens_seen": 2308571136 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 840762, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2229909896850586, + "objective/train/theoretical_loss": 3.4186276102310016, + "objective/train/tokens_used": 679095776, + "theoretical_loss": 3.4186276102310016, + "tokens_seen": 2308636672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030347041123370114, + "loss": 1.3159, + "theoretical_loss": 3.4186276102310016, + "tokens_seen": 2308636672 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034503510531595, + "loss": 1.2617, + "theoretical_loss": 3.41861981085928, + "tokens_seen": 2308702208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030343029087261786, + "loss": 1.3382, + "theoretical_loss": 3.418612011770942, + "tokens_seen": 2308767744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034102306920762, + "loss": 1.3178, + "theoretical_loss": 3.418604212965968, + "tokens_seen": 2308833280 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033901705115346, + "loss": 1.3188, + "theoretical_loss": 3.4185964144443406, + "tokens_seen": 2308898816 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303370110330993, + "loss": 1.3796, + "theoretical_loss": 3.418588616206041, + "tokens_seen": 2308964352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030335005015045137, + "loss": 1.2994, + "theoretical_loss": 3.4185808182510513, + "tokens_seen": 2309029888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030332998996990973, + "loss": 1.2656, + "theoretical_loss": 3.4185730205793528, + "tokens_seen": 2309095424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033099297893681, + "loss": 1.2961, + "theoretical_loss": 3.418565223190927, + "tokens_seen": 2309160960 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032898696088265, + "loss": 1.3741, + "theoretical_loss": 3.418557426085756, + "tokens_seen": 2309226496 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030326980942828487, + "loss": 1.2641, + "theoretical_loss": 3.418549629263821, + "tokens_seen": 2309292032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030324974924774323, + "loss": 1.3316, + "theoretical_loss": 3.4185418327251043, + "tokens_seen": 2309357568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032296890672016, + "loss": 1.2497, + "theoretical_loss": 3.4185340364695866, + "tokens_seen": 2309423104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030320962888666, + "loss": 1.3562, + "theoretical_loss": 3.418526240497251, + "tokens_seen": 2309488640 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030318956870611837, + "loss": 1.3177, + "theoretical_loss": 3.418518444808078, + "tokens_seen": 2309554176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030316950852557673, + "loss": 1.4123, + "theoretical_loss": 3.4185106494020494, + "tokens_seen": 2309619712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031494483450351, + "loss": 1.4084, + "theoretical_loss": 3.4185028542791476, + "tokens_seen": 2309685248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030312938816449346, + "loss": 1.3101, + "theoretical_loss": 3.4184950594393535, + "tokens_seen": 2309750784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031093279839519, + "loss": 1.3506, + "theoretical_loss": 3.4184872648826494, + "tokens_seen": 2309816320 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030308926780341024, + "loss": 1.3299, + "theoretical_loss": 3.4184794706090162, + "tokens_seen": 2309881856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030692076228686, + "loss": 1.4423, + "theoretical_loss": 3.4184716766184366, + "tokens_seen": 2309947392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030304914744232696, + "loss": 1.4018, + "theoretical_loss": 3.4184638829108915, + "tokens_seen": 2310012928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030290872617854, + "loss": 1.381, + "theoretical_loss": 3.418456089486363, + "tokens_seen": 2310078464 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030300902708124374, + "loss": 1.3378, + "theoretical_loss": 3.418448296344833, + "tokens_seen": 2310144000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029889669007021, + "loss": 1.34, + "theoretical_loss": 3.4184405034862824, + "tokens_seen": 2310209536 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030296890672016046, + "loss": 1.2839, + "theoretical_loss": 3.418432710910693, + "tokens_seen": 2310275072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029488465396188, + "loss": 1.3221, + "theoretical_loss": 3.4184249186180478, + "tokens_seen": 2310340608 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030292878635907724, + "loss": 1.3141, + "theoretical_loss": 3.418417126608327, + "tokens_seen": 2310406144 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029087261785356, + "loss": 1.3623, + "theoretical_loss": 3.4184093348815128, + "tokens_seen": 2310471680 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030288866599799397, + "loss": 1.25, + "theoretical_loss": 3.418401543437587, + "tokens_seen": 2310537216 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030286860581745233, + "loss": 1.4052, + "theoretical_loss": 3.4183937522765317, + "tokens_seen": 2310602752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030284854563691075, + "loss": 1.3798, + "theoretical_loss": 3.418385961398328, + "tokens_seen": 2310668288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028284854563691, + "loss": 1.2929, + "theoretical_loss": 3.4183781708029572, + "tokens_seen": 2310733824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030280842527582747, + "loss": 1.3549, + "theoretical_loss": 3.418370380490402, + "tokens_seen": 2310799360 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030278836509528583, + "loss": 1.2175, + "theoretical_loss": 3.418362590460644, + "tokens_seen": 2310864896 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027683049147442, + "loss": 1.3518, + "theoretical_loss": 3.418354800713664, + "tokens_seen": 2310930432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027482447342026, + "loss": 1.3995, + "theoretical_loss": 3.4183470112494447, + "tokens_seen": 2310995968 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302728184553661, + "loss": 1.3963, + "theoretical_loss": 3.4183392220679676, + "tokens_seen": 2311061504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030270812437311934, + "loss": 1.3102, + "theoretical_loss": 3.418331433169214, + "tokens_seen": 2311127040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026880641925777, + "loss": 1.3193, + "theoretical_loss": 3.4183236445531664, + "tokens_seen": 2311192576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026680040120361, + "loss": 1.3907, + "theoretical_loss": 3.4183158562198055, + "tokens_seen": 2311258112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026479438314945, + "loss": 1.368, + "theoretical_loss": 3.4183080681691136, + "tokens_seen": 2311323648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030262788365095284, + "loss": 1.2964, + "theoretical_loss": 3.4183002804010725, + "tokens_seen": 2311389184 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026078234704112, + "loss": 1.2629, + "theoretical_loss": 3.418292492915664, + "tokens_seen": 2311454720 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030258776328986956, + "loss": 1.4242, + "theoretical_loss": 3.4182847057128694, + "tokens_seen": 2311520256 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302567703109328, + "loss": 1.3809, + "theoretical_loss": 3.418276918792671, + "tokens_seen": 2311585792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025476429287864, + "loss": 1.2449, + "theoretical_loss": 3.4182691321550496, + "tokens_seen": 2311651328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030252758274824476, + "loss": 1.2899, + "theoretical_loss": 3.418261345799988, + "tokens_seen": 2311716864 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025075225677031, + "loss": 1.3976, + "theoretical_loss": 3.418253559727467, + "tokens_seen": 2311782400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030248746238716154, + "loss": 1.2174, + "theoretical_loss": 3.418245773937469, + "tokens_seen": 2311847936 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 841951, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3188968896865845, + "objective/train/theoretical_loss": 3.4182379884299756, + "objective/train/tokens_used": 682372576, + "theoretical_loss": 3.4182379884299756, + "tokens_seen": 2311913472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024674022066199, + "loss": 1.316, + "theoretical_loss": 3.4182379884299756, + "tokens_seen": 2311913472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030244734202607826, + "loss": 1.2904, + "theoretical_loss": 3.4182302032049687, + "tokens_seen": 2311979008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024272818455366, + "loss": 1.3186, + "theoretical_loss": 3.4182224182624297, + "tokens_seen": 2312044544 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302407221664995, + "loss": 1.3953, + "theoretical_loss": 3.4182146336023402, + "tokens_seen": 2312110080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023871614844534, + "loss": 1.3158, + "theoretical_loss": 3.4182068492246827, + "tokens_seen": 2312175616 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030236710130391177, + "loss": 1.3113, + "theoretical_loss": 3.4181990651294383, + "tokens_seen": 2312241152 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030234704112337013, + "loss": 1.3855, + "theoretical_loss": 3.418191281316589, + "tokens_seen": 2312306688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023269809428285, + "loss": 1.2326, + "theoretical_loss": 3.4181834977861163, + "tokens_seen": 2312372224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023069207622869, + "loss": 1.4527, + "theoretical_loss": 3.418175714538002, + "tokens_seen": 2312437760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030228686058174527, + "loss": 1.244, + "theoretical_loss": 3.418167931572228, + "tokens_seen": 2312503296 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030226680040120363, + "loss": 1.2672, + "theoretical_loss": 3.4181601488887763, + "tokens_seen": 2312568832 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302246740220662, + "loss": 1.3127, + "theoretical_loss": 3.4181523664876283, + "tokens_seen": 2312634368 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022266800401204, + "loss": 1.4019, + "theoretical_loss": 3.4181445843687657, + "tokens_seen": 2312699904 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030220661985957877, + "loss": 1.2384, + "theoretical_loss": 3.4181368025321706, + "tokens_seen": 2312765440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030218655967903713, + "loss": 1.1765, + "theoretical_loss": 3.418129020977825, + "tokens_seen": 2312830976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021664994984955, + "loss": 1.4129, + "theoretical_loss": 3.4181212397057097, + "tokens_seen": 2312896512 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030214643931795386, + "loss": 1.2611, + "theoretical_loss": 3.4181134587158066, + "tokens_seen": 2312962048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021263791374123, + "loss": 1.2861, + "theoretical_loss": 3.4181056780080987, + "tokens_seen": 2313027584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030210631895687064, + "loss": 1.4248, + "theoretical_loss": 3.418097897582567, + "tokens_seen": 2313093120 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302086258776329, + "loss": 1.3863, + "theoretical_loss": 3.418090117439193, + "tokens_seen": 2313158656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030206619859578736, + "loss": 1.3337, + "theoretical_loss": 3.418082337577958, + "tokens_seen": 2313224192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020461384152458, + "loss": 1.3666, + "theoretical_loss": 3.418074557998845, + "tokens_seen": 2313289728 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030202607823470414, + "loss": 1.4007, + "theoretical_loss": 3.4180667787018355, + "tokens_seen": 2313355264 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020060180541625, + "loss": 1.351, + "theoretical_loss": 3.418058999686911, + "tokens_seen": 2313420800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030198595787362087, + "loss": 1.3477, + "theoretical_loss": 3.418051220954053, + "tokens_seen": 2313486336 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030196589769307923, + "loss": 1.3481, + "theoretical_loss": 3.4180434425032438, + "tokens_seen": 2313551872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030194583751253764, + "loss": 1.3096, + "theoretical_loss": 3.418035664334465, + "tokens_seen": 2313617408 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301925777331996, + "loss": 1.3581, + "theoretical_loss": 3.4180278864476983, + "tokens_seen": 2313682944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030190571715145437, + "loss": 1.3355, + "theoretical_loss": 3.4180201088429256, + "tokens_seen": 2313748480 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030188565697091273, + "loss": 1.3508, + "theoretical_loss": 3.4180123315201287, + "tokens_seen": 2313814016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030186559679037115, + "loss": 1.3134, + "theoretical_loss": 3.418004554479289, + "tokens_seen": 2313879552 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018455366098295, + "loss": 1.3309, + "theoretical_loss": 3.417996777720389, + "tokens_seen": 2313945088 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030182547642928787, + "loss": 1.3344, + "theoretical_loss": 3.41798900124341, + "tokens_seen": 2314010624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030180541624874623, + "loss": 1.2217, + "theoretical_loss": 3.417981225048334, + "tokens_seen": 2314076160 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017853560682046, + "loss": 1.38, + "theoretical_loss": 3.417973449135143, + "tokens_seen": 2314141696 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301765295887663, + "loss": 1.2584, + "theoretical_loss": 3.4179656735038177, + "tokens_seen": 2314207232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017452357071214, + "loss": 1.3821, + "theoretical_loss": 3.4179578981543415, + "tokens_seen": 2314272768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030172517552657974, + "loss": 1.3986, + "theoretical_loss": 3.417950123086695, + "tokens_seen": 2314338304 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017051153460381, + "loss": 1.3918, + "theoretical_loss": 3.41794234830086, + "tokens_seen": 2314403840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016850551654965, + "loss": 1.2941, + "theoretical_loss": 3.4179345737968196, + "tokens_seen": 2314469376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016649949849549, + "loss": 1.3583, + "theoretical_loss": 3.4179267995745546, + "tokens_seen": 2314534912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030164493480441324, + "loss": 1.2991, + "theoretical_loss": 3.4179190256340464, + "tokens_seen": 2314600448 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016248746238716, + "loss": 1.2718, + "theoretical_loss": 3.4179112519752777, + "tokens_seen": 2314665984 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030160481444332996, + "loss": 1.3233, + "theoretical_loss": 3.41790347859823, + "tokens_seen": 2314731520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015847542627884, + "loss": 1.3227, + "theoretical_loss": 3.417895705502885, + "tokens_seen": 2314797056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030156469408224674, + "loss": 1.3716, + "theoretical_loss": 3.417887932689225, + "tokens_seen": 2314862592 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015446339017051, + "loss": 1.3808, + "theoretical_loss": 3.417880160157231, + "tokens_seen": 2314928128 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030152457372116347, + "loss": 1.382, + "theoretical_loss": 3.417872387906885, + "tokens_seen": 2314993664 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015045135406219, + "loss": 1.2498, + "theoretical_loss": 3.4178646159381696, + "tokens_seen": 2315059200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030148445336008025, + "loss": 1.3623, + "theoretical_loss": 3.4178568442510655, + "tokens_seen": 2315124736 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 843188, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.470449686050415, + "objective/train/theoretical_loss": 3.4178490728455557, + "objective/train/tokens_used": 685649376, + "theoretical_loss": 3.4178490728455557, + "tokens_seen": 2315190272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014643931795386, + "loss": 1.411, + "theoretical_loss": 3.4178490728455557, + "tokens_seen": 2315190272 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030144433299899697, + "loss": 1.3366, + "theoretical_loss": 3.417841301721621, + "tokens_seen": 2315255808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030142427281845533, + "loss": 1.3363, + "theoretical_loss": 3.4178335308792436, + "tokens_seen": 2315321344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030140421263791375, + "loss": 1.2642, + "theoretical_loss": 3.417825760318406, + "tokens_seen": 2315386880 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013841524573721, + "loss": 1.2778, + "theoretical_loss": 3.417817990039089, + "tokens_seen": 2315452416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013640922768305, + "loss": 1.2598, + "theoretical_loss": 3.4178102200412748, + "tokens_seen": 2315517952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030134403209628884, + "loss": 1.356, + "theoretical_loss": 3.4178024503249453, + "tokens_seen": 2315583488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030132397191574725, + "loss": 1.375, + "theoretical_loss": 3.4177946808900828, + "tokens_seen": 2315649024 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013039117352056, + "loss": 1.3849, + "theoretical_loss": 3.417786911736668, + "tokens_seen": 2315714560 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301283851554664, + "loss": 1.2668, + "theoretical_loss": 3.4177791428646835, + "tokens_seen": 2315780096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030126379137412234, + "loss": 1.4356, + "theoretical_loss": 3.4177713742741114, + "tokens_seen": 2315845632 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012437311935807, + "loss": 1.3242, + "theoretical_loss": 3.417763605964933, + "tokens_seen": 2315911168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012236710130391, + "loss": 1.1946, + "theoretical_loss": 3.4177558379371304, + "tokens_seen": 2315976704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012036108324975, + "loss": 1.3067, + "theoretical_loss": 3.4177480701906853, + "tokens_seen": 2316042240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030118355065195584, + "loss": 1.3339, + "theoretical_loss": 3.4177403027255795, + "tokens_seen": 2316107776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011634904714142, + "loss": 1.358, + "theoretical_loss": 3.4177325355417953, + "tokens_seen": 2316173312 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011434302908726, + "loss": 1.3194, + "theoretical_loss": 3.4177247686393137, + "tokens_seen": 2316238848 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301123370110331, + "loss": 1.4045, + "theoretical_loss": 3.417717002018118, + "tokens_seen": 2316304384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030110330992978935, + "loss": 1.2267, + "theoretical_loss": 3.4177092356781884, + "tokens_seen": 2316369920 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010832497492477, + "loss": 1.3221, + "theoretical_loss": 3.417701469619508, + "tokens_seen": 2316435456 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030106318956870607, + "loss": 1.3191, + "theoretical_loss": 3.4176937038420574, + "tokens_seen": 2316500992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030104312938816454, + "loss": 1.3367, + "theoretical_loss": 3.41768593834582, + "tokens_seen": 2316566528 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010230692076229, + "loss": 1.2884, + "theoretical_loss": 3.417678173130776, + "tokens_seen": 2316632064 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030100300902708127, + "loss": 1.2608, + "theoretical_loss": 3.417670408196909, + "tokens_seen": 2316697600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030098294884653963, + "loss": 1.4313, + "theoretical_loss": 3.4176626435441992, + "tokens_seen": 2316763136 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030096288866599804, + "loss": 1.3455, + "theoretical_loss": 3.41765487917263, + "tokens_seen": 2316828672 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003009428284854564, + "loss": 1.3167, + "theoretical_loss": 3.4176471150821826, + "tokens_seen": 2316894208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030092276830491477, + "loss": 1.3157, + "theoretical_loss": 3.4176393512728382, + "tokens_seen": 2316959744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030090270812437313, + "loss": 1.3507, + "theoretical_loss": 3.4176315877445793, + "tokens_seen": 2317025280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030088264794383155, + "loss": 1.3271, + "theoretical_loss": 3.4176238244973884, + "tokens_seen": 2317090816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008625877632899, + "loss": 1.4025, + "theoretical_loss": 3.417616061531246, + "tokens_seen": 2317156352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030084252758274827, + "loss": 1.285, + "theoretical_loss": 3.417608298846135, + "tokens_seen": 2317221888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030082246740220663, + "loss": 1.3636, + "theoretical_loss": 3.417600536442037, + "tokens_seen": 2317287424 + }, + { + "epoch": 0.4, + "learning_rate": 0.000300802407221665, + "loss": 1.2194, + "theoretical_loss": 3.4175927743189343, + "tokens_seen": 2317352960 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007823470411234, + "loss": 1.3609, + "theoretical_loss": 3.417585012476808, + "tokens_seen": 2317418496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007622868605818, + "loss": 1.3221, + "theoretical_loss": 3.41757725091564, + "tokens_seen": 2317484032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030074222668004014, + "loss": 1.3575, + "theoretical_loss": 3.4175694896354125, + "tokens_seen": 2317549568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007221664994985, + "loss": 1.3061, + "theoretical_loss": 3.417561728636108, + "tokens_seen": 2317615104 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007021063189569, + "loss": 1.3583, + "theoretical_loss": 3.4175539679177076, + "tokens_seen": 2317680640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006820461384153, + "loss": 1.3085, + "theoretical_loss": 3.4175462074801928, + "tokens_seen": 2317746176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030066198595787364, + "loss": 1.4035, + "theoretical_loss": 3.4175384473235466, + "tokens_seen": 2317811712 + }, + { + "epoch": 0.4, + "learning_rate": 0.000300641925777332, + "loss": 1.4512, + "theoretical_loss": 3.4175306874477505, + "tokens_seen": 2317877248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030062186559679036, + "loss": 1.3859, + "theoretical_loss": 3.417522927852786, + "tokens_seen": 2317942784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006018054162488, + "loss": 1.2733, + "theoretical_loss": 3.4175151685386354, + "tokens_seen": 2318008320 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030058174523570714, + "loss": 1.2964, + "theoretical_loss": 3.4175074095052804, + "tokens_seen": 2318073856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003005616850551655, + "loss": 1.3012, + "theoretical_loss": 3.4174996507527027, + "tokens_seen": 2318139392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030054162487462387, + "loss": 1.3892, + "theoretical_loss": 3.417491892280885, + "tokens_seen": 2318204928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003005215646940823, + "loss": 1.3756, + "theoretical_loss": 3.417484134089808, + "tokens_seen": 2318270464 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030050150451354065, + "loss": 1.329, + "theoretical_loss": 3.417476376179455, + "tokens_seen": 2318336000 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300481444332999, + "loss": 1.371, + "theoretical_loss": 3.4174686185498064, + "tokens_seen": 2318401536 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 844398, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2924747467041016, + "objective/train/theoretical_loss": 3.4174608612008455, + "objective/train/tokens_used": 688926176, + "theoretical_loss": 3.4174608612008455, + "tokens_seen": 2318467072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030046138415245737, + "loss": 1.3545, + "theoretical_loss": 3.4174608612008455, + "tokens_seen": 2318467072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030044132397191573, + "loss": 1.31, + "theoretical_loss": 3.4174531041325533, + "tokens_seen": 2318532608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030042126379137415, + "loss": 1.4708, + "theoretical_loss": 3.4174453473449122, + "tokens_seen": 2318598144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004012036108325, + "loss": 1.2725, + "theoretical_loss": 3.4174375908379035, + "tokens_seen": 2318663680 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003811434302909, + "loss": 1.3348, + "theoretical_loss": 3.41742983461151, + "tokens_seen": 2318729216 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030036108324974924, + "loss": 1.2673, + "theoretical_loss": 3.4174220786657132, + "tokens_seen": 2318794752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030034102306920765, + "loss": 1.3144, + "theoretical_loss": 3.4174143230004947, + "tokens_seen": 2318860288 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300320962888666, + "loss": 1.3886, + "theoretical_loss": 3.4174065676158367, + "tokens_seen": 2318925824 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003009027081244, + "loss": 1.2858, + "theoretical_loss": 3.4173988125117214, + "tokens_seen": 2318991360 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030028084252758274, + "loss": 1.3776, + "theoretical_loss": 3.41739105768813, + "tokens_seen": 2319056896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002607823470411, + "loss": 1.2683, + "theoretical_loss": 3.417383303145045, + "tokens_seen": 2319122432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002407221664995, + "loss": 1.3697, + "theoretical_loss": 3.4173755488824487, + "tokens_seen": 2319187968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002206619859579, + "loss": 1.3529, + "theoretical_loss": 3.417367794900322, + "tokens_seen": 2319253504 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030020060180541624, + "loss": 1.2769, + "theoretical_loss": 3.417360041198647, + "tokens_seen": 2319319040 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001805416248746, + "loss": 1.271, + "theoretical_loss": 3.4173522877774065, + "tokens_seen": 2319384576 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300160481444333, + "loss": 1.3926, + "theoretical_loss": 3.417344534636582, + "tokens_seen": 2319450112 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001404212637914, + "loss": 1.2467, + "theoretical_loss": 3.417336781776155, + "tokens_seen": 2319515648 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030012036108324975, + "loss": 1.3824, + "theoretical_loss": 3.4173290291961083, + "tokens_seen": 2319581184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001003009027081, + "loss": 1.3446, + "theoretical_loss": 3.417321276896423, + "tokens_seen": 2319646720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030008024072216647, + "loss": 1.4116, + "theoretical_loss": 3.4173135248770814, + "tokens_seen": 2319712256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000601805416249, + "loss": 1.2669, + "theoretical_loss": 3.4173057731380654, + "tokens_seen": 2319777792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030004012036108325, + "loss": 1.4348, + "theoretical_loss": 3.4172980216793567, + "tokens_seen": 2319843328 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000200601805416, + "loss": 1.3198, + "theoretical_loss": 3.417290270500938, + "tokens_seen": 2319908864 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003, + "loss": 1.3848, + "theoretical_loss": 3.4172825196027903, + "tokens_seen": 2319974400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999799398194584, + "loss": 1.4139, + "theoretical_loss": 3.417274768984896, + "tokens_seen": 2320039936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029995987963891675, + "loss": 1.3585, + "theoretical_loss": 3.4172670186472374, + "tokens_seen": 2320105472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999398194583751, + "loss": 1.312, + "theoretical_loss": 3.4172592685897962, + "tokens_seen": 2320171008 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999197592778335, + "loss": 1.3837, + "theoretical_loss": 3.417251518812554, + "tokens_seen": 2320236544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998996990972919, + "loss": 1.3585, + "theoretical_loss": 3.417243769315493, + "tokens_seen": 2320302080 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029987963891675026, + "loss": 1.4267, + "theoretical_loss": 3.417236020098595, + "tokens_seen": 2320367616 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998595787362086, + "loss": 1.1875, + "theoretical_loss": 3.4172282711618425, + "tokens_seen": 2320433152 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299839518555667, + "loss": 1.3223, + "theoretical_loss": 3.417220522505217, + "tokens_seen": 2320498688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029981945837512534, + "loss": 1.4236, + "theoretical_loss": 3.4172127741287004, + "tokens_seen": 2320564224 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029979939819458376, + "loss": 1.3446, + "theoretical_loss": 3.4172050260322746, + "tokens_seen": 2320629760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997793380140421, + "loss": 1.2907, + "theoretical_loss": 3.417197278215922, + "tokens_seen": 2320695296 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997592778335005, + "loss": 1.2979, + "theoretical_loss": 3.4171895306796243, + "tokens_seen": 2320760832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029973921765295885, + "loss": 1.326, + "theoretical_loss": 3.4171817834233638, + "tokens_seen": 2320826368 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029971915747241726, + "loss": 1.2653, + "theoretical_loss": 3.417174036447122, + "tokens_seen": 2320891904 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996990972918756, + "loss": 1.2683, + "theoretical_loss": 3.417166289750881, + "tokens_seen": 2320957440 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299679037111334, + "loss": 1.445, + "theoretical_loss": 3.417158543334623, + "tokens_seen": 2321022976 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029965897693079235, + "loss": 1.3592, + "theoretical_loss": 3.4171507971983295, + "tokens_seen": 2321088512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996389167502507, + "loss": 1.2946, + "theoretical_loss": 3.4171430513419834, + "tokens_seen": 2321154048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029961885656970913, + "loss": 1.3561, + "theoretical_loss": 3.4171353057655653, + "tokens_seen": 2321219584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995987963891675, + "loss": 1.422, + "theoretical_loss": 3.4171275604690585, + "tokens_seen": 2321285120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029957873620862585, + "loss": 1.2697, + "theoretical_loss": 3.417119815452444, + "tokens_seen": 2321350656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995586760280842, + "loss": 1.2822, + "theoretical_loss": 3.4171120707157043, + "tokens_seen": 2321416192 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995386158475427, + "loss": 1.2487, + "theoretical_loss": 3.4171043262588214, + "tokens_seen": 2321481728 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029951855566700105, + "loss": 1.4081, + "theoretical_loss": 3.4170965820817774, + "tokens_seen": 2321547264 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994984954864594, + "loss": 1.2559, + "theoretical_loss": 3.417088838184554, + "tokens_seen": 2321612800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029947843530591777, + "loss": 1.3436, + "theoretical_loss": 3.417081094567133, + "tokens_seen": 2321678336 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 845463, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3060812950134277, + "objective/train/theoretical_loss": 3.4170733512294964, + "objective/train/tokens_used": 692202976, + "theoretical_loss": 3.4170733512294964, + "tokens_seen": 2321743872 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029945837512537613, + "loss": 1.287, + "theoretical_loss": 3.4170733512294964, + "tokens_seen": 2321743872 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029943831494483455, + "loss": 1.226, + "theoretical_loss": 3.4170656081716273, + "tokens_seen": 2321809408 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994182547642929, + "loss": 1.2837, + "theoretical_loss": 3.417057865393506, + "tokens_seen": 2321874944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993981945837513, + "loss": 1.3648, + "theoretical_loss": 3.417050122895116, + "tokens_seen": 2321940480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029937813440320964, + "loss": 1.2569, + "theoretical_loss": 3.4170423806764383, + "tokens_seen": 2322006016 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029935807422266805, + "loss": 1.2698, + "theoretical_loss": 3.417034638737455, + "tokens_seen": 2322071552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993380140421264, + "loss": 1.3053, + "theoretical_loss": 3.417026897078149, + "tokens_seen": 2322137088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993179538615848, + "loss": 1.3126, + "theoretical_loss": 3.4170191556985015, + "tokens_seen": 2322202624 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029929789368104314, + "loss": 1.4164, + "theoretical_loss": 3.4170114145984942, + "tokens_seen": 2322268160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992778335005015, + "loss": 1.3429, + "theoretical_loss": 3.41700367377811, + "tokens_seen": 2322333696 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992577733199599, + "loss": 1.339, + "theoretical_loss": 3.41699593323733, + "tokens_seen": 2322399232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992377131394183, + "loss": 1.3387, + "theoretical_loss": 3.416988192976137, + "tokens_seen": 2322464768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029921765295887664, + "loss": 1.1831, + "theoretical_loss": 3.4169804529945127, + "tokens_seen": 2322530304 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299197592778335, + "loss": 1.3474, + "theoretical_loss": 3.4169727132924392, + "tokens_seen": 2322595840 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991775325977934, + "loss": 1.406, + "theoretical_loss": 3.416964973869898, + "tokens_seen": 2322661376 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991574724172518, + "loss": 1.3032, + "theoretical_loss": 3.416957234726872, + "tokens_seen": 2322726912 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029913741223671015, + "loss": 1.3625, + "theoretical_loss": 3.4169494958633426, + "tokens_seen": 2322792448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991173520561685, + "loss": 1.5022, + "theoretical_loss": 3.416941757279292, + "tokens_seen": 2322857984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029909729187562687, + "loss": 1.4387, + "theoretical_loss": 3.416934018974702, + "tokens_seen": 2322923520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990772316950853, + "loss": 1.3277, + "theoretical_loss": 3.4169262809495553, + "tokens_seen": 2322989056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029905717151454365, + "loss": 1.488, + "theoretical_loss": 3.4169185432038334, + "tokens_seen": 2323054592 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299037111334002, + "loss": 1.4867, + "theoretical_loss": 3.416910805737518, + "tokens_seen": 2323120128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990170511534604, + "loss": 1.2783, + "theoretical_loss": 3.4169030685505914, + "tokens_seen": 2323185664 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989969909729188, + "loss": 1.3284, + "theoretical_loss": 3.4168953316430364, + "tokens_seen": 2323251200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029897693079237715, + "loss": 1.2179, + "theoretical_loss": 3.416887595014834, + "tokens_seen": 2323316736 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989568706118355, + "loss": 1.3086, + "theoretical_loss": 3.4168798586659666, + "tokens_seen": 2323382272 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989368104312939, + "loss": 1.2476, + "theoretical_loss": 3.4168721225964163, + "tokens_seen": 2323447808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989167502507523, + "loss": 1.3361, + "theoretical_loss": 3.4168643868061648, + "tokens_seen": 2323513344 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029889669007021066, + "loss": 1.339, + "theoretical_loss": 3.416856651295195, + "tokens_seen": 2323578880 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298876629889669, + "loss": 1.2408, + "theoretical_loss": 3.4168489160634876, + "tokens_seen": 2323644416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988565697091274, + "loss": 1.332, + "theoretical_loss": 3.4168411811110255, + "tokens_seen": 2323709952 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029883650952858574, + "loss": 1.3044, + "theoretical_loss": 3.416833446437791, + "tokens_seen": 2323775488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029881644934804416, + "loss": 1.3418, + "theoretical_loss": 3.416825712043766, + "tokens_seen": 2323841024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987963891675025, + "loss": 1.3042, + "theoretical_loss": 3.416817977928932, + "tokens_seen": 2323906560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987763289869609, + "loss": 1.344, + "theoretical_loss": 3.4168102440932717, + "tokens_seen": 2323972096 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029875626880641925, + "loss": 1.338, + "theoretical_loss": 3.4168025105367663, + "tokens_seen": 2324037632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029873620862587766, + "loss": 1.3089, + "theoretical_loss": 3.4167947772593985, + "tokens_seen": 2324103168 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298716148445336, + "loss": 1.3484, + "theoretical_loss": 3.4167870442611505, + "tokens_seen": 2324168704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986960882647944, + "loss": 1.3548, + "theoretical_loss": 3.416779311542004, + "tokens_seen": 2324234240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029867602808425275, + "loss": 1.2672, + "theoretical_loss": 3.4167715791019413, + "tokens_seen": 2324299776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986559679037111, + "loss": 1.3502, + "theoretical_loss": 3.4167638469409445, + "tokens_seen": 2324365312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029863590772316953, + "loss": 1.3384, + "theoretical_loss": 3.4167561150589947, + "tokens_seen": 2324430848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986158475426279, + "loss": 1.2103, + "theoretical_loss": 3.4167483834560755, + "tokens_seen": 2324496384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029859578736208625, + "loss": 1.327, + "theoretical_loss": 3.4167406521321677, + "tokens_seen": 2324561920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985757271815446, + "loss": 1.2901, + "theoretical_loss": 3.416732921087254, + "tokens_seen": 2324627456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029855566700100303, + "loss": 1.2901, + "theoretical_loss": 3.416725190321317, + "tokens_seen": 2324692992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985356068204614, + "loss": 1.4099, + "theoretical_loss": 3.416717459834337, + "tokens_seen": 2324758528 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029851554663991976, + "loss": 1.2924, + "theoretical_loss": 3.416709729626298, + "tokens_seen": 2324824064 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984954864593781, + "loss": 1.3037, + "theoretical_loss": 3.416701999697181, + "tokens_seen": 2324889600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984754262788365, + "loss": 1.3325, + "theoretical_loss": 3.416694270046968, + "tokens_seen": 2324955136 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 846164, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3871805667877197, + "objective/train/theoretical_loss": 3.4166865406756415, + "objective/train/tokens_used": 695479776, + "theoretical_loss": 3.4166865406756415, + "tokens_seen": 2325020672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984553660982949, + "loss": 1.3233, + "theoretical_loss": 3.4166865406756415, + "tokens_seen": 2325020672 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029843530591775326, + "loss": 1.4494, + "theoretical_loss": 3.416678811583184, + "tokens_seen": 2325086208 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984152457372116, + "loss": 1.2388, + "theoretical_loss": 3.4166710827695765, + "tokens_seen": 2325151744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029839518555667, + "loss": 1.3266, + "theoretical_loss": 3.416663354234802, + "tokens_seen": 2325217280 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983751253761284, + "loss": 1.2436, + "theoretical_loss": 3.416655625978842, + "tokens_seen": 2325282816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029835506519558676, + "loss": 1.2568, + "theoretical_loss": 3.4166478980016786, + "tokens_seen": 2325348352 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983350050150451, + "loss": 1.2759, + "theoretical_loss": 3.4166401703032943, + "tokens_seen": 2325413888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983149448345035, + "loss": 1.3686, + "theoretical_loss": 3.416632442883671, + "tokens_seen": 2325479424 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029829488465396185, + "loss": 1.2813, + "theoretical_loss": 3.416624715742791, + "tokens_seen": 2325544960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029827482447342026, + "loss": 1.3076, + "theoretical_loss": 3.416616988880636, + "tokens_seen": 2325610496 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982547642928786, + "loss": 1.3844, + "theoretical_loss": 3.416609262297188, + "tokens_seen": 2325676032 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298234704112337, + "loss": 1.3185, + "theoretical_loss": 3.4166015359924296, + "tokens_seen": 2325741568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029821464393179535, + "loss": 1.2477, + "theoretical_loss": 3.416593809966342, + "tokens_seen": 2325807104 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029819458375125377, + "loss": 1.3701, + "theoretical_loss": 3.416586084218909, + "tokens_seen": 2325872640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029817452357071213, + "loss": 1.3564, + "theoretical_loss": 3.416578358750111, + "tokens_seen": 2325938176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981544633901705, + "loss": 1.3666, + "theoretical_loss": 3.416570633559931, + "tokens_seen": 2326003712 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029813440320962885, + "loss": 1.2992, + "theoretical_loss": 3.41656290864835, + "tokens_seen": 2326069248 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981143430290872, + "loss": 1.2307, + "theoretical_loss": 3.416555184015352, + "tokens_seen": 2326134784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029809428284854563, + "loss": 1.4657, + "theoretical_loss": 3.4165474596609173, + "tokens_seen": 2326200320 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298074222668004, + "loss": 1.3357, + "theoretical_loss": 3.416539735585029, + "tokens_seen": 2326265856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029805416248746236, + "loss": 1.2444, + "theoretical_loss": 3.416532011787669, + "tokens_seen": 2326331392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980341023069208, + "loss": 1.2933, + "theoretical_loss": 3.4165242882688194, + "tokens_seen": 2326396928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980140421263792, + "loss": 1.2904, + "theoretical_loss": 3.4165165650284623, + "tokens_seen": 2326462464 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029799398194583755, + "loss": 1.3447, + "theoretical_loss": 3.4165088420665795, + "tokens_seen": 2326528000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979739217652959, + "loss": 1.3363, + "theoretical_loss": 3.416501119383154, + "tokens_seen": 2326593536 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979538615847543, + "loss": 1.2475, + "theoretical_loss": 3.416493396978167, + "tokens_seen": 2326659072 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979338014042127, + "loss": 1.3534, + "theoretical_loss": 3.4164856748516006, + "tokens_seen": 2326724608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029791374122367106, + "loss": 1.1943, + "theoretical_loss": 3.4164779530034375, + "tokens_seen": 2326790144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978936810431294, + "loss": 1.3433, + "theoretical_loss": 3.4164702314336597, + "tokens_seen": 2326855680 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978736208625878, + "loss": 1.3027, + "theoretical_loss": 3.416462510142249, + "tokens_seen": 2326921216 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029785356068204614, + "loss": 1.3255, + "theoretical_loss": 3.4164547891291877, + "tokens_seen": 2326986752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029783350050150456, + "loss": 1.2712, + "theoretical_loss": 3.4164470683944583, + "tokens_seen": 2327052288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978134403209629, + "loss": 1.3842, + "theoretical_loss": 3.4164393479380424, + "tokens_seen": 2327117824 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977933801404213, + "loss": 1.3272, + "theoretical_loss": 3.4164316277599225, + "tokens_seen": 2327183360 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029777331995987965, + "loss": 1.3929, + "theoretical_loss": 3.4164239078600804, + "tokens_seen": 2327248896 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029775325977933806, + "loss": 1.275, + "theoretical_loss": 3.416416188238498, + "tokens_seen": 2327314432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977331995987964, + "loss": 1.3618, + "theoretical_loss": 3.4164084688951584, + "tokens_seen": 2327379968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977131394182548, + "loss": 1.3126, + "theoretical_loss": 3.416400749830043, + "tokens_seen": 2327445504 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029769307923771315, + "loss": 1.2169, + "theoretical_loss": 3.4163930310431336, + "tokens_seen": 2327511040 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976730190571715, + "loss": 1.3784, + "theoretical_loss": 3.4163853125344135, + "tokens_seen": 2327576576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029765295887662993, + "loss": 1.255, + "theoretical_loss": 3.4163775943038637, + "tokens_seen": 2327642112 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976328986960883, + "loss": 1.2552, + "theoretical_loss": 3.4163698763514674, + "tokens_seen": 2327707648 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029761283851554665, + "loss": 1.3461, + "theoretical_loss": 3.4163621586772055, + "tokens_seen": 2327773184 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297592778335005, + "loss": 1.3109, + "theoretical_loss": 3.416354441281061, + "tokens_seen": 2327838720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029757271815446343, + "loss": 1.2444, + "theoretical_loss": 3.416346724163016, + "tokens_seen": 2327904256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975526579739218, + "loss": 1.3298, + "theoretical_loss": 3.416339007323052, + "tokens_seen": 2327969792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029753259779338016, + "loss": 1.3681, + "theoretical_loss": 3.4163312907611525, + "tokens_seen": 2328035328 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975125376128385, + "loss": 1.3533, + "theoretical_loss": 3.4163235744772984, + "tokens_seen": 2328100864 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974924774322969, + "loss": 1.3243, + "theoretical_loss": 3.416315858471472, + "tokens_seen": 2328166400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974724172517553, + "loss": 1.2615, + "theoretical_loss": 3.4163081427436564, + "tokens_seen": 2328231936 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 847468, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.23801851272583, + "objective/train/theoretical_loss": 3.4163004272938324, + "objective/train/tokens_used": 698756576, + "theoretical_loss": 3.4163004272938324, + "tokens_seen": 2328297472 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029745235707121366, + "loss": 1.2983, + "theoretical_loss": 3.4163004272938324, + "tokens_seen": 2328297472 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297432296890672, + "loss": 1.322, + "theoretical_loss": 3.416292712121983, + "tokens_seen": 2328363008 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974122367101304, + "loss": 1.2374, + "theoretical_loss": 3.4162849972280904, + "tokens_seen": 2328428544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973921765295888, + "loss": 1.3498, + "theoretical_loss": 3.416277282612137, + "tokens_seen": 2328494080 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029737211634904716, + "loss": 1.3068, + "theoretical_loss": 3.4162695682741036, + "tokens_seen": 2328559616 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973520561685055, + "loss": 1.3514, + "theoretical_loss": 3.4162618542139738, + "tokens_seen": 2328625152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973319959879639, + "loss": 1.3814, + "theoretical_loss": 3.416254140431729, + "tokens_seen": 2328690688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029731193580742225, + "loss": 1.3495, + "theoretical_loss": 3.416246426927352, + "tokens_seen": 2328756224 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029729187562688067, + "loss": 1.2906, + "theoretical_loss": 3.416238713700824, + "tokens_seen": 2328821760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029727181544633903, + "loss": 1.3702, + "theoretical_loss": 3.4162310007521284, + "tokens_seen": 2328887296 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972517552657974, + "loss": 1.3749, + "theoretical_loss": 3.4162232880812464, + "tokens_seen": 2328952832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029723169508525575, + "loss": 1.3128, + "theoretical_loss": 3.4162155756881605, + "tokens_seen": 2329018368 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029721163490471417, + "loss": 1.4558, + "theoretical_loss": 3.416207863572853, + "tokens_seen": 2329083904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029719157472417253, + "loss": 1.3659, + "theoretical_loss": 3.4162001517353064, + "tokens_seen": 2329149440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971715145436309, + "loss": 1.3827, + "theoretical_loss": 3.416192440175502, + "tokens_seen": 2329214976 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029715145436308926, + "loss": 1.3374, + "theoretical_loss": 3.4161847288934224, + "tokens_seen": 2329280512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971313941825476, + "loss": 1.3199, + "theoretical_loss": 3.41617701788905, + "tokens_seen": 2329346048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029711133400200603, + "loss": 1.3217, + "theoretical_loss": 3.4161693071623667, + "tokens_seen": 2329411584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970912738214644, + "loss": 1.2676, + "theoretical_loss": 3.416161596713355, + "tokens_seen": 2329477120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029707121364092276, + "loss": 1.2554, + "theoretical_loss": 3.4161538865419967, + "tokens_seen": 2329542656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970511534603811, + "loss": 1.3127, + "theoretical_loss": 3.416146176648274, + "tokens_seen": 2329608192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029703109327983954, + "loss": 1.1493, + "theoretical_loss": 3.41613846703217, + "tokens_seen": 2329673728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970110330992979, + "loss": 1.3455, + "theoretical_loss": 3.416130757693666, + "tokens_seen": 2329739264 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029699097291875626, + "loss": 1.2408, + "theoretical_loss": 3.416123048632744, + "tokens_seen": 2329804800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969709127382146, + "loss": 1.3025, + "theoretical_loss": 3.4161153398493864, + "tokens_seen": 2329870336 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296950852557673, + "loss": 1.3278, + "theoretical_loss": 3.416107631343576, + "tokens_seen": 2329935872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969307923771314, + "loss": 1.2766, + "theoretical_loss": 3.4160999231152944, + "tokens_seen": 2330001408 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029691073219658976, + "loss": 1.4089, + "theoretical_loss": 3.416092215164524, + "tokens_seen": 2330066944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968906720160481, + "loss": 1.4085, + "theoretical_loss": 3.416084507491247, + "tokens_seen": 2330132480 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968706118355065, + "loss": 1.2834, + "theoretical_loss": 3.4160768000954453, + "tokens_seen": 2330198016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968505516549649, + "loss": 1.2586, + "theoretical_loss": 3.416069092977102, + "tokens_seen": 2330263552 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029683049147442327, + "loss": 1.3057, + "theoretical_loss": 3.4160613861361977, + "tokens_seen": 2330329088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029681043129388163, + "loss": 1.2852, + "theoretical_loss": 3.416053679572716, + "tokens_seen": 2330394624 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029679037111334, + "loss": 1.3628, + "theoretical_loss": 3.4160459732866393, + "tokens_seen": 2330460160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967703109327984, + "loss": 1.3531, + "theoretical_loss": 3.4160382672779486, + "tokens_seen": 2330525696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029675025075225677, + "loss": 1.2805, + "theoretical_loss": 3.416030561546627, + "tokens_seen": 2330591232 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029673019057171513, + "loss": 1.2983, + "theoretical_loss": 3.4160228560926567, + "tokens_seen": 2330656768 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967101303911735, + "loss": 1.2896, + "theoretical_loss": 3.4160151509160195, + "tokens_seen": 2330722304 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029669007021063186, + "loss": 1.4156, + "theoretical_loss": 3.416007446016698, + "tokens_seen": 2330787840 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966700100300903, + "loss": 1.3154, + "theoretical_loss": 3.415999741394674, + "tokens_seen": 2330853376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029664994984954864, + "loss": 1.2985, + "theoretical_loss": 3.4159920370499295, + "tokens_seen": 2330918912 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296629889669007, + "loss": 1.3793, + "theoretical_loss": 3.4159843329824477, + "tokens_seen": 2330984448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029660982948846536, + "loss": 1.2832, + "theoretical_loss": 3.41597662919221, + "tokens_seen": 2331049984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965897693079238, + "loss": 1.2897, + "theoretical_loss": 3.4159689256791994, + "tokens_seen": 2331115520 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029656970912738214, + "loss": 1.2209, + "theoretical_loss": 3.4159612224433973, + "tokens_seen": 2331181056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965496489468405, + "loss": 1.3581, + "theoretical_loss": 3.4159535194847868, + "tokens_seen": 2331246592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965295887662989, + "loss": 1.272, + "theoretical_loss": 3.4159458168033487, + "tokens_seen": 2331312128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965095285857573, + "loss": 1.4042, + "theoretical_loss": 3.415938114399067, + "tokens_seen": 2331377664 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964894684052157, + "loss": 1.319, + "theoretical_loss": 3.4159304122719227, + "tokens_seen": 2331443200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029646940822467406, + "loss": 1.2634, + "theoretical_loss": 3.4159227104218983, + "tokens_seen": 2331508736 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 848727, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.312225580215454, + "objective/train/theoretical_loss": 3.4159150088489767, + "objective/train/tokens_used": 702033376, + "theoretical_loss": 3.4159150088489767, + "tokens_seen": 2331574272 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964493480441324, + "loss": 1.3079, + "theoretical_loss": 3.4159150088489767, + "tokens_seen": 2331574272 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964292878635908, + "loss": 1.2808, + "theoretical_loss": 3.415907307553139, + "tokens_seen": 2331639808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964092276830492, + "loss": 1.4983, + "theoretical_loss": 3.4158996065343685, + "tokens_seen": 2331705344 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029638916750250756, + "loss": 1.2482, + "theoretical_loss": 3.4158919057926473, + "tokens_seen": 2331770880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963691073219659, + "loss": 1.362, + "theoretical_loss": 3.4158842053279566, + "tokens_seen": 2331836416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963490471414243, + "loss": 1.2402, + "theoretical_loss": 3.4158765051402797, + "tokens_seen": 2331901952 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029632898696088265, + "loss": 1.2778, + "theoretical_loss": 3.4158688052295987, + "tokens_seen": 2331967488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029630892678034107, + "loss": 1.2674, + "theoretical_loss": 3.4158611055958956, + "tokens_seen": 2332033024 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029628886659979943, + "loss": 1.3571, + "theoretical_loss": 3.415853406239153, + "tokens_seen": 2332098560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962688064192578, + "loss": 1.308, + "theoretical_loss": 3.4158457071593524, + "tokens_seen": 2332164096 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029624874623871615, + "loss": 1.3195, + "theoretical_loss": 3.415838008356477, + "tokens_seen": 2332229632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029622868605817457, + "loss": 1.3108, + "theoretical_loss": 3.4158303098305085, + "tokens_seen": 2332295168 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029620862587763293, + "loss": 1.3413, + "theoretical_loss": 3.4158226115814294, + "tokens_seen": 2332360704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961885656970913, + "loss": 1.2903, + "theoretical_loss": 3.4158149136092217, + "tokens_seen": 2332426240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029616850551654966, + "loss": 1.3983, + "theoretical_loss": 3.415807215913868, + "tokens_seen": 2332491776 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296148445336008, + "loss": 1.2348, + "theoretical_loss": 3.4157995184953505, + "tokens_seen": 2332557312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029612838515546643, + "loss": 1.249, + "theoretical_loss": 3.415791821353651, + "tokens_seen": 2332622848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961083249749248, + "loss": 1.3836, + "theoretical_loss": 3.415784124488752, + "tokens_seen": 2332688384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029608826479438316, + "loss": 1.3286, + "theoretical_loss": 3.415776427900636, + "tokens_seen": 2332753920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960682046138415, + "loss": 1.2731, + "theoretical_loss": 3.415768731589285, + "tokens_seen": 2332819456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029604814443329994, + "loss": 1.2743, + "theoretical_loss": 3.4157610355546817, + "tokens_seen": 2332884992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960280842527583, + "loss": 1.2831, + "theoretical_loss": 3.415753339796808, + "tokens_seen": 2332950528 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029600802407221666, + "loss": 1.412, + "theoretical_loss": 3.4157456443156464, + "tokens_seen": 2333016064 + }, + { + "epoch": 0.41, + "learning_rate": 0.000295987963891675, + "loss": 1.3577, + "theoretical_loss": 3.415737949111179, + "tokens_seen": 2333081600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959679037111334, + "loss": 1.3327, + "theoretical_loss": 3.4157302541833885, + "tokens_seen": 2333147136 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959478435305918, + "loss": 1.3875, + "theoretical_loss": 3.4157225595322562, + "tokens_seen": 2333212672 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029592778335005016, + "loss": 1.307, + "theoretical_loss": 3.4157148651577653, + "tokens_seen": 2333278208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029590772316950853, + "loss": 1.3359, + "theoretical_loss": 3.4157071710598976, + "tokens_seen": 2333343744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958876629889669, + "loss": 1.324, + "theoretical_loss": 3.4156994772386358, + "tokens_seen": 2333409280 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958676028084253, + "loss": 1.3911, + "theoretical_loss": 3.415691783693962, + "tokens_seen": 2333474816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029584754262788367, + "loss": 1.3705, + "theoretical_loss": 3.4156840904258585, + "tokens_seen": 2333540352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029582748244734203, + "loss": 1.2977, + "theoretical_loss": 3.415676397434307, + "tokens_seen": 2333605888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958074222668004, + "loss": 1.4023, + "theoretical_loss": 3.4156687047192906, + "tokens_seen": 2333671424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957873620862588, + "loss": 1.3715, + "theoretical_loss": 3.4156610122807916, + "tokens_seen": 2333736960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029576730190571717, + "loss": 1.3239, + "theoretical_loss": 3.4156533201187917, + "tokens_seen": 2333802496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029574724172517553, + "loss": 1.3011, + "theoretical_loss": 3.415645628233274, + "tokens_seen": 2333868032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957271815446339, + "loss": 1.3678, + "theoretical_loss": 3.4156379366242193, + "tokens_seen": 2333933568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029570712136409226, + "loss": 1.2218, + "theoretical_loss": 3.415630245291612, + "tokens_seen": 2333999104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956870611835507, + "loss": 1.3398, + "theoretical_loss": 3.4156225542354326, + "tokens_seen": 2334064640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029566700100300904, + "loss": 1.2375, + "theoretical_loss": 3.4156148634556645, + "tokens_seen": 2334130176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956469408224674, + "loss": 1.335, + "theoretical_loss": 3.4156071729522894, + "tokens_seen": 2334195712 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029562688064192576, + "loss": 1.2781, + "theoretical_loss": 3.4155994827252902, + "tokens_seen": 2334261248 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956068204613842, + "loss": 1.2541, + "theoretical_loss": 3.4155917927746486, + "tokens_seen": 2334326784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029558676028084254, + "loss": 1.3723, + "theoretical_loss": 3.415584103100347, + "tokens_seen": 2334392320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955667001003009, + "loss": 1.3626, + "theoretical_loss": 3.415576413702368, + "tokens_seen": 2334457856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029554663991975926, + "loss": 1.2342, + "theoretical_loss": 3.415568724580694, + "tokens_seen": 2334523392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955265797392176, + "loss": 1.4232, + "theoretical_loss": 3.415561035735307, + "tokens_seen": 2334588928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029550651955867604, + "loss": 1.34, + "theoretical_loss": 3.415553347166189, + "tokens_seen": 2334654464 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002954864593781344, + "loss": 1.3426, + "theoretical_loss": 3.4155456588733233, + "tokens_seen": 2334720000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029546639919759277, + "loss": 1.3654, + "theoretical_loss": 3.4155379708566915, + "tokens_seen": 2334785536 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 849886, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3592042922973633, + "objective/train/theoretical_loss": 3.4155302831162757, + "objective/train/tokens_used": 705310176, + "theoretical_loss": 3.4155302831162757, + "tokens_seen": 2334851072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029544633901705113, + "loss": 1.3117, + "theoretical_loss": 3.4155302831162757, + "tokens_seen": 2334851072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029542627883650955, + "loss": 1.398, + "theoretical_loss": 3.415522595652059, + "tokens_seen": 2334916608 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954062186559679, + "loss": 1.3174, + "theoretical_loss": 3.415514908464023, + "tokens_seen": 2334982144 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029538615847542627, + "loss": 1.2521, + "theoretical_loss": 3.4155072215521507, + "tokens_seen": 2335047680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029536609829488463, + "loss": 1.1583, + "theoretical_loss": 3.4154995349164237, + "tokens_seen": 2335113216 + }, + { + "epoch": 0.42, + "learning_rate": 0.000295346038114343, + "loss": 1.433, + "theoretical_loss": 3.415491848556825, + "tokens_seen": 2335178752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953259779338014, + "loss": 1.4057, + "theoretical_loss": 3.4154841624733367, + "tokens_seen": 2335244288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953059177532598, + "loss": 1.4043, + "theoretical_loss": 3.415476476665941, + "tokens_seen": 2335309824 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029528585757271814, + "loss": 1.3841, + "theoretical_loss": 3.41546879113462, + "tokens_seen": 2335375360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952657973921765, + "loss": 1.3145, + "theoretical_loss": 3.415461105879357, + "tokens_seen": 2335440896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952457372116349, + "loss": 1.3383, + "theoretical_loss": 3.415453420900133, + "tokens_seen": 2335506432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952256770310933, + "loss": 1.2907, + "theoretical_loss": 3.4154457361969315, + "tokens_seen": 2335571968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029520561685055164, + "loss": 1.2641, + "theoretical_loss": 3.4154380517697343, + "tokens_seen": 2335637504 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029518555667001, + "loss": 1.387, + "theoretical_loss": 3.4154303676185234, + "tokens_seen": 2335703040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029516549648946836, + "loss": 1.2435, + "theoretical_loss": 3.415422683743282, + "tokens_seen": 2335768576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951454363089268, + "loss": 1.338, + "theoretical_loss": 3.415415000143992, + "tokens_seen": 2335834112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029512537612838514, + "loss": 1.3855, + "theoretical_loss": 3.415407316820635, + "tokens_seen": 2335899648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951053159478435, + "loss": 1.3469, + "theoretical_loss": 3.4153996337731947, + "tokens_seen": 2335965184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029508525576730187, + "loss": 1.3466, + "theoretical_loss": 3.4153919510016526, + "tokens_seen": 2336030720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950651955867603, + "loss": 1.3578, + "theoretical_loss": 3.4153842685059916, + "tokens_seen": 2336096256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029504513540621865, + "loss": 1.2296, + "theoretical_loss": 3.415376586286194, + "tokens_seen": 2336161792 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029502507522567706, + "loss": 1.28, + "theoretical_loss": 3.415368904342241, + "tokens_seen": 2336227328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950050150451354, + "loss": 1.2837, + "theoretical_loss": 3.415361222674117, + "tokens_seen": 2336292864 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949849548645938, + "loss": 1.2328, + "theoretical_loss": 3.415353541281802, + "tokens_seen": 2336358400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949648946840522, + "loss": 1.3, + "theoretical_loss": 3.4153458601652806, + "tokens_seen": 2336423936 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029494483450351057, + "loss": 1.3962, + "theoretical_loss": 3.4153381793245337, + "tokens_seen": 2336489472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029492477432296893, + "loss": 1.3448, + "theoretical_loss": 3.415330498759544, + "tokens_seen": 2336555008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949047141424273, + "loss": 1.3352, + "theoretical_loss": 3.415322818470294, + "tokens_seen": 2336620544 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948846539618857, + "loss": 1.3367, + "theoretical_loss": 3.4153151384567666, + "tokens_seen": 2336686080 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029486459378134407, + "loss": 1.376, + "theoretical_loss": 3.415307458718943, + "tokens_seen": 2336751616 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029484453360080243, + "loss": 1.3997, + "theoretical_loss": 3.4152997792568063, + "tokens_seen": 2336817152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948244734202608, + "loss": 1.2599, + "theoretical_loss": 3.415292100070339, + "tokens_seen": 2336882688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948044132397192, + "loss": 1.3228, + "theoretical_loss": 3.415284421159523, + "tokens_seen": 2336948224 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029478435305917757, + "loss": 1.3112, + "theoretical_loss": 3.415276742524341, + "tokens_seen": 2337013760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029476429287863593, + "loss": 1.3172, + "theoretical_loss": 3.415269064164775, + "tokens_seen": 2337079296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947442326980943, + "loss": 1.2969, + "theoretical_loss": 3.4152613860808083, + "tokens_seen": 2337144832 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029472417251755266, + "loss": 1.3242, + "theoretical_loss": 3.415253708272422, + "tokens_seen": 2337210368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947041123370111, + "loss": 1.4255, + "theoretical_loss": 3.4152460307395995, + "tokens_seen": 2337275904 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029468405215646944, + "loss": 1.311, + "theoretical_loss": 3.415238353482323, + "tokens_seen": 2337341440 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946639919759278, + "loss": 1.354, + "theoretical_loss": 3.415230676500574, + "tokens_seen": 2337406976 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029464393179538616, + "loss": 1.4006, + "theoretical_loss": 3.415222999794336, + "tokens_seen": 2337472512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946238716148446, + "loss": 1.3561, + "theoretical_loss": 3.415215323363591, + "tokens_seen": 2337538048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029460381143430294, + "loss": 1.2781, + "theoretical_loss": 3.4152076472083213, + "tokens_seen": 2337603584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945837512537613, + "loss": 1.3327, + "theoretical_loss": 3.4151999713285095, + "tokens_seen": 2337669120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029456369107321966, + "loss": 1.1889, + "theoretical_loss": 3.4151922957241374, + "tokens_seen": 2337734656 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294543630892678, + "loss": 1.2396, + "theoretical_loss": 3.415184620395188, + "tokens_seen": 2337800192 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029452357071213644, + "loss": 1.2906, + "theoretical_loss": 3.415176945341644, + "tokens_seen": 2337865728 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945035105315948, + "loss": 1.3469, + "theoretical_loss": 3.415169270563487, + "tokens_seen": 2337931264 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029448345035105317, + "loss": 1.4277, + "theoretical_loss": 3.4151615960607, + "tokens_seen": 2337996800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029446339017051153, + "loss": 1.3448, + "theoretical_loss": 3.4151539218332645, + "tokens_seen": 2338062336 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 851124, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.309006690979004, + "objective/train/theoretical_loss": 3.4151462478811636, + "objective/train/tokens_used": 708586976, + "theoretical_loss": 3.4151462478811636, + "tokens_seen": 2338127872 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029444332998996995, + "loss": 1.3728, + "theoretical_loss": 3.4151462478811636, + "tokens_seen": 2338127872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944232698094283, + "loss": 1.3859, + "theoretical_loss": 3.4151385742043803, + "tokens_seen": 2338193408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029440320962888667, + "loss": 1.2727, + "theoretical_loss": 3.415130900802896, + "tokens_seen": 2338258944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029438314944834503, + "loss": 1.2918, + "theoretical_loss": 3.4151232276766934, + "tokens_seen": 2338324480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943630892678034, + "loss": 1.3248, + "theoretical_loss": 3.415115554825755, + "tokens_seen": 2338390016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943430290872618, + "loss": 1.1524, + "theoretical_loss": 3.415107882250063, + "tokens_seen": 2338455552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943229689067202, + "loss": 1.2893, + "theoretical_loss": 3.4151002099496, + "tokens_seen": 2338521088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029430290872617854, + "loss": 1.4178, + "theoretical_loss": 3.4150925379243486, + "tokens_seen": 2338586624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942828485456369, + "loss": 1.3956, + "theoretical_loss": 3.415084866174291, + "tokens_seen": 2338652160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942627883650953, + "loss": 1.3561, + "theoretical_loss": 3.4150771946994096, + "tokens_seen": 2338717696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942427281845537, + "loss": 1.338, + "theoretical_loss": 3.415069523499687, + "tokens_seen": 2338783232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029422266800401204, + "loss": 1.2536, + "theoretical_loss": 3.4150618525751053, + "tokens_seen": 2338848768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942026078234704, + "loss": 1.4008, + "theoretical_loss": 3.4150541819256475, + "tokens_seen": 2338914304 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029418254764292876, + "loss": 1.2825, + "theoretical_loss": 3.415046511551295, + "tokens_seen": 2338979840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941624874623872, + "loss": 1.2175, + "theoretical_loss": 3.415038841452031, + "tokens_seen": 2339045376 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029414242728184554, + "loss": 1.2598, + "theoretical_loss": 3.415031171627838, + "tokens_seen": 2339110912 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941223671013039, + "loss": 1.2531, + "theoretical_loss": 3.4150235020786983, + "tokens_seen": 2339176448 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029410230692076227, + "loss": 1.248, + "theoretical_loss": 3.4150158328045936, + "tokens_seen": 2339241984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940822467402207, + "loss": 1.3565, + "theoretical_loss": 3.4150081638055076, + "tokens_seen": 2339307520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029406218655967905, + "loss": 1.3412, + "theoretical_loss": 3.415000495081422, + "tokens_seen": 2339373056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940421263791374, + "loss": 1.3406, + "theoretical_loss": 3.414992826632319, + "tokens_seen": 2339438592 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029402206619859577, + "loss": 1.5103, + "theoretical_loss": 3.4149851584581814, + "tokens_seen": 2339504128 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029400200601805413, + "loss": 1.3801, + "theoretical_loss": 3.414977490558992, + "tokens_seen": 2339569664 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029398194583751255, + "loss": 1.2911, + "theoretical_loss": 3.414969822934733, + "tokens_seen": 2339635200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939618856569709, + "loss": 1.3391, + "theoretical_loss": 3.414962155585386, + "tokens_seen": 2339700736 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939418254764293, + "loss": 1.275, + "theoretical_loss": 3.4149544885109346, + "tokens_seen": 2339766272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029392176529588764, + "loss": 1.2562, + "theoretical_loss": 3.4149468217113608, + "tokens_seen": 2339831808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029390170511534605, + "loss": 1.3187, + "theoretical_loss": 3.4149391551866466, + "tokens_seen": 2339897344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938816449348044, + "loss": 1.3331, + "theoretical_loss": 3.4149314889367752, + "tokens_seen": 2339962880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938615847542628, + "loss": 1.3265, + "theoretical_loss": 3.414923822961729, + "tokens_seen": 2340028416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029384152457372114, + "loss": 1.2927, + "theoretical_loss": 3.4149161572614894, + "tokens_seen": 2340093952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938214643931795, + "loss": 1.3344, + "theoretical_loss": 3.4149084918360404, + "tokens_seen": 2340159488 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938014042126379, + "loss": 1.3062, + "theoretical_loss": 3.414900826685363, + "tokens_seen": 2340225024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937813440320963, + "loss": 1.3522, + "theoretical_loss": 3.414893161809441, + "tokens_seen": 2340290560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029376128385155464, + "loss": 1.3538, + "theoretical_loss": 3.414885497208256, + "tokens_seen": 2340356096 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293741223671013, + "loss": 1.3811, + "theoretical_loss": 3.4148778328817904, + "tokens_seen": 2340421632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937211634904714, + "loss": 1.3052, + "theoretical_loss": 3.414870168830027, + "tokens_seen": 2340487168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937011033099298, + "loss": 1.2729, + "theoretical_loss": 3.414862505052948, + "tokens_seen": 2340552704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029368104312938815, + "loss": 1.4063, + "theoretical_loss": 3.414854841550537, + "tokens_seen": 2340618240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936609829488465, + "loss": 1.2987, + "theoretical_loss": 3.4148471783227743, + "tokens_seen": 2340683776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029364092276830487, + "loss": 1.4054, + "theoretical_loss": 3.4148395153696445, + "tokens_seen": 2340749312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936208625877633, + "loss": 1.2776, + "theoretical_loss": 3.4148318526911288, + "tokens_seen": 2340814848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029360080240722165, + "loss": 1.3067, + "theoretical_loss": 3.41482419028721, + "tokens_seen": 2340880384 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029358074222668, + "loss": 1.346, + "theoretical_loss": 3.4148165281578704, + "tokens_seen": 2340945920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935606820461384, + "loss": 1.3869, + "theoretical_loss": 3.414808866303093, + "tokens_seen": 2341011456 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029354062186559684, + "loss": 1.2453, + "theoretical_loss": 3.4148012047228598, + "tokens_seen": 2341076992 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935205616850552, + "loss": 1.2647, + "theoretical_loss": 3.414793543417153, + "tokens_seen": 2341142528 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029350050150451357, + "loss": 1.3429, + "theoretical_loss": 3.414785882385956, + "tokens_seen": 2341208064 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029348044132397193, + "loss": 1.3021, + "theoretical_loss": 3.414778221629251, + "tokens_seen": 2341273600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029346038114343035, + "loss": 1.357, + "theoretical_loss": 3.41477056114702, + "tokens_seen": 2341339136 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 852174, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2691469192504883, + "objective/train/theoretical_loss": 3.4147629009392455, + "objective/train/tokens_used": 711863776, + "theoretical_loss": 3.4147629009392455, + "tokens_seen": 2341404672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934403209628887, + "loss": 1.2818, + "theoretical_loss": 3.4147629009392455, + "tokens_seen": 2341404672 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029342026078234707, + "loss": 1.3742, + "theoretical_loss": 3.4147552410059103, + "tokens_seen": 2341470208 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029340020060180543, + "loss": 1.2507, + "theoretical_loss": 3.414747581346997, + "tokens_seen": 2341535744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933801404212638, + "loss": 1.3375, + "theoretical_loss": 3.4147399219624877, + "tokens_seen": 2341601280 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933600802407222, + "loss": 1.3167, + "theoretical_loss": 3.4147322628523655, + "tokens_seen": 2341666816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933400200601806, + "loss": 1.2899, + "theoretical_loss": 3.414724604016612, + "tokens_seen": 2341732352 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029331995987963894, + "loss": 1.3729, + "theoretical_loss": 3.4147169454552104, + "tokens_seen": 2341797888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932998996990973, + "loss": 1.3274, + "theoretical_loss": 3.414709287168143, + "tokens_seen": 2341863424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932798395185557, + "loss": 1.2407, + "theoretical_loss": 3.4147016291553918, + "tokens_seen": 2341928960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932597793380141, + "loss": 1.323, + "theoretical_loss": 3.4146939714169404, + "tokens_seen": 2341994496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029323971915747244, + "loss": 1.3038, + "theoretical_loss": 3.4146863139527706, + "tokens_seen": 2342060032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932196589769308, + "loss": 1.3204, + "theoretical_loss": 3.414678656762865, + "tokens_seen": 2342125568 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029319959879638916, + "loss": 1.3188, + "theoretical_loss": 3.4146709998472056, + "tokens_seen": 2342191104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931795386158476, + "loss": 1.4668, + "theoretical_loss": 3.414663343205776, + "tokens_seen": 2342256640 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029315947843530594, + "loss": 1.3487, + "theoretical_loss": 3.4146556868385574, + "tokens_seen": 2342322176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931394182547643, + "loss": 1.3487, + "theoretical_loss": 3.4146480307455334, + "tokens_seen": 2342387712 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029311935807422267, + "loss": 1.3095, + "theoretical_loss": 3.414640374926686, + "tokens_seen": 2342453248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930992978936811, + "loss": 1.3843, + "theoretical_loss": 3.4146327193819976, + "tokens_seen": 2342518784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029307923771313945, + "loss": 1.2895, + "theoretical_loss": 3.4146250641114513, + "tokens_seen": 2342584320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930591775325978, + "loss": 1.2366, + "theoretical_loss": 3.414617409115029, + "tokens_seen": 2342649856 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029303911735205617, + "loss": 1.4446, + "theoretical_loss": 3.414609754392714, + "tokens_seen": 2342715392 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029301905717151453, + "loss": 1.4213, + "theoretical_loss": 3.4146020999444877, + "tokens_seen": 2342780928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029299899699097295, + "loss": 1.3561, + "theoretical_loss": 3.414594445770333, + "tokens_seen": 2342846464 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929789368104313, + "loss": 1.3572, + "theoretical_loss": 3.4145867918702333, + "tokens_seen": 2342912000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929588766298897, + "loss": 1.2555, + "theoretical_loss": 3.41457913824417, + "tokens_seen": 2342977536 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029293881644934804, + "loss": 1.1866, + "theoretical_loss": 3.414571484892126, + "tokens_seen": 2343043072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029291875626880645, + "loss": 1.4035, + "theoretical_loss": 3.414563831814084, + "tokens_seen": 2343108608 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928986960882648, + "loss": 1.395, + "theoretical_loss": 3.4145561790100265, + "tokens_seen": 2343174144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928786359077232, + "loss": 1.3584, + "theoretical_loss": 3.414548526479936, + "tokens_seen": 2343239680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029285857572718154, + "loss": 1.2548, + "theoretical_loss": 3.414540874223795, + "tokens_seen": 2343305216 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928385155466399, + "loss": 1.3288, + "theoretical_loss": 3.4145332222415856, + "tokens_seen": 2343370752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928184553660983, + "loss": 1.2832, + "theoretical_loss": 3.414525570533291, + "tokens_seen": 2343436288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927983951855567, + "loss": 1.3175, + "theoretical_loss": 3.4145179190988935, + "tokens_seen": 2343501824 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029277833500501504, + "loss": 1.4078, + "theoretical_loss": 3.4145102679383754, + "tokens_seen": 2343567360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927582748244734, + "loss": 1.4082, + "theoretical_loss": 3.4145026170517196, + "tokens_seen": 2343632896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927382146439318, + "loss": 1.3392, + "theoretical_loss": 3.4144949664389084, + "tokens_seen": 2343698432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927181544633902, + "loss": 1.3127, + "theoretical_loss": 3.4144873160999243, + "tokens_seen": 2343763968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029269809428284855, + "loss": 1.3112, + "theoretical_loss": 3.4144796660347505, + "tokens_seen": 2343829504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926780341023069, + "loss": 1.3139, + "theoretical_loss": 3.4144720162433684, + "tokens_seen": 2343895040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029265797392176527, + "loss": 1.3309, + "theoretical_loss": 3.4144643667257615, + "tokens_seen": 2343960576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926379137412237, + "loss": 1.337, + "theoretical_loss": 3.414456717481912, + "tokens_seen": 2344026112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029261785356068205, + "loss": 1.3388, + "theoretical_loss": 3.4144490685118023, + "tokens_seen": 2344091648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925977933801404, + "loss": 1.3224, + "theoretical_loss": 3.4144414198154154, + "tokens_seen": 2344157184 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925777331995988, + "loss": 1.2667, + "theoretical_loss": 3.414433771392733, + "tokens_seen": 2344222720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925576730190572, + "loss": 1.3927, + "theoretical_loss": 3.4144261232437385, + "tokens_seen": 2344288256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029253761283851555, + "loss": 1.3305, + "theoretical_loss": 3.4144184753684144, + "tokens_seen": 2344353792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925175526579739, + "loss": 1.2647, + "theoretical_loss": 3.4144108277667424, + "tokens_seen": 2344419328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924974924774323, + "loss": 1.3053, + "theoretical_loss": 3.4144031804387067, + "tokens_seen": 2344484864 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924774322968907, + "loss": 1.2303, + "theoretical_loss": 3.4143955333842877, + "tokens_seen": 2344550400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029245737211634906, + "loss": 1.3762, + "theoretical_loss": 3.41438788660347, + "tokens_seen": 2344615936 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 853299, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2306807041168213, + "objective/train/theoretical_loss": 3.414380240096235, + "objective/train/tokens_used": 715140576, + "theoretical_loss": 3.414380240096235, + "tokens_seen": 2344681472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924373119358074, + "loss": 1.2449, + "theoretical_loss": 3.414380240096235, + "tokens_seen": 2344681472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924172517552658, + "loss": 1.2801, + "theoretical_loss": 3.4143725938625655, + "tokens_seen": 2344747008 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029239719157472414, + "loss": 1.3608, + "theoretical_loss": 3.414364947902444, + "tokens_seen": 2344812544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029237713139418256, + "loss": 1.2909, + "theoretical_loss": 3.414357302215853, + "tokens_seen": 2344878080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923570712136409, + "loss": 1.2538, + "theoretical_loss": 3.4143496568027754, + "tokens_seen": 2344943616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923370110330993, + "loss": 1.4334, + "theoretical_loss": 3.414342011663194, + "tokens_seen": 2345009152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029231695085255765, + "loss": 1.4027, + "theoretical_loss": 3.4143343667970907, + "tokens_seen": 2345074688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029229689067201606, + "loss": 1.3883, + "theoretical_loss": 3.414326722204448, + "tokens_seen": 2345140224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922768304914744, + "loss": 1.2721, + "theoretical_loss": 3.4143190778852492, + "tokens_seen": 2345205760 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922567703109328, + "loss": 1.2336, + "theoretical_loss": 3.414311433839477, + "tokens_seen": 2345271296 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029223671013039115, + "loss": 1.2408, + "theoretical_loss": 3.4143037900671125, + "tokens_seen": 2345336832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922166499498495, + "loss": 1.3427, + "theoretical_loss": 3.41429614656814, + "tokens_seen": 2345402368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921965897693079, + "loss": 1.4586, + "theoretical_loss": 3.414288503342541, + "tokens_seen": 2345467904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921765295887663, + "loss": 1.3059, + "theoretical_loss": 3.4142808603902988, + "tokens_seen": 2345533440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029215646940822465, + "loss": 1.3491, + "theoretical_loss": 3.4142732177113952, + "tokens_seen": 2345598976 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292136409227683, + "loss": 1.2738, + "theoretical_loss": 3.4142655753058135, + "tokens_seen": 2345664512 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029211634904714143, + "loss": 1.1786, + "theoretical_loss": 3.414257933173536, + "tokens_seen": 2345730048 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920962888665998, + "loss": 1.347, + "theoretical_loss": 3.4142502913145454, + "tokens_seen": 2345795584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029207622868605815, + "loss": 1.2846, + "theoretical_loss": 3.414242649728824, + "tokens_seen": 2345861120 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920561685055165, + "loss": 1.301, + "theoretical_loss": 3.4142350084163544, + "tokens_seen": 2345926656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029203610832497493, + "loss": 1.3233, + "theoretical_loss": 3.4142273673771197, + "tokens_seen": 2345992192 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029201604814443335, + "loss": 1.2772, + "theoretical_loss": 3.414219726611102, + "tokens_seen": 2346057728 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919959879638917, + "loss": 1.3851, + "theoretical_loss": 3.414212086118284, + "tokens_seen": 2346123264 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919759277833501, + "loss": 1.3292, + "theoretical_loss": 3.414204445898649, + "tokens_seen": 2346188800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029195586760280844, + "loss": 1.2977, + "theoretical_loss": 3.4141968059521783, + "tokens_seen": 2346254336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029193580742226685, + "loss": 1.279, + "theoretical_loss": 3.4141891662788555, + "tokens_seen": 2346319872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919157472417252, + "loss": 1.3303, + "theoretical_loss": 3.4141815268786626, + "tokens_seen": 2346385408 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918956870611836, + "loss": 1.3512, + "theoretical_loss": 3.414173887751583, + "tokens_seen": 2346450944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029187562688064194, + "loss": 1.3343, + "theoretical_loss": 3.4141662488975983, + "tokens_seen": 2346516480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918555667001003, + "loss": 1.2663, + "theoretical_loss": 3.414158610316692, + "tokens_seen": 2346582016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918355065195587, + "loss": 1.3676, + "theoretical_loss": 3.414150972008846, + "tokens_seen": 2346647552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918154463390171, + "loss": 1.4228, + "theoretical_loss": 3.4141433339740432, + "tokens_seen": 2346713088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029179538615847544, + "loss": 1.2755, + "theoretical_loss": 3.4141356962122664, + "tokens_seen": 2346778624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917753259779338, + "loss": 1.3381, + "theoretical_loss": 3.4141280587234983, + "tokens_seen": 2346844160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917552657973922, + "loss": 1.2599, + "theoretical_loss": 3.4141204215077208, + "tokens_seen": 2346909696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917352056168506, + "loss": 1.3679, + "theoretical_loss": 3.4141127845649173, + "tokens_seen": 2346975232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029171514543630895, + "loss": 1.2848, + "theoretical_loss": 3.4141051478950706, + "tokens_seen": 2347040768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916950852557673, + "loss": 1.356, + "theoretical_loss": 3.414097511498162, + "tokens_seen": 2347106304 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029167502507522567, + "loss": 1.3514, + "theoretical_loss": 3.4140898753741755, + "tokens_seen": 2347171840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916549648946841, + "loss": 1.3511, + "theoretical_loss": 3.414082239523093, + "tokens_seen": 2347237376 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029163490471414245, + "loss": 1.3514, + "theoretical_loss": 3.4140746039448975, + "tokens_seen": 2347302912 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916148445336008, + "loss": 1.3456, + "theoretical_loss": 3.4140669686395713, + "tokens_seen": 2347368448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915947843530592, + "loss": 1.2719, + "theoretical_loss": 3.4140593336070975, + "tokens_seen": 2347433984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915747241725176, + "loss": 1.3268, + "theoretical_loss": 3.414051698847458, + "tokens_seen": 2347499520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029155466399197595, + "loss": 1.2536, + "theoretical_loss": 3.414044064360636, + "tokens_seen": 2347565056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915346038114343, + "loss": 1.4202, + "theoretical_loss": 3.4140364301466137, + "tokens_seen": 2347630592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915145436308927, + "loss": 1.3299, + "theoretical_loss": 3.4140287962053746, + "tokens_seen": 2347696128 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914944834503511, + "loss": 1.3332, + "theoretical_loss": 3.4140211625369004, + "tokens_seen": 2347761664 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029147442326980946, + "loss": 1.3181, + "theoretical_loss": 3.414013529141174, + "tokens_seen": 2347827200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914543630892678, + "loss": 1.271, + "theoretical_loss": 3.4140058960181787, + "tokens_seen": 2347892736 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 854534, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3929357528686523, + "objective/train/theoretical_loss": 3.4139982631678962, + "objective/train/tokens_used": 718417376, + "theoretical_loss": 3.4139982631678962, + "tokens_seen": 2347958272 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914343029087262, + "loss": 1.3423, + "theoretical_loss": 3.4139982631678962, + "tokens_seen": 2347958272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029141424272818454, + "loss": 1.3504, + "theoretical_loss": 3.41399063059031, + "tokens_seen": 2348023808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029139418254764296, + "loss": 1.3376, + "theoretical_loss": 3.4139829982854017, + "tokens_seen": 2348089344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913741223671013, + "loss": 1.2886, + "theoretical_loss": 3.4139753662531547, + "tokens_seen": 2348154880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913540621865597, + "loss": 1.1705, + "theoretical_loss": 3.4139677344935517, + "tokens_seen": 2348220416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029133400200601805, + "loss": 1.2306, + "theoretical_loss": 3.4139601030065747, + "tokens_seen": 2348285952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029131394182547646, + "loss": 1.2209, + "theoretical_loss": 3.413952471792207, + "tokens_seen": 2348351488 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912938816449348, + "loss": 1.3183, + "theoretical_loss": 3.4139448408504314, + "tokens_seen": 2348417024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912738214643932, + "loss": 1.3027, + "theoretical_loss": 3.4139372101812295, + "tokens_seen": 2348482560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029125376128385155, + "loss": 1.4091, + "theoretical_loss": 3.4139295797845852, + "tokens_seen": 2348548096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912337011033099, + "loss": 1.2429, + "theoretical_loss": 3.4139219496604802, + "tokens_seen": 2348613632 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029121364092276833, + "loss": 1.4514, + "theoretical_loss": 3.413914319808898, + "tokens_seen": 2348679168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911935807422267, + "loss": 1.3416, + "theoretical_loss": 3.4139066902298207, + "tokens_seen": 2348744704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029117352056168505, + "loss": 1.1761, + "theoretical_loss": 3.413899060923231, + "tokens_seen": 2348810240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911534603811434, + "loss": 1.2629, + "theoretical_loss": 3.4138914318891116, + "tokens_seen": 2348875776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029113340020060183, + "loss": 1.4038, + "theoretical_loss": 3.4138838031274457, + "tokens_seen": 2348941312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911133400200602, + "loss": 1.2231, + "theoretical_loss": 3.413876174638215, + "tokens_seen": 2349006848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029109327983951855, + "loss": 1.41, + "theoretical_loss": 3.413868546421403, + "tokens_seen": 2349072384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910732196589769, + "loss": 1.2798, + "theoretical_loss": 3.413860918476992, + "tokens_seen": 2349137920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910531594784353, + "loss": 1.3151, + "theoretical_loss": 3.4138532908049646, + "tokens_seen": 2349203456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910330992978937, + "loss": 1.389, + "theoretical_loss": 3.4138456634053034, + "tokens_seen": 2349268992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029101303911735206, + "loss": 1.378, + "theoretical_loss": 3.413838036277992, + "tokens_seen": 2349334528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909929789368104, + "loss": 1.2532, + "theoretical_loss": 3.4138304094230114, + "tokens_seen": 2349400064 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909729187562688, + "loss": 1.315, + "theoretical_loss": 3.413822782840346, + "tokens_seen": 2349465600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909528585757272, + "loss": 1.375, + "theoretical_loss": 3.4138151565299775, + "tokens_seen": 2349531136 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029093279839518556, + "loss": 1.3639, + "theoretical_loss": 3.4138075304918885, + "tokens_seen": 2349596672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909127382146439, + "loss": 1.2942, + "theoretical_loss": 3.413799904726062, + "tokens_seen": 2349662208 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908926780341023, + "loss": 1.2694, + "theoretical_loss": 3.413792279232481, + "tokens_seen": 2349727744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029087261785356065, + "loss": 1.2612, + "theoretical_loss": 3.4137846540111276, + "tokens_seen": 2349793280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029085255767301906, + "loss": 1.3823, + "theoretical_loss": 3.413777029061985, + "tokens_seen": 2349858816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908324974924774, + "loss": 1.3357, + "theoretical_loss": 3.4137694043850355, + "tokens_seen": 2349924352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908124373119358, + "loss": 1.3077, + "theoretical_loss": 3.4137617799802618, + "tokens_seen": 2349989888 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029079237713139415, + "loss": 1.3298, + "theoretical_loss": 3.413754155847647, + "tokens_seen": 2350055424 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029077231695085257, + "loss": 1.3167, + "theoretical_loss": 3.413746531987173, + "tokens_seen": 2350120960 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029075225677031093, + "loss": 1.3132, + "theoretical_loss": 3.4137389083988237, + "tokens_seen": 2350186496 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907321965897693, + "loss": 1.4474, + "theoretical_loss": 3.413731285082581, + "tokens_seen": 2350252032 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029071213640922765, + "loss": 1.2245, + "theoretical_loss": 3.4137236620384277, + "tokens_seen": 2350317568 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290692076228686, + "loss": 1.324, + "theoretical_loss": 3.413716039266346, + "tokens_seen": 2350383104 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029067201604814443, + "loss": 1.3917, + "theoretical_loss": 3.4137084167663194, + "tokens_seen": 2350448640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906519558676028, + "loss": 1.3806, + "theoretical_loss": 3.4137007945383306, + "tokens_seen": 2350514176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029063189568706116, + "loss": 1.311, + "theoretical_loss": 3.4136931725823616, + "tokens_seen": 2350579712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906118355065195, + "loss": 1.3925, + "theoretical_loss": 3.413685550898396, + "tokens_seen": 2350645248 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029059177532597794, + "loss": 1.4006, + "theoretical_loss": 3.4136779294864157, + "tokens_seen": 2350710784 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905717151454363, + "loss": 1.3693, + "theoretical_loss": 3.413670308346404, + "tokens_seen": 2350776320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029055165496489466, + "loss": 1.3237, + "theoretical_loss": 3.413662687478343, + "tokens_seen": 2350841856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905315947843531, + "loss": 1.4029, + "theoretical_loss": 3.413655066882216, + "tokens_seen": 2350907392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905115346038115, + "loss": 1.3074, + "theoretical_loss": 3.413647446558006, + "tokens_seen": 2350972928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029049147442326986, + "loss": 1.2797, + "theoretical_loss": 3.4136398265056944, + "tokens_seen": 2351038464 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904714142427282, + "loss": 1.3426, + "theoretical_loss": 3.413632206725265, + "tokens_seen": 2351104000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904513540621866, + "loss": 1.3192, + "theoretical_loss": 3.4136245872167006, + "tokens_seen": 2351169536 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 855824, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.311852216720581, + "objective/train/theoretical_loss": 3.4136169679799835, + "objective/train/tokens_used": 721694176, + "theoretical_loss": 3.4136169679799835, + "tokens_seen": 2351235072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029043129388164494, + "loss": 1.3349, + "theoretical_loss": 3.4136169679799835, + "tokens_seen": 2351235072 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029041123370110336, + "loss": 1.3626, + "theoretical_loss": 3.413609349015096, + "tokens_seen": 2351300608 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903911735205617, + "loss": 1.3525, + "theoretical_loss": 3.4136017303220223, + "tokens_seen": 2351366144 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903711133400201, + "loss": 1.4367, + "theoretical_loss": 3.4135941119007436, + "tokens_seen": 2351431680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029035105315947845, + "loss": 1.2782, + "theoretical_loss": 3.413586493751243, + "tokens_seen": 2351497216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029033099297893686, + "loss": 1.3611, + "theoretical_loss": 3.4135788758735037, + "tokens_seen": 2351562752 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903109327983952, + "loss": 1.1881, + "theoretical_loss": 3.413571258267508, + "tokens_seen": 2351628288 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902908726178536, + "loss": 1.3276, + "theoretical_loss": 3.413563640933239, + "tokens_seen": 2351693824 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029027081243731195, + "loss": 1.3406, + "theoretical_loss": 3.413556023870679, + "tokens_seen": 2351759360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902507522567703, + "loss": 1.2682, + "theoretical_loss": 3.413548407079811, + "tokens_seen": 2351824896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029023069207622873, + "loss": 1.2737, + "theoretical_loss": 3.413540790560618, + "tokens_seen": 2351890432 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902106318956871, + "loss": 1.3827, + "theoretical_loss": 3.4135331743130823, + "tokens_seen": 2351955968 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029019057171514545, + "loss": 1.3987, + "theoretical_loss": 3.4135255583371866, + "tokens_seen": 2352021504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901705115346038, + "loss": 1.2845, + "theoretical_loss": 3.413517942632914, + "tokens_seen": 2352087040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029015045135406223, + "loss": 1.3382, + "theoretical_loss": 3.413510327200247, + "tokens_seen": 2352152576 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901303911735206, + "loss": 1.357, + "theoretical_loss": 3.4135027120391683, + "tokens_seen": 2352218112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029011033099297896, + "loss": 1.3524, + "theoretical_loss": 3.413495097149661, + "tokens_seen": 2352283648 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900902708124373, + "loss": 1.3853, + "theoretical_loss": 3.413487482531708, + "tokens_seen": 2352349184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900702106318957, + "loss": 1.4829, + "theoretical_loss": 3.413479868185291, + "tokens_seen": 2352414720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900501504513541, + "loss": 1.3206, + "theoretical_loss": 3.4134722541103937, + "tokens_seen": 2352480256 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029003009027081246, + "loss": 1.3706, + "theoretical_loss": 3.4134646403069984, + "tokens_seen": 2352545792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900100300902708, + "loss": 1.3216, + "theoretical_loss": 3.413457026775088, + "tokens_seen": 2352611328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899899699097292, + "loss": 1.3078, + "theoretical_loss": 3.4134494135146456, + "tokens_seen": 2352676864 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899699097291876, + "loss": 1.2753, + "theoretical_loss": 3.4134418005256535, + "tokens_seen": 2352742400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028994984954864596, + "loss": 1.3278, + "theoretical_loss": 3.4134341878080945, + "tokens_seen": 2352807936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899297893681043, + "loss": 1.3278, + "theoretical_loss": 3.4134265753619513, + "tokens_seen": 2352873472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899097291875627, + "loss": 1.2864, + "theoretical_loss": 3.4134189631872074, + "tokens_seen": 2352939008 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028988966900702105, + "loss": 1.3791, + "theoretical_loss": 3.4134113512838447, + "tokens_seen": 2353004544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028986960882647946, + "loss": 1.2683, + "theoretical_loss": 3.413403739651846, + "tokens_seen": 2353070080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898495486459378, + "loss": 1.2813, + "theoretical_loss": 3.4133961282911947, + "tokens_seen": 2353135616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898294884653962, + "loss": 1.2796, + "theoretical_loss": 3.4133885172018728, + "tokens_seen": 2353201152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028980942828485455, + "loss": 1.3621, + "theoretical_loss": 3.413380906383864, + "tokens_seen": 2353266688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028978936810431297, + "loss": 1.3153, + "theoretical_loss": 3.4133732958371503, + "tokens_seen": 2353332224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028976930792377133, + "loss": 1.294, + "theoretical_loss": 3.4133656855617147, + "tokens_seen": 2353397760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897492477432297, + "loss": 1.3717, + "theoretical_loss": 3.4133580755575403, + "tokens_seen": 2353463296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028972918756268805, + "loss": 1.3535, + "theoretical_loss": 3.413350465824609, + "tokens_seen": 2353528832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897091273821464, + "loss": 1.331, + "theoretical_loss": 3.4133428563629047, + "tokens_seen": 2353594368 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028968906720160483, + "loss": 1.3378, + "theoretical_loss": 3.4133352471724097, + "tokens_seen": 2353659904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896690070210632, + "loss": 1.3394, + "theoretical_loss": 3.413327638253106, + "tokens_seen": 2353725440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028964894684052156, + "loss": 1.3659, + "theoretical_loss": 3.4133200296049777, + "tokens_seen": 2353790976 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896288866599799, + "loss": 1.3217, + "theoretical_loss": 3.4133124212280066, + "tokens_seen": 2353856512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028960882647943834, + "loss": 1.327, + "theoretical_loss": 3.4133048131221764, + "tokens_seen": 2353922048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895887662988967, + "loss": 1.2518, + "theoretical_loss": 3.413297205287469, + "tokens_seen": 2353987584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028956870611835506, + "loss": 1.2116, + "theoretical_loss": 3.4132895977238675, + "tokens_seen": 2354053120 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895486459378134, + "loss": 1.3201, + "theoretical_loss": 3.413281990431355, + "tokens_seen": 2354118656 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895285857572718, + "loss": 1.3274, + "theoretical_loss": 3.413274383409914, + "tokens_seen": 2354184192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895085255767302, + "loss": 1.2534, + "theoretical_loss": 3.413266776659527, + "tokens_seen": 2354249728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028948846539618856, + "loss": 1.3595, + "theoretical_loss": 3.4132591701801775, + "tokens_seen": 2354315264 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894684052156469, + "loss": 1.3126, + "theoretical_loss": 3.413251563971848, + "tokens_seen": 2354380800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894483450351053, + "loss": 1.3224, + "theoretical_loss": 3.413243958034521, + "tokens_seen": 2354446336 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 856929, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.329018235206604, + "objective/train/theoretical_loss": 3.4132363523681795, + "objective/train/tokens_used": 724970976, + "theoretical_loss": 3.4132363523681795, + "tokens_seen": 2354511872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894282848545637, + "loss": 1.323, + "theoretical_loss": 3.4132363523681795, + "tokens_seen": 2354511872 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028940822467402207, + "loss": 1.366, + "theoretical_loss": 3.413228746972807, + "tokens_seen": 2354577408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028938816449348043, + "loss": 1.3172, + "theoretical_loss": 3.4132211418483847, + "tokens_seen": 2354642944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893681043129388, + "loss": 1.227, + "theoretical_loss": 3.413213536994897, + "tokens_seen": 2354708480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893480441323972, + "loss": 1.3933, + "theoretical_loss": 3.4132059324123256, + "tokens_seen": 2354774016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028932798395185557, + "loss": 1.2554, + "theoretical_loss": 3.4131983281006537, + "tokens_seen": 2354839552 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028930792377131393, + "loss": 1.2502, + "theoretical_loss": 3.4131907240598647, + "tokens_seen": 2354905088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892878635907723, + "loss": 1.2954, + "theoretical_loss": 3.4131831202899408, + "tokens_seen": 2354970624 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028926780341023066, + "loss": 1.352, + "theoretical_loss": 3.4131755167908646, + "tokens_seen": 2355036160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892477432296891, + "loss": 1.2774, + "theoretical_loss": 3.4131679135626194, + "tokens_seen": 2355101696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028922768304914744, + "loss": 1.3855, + "theoretical_loss": 3.4131603106051878, + "tokens_seen": 2355167232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892076228686058, + "loss": 1.3631, + "theoretical_loss": 3.4131527079185524, + "tokens_seen": 2355232768 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028918756268806416, + "loss": 1.2602, + "theoretical_loss": 3.413145105502697, + "tokens_seen": 2355298304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891675025075226, + "loss": 1.2464, + "theoretical_loss": 3.413137503357603, + "tokens_seen": 2355363840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028914744232698094, + "loss": 1.3946, + "theoretical_loss": 3.4131299014832543, + "tokens_seen": 2355429376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891273821464393, + "loss": 1.3346, + "theoretical_loss": 3.413122299879633, + "tokens_seen": 2355494912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028910732196589766, + "loss": 1.3224, + "theoretical_loss": 3.4131146985467224, + "tokens_seen": 2355560448 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289087261785356, + "loss": 1.3819, + "theoretical_loss": 3.4131070974845055, + "tokens_seen": 2355625984 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028906720160481444, + "loss": 1.244, + "theoretical_loss": 3.4130994966929644, + "tokens_seen": 2355691520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890471414242728, + "loss": 1.2415, + "theoretical_loss": 3.413091896172083, + "tokens_seen": 2355757056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890270812437312, + "loss": 1.3256, + "theoretical_loss": 3.4130842959218426, + "tokens_seen": 2355822592 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890070210631896, + "loss": 1.394, + "theoretical_loss": 3.4130766959422276, + "tokens_seen": 2355888128 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288986960882648, + "loss": 1.2872, + "theoretical_loss": 3.4130690962332197, + "tokens_seen": 2355953664 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028896690070210636, + "loss": 1.3187, + "theoretical_loss": 3.4130614967948025, + "tokens_seen": 2356019200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889468405215647, + "loss": 1.3057, + "theoretical_loss": 3.4130538976269587, + "tokens_seen": 2356084736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889267803410231, + "loss": 1.2971, + "theoretical_loss": 3.413046298729671, + "tokens_seen": 2356150272 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028890672016048145, + "loss": 1.3528, + "theoretical_loss": 3.4130387001029217, + "tokens_seen": 2356215808 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028888665997993986, + "loss": 1.2862, + "theoretical_loss": 3.4130311017466943, + "tokens_seen": 2356281344 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028886659979939823, + "loss": 1.3092, + "theoretical_loss": 3.4130235036609715, + "tokens_seen": 2356346880 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888465396188566, + "loss": 1.2889, + "theoretical_loss": 3.4130159058457368, + "tokens_seen": 2356412416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028882647943831495, + "loss": 1.357, + "theoretical_loss": 3.413008308300972, + "tokens_seen": 2356477952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028880641925777337, + "loss": 1.2521, + "theoretical_loss": 3.41300071102666, + "tokens_seen": 2356543488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028878635907723173, + "loss": 1.3158, + "theoretical_loss": 3.4129931140227843, + "tokens_seen": 2356609024 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887662988966901, + "loss": 1.3073, + "theoretical_loss": 3.4129855172893278, + "tokens_seen": 2356674560 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028874623871614845, + "loss": 1.2432, + "theoretical_loss": 3.4129779208262723, + "tokens_seen": 2356740096 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887261785356068, + "loss": 1.3838, + "theoretical_loss": 3.412970324633602, + "tokens_seen": 2356805632 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028870611835506523, + "loss": 1.3482, + "theoretical_loss": 3.4129627287112987, + "tokens_seen": 2356871168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886860581745236, + "loss": 1.325, + "theoretical_loss": 3.412955133059346, + "tokens_seen": 2356936704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028866599799398196, + "loss": 1.2589, + "theoretical_loss": 3.4129475376777263, + "tokens_seen": 2357002240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886459378134403, + "loss": 1.3432, + "theoretical_loss": 3.412939942566423, + "tokens_seen": 2357067776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028862587763289874, + "loss": 1.2777, + "theoretical_loss": 3.412932347725418, + "tokens_seen": 2357133312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886058174523571, + "loss": 1.252, + "theoretical_loss": 3.412924753154695, + "tokens_seen": 2357198848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028858575727181546, + "loss": 1.3248, + "theoretical_loss": 3.4129171588542366, + "tokens_seen": 2357264384 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885656970912738, + "loss": 1.3245, + "theoretical_loss": 3.4129095648240257, + "tokens_seen": 2357329920 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885456369107322, + "loss": 1.4255, + "theoretical_loss": 3.412901971064045, + "tokens_seen": 2357395456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885255767301906, + "loss": 1.419, + "theoretical_loss": 3.4128943775742773, + "tokens_seen": 2357460992 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028850551654964896, + "loss": 1.3005, + "theoretical_loss": 3.4128867843547064, + "tokens_seen": 2357526528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884854563691073, + "loss": 1.318, + "theoretical_loss": 3.412879191405314, + "tokens_seen": 2357592064 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884653961885657, + "loss": 1.319, + "theoretical_loss": 3.412871598726084, + "tokens_seen": 2357657600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884453360080241, + "loss": 1.2678, + "theoretical_loss": 3.4128640063169975, + "tokens_seen": 2357723136 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 858137, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2626562118530273, + "objective/train/theoretical_loss": 3.41285641417804, + "objective/train/tokens_used": 728247776, + "theoretical_loss": 3.41285641417804, + "tokens_seen": 2357788672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028842527582748247, + "loss": 1.2446, + "theoretical_loss": 3.41285641417804, + "tokens_seen": 2357788672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028840521564694083, + "loss": 1.2647, + "theoretical_loss": 3.412848822309192, + "tokens_seen": 2357854208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883851554663992, + "loss": 1.2956, + "theoretical_loss": 3.4128412307104377, + "tokens_seen": 2357919744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883650952858576, + "loss": 1.386, + "theoretical_loss": 3.41283363938176, + "tokens_seen": 2357985280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028834503510531597, + "loss": 1.2778, + "theoretical_loss": 3.412826048323141, + "tokens_seen": 2358050816 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028832497492477433, + "loss": 1.261, + "theoretical_loss": 3.412818457534564, + "tokens_seen": 2358116352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883049147442327, + "loss": 1.3761, + "theoretical_loss": 3.412810867016012, + "tokens_seen": 2358181888 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028828485456369106, + "loss": 1.3664, + "theoretical_loss": 3.4128032767674683, + "tokens_seen": 2358247424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882647943831495, + "loss": 1.3789, + "theoretical_loss": 3.4127956867889147, + "tokens_seen": 2358312960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028824473420260784, + "loss": 1.3448, + "theoretical_loss": 3.412788097080335, + "tokens_seen": 2358378496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882246740220662, + "loss": 1.2886, + "theoretical_loss": 3.4127805076417115, + "tokens_seen": 2358444032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028820461384152456, + "loss": 1.3138, + "theoretical_loss": 3.4127729184730278, + "tokens_seen": 2358509568 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288184553660983, + "loss": 1.2613, + "theoretical_loss": 3.412765329574266, + "tokens_seen": 2358575104 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028816449348044134, + "loss": 1.342, + "theoretical_loss": 3.4127577409454095, + "tokens_seen": 2358640640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881444332998997, + "loss": 1.205, + "theoretical_loss": 3.412750152586441, + "tokens_seen": 2358706176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028812437311935806, + "loss": 1.3256, + "theoretical_loss": 3.4127425644973437, + "tokens_seen": 2358771712 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881043129388164, + "loss": 1.4523, + "theoretical_loss": 3.4127349766781, + "tokens_seen": 2358837248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028808425275827484, + "loss": 1.3606, + "theoretical_loss": 3.4127273891286936, + "tokens_seen": 2358902784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880641925777332, + "loss": 1.2846, + "theoretical_loss": 3.4127198018491067, + "tokens_seen": 2358968320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028804413239719157, + "loss": 1.3933, + "theoretical_loss": 3.4127122148393223, + "tokens_seen": 2359033856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028802407221664993, + "loss": 1.3951, + "theoretical_loss": 3.4127046280993234, + "tokens_seen": 2359099392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028800401203610835, + "loss": 1.2757, + "theoretical_loss": 3.4126970416290927, + "tokens_seen": 2359164928 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879839518555667, + "loss": 1.3559, + "theoretical_loss": 3.412689455428614, + "tokens_seen": 2359230464 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028796389167502507, + "loss": 1.3227, + "theoretical_loss": 3.412681869497869, + "tokens_seen": 2359296000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028794383149448343, + "loss": 1.2318, + "theoretical_loss": 3.4126742838368416, + "tokens_seen": 2359361536 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879237713139418, + "loss": 1.2728, + "theoretical_loss": 3.412666698445514, + "tokens_seen": 2359427072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879037111334002, + "loss": 1.3249, + "theoretical_loss": 3.4126591133238695, + "tokens_seen": 2359492608 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878836509528586, + "loss": 1.3723, + "theoretical_loss": 3.412651528471891, + "tokens_seen": 2359558144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028786359077231694, + "loss": 1.2988, + "theoretical_loss": 3.4126439438895613, + "tokens_seen": 2359623680 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878435305917753, + "loss": 1.2978, + "theoretical_loss": 3.4126363595768634, + "tokens_seen": 2359689216 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878234704112337, + "loss": 1.2914, + "theoretical_loss": 3.41262877553378, + "tokens_seen": 2359754752 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878034102306921, + "loss": 1.4378, + "theoretical_loss": 3.412621191760295, + "tokens_seen": 2359820288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028778335005015044, + "loss": 1.32, + "theoretical_loss": 3.41261360825639, + "tokens_seen": 2359885824 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877632898696088, + "loss": 1.2968, + "theoretical_loss": 3.4126060250220487, + "tokens_seen": 2359951360 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028774322968906716, + "loss": 1.3152, + "theoretical_loss": 3.412598442057254, + "tokens_seen": 2360016896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877231695085256, + "loss": 1.2884, + "theoretical_loss": 3.412590859361988, + "tokens_seen": 2360082432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028770310932798394, + "loss": 1.3062, + "theoretical_loss": 3.4125832769362345, + "tokens_seen": 2360147968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876830491474423, + "loss": 1.3748, + "theoretical_loss": 3.412575694779977, + "tokens_seen": 2360213504 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028766298896690067, + "loss": 1.2681, + "theoretical_loss": 3.4125681128931973, + "tokens_seen": 2360279040 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876429287863591, + "loss": 1.2125, + "theoretical_loss": 3.4125605312758784, + "tokens_seen": 2360344576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028762286860581745, + "loss": 1.2953, + "theoretical_loss": 3.4125529499280036, + "tokens_seen": 2360410112 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876028084252758, + "loss": 1.3048, + "theoretical_loss": 3.412545368849556, + "tokens_seen": 2360475648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028758274824473417, + "loss": 1.3188, + "theoretical_loss": 3.4125377880405185, + "tokens_seen": 2360541184 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028756268806419253, + "loss": 1.3008, + "theoretical_loss": 3.4125302075008737, + "tokens_seen": 2360606720 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028754262788365095, + "loss": 1.3116, + "theoretical_loss": 3.412522627230605, + "tokens_seen": 2360672256 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028752256770310936, + "loss": 1.3642, + "theoretical_loss": 3.412515047229695, + "tokens_seen": 2360737792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875025075225677, + "loss": 1.2148, + "theoretical_loss": 3.4125074674981266, + "tokens_seen": 2360803328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874824473420261, + "loss": 1.3244, + "theoretical_loss": 3.412499888035883, + "tokens_seen": 2360868864 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874623871614845, + "loss": 1.3952, + "theoretical_loss": 3.412492308842947, + "tokens_seen": 2360934400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028744232698094287, + "loss": 1.3934, + "theoretical_loss": 3.412484729919302, + "tokens_seen": 2360999936 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 859369, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2805547714233398, + "objective/train/theoretical_loss": 3.41247715126493, + "objective/train/tokens_used": 731524576, + "theoretical_loss": 3.41247715126493, + "tokens_seen": 2361065472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028742226680040123, + "loss": 1.241, + "theoretical_loss": 3.41247715126493, + "tokens_seen": 2361065472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874022066198596, + "loss": 1.3405, + "theoretical_loss": 3.412469572879815, + "tokens_seen": 2361131008 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287382146439318, + "loss": 1.3676, + "theoretical_loss": 3.412461994763939, + "tokens_seen": 2361196544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028736208625877637, + "loss": 1.3654, + "theoretical_loss": 3.412454416917286, + "tokens_seen": 2361262080 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028734202607823473, + "loss": 1.229, + "theoretical_loss": 3.412446839339838, + "tokens_seen": 2361327616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873219658976931, + "loss": 1.2543, + "theoretical_loss": 3.4124392620315787, + "tokens_seen": 2361393152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028730190571715146, + "loss": 1.2465, + "theoretical_loss": 3.4124316849924905, + "tokens_seen": 2361458688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872818455366099, + "loss": 1.353, + "theoretical_loss": 3.412424108222557, + "tokens_seen": 2361524224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028726178535606824, + "loss": 1.3166, + "theoretical_loss": 3.4124165317217603, + "tokens_seen": 2361589760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872417251755266, + "loss": 1.2422, + "theoretical_loss": 3.4124089554900845, + "tokens_seen": 2361655296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028722166499498496, + "loss": 1.2223, + "theoretical_loss": 3.4124013795275117, + "tokens_seen": 2361720832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872016048144434, + "loss": 1.2965, + "theoretical_loss": 3.412393803834025, + "tokens_seen": 2361786368 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028718154463390174, + "loss": 1.2567, + "theoretical_loss": 3.4123862284096074, + "tokens_seen": 2361851904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871614844533601, + "loss": 1.3686, + "theoretical_loss": 3.4123786532542417, + "tokens_seen": 2361917440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028714142427281846, + "loss": 1.3152, + "theoretical_loss": 3.4123710783679115, + "tokens_seen": 2361982976 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871213640922768, + "loss": 1.3515, + "theoretical_loss": 3.4123635037505995, + "tokens_seen": 2362048512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028710130391173524, + "loss": 1.3211, + "theoretical_loss": 3.4123559294022883, + "tokens_seen": 2362114048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870812437311936, + "loss": 1.3728, + "theoretical_loss": 3.412348355322962, + "tokens_seen": 2362179584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028706118355065197, + "loss": 1.3758, + "theoretical_loss": 3.4123407815126017, + "tokens_seen": 2362245120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028704112337011033, + "loss": 1.1617, + "theoretical_loss": 3.412333207971192, + "tokens_seen": 2362310656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028702106318956875, + "loss": 1.3347, + "theoretical_loss": 3.4123256346987154, + "tokens_seen": 2362376192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870010030090271, + "loss": 1.4538, + "theoretical_loss": 3.4123180616951547, + "tokens_seen": 2362441728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028698094282848547, + "loss": 1.2873, + "theoretical_loss": 3.412310488960493, + "tokens_seen": 2362507264 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028696088264794383, + "loss": 1.3706, + "theoretical_loss": 3.4123029164947134, + "tokens_seen": 2362572800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869408224674022, + "loss": 1.2547, + "theoretical_loss": 3.412295344297799, + "tokens_seen": 2362638336 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869207622868606, + "loss": 1.2416, + "theoretical_loss": 3.4122877723697327, + "tokens_seen": 2362703872 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286900702106319, + "loss": 1.3231, + "theoretical_loss": 3.412280200710497, + "tokens_seen": 2362769408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028688064192577734, + "loss": 1.3412, + "theoretical_loss": 3.4122726293200754, + "tokens_seen": 2362834944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868605817452357, + "loss": 1.4128, + "theoretical_loss": 3.412265058198451, + "tokens_seen": 2362900480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868405215646941, + "loss": 1.3395, + "theoretical_loss": 3.4122574873456064, + "tokens_seen": 2362966016 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868204613841525, + "loss": 1.3915, + "theoretical_loss": 3.412249916761525, + "tokens_seen": 2363031552 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028680040120361084, + "loss": 1.3195, + "theoretical_loss": 3.41224234644619, + "tokens_seen": 2363097088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867803410230692, + "loss": 1.2825, + "theoretical_loss": 3.4122347763995835, + "tokens_seen": 2363162624 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028676028084252756, + "loss": 1.3586, + "theoretical_loss": 3.412227206621689, + "tokens_seen": 2363228160 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286740220661986, + "loss": 1.2506, + "theoretical_loss": 3.4122196371124898, + "tokens_seen": 2363293696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028672016048144434, + "loss": 1.2937, + "theoretical_loss": 3.4122120678719687, + "tokens_seen": 2363359232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867001003009027, + "loss": 1.3666, + "theoretical_loss": 3.4122044989001084, + "tokens_seen": 2363424768 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028668004012036107, + "loss": 1.4238, + "theoretical_loss": 3.4121969301968926, + "tokens_seen": 2363490304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866599799398195, + "loss": 1.3419, + "theoretical_loss": 3.4121893617623034, + "tokens_seen": 2363555840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028663991975927785, + "loss": 1.2763, + "theoretical_loss": 3.4121817935963246, + "tokens_seen": 2363621376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866198595787362, + "loss": 1.2385, + "theoretical_loss": 3.412174225698939, + "tokens_seen": 2363686912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028659979939819457, + "loss": 1.3064, + "theoretical_loss": 3.41216665807013, + "tokens_seen": 2363752448 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028657973921765293, + "loss": 1.4066, + "theoretical_loss": 3.4121590907098795, + "tokens_seen": 2363817984 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028655967903711135, + "loss": 1.2873, + "theoretical_loss": 3.4121515236181716, + "tokens_seen": 2363883520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865396188565697, + "loss": 1.3557, + "theoretical_loss": 3.4121439567949885, + "tokens_seen": 2363949056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865195586760281, + "loss": 1.3409, + "theoretical_loss": 3.412136390240314, + "tokens_seen": 2364014592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028649949849548644, + "loss": 1.2567, + "theoretical_loss": 3.4121288239541308, + "tokens_seen": 2364080128 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028647943831494485, + "loss": 1.2833, + "theoretical_loss": 3.412121257936422, + "tokens_seen": 2364145664 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864593781344032, + "loss": 1.2844, + "theoretical_loss": 3.4121136921871704, + "tokens_seen": 2364211200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864393179538616, + "loss": 1.3008, + "theoretical_loss": 3.4121061267063593, + "tokens_seen": 2364276736 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 860660, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3390905857086182, + "objective/train/theoretical_loss": 3.4120985614939716, + "objective/train/tokens_used": 734801376, + "theoretical_loss": 3.4120985614939716, + "tokens_seen": 2364342272 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028641925777331994, + "loss": 1.2884, + "theoretical_loss": 3.4120985614939716, + "tokens_seen": 2364342272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863991975927783, + "loss": 1.4302, + "theoretical_loss": 3.41209099654999, + "tokens_seen": 2364407808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863791374122367, + "loss": 1.3953, + "theoretical_loss": 3.4120834318743984, + "tokens_seen": 2364473344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863590772316951, + "loss": 1.3037, + "theoretical_loss": 3.4120758674671796, + "tokens_seen": 2364538880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028633901705115344, + "loss": 1.2538, + "theoretical_loss": 3.412068303328316, + "tokens_seen": 2364604416 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863189568706118, + "loss": 1.3618, + "theoretical_loss": 3.412060739457791, + "tokens_seen": 2364669952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862988966900702, + "loss": 1.2846, + "theoretical_loss": 3.4120531758555877, + "tokens_seen": 2364735488 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862788365095286, + "loss": 1.3461, + "theoretical_loss": 3.412045612521689, + "tokens_seen": 2364801024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028625877632898694, + "loss": 1.3408, + "theoretical_loss": 3.4120380494560782, + "tokens_seen": 2364866560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862387161484453, + "loss": 1.3207, + "theoretical_loss": 3.412030486658738, + "tokens_seen": 2364932096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028621865596790367, + "loss": 1.3847, + "theoretical_loss": 3.412022924129652, + "tokens_seen": 2364997632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861985957873621, + "loss": 1.2924, + "theoretical_loss": 3.4120153618688027, + "tokens_seen": 2365063168 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028617853560682045, + "loss": 1.3711, + "theoretical_loss": 3.4120077998761733, + "tokens_seen": 2365128704 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861584754262788, + "loss": 1.2808, + "theoretical_loss": 3.412000238151747, + "tokens_seen": 2365194240 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028613841524573717, + "loss": 1.3641, + "theoretical_loss": 3.411992676695507, + "tokens_seen": 2365259776 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861183550651956, + "loss": 1.2874, + "theoretical_loss": 3.411985115507436, + "tokens_seen": 2365325312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028609829488465395, + "loss": 1.3244, + "theoretical_loss": 3.411977554587517, + "tokens_seen": 2365390848 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860782347041123, + "loss": 1.3673, + "theoretical_loss": 3.4119699939357337, + "tokens_seen": 2365456384 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860581745235707, + "loss": 1.3912, + "theoretical_loss": 3.4119624335520684, + "tokens_seen": 2365521920 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860381143430291, + "loss": 1.3915, + "theoretical_loss": 3.4119548734365046, + "tokens_seen": 2365587456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860180541624875, + "loss": 1.351, + "theoretical_loss": 3.411947313589025, + "tokens_seen": 2365652992 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028599799398194587, + "loss": 1.1967, + "theoretical_loss": 3.4119397540096132, + "tokens_seen": 2365718528 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028597793380140423, + "loss": 1.2212, + "theoretical_loss": 3.411932194698252, + "tokens_seen": 2365784064 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859578736208626, + "loss": 1.3651, + "theoretical_loss": 3.411924635654924, + "tokens_seen": 2365849600 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285937813440321, + "loss": 1.2923, + "theoretical_loss": 3.411917076879613, + "tokens_seen": 2365915136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859177532597794, + "loss": 1.2929, + "theoretical_loss": 3.411909518372302, + "tokens_seen": 2365980672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028589769307923774, + "loss": 1.2801, + "theoretical_loss": 3.411901960132974, + "tokens_seen": 2366046208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858776328986961, + "loss": 1.3, + "theoretical_loss": 3.4118944021616118, + "tokens_seen": 2366111744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858575727181545, + "loss": 1.3226, + "theoretical_loss": 3.4118868444581985, + "tokens_seen": 2366177280 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858375125376129, + "loss": 1.4061, + "theoretical_loss": 3.4118792870227175, + "tokens_seen": 2366242816 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028581745235707124, + "loss": 1.3107, + "theoretical_loss": 3.411871729855151, + "tokens_seen": 2366308352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857973921765296, + "loss": 1.2826, + "theoretical_loss": 3.4118641729554837, + "tokens_seen": 2366373888 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028577733199598796, + "loss": 1.4263, + "theoretical_loss": 3.411856616323697, + "tokens_seen": 2366439424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857572718154464, + "loss": 1.3163, + "theoretical_loss": 3.4118490599597755, + "tokens_seen": 2366504960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028573721163490474, + "loss": 1.3527, + "theoretical_loss": 3.411841503863701, + "tokens_seen": 2366570496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857171514543631, + "loss": 1.3231, + "theoretical_loss": 3.4118339480354574, + "tokens_seen": 2366636032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028569709127382147, + "loss": 1.2775, + "theoretical_loss": 3.411826392475027, + "tokens_seen": 2366701568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856770310932799, + "loss": 1.2985, + "theoretical_loss": 3.4118188371823943, + "tokens_seen": 2366767104 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028565697091273825, + "loss": 1.3201, + "theoretical_loss": 3.411811282157541, + "tokens_seen": 2366832640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856369107321966, + "loss": 1.2818, + "theoretical_loss": 3.4118037274004505, + "tokens_seen": 2366898176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028561685055165497, + "loss": 1.3636, + "theoretical_loss": 3.411796172911106, + "tokens_seen": 2366963712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028559679037111333, + "loss": 1.2351, + "theoretical_loss": 3.4117886186894912, + "tokens_seen": 2367029248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028557673019057175, + "loss": 1.3674, + "theoretical_loss": 3.4117810647355884, + "tokens_seen": 2367094784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855566700100301, + "loss": 1.2995, + "theoretical_loss": 3.411773511049381, + "tokens_seen": 2367160320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855366098294885, + "loss": 1.3726, + "theoretical_loss": 3.4117659576308523, + "tokens_seen": 2367225856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028551654964894684, + "loss": 1.3567, + "theoretical_loss": 3.411758404479985, + "tokens_seen": 2367291392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028549648946840525, + "loss": 1.3031, + "theoretical_loss": 3.4117508515967625, + "tokens_seen": 2367356928 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854764292878636, + "loss": 1.2887, + "theoretical_loss": 3.411743298981168, + "tokens_seen": 2367422464 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285456369107322, + "loss": 1.1462, + "theoretical_loss": 3.411735746633184, + "tokens_seen": 2367488000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028543630892678034, + "loss": 1.3338, + "theoretical_loss": 3.411728194552794, + "tokens_seen": 2367553536 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 861837, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3536452054977417, + "objective/train/theoretical_loss": 3.411720642739981, + "objective/train/tokens_used": 738078176, + "theoretical_loss": 3.411720642739981, + "tokens_seen": 2367619072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854162487462387, + "loss": 1.3665, + "theoretical_loss": 3.411720642739981, + "tokens_seen": 2367619072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002853961885656971, + "loss": 1.2447, + "theoretical_loss": 3.411713091194729, + "tokens_seen": 2367684608 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853761283851555, + "loss": 1.3943, + "theoretical_loss": 3.41170553991702, + "tokens_seen": 2367750144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028535606820461384, + "loss": 1.2777, + "theoretical_loss": 3.411697988906837, + "tokens_seen": 2367815680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853360080240722, + "loss": 1.2979, + "theoretical_loss": 3.411690438164164, + "tokens_seen": 2367881216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853159478435306, + "loss": 1.4002, + "theoretical_loss": 3.411682887688984, + "tokens_seen": 2367946752 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285295887662989, + "loss": 1.2942, + "theoretical_loss": 3.411675337481279, + "tokens_seen": 2368012288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028527582748244735, + "loss": 1.1875, + "theoretical_loss": 3.4116677875410337, + "tokens_seen": 2368077824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852557673019057, + "loss": 1.2492, + "theoretical_loss": 3.4116602378682304, + "tokens_seen": 2368143360 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028523570712136407, + "loss": 1.4297, + "theoretical_loss": 3.411652688462852, + "tokens_seen": 2368208896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852156469408225, + "loss": 1.4129, + "theoretical_loss": 3.4116451393248823, + "tokens_seen": 2368274432 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028519558676028085, + "loss": 1.3353, + "theoretical_loss": 3.411637590454304, + "tokens_seen": 2368339968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851755265797392, + "loss": 1.2396, + "theoretical_loss": 3.4116300418511, + "tokens_seen": 2368405504 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028515546639919757, + "loss": 1.4334, + "theoretical_loss": 3.4116224935152544, + "tokens_seen": 2368471040 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285135406218656, + "loss": 1.2243, + "theoretical_loss": 3.411614945446749, + "tokens_seen": 2368536576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028511534603811435, + "loss": 1.3635, + "theoretical_loss": 3.411607397645568, + "tokens_seen": 2368602112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850952858575727, + "loss": 1.288, + "theoretical_loss": 3.4115998501116938, + "tokens_seen": 2368667648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850752256770311, + "loss": 1.4284, + "theoretical_loss": 3.41159230284511, + "tokens_seen": 2368733184 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850551654964895, + "loss": 1.3042, + "theoretical_loss": 3.4115847558457997, + "tokens_seen": 2368798720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028503510531594785, + "loss": 1.3574, + "theoretical_loss": 3.4115772091137457, + "tokens_seen": 2368864256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850150451354062, + "loss": 1.4121, + "theoretical_loss": 3.411569662648932, + "tokens_seen": 2368929792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849949849548646, + "loss": 1.3017, + "theoretical_loss": 3.4115621164513406, + "tokens_seen": 2368995328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028497492477432294, + "loss": 1.3189, + "theoretical_loss": 3.411554570520955, + "tokens_seen": 2369060864 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028495486459378136, + "loss": 1.2421, + "theoretical_loss": 3.411547024857759, + "tokens_seen": 2369126400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849348044132397, + "loss": 1.1777, + "theoretical_loss": 3.411539479461735, + "tokens_seen": 2369191936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849147442326981, + "loss": 1.3888, + "theoretical_loss": 3.4115319343328667, + "tokens_seen": 2369257472 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028489468405215644, + "loss": 1.3279, + "theoretical_loss": 3.4115243894711367, + "tokens_seen": 2369323008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028487462387161486, + "loss": 1.2475, + "theoretical_loss": 3.4115168448765285, + "tokens_seen": 2369388544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848545636910732, + "loss": 1.3397, + "theoretical_loss": 3.4115093005490253, + "tokens_seen": 2369454080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848345035105316, + "loss": 1.3847, + "theoretical_loss": 3.41150175648861, + "tokens_seen": 2369519616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028481444332998995, + "loss": 1.303, + "theoretical_loss": 3.411494212695266, + "tokens_seen": 2369585152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847943831494483, + "loss": 1.2856, + "theoretical_loss": 3.4114866691689762, + "tokens_seen": 2369650688 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847743229689067, + "loss": 1.2327, + "theoretical_loss": 3.4114791259097244, + "tokens_seen": 2369716224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847542627883651, + "loss": 1.3176, + "theoretical_loss": 3.4114715829174926, + "tokens_seen": 2369781760 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028473420260782345, + "loss": 1.3408, + "theoretical_loss": 3.411464040192265, + "tokens_seen": 2369847296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847141424272818, + "loss": 1.3041, + "theoretical_loss": 3.411456497734025, + "tokens_seen": 2369912832 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028469408224674023, + "loss": 1.3031, + "theoretical_loss": 3.4114489555427543, + "tokens_seen": 2369978368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846740220661986, + "loss": 1.3527, + "theoretical_loss": 3.4114414136184372, + "tokens_seen": 2370043904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028465396188565695, + "loss": 1.3059, + "theoretical_loss": 3.411433871961057, + "tokens_seen": 2370109440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846339017051153, + "loss": 1.3545, + "theoretical_loss": 3.411426330570596, + "tokens_seen": 2370174976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846138415245737, + "loss": 1.3494, + "theoretical_loss": 3.411418789447038, + "tokens_seen": 2370240512 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845937813440321, + "loss": 1.3317, + "theoretical_loss": 3.4114112485903663, + "tokens_seen": 2370306048 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028457372116349046, + "loss": 1.3315, + "theoretical_loss": 3.4114037080005635, + "tokens_seen": 2370371584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845536609829488, + "loss": 1.2688, + "theoretical_loss": 3.411396167677613, + "tokens_seen": 2370437120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028453360080240724, + "loss": 1.216, + "theoretical_loss": 3.4113886276214984, + "tokens_seen": 2370502656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028451354062186565, + "loss": 1.2913, + "theoretical_loss": 3.4113810878322024, + "tokens_seen": 2370568192 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284493480441324, + "loss": 1.3825, + "theoretical_loss": 3.411373548309708, + "tokens_seen": 2370633728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844734202607824, + "loss": 1.3224, + "theoretical_loss": 3.411366009053999, + "tokens_seen": 2370699264 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028445336008024074, + "loss": 1.2686, + "theoretical_loss": 3.4113584700650588, + "tokens_seen": 2370764800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844332998996991, + "loss": 1.3098, + "theoretical_loss": 3.411350931342869, + "tokens_seen": 2370830336 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 863154, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3327744007110596, + "objective/train/theoretical_loss": 3.4113433928874146, + "objective/train/tokens_used": 741354976, + "theoretical_loss": 3.4113433928874146, + "tokens_seen": 2370895872 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844132397191575, + "loss": 1.2354, + "theoretical_loss": 3.4113433928874146, + "tokens_seen": 2370895872 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843931795386159, + "loss": 1.3593, + "theoretical_loss": 3.4113358546986783, + "tokens_seen": 2370961408 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028437311935807424, + "loss": 1.3502, + "theoretical_loss": 3.411328316776643, + "tokens_seen": 2371026944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843530591775326, + "loss": 1.4473, + "theoretical_loss": 3.4113207791212914, + "tokens_seen": 2371092480 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284332998996991, + "loss": 1.3367, + "theoretical_loss": 3.4113132417326075, + "tokens_seen": 2371158016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843129388164494, + "loss": 1.3558, + "theoretical_loss": 3.4113057046105744, + "tokens_seen": 2371223552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028429287863590775, + "loss": 1.3908, + "theoretical_loss": 3.411298167755175, + "tokens_seen": 2371289088 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842728184553661, + "loss": 1.3502, + "theoretical_loss": 3.4112906311663926, + "tokens_seen": 2371354624 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028425275827482447, + "loss": 1.3454, + "theoretical_loss": 3.4112830948442108, + "tokens_seen": 2371420160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842326980942829, + "loss": 1.3213, + "theoretical_loss": 3.4112755587886117, + "tokens_seen": 2371485696 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028421263791374125, + "loss": 1.4537, + "theoretical_loss": 3.4112680229995798, + "tokens_seen": 2371551232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841925777331996, + "loss": 1.3642, + "theoretical_loss": 3.4112604874770978, + "tokens_seen": 2371616768 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284172517552658, + "loss": 1.3696, + "theoretical_loss": 3.4112529522211483, + "tokens_seen": 2371682304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841524573721164, + "loss": 1.3731, + "theoretical_loss": 3.4112454172317155, + "tokens_seen": 2371747840 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028413239719157475, + "loss": 1.2867, + "theoretical_loss": 3.4112378825087823, + "tokens_seen": 2371813376 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841123370110331, + "loss": 1.3119, + "theoretical_loss": 3.4112303480523316, + "tokens_seen": 2371878912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840922768304915, + "loss": 1.3483, + "theoretical_loss": 3.411222813862347, + "tokens_seen": 2371944448 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840722166499499, + "loss": 1.3446, + "theoretical_loss": 3.411215279938811, + "tokens_seen": 2372009984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028405215646940825, + "loss": 1.2628, + "theoretical_loss": 3.4112077462817076, + "tokens_seen": 2372075520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840320962888666, + "loss": 1.2579, + "theoretical_loss": 3.41120021289102, + "tokens_seen": 2372141056 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284012036108325, + "loss": 1.3252, + "theoretical_loss": 3.4111926797667307, + "tokens_seen": 2372206592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028399197592778334, + "loss": 1.2872, + "theoretical_loss": 3.4111851469088235, + "tokens_seen": 2372272128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028397191574724176, + "loss": 1.2153, + "theoretical_loss": 3.411177614317282, + "tokens_seen": 2372337664 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839518555667001, + "loss": 1.349, + "theoretical_loss": 3.4111700819920885, + "tokens_seen": 2372403200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839317953861585, + "loss": 1.1373, + "theoretical_loss": 3.4111625499332265, + "tokens_seen": 2372468736 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028391173520561684, + "loss": 1.3837, + "theoretical_loss": 3.4111550181406796, + "tokens_seen": 2372534272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028389167502507526, + "loss": 1.3397, + "theoretical_loss": 3.4111474866144307, + "tokens_seen": 2372599808 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838716148445336, + "loss": 1.3024, + "theoretical_loss": 3.4111399553544635, + "tokens_seen": 2372665344 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283851554663992, + "loss": 1.3625, + "theoretical_loss": 3.4111324243607606, + "tokens_seen": 2372730880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028383149448345035, + "loss": 1.367, + "theoretical_loss": 3.411124893633305, + "tokens_seen": 2372796416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838114343029087, + "loss": 1.3969, + "theoretical_loss": 3.4111173631720813, + "tokens_seen": 2372861952 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837913741223671, + "loss": 1.196, + "theoretical_loss": 3.4111098329770715, + "tokens_seen": 2372927488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837713139418255, + "loss": 1.3239, + "theoretical_loss": 3.411102303048259, + "tokens_seen": 2372993024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028375125376128385, + "loss": 1.3059, + "theoretical_loss": 3.411094773385628, + "tokens_seen": 2373058560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837311935807422, + "loss": 1.5196, + "theoretical_loss": 3.4110872439891597, + "tokens_seen": 2373124096 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028371113340020063, + "loss": 1.383, + "theoretical_loss": 3.4110797148588397, + "tokens_seen": 2373189632 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283691073219659, + "loss": 1.466, + "theoretical_loss": 3.41107218599465, + "tokens_seen": 2373255168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028367101303911735, + "loss": 1.2908, + "theoretical_loss": 3.4110646573965737, + "tokens_seen": 2373320704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836509528585757, + "loss": 1.3131, + "theoretical_loss": 3.411057129064594, + "tokens_seen": 2373386240 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836308926780341, + "loss": 1.3103, + "theoretical_loss": 3.4110496009986955, + "tokens_seen": 2373451776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836108324974925, + "loss": 1.3384, + "theoretical_loss": 3.4110420731988595, + "tokens_seen": 2373517312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028359077231695086, + "loss": 1.3036, + "theoretical_loss": 3.411034545665071, + "tokens_seen": 2373582848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835707121364092, + "loss": 1.3782, + "theoretical_loss": 3.411027018397312, + "tokens_seen": 2373648384 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835506519558676, + "loss": 1.2992, + "theoretical_loss": 3.4110194913955665, + "tokens_seen": 2373713920 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283530591775326, + "loss": 1.2488, + "theoretical_loss": 3.411011964659817, + "tokens_seen": 2373779456 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028351053159478436, + "loss": 1.3812, + "theoretical_loss": 3.4110044381900475, + "tokens_seen": 2373844992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834904714142427, + "loss": 1.212, + "theoretical_loss": 3.410996911986241, + "tokens_seen": 2373910528 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834704112337011, + "loss": 1.304, + "theoretical_loss": 3.4109893860483806, + "tokens_seen": 2373976064 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028345035105315945, + "loss": 1.3324, + "theoretical_loss": 3.41098186037645, + "tokens_seen": 2374041600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028343029087261786, + "loss": 1.1888, + "theoretical_loss": 3.4109743349704322, + "tokens_seen": 2374107136 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 864337, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4003491401672363, + "objective/train/theoretical_loss": 3.41096680983031, + "objective/train/tokens_used": 744631776, + "theoretical_loss": 3.41096680983031, + "tokens_seen": 2374172672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834102306920762, + "loss": 1.3379, + "theoretical_loss": 3.41096680983031, + "tokens_seen": 2374172672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833901705115346, + "loss": 1.3047, + "theoretical_loss": 3.4109592849560673, + "tokens_seen": 2374238208 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028337011033099295, + "loss": 1.2141, + "theoretical_loss": 3.4109517603476873, + "tokens_seen": 2374303744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028335005015045137, + "loss": 1.3176, + "theoretical_loss": 3.410944236005153, + "tokens_seen": 2374369280 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028332998996990973, + "loss": 1.3421, + "theoretical_loss": 3.4109367119284477, + "tokens_seen": 2374434816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833099297893681, + "loss": 1.3578, + "theoretical_loss": 3.4109291881175547, + "tokens_seen": 2374500352 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028328986960882645, + "loss": 1.3424, + "theoretical_loss": 3.4109216645724576, + "tokens_seen": 2374565888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832698094282848, + "loss": 1.2745, + "theoretical_loss": 3.4109141412931394, + "tokens_seen": 2374631424 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028324974924774323, + "loss": 1.3234, + "theoretical_loss": 3.4109066182795833, + "tokens_seen": 2374696960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832296890672016, + "loss": 1.2722, + "theoretical_loss": 3.410899095531773, + "tokens_seen": 2374762496 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028320962888665996, + "loss": 1.3156, + "theoretical_loss": 3.410891573049691, + "tokens_seen": 2374828032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831895687061183, + "loss": 1.3126, + "theoretical_loss": 3.410884050833321, + "tokens_seen": 2374893568 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028316950852557674, + "loss": 1.3172, + "theoretical_loss": 3.4108765288826466, + "tokens_seen": 2374959104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831494483450351, + "loss": 1.387, + "theoretical_loss": 3.410869007197651, + "tokens_seen": 2375024640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028312938816449346, + "loss": 1.4503, + "theoretical_loss": 3.410861485778317, + "tokens_seen": 2375090176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831093279839518, + "loss": 1.3157, + "theoretical_loss": 3.410853964624628, + "tokens_seen": 2375155712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830892678034102, + "loss": 1.3085, + "theoretical_loss": 3.4108464437365678, + "tokens_seen": 2375221248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830692076228686, + "loss": 1.3225, + "theoretical_loss": 3.410838923114119, + "tokens_seen": 2375286784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028304914744232696, + "loss": 1.2802, + "theoretical_loss": 3.4108314027572657, + "tokens_seen": 2375352320 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830290872617854, + "loss": 1.2924, + "theoretical_loss": 3.4108238826659902, + "tokens_seen": 2375417856 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028300902708124374, + "loss": 1.3533, + "theoretical_loss": 3.4108163628402766, + "tokens_seen": 2375483392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028298896690070216, + "loss": 1.338, + "theoretical_loss": 3.4108088432801082, + "tokens_seen": 2375548928 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829689067201605, + "loss": 1.2492, + "theoretical_loss": 3.4108013239854675, + "tokens_seen": 2375614464 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829488465396189, + "loss": 1.315, + "theoretical_loss": 3.410793804956339, + "tokens_seen": 2375680000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028292878635907725, + "loss": 1.4085, + "theoretical_loss": 3.4107862861927045, + "tokens_seen": 2375745536 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028290872617853566, + "loss": 1.3728, + "theoretical_loss": 3.4107787676945485, + "tokens_seen": 2375811072 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282888665997994, + "loss": 1.2542, + "theoretical_loss": 3.4107712494618543, + "tokens_seen": 2375876608 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828686058174524, + "loss": 1.3712, + "theoretical_loss": 3.410763731494604, + "tokens_seen": 2375942144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028284854563691075, + "loss": 1.3305, + "theoretical_loss": 3.4107562137927827, + "tokens_seen": 2376007680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828284854563691, + "loss": 1.316, + "theoretical_loss": 3.410748696356372, + "tokens_seen": 2376073216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828084252758275, + "loss": 1.3245, + "theoretical_loss": 3.410741179185356, + "tokens_seen": 2376138752 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827883650952859, + "loss": 1.247, + "theoretical_loss": 3.4107336622797186, + "tokens_seen": 2376204288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028276830491474425, + "loss": 1.3875, + "theoretical_loss": 3.410726145639442, + "tokens_seen": 2376269824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827482447342026, + "loss": 1.2788, + "theoretical_loss": 3.41071862926451, + "tokens_seen": 2376335360 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028272818455366103, + "loss": 1.2089, + "theoretical_loss": 3.410711113154906, + "tokens_seen": 2376400896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827081243731194, + "loss": 1.3753, + "theoretical_loss": 3.4107035973106132, + "tokens_seen": 2376466432 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028268806419257775, + "loss": 1.3201, + "theoretical_loss": 3.410696081731615, + "tokens_seen": 2376531968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826680040120361, + "loss": 1.3418, + "theoretical_loss": 3.4106885664178948, + "tokens_seen": 2376597504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826479438314945, + "loss": 1.3258, + "theoretical_loss": 3.4106810513694357, + "tokens_seen": 2376663040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826278836509529, + "loss": 1.2971, + "theoretical_loss": 3.410673536586221, + "tokens_seen": 2376728576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028260782347041126, + "loss": 1.2274, + "theoretical_loss": 3.4106660220682343, + "tokens_seen": 2376794112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825877632898696, + "loss": 1.2881, + "theoretical_loss": 3.4106585078154583, + "tokens_seen": 2376859648 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282567703109328, + "loss": 1.3114, + "theoretical_loss": 3.4106509938278773, + "tokens_seen": 2376925184 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825476429287864, + "loss": 1.2747, + "theoretical_loss": 3.410643480105474, + "tokens_seen": 2376990720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028252758274824476, + "loss": 1.3043, + "theoretical_loss": 3.4106359666482318, + "tokens_seen": 2377056256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825075225677031, + "loss": 1.3016, + "theoretical_loss": 3.410628453456134, + "tokens_seen": 2377121792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824874623871615, + "loss": 1.3818, + "theoretical_loss": 3.4106209405291645, + "tokens_seen": 2377187328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028246740220661985, + "loss": 1.3147, + "theoretical_loss": 3.4106134278673057, + "tokens_seen": 2377252864 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028244734202607826, + "loss": 1.3639, + "theoretical_loss": 3.4106059154705415, + "tokens_seen": 2377318400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824272818455366, + "loss": 1.3316, + "theoretical_loss": 3.410598403338855, + "tokens_seen": 2377383936 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 865580, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3770675659179688, + "objective/train/theoretical_loss": 3.41059089147223, + "objective/train/tokens_used": 747908576, + "theoretical_loss": 3.41059089147223, + "tokens_seen": 2377449472 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282407221664995, + "loss": 1.3573, + "theoretical_loss": 3.41059089147223, + "tokens_seen": 2377449472 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028238716148445335, + "loss": 1.3974, + "theoretical_loss": 3.410583379870649, + "tokens_seen": 2377515008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028236710130391177, + "loss": 1.2249, + "theoretical_loss": 3.410575868534096, + "tokens_seen": 2377580544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028234704112337013, + "loss": 1.2719, + "theoretical_loss": 3.4105683574625543, + "tokens_seen": 2377646080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823269809428285, + "loss": 1.2301, + "theoretical_loss": 3.4105608466560073, + "tokens_seen": 2377711616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028230692076228685, + "loss": 1.3313, + "theoretical_loss": 3.410553336114438, + "tokens_seen": 2377777152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822868605817452, + "loss": 1.2619, + "theoretical_loss": 3.41054582583783, + "tokens_seen": 2377842688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028226680040120363, + "loss": 1.3882, + "theoretical_loss": 3.4105383158261664, + "tokens_seen": 2377908224 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282246740220662, + "loss": 1.2948, + "theoretical_loss": 3.4105308060794313, + "tokens_seen": 2377973760 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028222668004012036, + "loss": 1.3541, + "theoretical_loss": 3.410523296597607, + "tokens_seen": 2378039296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822066198595787, + "loss": 1.3207, + "theoretical_loss": 3.4105157873806773, + "tokens_seen": 2378104832 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028218655967903714, + "loss": 1.3738, + "theoretical_loss": 3.410508278428626, + "tokens_seen": 2378170368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821664994984955, + "loss": 1.2812, + "theoretical_loss": 3.410500769741436, + "tokens_seen": 2378235904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028214643931795386, + "loss": 1.3414, + "theoretical_loss": 3.41049326131909, + "tokens_seen": 2378301440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821263791374122, + "loss": 1.3073, + "theoretical_loss": 3.4104857531615727, + "tokens_seen": 2378366976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821063189568706, + "loss": 1.3326, + "theoretical_loss": 3.410478245268867, + "tokens_seen": 2378432512 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282086258776329, + "loss": 1.3411, + "theoretical_loss": 3.410470737640956, + "tokens_seen": 2378498048 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028206619859578736, + "loss": 1.316, + "theoretical_loss": 3.410463230277823, + "tokens_seen": 2378563584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820461384152457, + "loss": 1.3343, + "theoretical_loss": 3.4104557231794512, + "tokens_seen": 2378629120 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820260782347041, + "loss": 1.2912, + "theoretical_loss": 3.410448216345825, + "tokens_seen": 2378694656 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820060180541625, + "loss": 1.2536, + "theoretical_loss": 3.4104407097769265, + "tokens_seen": 2378760192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028198595787362087, + "loss": 1.2707, + "theoretical_loss": 3.4104332034727403, + "tokens_seen": 2378825728 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028196589769307923, + "loss": 1.3409, + "theoretical_loss": 3.4104256974332485, + "tokens_seen": 2378891264 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819458375125376, + "loss": 1.3658, + "theoretical_loss": 3.4104181916584357, + "tokens_seen": 2378956800 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281925777331996, + "loss": 1.3991, + "theoretical_loss": 3.410410686148284, + "tokens_seen": 2379022336 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028190571715145437, + "loss": 1.4677, + "theoretical_loss": 3.4104031809027777, + "tokens_seen": 2379087872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028188565697091273, + "loss": 1.3324, + "theoretical_loss": 3.4103956759219, + "tokens_seen": 2379153408 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818655967903711, + "loss": 1.3083, + "theoretical_loss": 3.4103881712056343, + "tokens_seen": 2379218944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028184553660982946, + "loss": 1.2914, + "theoretical_loss": 3.410380666753964, + "tokens_seen": 2379284480 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818254764292879, + "loss": 1.4032, + "theoretical_loss": 3.4103731625668723, + "tokens_seen": 2379350016 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028180541624874624, + "loss": 1.2813, + "theoretical_loss": 3.4103656586443423, + "tokens_seen": 2379415552 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817853560682046, + "loss": 1.2874, + "theoretical_loss": 3.410358154986358, + "tokens_seen": 2379481088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028176529588766296, + "loss": 1.356, + "theoretical_loss": 3.4103506515929025, + "tokens_seen": 2379546624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817452357071214, + "loss": 1.2697, + "theoretical_loss": 3.4103431484639595, + "tokens_seen": 2379612160 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028172517552657974, + "loss": 1.3486, + "theoretical_loss": 3.4103356455995115, + "tokens_seen": 2379677696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817051153460381, + "loss": 1.3395, + "theoretical_loss": 3.410328142999543, + "tokens_seen": 2379743232 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028168505516549646, + "loss": 1.3587, + "theoretical_loss": 3.410320640664037, + "tokens_seen": 2379808768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816649949849548, + "loss": 1.2731, + "theoretical_loss": 3.4103131385929766, + "tokens_seen": 2379874304 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028164493480441324, + "loss": 1.392, + "theoretical_loss": 3.410305636786345, + "tokens_seen": 2379939840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816248746238716, + "loss": 1.3405, + "theoretical_loss": 3.4102981352441266, + "tokens_seen": 2380005376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028160481444332997, + "loss": 1.2791, + "theoretical_loss": 3.410290633966304, + "tokens_seen": 2380070912 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028158475426278833, + "loss": 1.3354, + "theoretical_loss": 3.410283132952861, + "tokens_seen": 2380136448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028156469408224674, + "loss": 1.1994, + "theoretical_loss": 3.4102756322037804, + "tokens_seen": 2380201984 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815446339017051, + "loss": 1.3435, + "theoretical_loss": 3.4102681317190457, + "tokens_seen": 2380267520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815245737211635, + "loss": 1.2966, + "theoretical_loss": 3.4102606314986414, + "tokens_seen": 2380333056 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815045135406219, + "loss": 1.4798, + "theoretical_loss": 3.4102531315425497, + "tokens_seen": 2380398592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028148445336008025, + "loss": 1.2646, + "theoretical_loss": 3.4102456318507546, + "tokens_seen": 2380464128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028146439317953866, + "loss": 1.3102, + "theoretical_loss": 3.410238132423239, + "tokens_seen": 2380529664 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281444332998997, + "loss": 1.3332, + "theoretical_loss": 3.410230633259987, + "tokens_seen": 2380595200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814242728184554, + "loss": 1.3754, + "theoretical_loss": 3.4102231343609812, + "tokens_seen": 2380660736 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 866853, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4650294780731201, + "objective/train/theoretical_loss": 3.410215635726206, + "objective/train/tokens_used": 751185376, + "theoretical_loss": 3.410215635726206, + "tokens_seen": 2380726272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028140421263791375, + "loss": 1.3166, + "theoretical_loss": 3.410215635726206, + "tokens_seen": 2380726272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028138415245737217, + "loss": 1.3479, + "theoretical_loss": 3.410208137355644, + "tokens_seen": 2380791808 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028136409227683053, + "loss": 1.356, + "theoretical_loss": 3.410200639249279, + "tokens_seen": 2380857344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813440320962889, + "loss": 1.3532, + "theoretical_loss": 3.4101931414070945, + "tokens_seen": 2380922880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028132397191574725, + "loss": 1.2532, + "theoretical_loss": 3.4101856438290734, + "tokens_seen": 2380988416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813039117352056, + "loss": 1.2538, + "theoretical_loss": 3.4101781465151997, + "tokens_seen": 2381053952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028128385155466403, + "loss": 1.3647, + "theoretical_loss": 3.4101706494654564, + "tokens_seen": 2381119488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812637913741224, + "loss": 1.3917, + "theoretical_loss": 3.4101631526798273, + "tokens_seen": 2381185024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028124373119358076, + "loss": 1.3517, + "theoretical_loss": 3.4101556561582957, + "tokens_seen": 2381250560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812236710130391, + "loss": 1.327, + "theoretical_loss": 3.410148159900845, + "tokens_seen": 2381316096 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028120361083249754, + "loss": 1.4366, + "theoretical_loss": 3.410140663907458, + "tokens_seen": 2381381632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811835506519559, + "loss": 1.2831, + "theoretical_loss": 3.4101331681781195, + "tokens_seen": 2381447168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028116349047141426, + "loss": 1.3691, + "theoretical_loss": 3.4101256727128115, + "tokens_seen": 2381512704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811434302908726, + "loss": 1.2812, + "theoretical_loss": 3.410118177511519, + "tokens_seen": 2381578240 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281123370110331, + "loss": 1.2811, + "theoretical_loss": 3.410110682574224, + "tokens_seen": 2381643776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811033099297894, + "loss": 1.4121, + "theoretical_loss": 3.4101031879009103, + "tokens_seen": 2381709312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028108324974924776, + "loss": 1.3692, + "theoretical_loss": 3.4100956934915616, + "tokens_seen": 2381774848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810631895687061, + "loss": 1.3754, + "theoretical_loss": 3.4100881993461614, + "tokens_seen": 2381840384 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810431293881645, + "loss": 1.2967, + "theoretical_loss": 3.4100807054646927, + "tokens_seen": 2381905920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810230692076229, + "loss": 1.3883, + "theoretical_loss": 3.4100732118471395, + "tokens_seen": 2381971456 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028100300902708127, + "loss": 1.2094, + "theoretical_loss": 3.410065718493485, + "tokens_seen": 2382036992 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028098294884653963, + "loss": 1.3485, + "theoretical_loss": 3.410058225403713, + "tokens_seen": 2382102528 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280962888665998, + "loss": 1.3817, + "theoretical_loss": 3.410050732577806, + "tokens_seen": 2382168064 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028094282848545635, + "loss": 1.4285, + "theoretical_loss": 3.410043240015748, + "tokens_seen": 2382233600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028092276830491477, + "loss": 1.3323, + "theoretical_loss": 3.4100357477175227, + "tokens_seen": 2382299136 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028090270812437313, + "loss": 1.2598, + "theoretical_loss": 3.4100282556831134, + "tokens_seen": 2382364672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808826479438315, + "loss": 1.3228, + "theoretical_loss": 3.4100207639125033, + "tokens_seen": 2382430208 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028086258776328986, + "loss": 1.4418, + "theoretical_loss": 3.4100132724056764, + "tokens_seen": 2382495744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808425275827483, + "loss": 1.308, + "theoretical_loss": 3.4100057811626154, + "tokens_seen": 2382561280 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028082246740220664, + "loss": 1.3924, + "theoretical_loss": 3.4099982901833044, + "tokens_seen": 2382626816 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280802407221665, + "loss": 1.3129, + "theoretical_loss": 3.4099907994677263, + "tokens_seen": 2382692352 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028078234704112336, + "loss": 1.3365, + "theoretical_loss": 3.409983309015865, + "tokens_seen": 2382757888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807622868605818, + "loss": 1.3387, + "theoretical_loss": 3.409975818827704, + "tokens_seen": 2382823424 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028074222668004014, + "loss": 1.3871, + "theoretical_loss": 3.4099683289032265, + "tokens_seen": 2382888960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807221664994985, + "loss": 1.3555, + "theoretical_loss": 3.409960839242416, + "tokens_seen": 2382954496 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028070210631895686, + "loss": 1.2642, + "theoretical_loss": 3.409953349845256, + "tokens_seen": 2383020032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806820461384152, + "loss": 1.2793, + "theoretical_loss": 3.4099458607117303, + "tokens_seen": 2383085568 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028066198595787364, + "loss": 1.3662, + "theoretical_loss": 3.409938371841822, + "tokens_seen": 2383151104 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280641925777332, + "loss": 1.3333, + "theoretical_loss": 3.4099308832355146, + "tokens_seen": 2383216640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028062186559679037, + "loss": 1.4395, + "theoretical_loss": 3.4099233948927914, + "tokens_seen": 2383282176 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028060180541624873, + "loss": 1.3788, + "theoretical_loss": 3.4099159068136364, + "tokens_seen": 2383347712 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028058174523570715, + "loss": 1.2366, + "theoretical_loss": 3.4099084189980324, + "tokens_seen": 2383413248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805616850551655, + "loss": 1.2712, + "theoretical_loss": 3.4099009314459634, + "tokens_seen": 2383478784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028054162487462387, + "loss": 1.2927, + "theoretical_loss": 3.409893444157413, + "tokens_seen": 2383544320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028052156469408223, + "loss": 1.2963, + "theoretical_loss": 3.409885957132364, + "tokens_seen": 2383609856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805015045135406, + "loss": 1.4209, + "theoretical_loss": 3.409878470370801, + "tokens_seen": 2383675392 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280481444332999, + "loss": 1.39, + "theoretical_loss": 3.4098709838727057, + "tokens_seen": 2383740928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028046138415245737, + "loss": 1.3058, + "theoretical_loss": 3.4098634976380633, + "tokens_seen": 2383806464 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028044132397191574, + "loss": 1.2907, + "theoretical_loss": 3.409856011666857, + "tokens_seen": 2383872000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804212637913741, + "loss": 1.285, + "theoretical_loss": 3.4098485259590694, + "tokens_seen": 2383937536 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 868029, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2294628620147705, + "objective/train/theoretical_loss": 3.409841040514684, + "objective/train/tokens_used": 754462176, + "theoretical_loss": 3.409841040514684, + "tokens_seen": 2384003072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804012036108325, + "loss": 1.2541, + "theoretical_loss": 3.409841040514684, + "tokens_seen": 2384003072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002803811434302909, + "loss": 1.2648, + "theoretical_loss": 3.4098335553336856, + "tokens_seen": 2384068608 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028036108324974924, + "loss": 1.3008, + "theoretical_loss": 3.409826070416057, + "tokens_seen": 2384134144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002803410230692076, + "loss": 1.2395, + "theoretical_loss": 3.4098185857617813, + "tokens_seen": 2384199680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028032096288866596, + "loss": 1.3242, + "theoretical_loss": 3.4098111013708423, + "tokens_seen": 2384265216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803009027081244, + "loss": 1.3508, + "theoretical_loss": 3.409803617243224, + "tokens_seen": 2384330752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028028084252758274, + "loss": 1.3576, + "theoretical_loss": 3.4097961333789084, + "tokens_seen": 2384396288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802607823470411, + "loss": 1.2337, + "theoretical_loss": 3.4097886497778807, + "tokens_seen": 2384461824 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028024072216649947, + "loss": 1.3077, + "theoretical_loss": 3.4097811664401236, + "tokens_seen": 2384527360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802206619859579, + "loss": 1.3668, + "theoretical_loss": 3.4097736833656205, + "tokens_seen": 2384592896 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028020060180541624, + "loss": 1.3489, + "theoretical_loss": 3.409766200554355, + "tokens_seen": 2384658432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801805416248746, + "loss": 1.3607, + "theoretical_loss": 3.409758718006311, + "tokens_seen": 2384723968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028016048144433297, + "loss": 1.3002, + "theoretical_loss": 3.4097512357214717, + "tokens_seen": 2384789504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028014042126379133, + "loss": 1.3373, + "theoretical_loss": 3.4097437536998205, + "tokens_seen": 2384855040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028012036108324975, + "loss": 1.2911, + "theoretical_loss": 3.409736271941341, + "tokens_seen": 2384920576 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801003009027081, + "loss": 1.2898, + "theoretical_loss": 3.4097287904460165, + "tokens_seen": 2384986112 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028008024072216647, + "loss": 1.3524, + "theoretical_loss": 3.409721309213831, + "tokens_seen": 2385051648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028006018054162483, + "loss": 1.3748, + "theoretical_loss": 3.409713828244768, + "tokens_seen": 2385117184 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028004012036108325, + "loss": 1.2955, + "theoretical_loss": 3.4097063475388105, + "tokens_seen": 2385182720 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028002006018054167, + "loss": 1.269, + "theoretical_loss": 3.4096988670959427, + "tokens_seen": 2385248256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028000000000000003, + "loss": 1.2192, + "theoretical_loss": 3.409691386916147, + "tokens_seen": 2385313792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799799398194584, + "loss": 1.292, + "theoretical_loss": 3.4096839069994083, + "tokens_seen": 2385379328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027995987963891675, + "loss": 1.3055, + "theoretical_loss": 3.409676427345709, + "tokens_seen": 2385444864 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027993981945837517, + "loss": 1.2948, + "theoretical_loss": 3.4096689479550335, + "tokens_seen": 2385510400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027991975927783353, + "loss": 1.2718, + "theoretical_loss": 3.4096614688273648, + "tokens_seen": 2385575936 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798996990972919, + "loss": 1.3102, + "theoretical_loss": 3.4096539899626865, + "tokens_seen": 2385641472 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027987963891675026, + "loss": 1.3307, + "theoretical_loss": 3.4096465113609824, + "tokens_seen": 2385707008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798595787362087, + "loss": 1.3586, + "theoretical_loss": 3.4096390330222355, + "tokens_seen": 2385772544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027983951855566704, + "loss": 1.3346, + "theoretical_loss": 3.4096315549464298, + "tokens_seen": 2385838080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798194583751254, + "loss": 1.2735, + "theoretical_loss": 3.4096240771335484, + "tokens_seen": 2385903616 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027979939819458376, + "loss": 1.2501, + "theoretical_loss": 3.4096165995835754, + "tokens_seen": 2385969152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797793380140422, + "loss": 1.3442, + "theoretical_loss": 3.409609122296494, + "tokens_seen": 2386034688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027975927783350054, + "loss": 1.3476, + "theoretical_loss": 3.4096016452722875, + "tokens_seen": 2386100224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797392176529589, + "loss": 1.3751, + "theoretical_loss": 3.4095941685109397, + "tokens_seen": 2386165760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027971915747241726, + "loss": 1.3278, + "theoretical_loss": 3.4095866920124345, + "tokens_seen": 2386231296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796990972918756, + "loss": 1.3246, + "theoretical_loss": 3.4095792157767546, + "tokens_seen": 2386296832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027967903711133404, + "loss": 1.2611, + "theoretical_loss": 3.4095717398038845, + "tokens_seen": 2386362368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796589769307924, + "loss": 1.372, + "theoretical_loss": 3.4095642640938073, + "tokens_seen": 2386427904 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027963891675025077, + "loss": 1.3916, + "theoretical_loss": 3.409556788646506, + "tokens_seen": 2386493440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027961885656970913, + "loss": 1.3651, + "theoretical_loss": 3.4095493134619654, + "tokens_seen": 2386558976 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027959879638916755, + "loss": 1.4094, + "theoretical_loss": 3.409541838540168, + "tokens_seen": 2386624512 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795787362086259, + "loss": 1.3346, + "theoretical_loss": 3.4095343638810975, + "tokens_seen": 2386690048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027955867602808427, + "loss": 1.2631, + "theoretical_loss": 3.4095268894847375, + "tokens_seen": 2386755584 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027953861584754263, + "loss": 1.4029, + "theoretical_loss": 3.409519415351072, + "tokens_seen": 2386821120 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279518555667001, + "loss": 1.3067, + "theoretical_loss": 3.4095119414800843, + "tokens_seen": 2386886656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794984954864594, + "loss": 1.223, + "theoretical_loss": 3.4095044678717574, + "tokens_seen": 2386952192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794784353059178, + "loss": 1.2985, + "theoretical_loss": 3.409496994526076, + "tokens_seen": 2387017728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027945837512537614, + "loss": 1.2422, + "theoretical_loss": 3.4094895214430228, + "tokens_seen": 2387083264 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794383149448345, + "loss": 1.4276, + "theoretical_loss": 3.409482048622581, + "tokens_seen": 2387148800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794182547642929, + "loss": 1.3104, + "theoretical_loss": 3.4094745760647354, + "tokens_seen": 2387214336 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 869259, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3626919984817505, + "objective/train/theoretical_loss": 3.409467103769469, + "objective/train/tokens_used": 757738976, + "theoretical_loss": 3.409467103769469, + "tokens_seen": 2387279872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793981945837513, + "loss": 1.2977, + "theoretical_loss": 3.409467103769469, + "tokens_seen": 2387279872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027937813440320964, + "loss": 1.4006, + "theoretical_loss": 3.4094596317367647, + "tokens_seen": 2387345408 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279358074222668, + "loss": 1.185, + "theoretical_loss": 3.409452159966607, + "tokens_seen": 2387410944 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027933801404212636, + "loss": 1.4117, + "theoretical_loss": 3.409444688458979, + "tokens_seen": 2387476480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793179538615848, + "loss": 1.2789, + "theoretical_loss": 3.4094372172138643, + "tokens_seen": 2387542016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027929789368104314, + "loss": 1.4358, + "theoretical_loss": 3.4094297462312464, + "tokens_seen": 2387607552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792778335005015, + "loss": 1.4048, + "theoretical_loss": 3.409422275511109, + "tokens_seen": 2387673088 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027925777331995987, + "loss": 1.2656, + "theoretical_loss": 3.409414805053436, + "tokens_seen": 2387738624 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792377131394183, + "loss": 1.2627, + "theoretical_loss": 3.4094073348582103, + "tokens_seen": 2387804160 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027921765295887664, + "loss": 1.3163, + "theoretical_loss": 3.409399864925416, + "tokens_seen": 2387869696 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279197592778335, + "loss": 1.3376, + "theoretical_loss": 3.4093923952550367, + "tokens_seen": 2387935232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027917753259779337, + "loss": 1.3544, + "theoretical_loss": 3.409384925847055, + "tokens_seen": 2388000768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027915747241725173, + "loss": 1.3194, + "theoretical_loss": 3.4093774567014563, + "tokens_seen": 2388066304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027913741223671015, + "loss": 1.2566, + "theoretical_loss": 3.4093699878182226, + "tokens_seen": 2388131840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791173520561685, + "loss": 1.3513, + "theoretical_loss": 3.4093625191973382, + "tokens_seen": 2388197376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027909729187562687, + "loss": 1.4182, + "theoretical_loss": 3.4093550508387866, + "tokens_seen": 2388262912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027907723169508523, + "loss": 1.3653, + "theoretical_loss": 3.4093475827425515, + "tokens_seen": 2388328448 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027905717151454365, + "loss": 1.2908, + "theoretical_loss": 3.4093401149086158, + "tokens_seen": 2388393984 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279037111334002, + "loss": 1.4036, + "theoretical_loss": 3.4093326473369636, + "tokens_seen": 2388459520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790170511534604, + "loss": 1.3354, + "theoretical_loss": 3.409325180027579, + "tokens_seen": 2388525056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027899699097291874, + "loss": 1.3056, + "theoretical_loss": 3.4093177129804446, + "tokens_seen": 2388590592 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789769307923771, + "loss": 1.2023, + "theoretical_loss": 3.4093102461955445, + "tokens_seen": 2388656128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789568706118355, + "loss": 1.2744, + "theoretical_loss": 3.4093027796728625, + "tokens_seen": 2388721664 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789368104312939, + "loss": 1.361, + "theoretical_loss": 3.409295313412382, + "tokens_seen": 2388787200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027891675025075224, + "loss": 1.3486, + "theoretical_loss": 3.4092878474140864, + "tokens_seen": 2388852736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788966900702106, + "loss": 1.248, + "theoretical_loss": 3.4092803816779593, + "tokens_seen": 2388918272 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278876629889669, + "loss": 1.2359, + "theoretical_loss": 3.4092729162039848, + "tokens_seen": 2388983808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788565697091274, + "loss": 1.3306, + "theoretical_loss": 3.4092654509921463, + "tokens_seen": 2389049344 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027883650952858574, + "loss": 1.3045, + "theoretical_loss": 3.4092579860424266, + "tokens_seen": 2389114880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788164493480441, + "loss": 1.4198, + "theoretical_loss": 3.4092505213548105, + "tokens_seen": 2389180416 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027879638916750247, + "loss": 1.2892, + "theoretical_loss": 3.4092430569292813, + "tokens_seen": 2389245952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787763289869609, + "loss": 1.3572, + "theoretical_loss": 3.409235592765822, + "tokens_seen": 2389311488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027875626880641925, + "loss": 1.2506, + "theoretical_loss": 3.4092281288644166, + "tokens_seen": 2389377024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787362086258776, + "loss": 1.3228, + "theoretical_loss": 3.4092206652250487, + "tokens_seen": 2389442560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027871614844533597, + "loss": 1.2606, + "theoretical_loss": 3.4092132018477024, + "tokens_seen": 2389508096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786960882647944, + "loss": 1.2479, + "theoretical_loss": 3.40920573873236, + "tokens_seen": 2389573632 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027867602808425275, + "loss": 1.4392, + "theoretical_loss": 3.4091982758790067, + "tokens_seen": 2389639168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786559679037111, + "loss": 1.3376, + "theoretical_loss": 3.4091908132876254, + "tokens_seen": 2389704704 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786359077231695, + "loss": 1.3837, + "theoretical_loss": 3.4091833509581995, + "tokens_seen": 2389770240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786158475426279, + "loss": 1.379, + "theoretical_loss": 3.409175888890713, + "tokens_seen": 2389835776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027859578736208625, + "loss": 1.3274, + "theoretical_loss": 3.409168427085149, + "tokens_seen": 2389901312 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785757271815446, + "loss": 1.3027, + "theoretical_loss": 3.4091609655414916, + "tokens_seen": 2389966848 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278555667001003, + "loss": 1.2772, + "theoretical_loss": 3.4091535042597245, + "tokens_seen": 2390032384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027853560682046134, + "loss": 1.3184, + "theoretical_loss": 3.409146043239831, + "tokens_seen": 2390097920 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785155466399198, + "loss": 1.328, + "theoretical_loss": 3.409138582481795, + "tokens_seen": 2390163456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784954864593782, + "loss": 1.2777, + "theoretical_loss": 3.4091311219855998, + "tokens_seen": 2390228992 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027847542627883654, + "loss": 1.2834, + "theoretical_loss": 3.409123661751229, + "tokens_seen": 2390294528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784553660982949, + "loss": 1.3037, + "theoretical_loss": 3.4091162017786667, + "tokens_seen": 2390360064 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784353059177533, + "loss": 1.3447, + "theoretical_loss": 3.4091087420678963, + "tokens_seen": 2390425600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784152457372117, + "loss": 1.3253, + "theoretical_loss": 3.4091012826189013, + "tokens_seen": 2390491136 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 870459, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.38811194896698, + "objective/train/theoretical_loss": 3.4090938234316654, + "objective/train/tokens_used": 761015776, + "theoretical_loss": 3.4090938234316654, + "tokens_seen": 2390556672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027839518555667004, + "loss": 1.2951, + "theoretical_loss": 3.4090938234316654, + "tokens_seen": 2390556672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783751253761284, + "loss": 1.3781, + "theoretical_loss": 3.4090863645061726, + "tokens_seen": 2390622208 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027835506519558676, + "loss": 1.1978, + "theoretical_loss": 3.409078905842406, + "tokens_seen": 2390687744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783350050150452, + "loss": 1.3197, + "theoretical_loss": 3.4090714474403496, + "tokens_seen": 2390753280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027831494483450354, + "loss": 1.3465, + "theoretical_loss": 3.409063989299987, + "tokens_seen": 2390818816 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782948846539619, + "loss": 1.3921, + "theoretical_loss": 3.409056531421302, + "tokens_seen": 2390884352 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027827482447342027, + "loss": 1.3296, + "theoretical_loss": 3.409049073804278, + "tokens_seen": 2390949888 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782547642928787, + "loss": 1.4247, + "theoretical_loss": 3.409041616448898, + "tokens_seen": 2391015424 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027823470411233705, + "loss": 1.2734, + "theoretical_loss": 3.4090341593551465, + "tokens_seen": 2391080960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782146439317954, + "loss": 1.2949, + "theoretical_loss": 3.409026702523007, + "tokens_seen": 2391146496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027819458375125377, + "loss": 1.4054, + "theoretical_loss": 3.4090192459524635, + "tokens_seen": 2391212032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027817452357071213, + "loss": 1.3173, + "theoretical_loss": 3.4090117896434995, + "tokens_seen": 2391277568 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027815446339017055, + "loss": 1.2453, + "theoretical_loss": 3.409004333596098, + "tokens_seen": 2391343104 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781344032096289, + "loss": 1.3776, + "theoretical_loss": 3.4089968778102424, + "tokens_seen": 2391408640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781143430290873, + "loss": 1.2901, + "theoretical_loss": 3.4089894222859183, + "tokens_seen": 2391474176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027809428284854564, + "loss": 1.4012, + "theoretical_loss": 3.408981967023107, + "tokens_seen": 2391539712 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027807422266800405, + "loss": 1.2961, + "theoretical_loss": 3.408974512021794, + "tokens_seen": 2391605248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780541624874624, + "loss": 1.2493, + "theoretical_loss": 3.408967057281962, + "tokens_seen": 2391670784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780341023069208, + "loss": 1.3881, + "theoretical_loss": 3.408959602803595, + "tokens_seen": 2391736320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027801404212637914, + "loss": 1.5254, + "theoretical_loss": 3.4089521485866765, + "tokens_seen": 2391801856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779939819458375, + "loss": 1.2621, + "theoretical_loss": 3.40894469463119, + "tokens_seen": 2391867392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779739217652959, + "loss": 1.3712, + "theoretical_loss": 3.4089372409371195, + "tokens_seen": 2391932928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779538615847543, + "loss": 1.3463, + "theoretical_loss": 3.408929787504449, + "tokens_seen": 2391998464 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027793380140421264, + "loss": 1.2377, + "theoretical_loss": 3.408922334333161, + "tokens_seen": 2392064000 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277913741223671, + "loss": 1.3519, + "theoretical_loss": 3.4089148814232404, + "tokens_seen": 2392129536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778936810431294, + "loss": 1.3265, + "theoretical_loss": 3.40890742877467, + "tokens_seen": 2392195072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778736208625878, + "loss": 1.4015, + "theoretical_loss": 3.408899976387434, + "tokens_seen": 2392260608 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027785356068204614, + "loss": 1.3939, + "theoretical_loss": 3.4088925242615162, + "tokens_seen": 2392326144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778335005015045, + "loss": 1.321, + "theoretical_loss": 3.4088850723968998, + "tokens_seen": 2392391680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027781344032096287, + "loss": 1.3623, + "theoretical_loss": 3.408877620793569, + "tokens_seen": 2392457216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777933801404213, + "loss": 1.3018, + "theoretical_loss": 3.4088701694515064, + "tokens_seen": 2392522752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027777331995987965, + "loss": 1.3178, + "theoretical_loss": 3.408862718370697, + "tokens_seen": 2392588288 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277753259779338, + "loss": 1.2818, + "theoretical_loss": 3.408855267551124, + "tokens_seen": 2392653824 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027773319959879637, + "loss": 1.4012, + "theoretical_loss": 3.408847816992771, + "tokens_seen": 2392719360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777131394182548, + "loss": 1.392, + "theoretical_loss": 3.4088403666956215, + "tokens_seen": 2392784896 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027769307923771315, + "loss": 1.3222, + "theoretical_loss": 3.4088329166596596, + "tokens_seen": 2392850432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776730190571715, + "loss": 1.2379, + "theoretical_loss": 3.4088254668848683, + "tokens_seen": 2392915968 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776529588766299, + "loss": 1.2514, + "theoretical_loss": 3.4088180173712326, + "tokens_seen": 2392981504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776328986960883, + "loss": 1.3376, + "theoretical_loss": 3.4088105681187346, + "tokens_seen": 2393047040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027761283851554665, + "loss": 1.3743, + "theoretical_loss": 3.4088031191273593, + "tokens_seen": 2393112576 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277592778335005, + "loss": 1.2103, + "theoretical_loss": 3.40879567039709, + "tokens_seen": 2393178112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775727181544634, + "loss": 1.3838, + "theoretical_loss": 3.4087882219279098, + "tokens_seen": 2393243648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027755265797392174, + "loss": 1.4406, + "theoretical_loss": 3.408780773719803, + "tokens_seen": 2393309184 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027753259779338016, + "loss": 1.4041, + "theoretical_loss": 3.408773325772753, + "tokens_seen": 2393374720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775125376128385, + "loss": 1.1663, + "theoretical_loss": 3.408765878086744, + "tokens_seen": 2393440256 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774924774322969, + "loss": 1.1972, + "theoretical_loss": 3.4087584306617593, + "tokens_seen": 2393505792 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027747241725175524, + "loss": 1.3388, + "theoretical_loss": 3.4087509834977823, + "tokens_seen": 2393571328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027745235707121366, + "loss": 1.2026, + "theoretical_loss": 3.4087435365947973, + "tokens_seen": 2393636864 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277432296890672, + "loss": 1.3068, + "theoretical_loss": 3.408736089952788, + "tokens_seen": 2393702400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774122367101304, + "loss": 1.2507, + "theoretical_loss": 3.4087286435717377, + "tokens_seen": 2393767936 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 871588, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3815319538116455, + "objective/train/theoretical_loss": 3.4087211974516305, + "objective/train/tokens_used": 764292576, + "theoretical_loss": 3.4087211974516305, + "tokens_seen": 2393833472 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027739217652958875, + "loss": 1.2661, + "theoretical_loss": 3.4087211974516305, + "tokens_seen": 2393833472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773721163490471, + "loss": 1.2545, + "theoretical_loss": 3.4087137515924497, + "tokens_seen": 2393899008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773520561685055, + "loss": 1.2263, + "theoretical_loss": 3.4087063059941793, + "tokens_seen": 2393964544 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773319959879639, + "loss": 1.3182, + "theoretical_loss": 3.408698860656803, + "tokens_seen": 2394030080 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027731193580742225, + "loss": 1.3328, + "theoretical_loss": 3.4086914155803045, + "tokens_seen": 2394095616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772918756268806, + "loss": 1.421, + "theoretical_loss": 3.4086839707646677, + "tokens_seen": 2394161152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027727181544633903, + "loss": 1.2766, + "theoretical_loss": 3.408676526209876, + "tokens_seen": 2394226688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772517552657974, + "loss": 1.4351, + "theoretical_loss": 3.408669081915913, + "tokens_seen": 2394292224 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027723169508525575, + "loss": 1.2423, + "theoretical_loss": 3.4086616378827626, + "tokens_seen": 2394357760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772116349047141, + "loss": 1.323, + "theoretical_loss": 3.4086541941104085, + "tokens_seen": 2394423296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771915747241725, + "loss": 1.3549, + "theoretical_loss": 3.4086467505988347, + "tokens_seen": 2394488832 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771715145436309, + "loss": 1.2289, + "theoretical_loss": 3.408639307348025, + "tokens_seen": 2394554368 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027715145436308926, + "loss": 1.2752, + "theoretical_loss": 3.4086318643579627, + "tokens_seen": 2394619904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771313941825476, + "loss": 1.2042, + "theoretical_loss": 3.408624421628631, + "tokens_seen": 2394685440 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277111334002006, + "loss": 1.2713, + "theoretical_loss": 3.408616979160015, + "tokens_seen": 2394750976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770912738214644, + "loss": 1.206, + "theoretical_loss": 3.408609536952098, + "tokens_seen": 2394816512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027707121364092276, + "loss": 1.3655, + "theoretical_loss": 3.408602095004863, + "tokens_seen": 2394882048 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770511534603811, + "loss": 1.3771, + "theoretical_loss": 3.4085946533182945, + "tokens_seen": 2394947584 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770310932798395, + "loss": 1.3121, + "theoretical_loss": 3.4085872118923755, + "tokens_seen": 2395013120 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770110330992979, + "loss": 1.3309, + "theoretical_loss": 3.4085797707270906, + "tokens_seen": 2395078656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769909729187563, + "loss": 1.3682, + "theoretical_loss": 3.408572329822423, + "tokens_seen": 2395144192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769709127382147, + "loss": 1.3539, + "theoretical_loss": 3.408564889178357, + "tokens_seen": 2395209728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027695085255767304, + "loss": 1.3183, + "theoretical_loss": 3.408557448794875, + "tokens_seen": 2395275264 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769307923771314, + "loss": 1.4011, + "theoretical_loss": 3.4085500086719622, + "tokens_seen": 2395340800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769107321965898, + "loss": 1.2749, + "theoretical_loss": 3.408542568809602, + "tokens_seen": 2395406336 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768906720160482, + "loss": 1.3038, + "theoretical_loss": 3.408535129207778, + "tokens_seen": 2395471872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027687061183550655, + "loss": 1.4534, + "theoretical_loss": 3.4085276898664736, + "tokens_seen": 2395537408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768505516549649, + "loss": 1.3297, + "theoretical_loss": 3.4085202507856724, + "tokens_seen": 2395602944 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027683049147442327, + "loss": 1.1877, + "theoretical_loss": 3.4085128119653594, + "tokens_seen": 2395668480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768104312938817, + "loss": 1.258, + "theoretical_loss": 3.408505373405517, + "tokens_seen": 2395734016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027679037111334005, + "loss": 1.2916, + "theoretical_loss": 3.40849793510613, + "tokens_seen": 2395799552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767703109327984, + "loss": 1.3794, + "theoretical_loss": 3.4084904970671817, + "tokens_seen": 2395865088 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027675025075225677, + "loss": 1.3948, + "theoretical_loss": 3.4084830592886552, + "tokens_seen": 2395930624 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767301905717152, + "loss": 1.3623, + "theoretical_loss": 3.4084756217705356, + "tokens_seen": 2395996160 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027671013039117355, + "loss": 1.2729, + "theoretical_loss": 3.4084681845128055, + "tokens_seen": 2396061696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766900702106319, + "loss": 1.2556, + "theoretical_loss": 3.4084607475154494, + "tokens_seen": 2396127232 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766700100300903, + "loss": 1.323, + "theoretical_loss": 3.4084533107784507, + "tokens_seen": 2396192768 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766499498495487, + "loss": 1.3776, + "theoretical_loss": 3.408445874301793, + "tokens_seen": 2396258304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027662988966900705, + "loss": 1.3659, + "theoretical_loss": 3.4084384380854607, + "tokens_seen": 2396323840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766098294884654, + "loss": 1.3252, + "theoretical_loss": 3.408431002129437, + "tokens_seen": 2396389376 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765897693079238, + "loss": 1.3118, + "theoretical_loss": 3.4084235664337057, + "tokens_seen": 2396454912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027656970912738214, + "loss": 1.3606, + "theoretical_loss": 3.4084161309982504, + "tokens_seen": 2396520448 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027654964894684056, + "loss": 1.3327, + "theoretical_loss": 3.4084086958230557, + "tokens_seen": 2396585984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765295887662989, + "loss": 1.2386, + "theoretical_loss": 3.408401260908105, + "tokens_seen": 2396651520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765095285857573, + "loss": 1.3603, + "theoretical_loss": 3.4083938262533815, + "tokens_seen": 2396717056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027648946840521564, + "loss": 1.3089, + "theoretical_loss": 3.40838639185887, + "tokens_seen": 2396782592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027646940822467406, + "loss": 1.2454, + "theoretical_loss": 3.408378957724553, + "tokens_seen": 2396848128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764493480441324, + "loss": 1.3112, + "theoretical_loss": 3.4083715238504153, + "tokens_seen": 2396913664 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764292878635908, + "loss": 1.3484, + "theoretical_loss": 3.4083640902364403, + "tokens_seen": 2396979200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027640922768304915, + "loss": 1.367, + "theoretical_loss": 3.408356656882612, + "tokens_seen": 2397044736 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 872754, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.360323190689087, + "objective/train/theoretical_loss": 3.4083492237889135, + "objective/train/tokens_used": 767569376, + "theoretical_loss": 3.4083492237889135, + "tokens_seen": 2397110272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763891675025075, + "loss": 1.3887, + "theoretical_loss": 3.4083492237889135, + "tokens_seen": 2397110272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763691073219659, + "loss": 1.3327, + "theoretical_loss": 3.4083417909553297, + "tokens_seen": 2397175808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763490471414243, + "loss": 1.2866, + "theoretical_loss": 3.4083343583818433, + "tokens_seen": 2397241344 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027632898696088265, + "loss": 1.4575, + "theoretical_loss": 3.408326926068439, + "tokens_seen": 2397306880 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276308926780341, + "loss": 1.2744, + "theoretical_loss": 3.4083194940150996, + "tokens_seen": 2397372416 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027628886659979943, + "loss": 1.3538, + "theoretical_loss": 3.40831206222181, + "tokens_seen": 2397437952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762688064192578, + "loss": 1.3931, + "theoretical_loss": 3.408304630688553, + "tokens_seen": 2397503488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027624874623871615, + "loss": 1.2701, + "theoretical_loss": 3.4082971994153133, + "tokens_seen": 2397569024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762286860581745, + "loss": 1.2899, + "theoretical_loss": 3.4082897684020743, + "tokens_seen": 2397634560 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762086258776329, + "loss": 1.3303, + "theoretical_loss": 3.4082823376488194, + "tokens_seen": 2397700096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761885656970913, + "loss": 1.331, + "theoretical_loss": 3.4082749071555325, + "tokens_seen": 2397765632 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027616850551654966, + "loss": 1.2312, + "theoretical_loss": 3.408267476922198, + "tokens_seen": 2397831168 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276148445336008, + "loss": 1.337, + "theoretical_loss": 3.4082600469488, + "tokens_seen": 2397896704 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761283851554664, + "loss": 1.3525, + "theoretical_loss": 3.4082526172353207, + "tokens_seen": 2397962240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761083249749248, + "loss": 1.3213, + "theoretical_loss": 3.408245187781745, + "tokens_seen": 2398027776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027608826479438316, + "loss": 1.3443, + "theoretical_loss": 3.4082377585880566, + "tokens_seen": 2398093312 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760682046138415, + "loss": 1.3525, + "theoretical_loss": 3.408230329654239, + "tokens_seen": 2398158848 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760481444332999, + "loss": 1.3702, + "theoretical_loss": 3.4082229009802765, + "tokens_seen": 2398224384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027602808425275825, + "loss": 1.2997, + "theoretical_loss": 3.4082154725661526, + "tokens_seen": 2398289920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027600802407221666, + "loss": 1.3106, + "theoretical_loss": 3.4082080444118517, + "tokens_seen": 2398355456 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275987963891675, + "loss": 1.2634, + "theoretical_loss": 3.4082006165173566, + "tokens_seen": 2398420992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759679037111334, + "loss": 1.313, + "theoretical_loss": 3.4081931888826515, + "tokens_seen": 2398486528 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027594784353059175, + "loss": 1.438, + "theoretical_loss": 3.4081857615077205, + "tokens_seen": 2398552064 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027592778335005017, + "loss": 1.3885, + "theoretical_loss": 3.4081783343925474, + "tokens_seen": 2398617600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027590772316950853, + "loss": 1.3855, + "theoretical_loss": 3.408170907537116, + "tokens_seen": 2398683136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758876629889669, + "loss": 1.3965, + "theoretical_loss": 3.4081634809414094, + "tokens_seen": 2398748672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027586760280842525, + "loss": 1.3221, + "theoretical_loss": 3.4081560546054126, + "tokens_seen": 2398814208 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758475426278836, + "loss": 1.3831, + "theoretical_loss": 3.4081486285291085, + "tokens_seen": 2398879744 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027582748244734203, + "loss": 1.2525, + "theoretical_loss": 3.408141202712481, + "tokens_seen": 2398945280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758074222668004, + "loss": 1.2654, + "theoretical_loss": 3.4081337771555145, + "tokens_seen": 2399010816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027578736208625876, + "loss": 1.3737, + "theoretical_loss": 3.4081263518581926, + "tokens_seen": 2399076352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757673019057171, + "loss": 1.3749, + "theoretical_loss": 3.408118926820499, + "tokens_seen": 2399141888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027574724172517554, + "loss": 1.2796, + "theoretical_loss": 3.4081115020424173, + "tokens_seen": 2399207424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757271815446339, + "loss": 1.2403, + "theoretical_loss": 3.408104077523932, + "tokens_seen": 2399272960 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027570712136409226, + "loss": 1.402, + "theoretical_loss": 3.408096653265026, + "tokens_seen": 2399338496 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756870611835506, + "loss": 1.3959, + "theoretical_loss": 3.408089229265684, + "tokens_seen": 2399404032 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275667001003009, + "loss": 1.2776, + "theoretical_loss": 3.4080818055258897, + "tokens_seen": 2399469568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756469408224674, + "loss": 1.3649, + "theoretical_loss": 3.4080743820456263, + "tokens_seen": 2399535104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027562688064192576, + "loss": 1.194, + "theoretical_loss": 3.4080669588248784, + "tokens_seen": 2399600640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756068204613841, + "loss": 1.3388, + "theoretical_loss": 3.4080595358636296, + "tokens_seen": 2399666176 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755867602808425, + "loss": 1.1689, + "theoretical_loss": 3.4080521131618635, + "tokens_seen": 2399731712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755667001003009, + "loss": 1.2937, + "theoretical_loss": 3.4080446907195636, + "tokens_seen": 2399797248 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027554663991975927, + "loss": 1.3176, + "theoretical_loss": 3.408037268536715, + "tokens_seen": 2399862784 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027552657973921763, + "loss": 1.2586, + "theoretical_loss": 3.4080298466133003, + "tokens_seen": 2399928320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027550651955867604, + "loss": 1.3707, + "theoretical_loss": 3.408022424949304, + "tokens_seen": 2399993856 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027548645937813446, + "loss": 1.3295, + "theoretical_loss": 3.40801500354471, + "tokens_seen": 2400059392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754663991975928, + "loss": 1.2894, + "theoretical_loss": 3.408007582399502, + "tokens_seen": 2400124928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754463390170512, + "loss": 1.4401, + "theoretical_loss": 3.4080001615136633, + "tokens_seen": 2400190464 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027542627883650955, + "loss": 1.3138, + "theoretical_loss": 3.4079927408871784, + "tokens_seen": 2400256000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754062186559679, + "loss": 1.2888, + "theoretical_loss": 3.407985320520031, + "tokens_seen": 2400321536 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 873951, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4510786533355713, + "objective/train/theoretical_loss": 3.407977900412205, + "objective/train/tokens_used": 770846176, + "theoretical_loss": 3.407977900412205, + "tokens_seen": 2400387072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753861584754263, + "loss": 1.4135, + "theoretical_loss": 3.407977900412205, + "tokens_seen": 2400387072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753660982948847, + "loss": 1.2497, + "theoretical_loss": 3.4079704805636846, + "tokens_seen": 2400452608 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027534603811434305, + "loss": 1.4125, + "theoretical_loss": 3.4079630609744527, + "tokens_seen": 2400518144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753259779338014, + "loss": 1.46, + "theoretical_loss": 3.4079556416444943, + "tokens_seen": 2400583680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027530591775325983, + "loss": 1.3364, + "theoretical_loss": 3.407948222573792, + "tokens_seen": 2400649216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002752858575727182, + "loss": 1.297, + "theoretical_loss": 3.407940803762331, + "tokens_seen": 2400714752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027526579739217655, + "loss": 1.2674, + "theoretical_loss": 3.4079333852100944, + "tokens_seen": 2400780288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752457372116349, + "loss": 1.2989, + "theoretical_loss": 3.407925966917066, + "tokens_seen": 2400845824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752256770310933, + "loss": 1.2649, + "theoretical_loss": 3.40791854888323, + "tokens_seen": 2400911360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752056168505517, + "loss": 1.3714, + "theoretical_loss": 3.4079111311085697, + "tokens_seen": 2400976896 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752056168505517, + "loss": 1.392, + "theoretical_loss": 3.4079037135930697, + "tokens_seen": 2401042432 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027518555667001006, + "loss": 1.4094, + "theoretical_loss": 3.4078962963367134, + "tokens_seen": 2401107968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751654964894684, + "loss": 1.1846, + "theoretical_loss": 3.4078888793394855, + "tokens_seen": 2401173504 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751454363089268, + "loss": 1.3237, + "theoretical_loss": 3.4078814626013685, + "tokens_seen": 2401239040 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751253761283852, + "loss": 1.3889, + "theoretical_loss": 3.4078740461223473, + "tokens_seen": 2401304576 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027510531594784356, + "loss": 1.4075, + "theoretical_loss": 3.407866629902405, + "tokens_seen": 2401370112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750852557673019, + "loss": 1.4103, + "theoretical_loss": 3.407859213941527, + "tokens_seen": 2401435648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750651955867603, + "loss": 1.2524, + "theoretical_loss": 3.4078517982396956, + "tokens_seen": 2401501184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027504513540621865, + "loss": 1.2593, + "theoretical_loss": 3.407844382796895, + "tokens_seen": 2401566720 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027502507522567706, + "loss": 1.4134, + "theoretical_loss": 3.4078369676131093, + "tokens_seen": 2401632256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750050150451354, + "loss": 1.3256, + "theoretical_loss": 3.4078295526883227, + "tokens_seen": 2401697792 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749849548645938, + "loss": 1.3262, + "theoretical_loss": 3.407822138022518, + "tokens_seen": 2401763328 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027496489468405215, + "loss": 1.3199, + "theoretical_loss": 3.407814723615681, + "tokens_seen": 2401828864 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027494483450351057, + "loss": 1.1812, + "theoretical_loss": 3.4078073094677936, + "tokens_seen": 2401894400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027492477432296893, + "loss": 1.3434, + "theoretical_loss": 3.407799895578841, + "tokens_seen": 2401959936 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749047141424273, + "loss": 1.2822, + "theoretical_loss": 3.4077924819488064, + "tokens_seen": 2402025472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027488465396188565, + "loss": 1.298, + "theoretical_loss": 3.4077850685776743, + "tokens_seen": 2402091008 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274864593781344, + "loss": 1.2224, + "theoretical_loss": 3.407777655465428, + "tokens_seen": 2402156544 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027484453360080243, + "loss": 1.4537, + "theoretical_loss": 3.4077702426120515, + "tokens_seen": 2402222080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002748244734202608, + "loss": 1.4607, + "theoretical_loss": 3.4077628300175284, + "tokens_seen": 2402287616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027480441323971916, + "loss": 1.3727, + "theoretical_loss": 3.4077554176818436, + "tokens_seen": 2402353152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747843530591775, + "loss": 1.2401, + "theoretical_loss": 3.4077480056049803, + "tokens_seen": 2402418688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027476429287863594, + "loss": 1.2116, + "theoretical_loss": 3.4077405937869227, + "tokens_seen": 2402484224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747442326980943, + "loss": 1.3508, + "theoretical_loss": 3.407733182227654, + "tokens_seen": 2402549760 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027472417251755266, + "loss": 1.4278, + "theoretical_loss": 3.407725770927159, + "tokens_seen": 2402615296 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274704112337011, + "loss": 1.2559, + "theoretical_loss": 3.4077183598854215, + "tokens_seen": 2402680832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746840521564694, + "loss": 1.389, + "theoretical_loss": 3.407710949102425, + "tokens_seen": 2402746368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746639919759278, + "loss": 1.3656, + "theoretical_loss": 3.4077035385781533, + "tokens_seen": 2402811904 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027464393179538616, + "loss": 1.3845, + "theoretical_loss": 3.4076961283125904, + "tokens_seen": 2402877440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746238716148445, + "loss": 1.2302, + "theoretical_loss": 3.407688718305721, + "tokens_seen": 2402942976 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746038114343029, + "loss": 1.4383, + "theoretical_loss": 3.4076813085575277, + "tokens_seen": 2403008512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745837512537613, + "loss": 1.3165, + "theoretical_loss": 3.4076738990679956, + "tokens_seen": 2403074048 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027456369107321967, + "loss": 1.2525, + "theoretical_loss": 3.4076664898371076, + "tokens_seen": 2403139584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027454363089267803, + "loss": 1.2551, + "theoretical_loss": 3.407659080864849, + "tokens_seen": 2403205120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745235707121364, + "loss": 1.3652, + "theoretical_loss": 3.4076516721512022, + "tokens_seen": 2403270656 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027450351053159475, + "loss": 1.2601, + "theoretical_loss": 3.407644263696152, + "tokens_seen": 2403336192 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027448345035105317, + "loss": 1.2774, + "theoretical_loss": 3.407636855499682, + "tokens_seen": 2403401728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027446339017051153, + "loss": 1.3591, + "theoretical_loss": 3.407629447561776, + "tokens_seen": 2403467264 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744433299899699, + "loss": 1.3509, + "theoretical_loss": 3.4076220398824186, + "tokens_seen": 2403532800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027442326980942826, + "loss": 1.3731, + "theoretical_loss": 3.4076146324615935, + "tokens_seen": 2403598336 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 875100, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3885475397109985, + "objective/train/theoretical_loss": 3.4076072252992837, + "objective/train/tokens_used": 774122976, + "theoretical_loss": 3.4076072252992837, + "tokens_seen": 2403663872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027440320962888667, + "loss": 1.3443, + "theoretical_loss": 3.4076072252992837, + "tokens_seen": 2403663872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027438314944834503, + "loss": 1.4259, + "theoretical_loss": 3.4075998183954743, + "tokens_seen": 2403729408 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743630892678034, + "loss": 1.338, + "theoretical_loss": 3.407592411750149, + "tokens_seen": 2403794944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027434302908726176, + "loss": 1.3654, + "theoretical_loss": 3.407585005363291, + "tokens_seen": 2403860480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743229689067202, + "loss": 1.3199, + "theoretical_loss": 3.4075775992348847, + "tokens_seen": 2403926016 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027430290872617854, + "loss": 1.3459, + "theoretical_loss": 3.407570193364914, + "tokens_seen": 2403991552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742828485456369, + "loss": 1.4096, + "theoretical_loss": 3.4075627877533634, + "tokens_seen": 2404057088 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027426278836509526, + "loss": 1.253, + "theoretical_loss": 3.4075553824002163, + "tokens_seen": 2404122624 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742427281845536, + "loss": 1.4278, + "theoretical_loss": 3.407547977305456, + "tokens_seen": 2404188160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027422266800401204, + "loss": 1.4905, + "theoretical_loss": 3.407540572469068, + "tokens_seen": 2404253696 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742026078234704, + "loss": 1.3587, + "theoretical_loss": 3.407533167891035, + "tokens_seen": 2404319232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027418254764292877, + "loss": 1.3744, + "theoretical_loss": 3.4075257635713414, + "tokens_seen": 2404384768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027416248746238713, + "loss": 1.5, + "theoretical_loss": 3.407518359509971, + "tokens_seen": 2404450304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027414242728184554, + "loss": 1.2847, + "theoretical_loss": 3.4075109557069077, + "tokens_seen": 2404515840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741223671013039, + "loss": 1.3629, + "theoretical_loss": 3.4075035521621357, + "tokens_seen": 2404581376 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027410230692076227, + "loss": 1.1964, + "theoretical_loss": 3.407496148875639, + "tokens_seen": 2404646912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027408224674022063, + "loss": 1.2677, + "theoretical_loss": 3.407488745847401, + "tokens_seen": 2404712448 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274062186559679, + "loss": 1.3067, + "theoretical_loss": 3.4074813430774062, + "tokens_seen": 2404777984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740421263791374, + "loss": 1.3459, + "theoretical_loss": 3.4074739405656382, + "tokens_seen": 2404843520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740220661985958, + "loss": 1.399, + "theoretical_loss": 3.4074665383120815, + "tokens_seen": 2404909056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740020060180542, + "loss": 1.3337, + "theoretical_loss": 3.407459136316719, + "tokens_seen": 2404974592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027398194583751255, + "loss": 1.306, + "theoretical_loss": 3.407451734579536, + "tokens_seen": 2405040128 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027396188565697097, + "loss": 1.3503, + "theoretical_loss": 3.4074443331005155, + "tokens_seen": 2405105664 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027394182547642933, + "loss": 1.5323, + "theoretical_loss": 3.407436931879642, + "tokens_seen": 2405171200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002739217652958877, + "loss": 1.3805, + "theoretical_loss": 3.407429530916899, + "tokens_seen": 2405236736 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027390170511534605, + "loss": 1.3784, + "theoretical_loss": 3.4074221302122707, + "tokens_seen": 2405302272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738816449348044, + "loss": 1.5019, + "theoretical_loss": 3.407414729765741, + "tokens_seen": 2405367808 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027386158475426283, + "loss": 1.2801, + "theoretical_loss": 3.407407329577294, + "tokens_seen": 2405433344 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738415245737212, + "loss": 1.3146, + "theoretical_loss": 3.4073999296469135, + "tokens_seen": 2405498880 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027382146439317956, + "loss": 1.3977, + "theoretical_loss": 3.407392529974584, + "tokens_seen": 2405564416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738014042126379, + "loss": 1.2694, + "theoretical_loss": 3.4073851305602885, + "tokens_seen": 2405629952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027378134403209634, + "loss": 1.386, + "theoretical_loss": 3.4073777314040115, + "tokens_seen": 2405695488 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737612838515547, + "loss": 1.306, + "theoretical_loss": 3.407370332505737, + "tokens_seen": 2405761024 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027374122367101306, + "loss": 1.2434, + "theoretical_loss": 3.4073629338654494, + "tokens_seen": 2405826560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737211634904714, + "loss": 1.3107, + "theoretical_loss": 3.407355535483132, + "tokens_seen": 2405892096 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737011033099298, + "loss": 1.4423, + "theoretical_loss": 3.4073481373587686, + "tokens_seen": 2405957632 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736810431293882, + "loss": 1.4299, + "theoretical_loss": 3.4073407394923443, + "tokens_seen": 2406023168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027366098294884656, + "loss": 1.3723, + "theoretical_loss": 3.4073333418838416, + "tokens_seen": 2406088704 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736409227683049, + "loss": 1.3525, + "theoretical_loss": 3.407325944533246, + "tokens_seen": 2406154240 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736208625877633, + "loss": 1.2994, + "theoretical_loss": 3.40731854744054, + "tokens_seen": 2406219776 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736008024072217, + "loss": 1.3442, + "theoretical_loss": 3.407311150605709, + "tokens_seen": 2406285312 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027358074222668007, + "loss": 1.3116, + "theoretical_loss": 3.407303754028736, + "tokens_seen": 2406350848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027356068204613843, + "loss": 1.4537, + "theoretical_loss": 3.4072963577096056, + "tokens_seen": 2406416384 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735406218655968, + "loss": 1.4134, + "theoretical_loss": 3.407288961648301, + "tokens_seen": 2406481920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027352056168505515, + "loss": 1.398, + "theoretical_loss": 3.407281565844807, + "tokens_seen": 2406547456 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027350050150451357, + "loss": 1.3026, + "theoretical_loss": 3.407274170299107, + "tokens_seen": 2406612992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027348044132397193, + "loss": 1.3736, + "theoretical_loss": 3.4072667750111854, + "tokens_seen": 2406678528 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734603811434303, + "loss": 1.4851, + "theoretical_loss": 3.407259379981026, + "tokens_seen": 2406744064 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027344032096288866, + "loss": 1.2229, + "theoretical_loss": 3.407251985208613, + "tokens_seen": 2406809600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734202607823471, + "loss": 1.3228, + "theoretical_loss": 3.4072445906939306, + "tokens_seen": 2406875136 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 876268, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4658091068267822, + "objective/train/theoretical_loss": 3.407237196436962, + "objective/train/tokens_used": 777399776, + "theoretical_loss": 3.407237196436962, + "tokens_seen": 2406940672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027340020060180544, + "loss": 1.4443, + "theoretical_loss": 3.407237196436962, + "tokens_seen": 2406940672 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733801404212638, + "loss": 1.2755, + "theoretical_loss": 3.4072298024376915, + "tokens_seen": 2407006208 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027336008024072216, + "loss": 1.2804, + "theoretical_loss": 3.4072224086961036, + "tokens_seen": 2407071744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733400200601806, + "loss": 1.3005, + "theoretical_loss": 3.4072150152121816, + "tokens_seen": 2407137280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027331995987963894, + "loss": 1.3541, + "theoretical_loss": 3.4072076219859104, + "tokens_seen": 2407202816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732998996990973, + "loss": 1.5378, + "theoretical_loss": 3.407200229017273, + "tokens_seen": 2407268352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027327983951855566, + "loss": 1.366, + "theoretical_loss": 3.4071928363062542, + "tokens_seen": 2407333888 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273259779338014, + "loss": 1.3577, + "theoretical_loss": 3.4071854438528377, + "tokens_seen": 2407399424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027323971915747244, + "loss": 1.3391, + "theoretical_loss": 3.407178051657007, + "tokens_seen": 2407464960 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732196589769308, + "loss": 1.3225, + "theoretical_loss": 3.4071706597187474, + "tokens_seen": 2407530496 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027319959879638917, + "loss": 1.2143, + "theoretical_loss": 3.4071632680380417, + "tokens_seen": 2407596032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027317953861584753, + "loss": 1.3921, + "theoretical_loss": 3.4071558766148744, + "tokens_seen": 2407661568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027315947843530594, + "loss": 1.272, + "theoretical_loss": 3.4071484854492295, + "tokens_seen": 2407727104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731394182547643, + "loss": 1.3395, + "theoretical_loss": 3.407141094541091, + "tokens_seen": 2407792640 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027311935807422267, + "loss": 1.3014, + "theoretical_loss": 3.407133703890443, + "tokens_seen": 2407858176 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027309929789368103, + "loss": 1.3823, + "theoretical_loss": 3.4071263134972694, + "tokens_seen": 2407923712 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002730792377131394, + "loss": 1.3122, + "theoretical_loss": 3.407118923361554, + "tokens_seen": 2407989248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002730591775325978, + "loss": 1.3595, + "theoretical_loss": 3.407111533483281, + "tokens_seen": 2408054784 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027303911735205617, + "loss": 1.2864, + "theoretical_loss": 3.407104143862435, + "tokens_seen": 2408120320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027301905717151453, + "loss": 1.2997, + "theoretical_loss": 3.4070967544989994, + "tokens_seen": 2408185856 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729989969909729, + "loss": 1.4726, + "theoretical_loss": 3.407089365392958, + "tokens_seen": 2408251392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729789368104313, + "loss": 1.3011, + "theoretical_loss": 3.4070819765442955, + "tokens_seen": 2408316928 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729588766298897, + "loss": 1.3438, + "theoretical_loss": 3.407074587952996, + "tokens_seen": 2408382464 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027293881644934804, + "loss": 1.2962, + "theoretical_loss": 3.4070671996190423, + "tokens_seen": 2408448000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729187562688064, + "loss": 1.4236, + "theoretical_loss": 3.4070598115424198, + "tokens_seen": 2408513536 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027289869608826476, + "loss": 1.3333, + "theoretical_loss": 3.407052423723112, + "tokens_seen": 2408579072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728786359077232, + "loss": 1.2338, + "theoretical_loss": 3.4070450361611027, + "tokens_seen": 2408644608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027285857572718154, + "loss": 1.2131, + "theoretical_loss": 3.4070376488563765, + "tokens_seen": 2408710144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728385155466399, + "loss": 1.4353, + "theoretical_loss": 3.407030261808917, + "tokens_seen": 2408775680 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027281845536609827, + "loss": 1.2525, + "theoretical_loss": 3.407022875018708, + "tokens_seen": 2408841216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727983951855567, + "loss": 1.2966, + "theoretical_loss": 3.407015488485735, + "tokens_seen": 2408906752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027277833500501504, + "loss": 1.2344, + "theoretical_loss": 3.40700810220998, + "tokens_seen": 2408972288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727582748244734, + "loss": 1.3516, + "theoretical_loss": 3.407000716191428, + "tokens_seen": 2409037824 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027273821464393177, + "loss": 1.2706, + "theoretical_loss": 3.4069933304300637, + "tokens_seen": 2409103360 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027271815446339013, + "loss": 1.276, + "theoretical_loss": 3.40698594492587, + "tokens_seen": 2409168896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027269809428284855, + "loss": 1.3287, + "theoretical_loss": 3.406978559678832, + "tokens_seen": 2409234432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002726780341023069, + "loss": 1.3735, + "theoretical_loss": 3.4069711746889326, + "tokens_seen": 2409299968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027265797392176527, + "loss": 1.3452, + "theoretical_loss": 3.4069637899561567, + "tokens_seen": 2409365504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027263791374122363, + "loss": 1.4255, + "theoretical_loss": 3.406956405480488, + "tokens_seen": 2409431040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027261785356068205, + "loss": 1.2694, + "theoretical_loss": 3.406949021261911, + "tokens_seen": 2409496576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725977933801404, + "loss": 1.4514, + "theoretical_loss": 3.406941637300409, + "tokens_seen": 2409562112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725777331995988, + "loss": 1.4954, + "theoretical_loss": 3.406934253595967, + "tokens_seen": 2409627648 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027255767301905714, + "loss": 1.4562, + "theoretical_loss": 3.406926870148568, + "tokens_seen": 2409693184 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725376128385155, + "loss": 1.479, + "theoretical_loss": 3.4069194869581967, + "tokens_seen": 2409758720 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027251755265797397, + "loss": 1.2468, + "theoretical_loss": 3.4069121040248374, + "tokens_seen": 2409824256 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027249749247743233, + "loss": 1.27, + "theoretical_loss": 3.4069047213484738, + "tokens_seen": 2409889792 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724774322968907, + "loss": 1.3293, + "theoretical_loss": 3.40689733892909, + "tokens_seen": 2409955328 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027245737211634906, + "loss": 1.3974, + "theoretical_loss": 3.40688995676667, + "tokens_seen": 2410020864 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724373119358075, + "loss": 1.4021, + "theoretical_loss": 3.4068825748611977, + "tokens_seen": 2410086400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027241725175526584, + "loss": 1.2945, + "theoretical_loss": 3.4068751932126577, + "tokens_seen": 2410151936 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 877499, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2382419109344482, + "objective/train/theoretical_loss": 3.4068678118210336, + "objective/train/tokens_used": 780676576, + "theoretical_loss": 3.4068678118210336, + "tokens_seen": 2410217472 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723971915747242, + "loss": 1.3304, + "theoretical_loss": 3.4068678118210336, + "tokens_seen": 2410217472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027237713139418256, + "loss": 1.3771, + "theoretical_loss": 3.4068604306863097, + "tokens_seen": 2410283008 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272357071213641, + "loss": 1.3283, + "theoretical_loss": 3.40685304980847, + "tokens_seen": 2410348544 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027233701103309934, + "loss": 1.4518, + "theoretical_loss": 3.406845669187499, + "tokens_seen": 2410414080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723169508525577, + "loss": 1.2622, + "theoretical_loss": 3.40683828882338, + "tokens_seen": 2410479616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027229689067201606, + "loss": 1.376, + "theoretical_loss": 3.4068309087160973, + "tokens_seen": 2410545152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722768304914744, + "loss": 1.2939, + "theoretical_loss": 3.406823528865635, + "tokens_seen": 2410610688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027225677031093284, + "loss": 1.3564, + "theoretical_loss": 3.406816149271978, + "tokens_seen": 2410676224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722367101303912, + "loss": 1.3474, + "theoretical_loss": 3.4068087699351093, + "tokens_seen": 2410741760 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027221664994984957, + "loss": 1.3195, + "theoretical_loss": 3.406801390855013, + "tokens_seen": 2410807296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027219658976930793, + "loss": 1.3918, + "theoretical_loss": 3.406794012031674, + "tokens_seen": 2410872832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027217652958876635, + "loss": 1.347, + "theoretical_loss": 3.4067866334650763, + "tokens_seen": 2410938368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721564694082247, + "loss": 1.3931, + "theoretical_loss": 3.406779255155203, + "tokens_seen": 2411003904 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027213640922768307, + "loss": 1.3192, + "theoretical_loss": 3.406771877102039, + "tokens_seen": 2411069440 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027211634904714143, + "loss": 1.3234, + "theoretical_loss": 3.406764499305569, + "tokens_seen": 2411134976 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720962888665998, + "loss": 1.2957, + "theoretical_loss": 3.4067571217657755, + "tokens_seen": 2411200512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720762286860582, + "loss": 1.3583, + "theoretical_loss": 3.4067497444826436, + "tokens_seen": 2411266048 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027205616850551657, + "loss": 1.3308, + "theoretical_loss": 3.406742367456157, + "tokens_seen": 2411331584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027203610832497493, + "loss": 1.3276, + "theoretical_loss": 3.4067349906863003, + "tokens_seen": 2411397120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720160481444333, + "loss": 1.3166, + "theoretical_loss": 3.406727614173057, + "tokens_seen": 2411462656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719959879638917, + "loss": 1.3315, + "theoretical_loss": 3.406720237916412, + "tokens_seen": 2411528192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719759277833501, + "loss": 1.5321, + "theoretical_loss": 3.4067128619163487, + "tokens_seen": 2411593728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027195586760280844, + "loss": 1.3522, + "theoretical_loss": 3.406705486172851, + "tokens_seen": 2411659264 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719358074222668, + "loss": 1.3515, + "theoretical_loss": 3.4066981106859036, + "tokens_seen": 2411724800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027191574724172516, + "loss": 1.1903, + "theoretical_loss": 3.4066907354554905, + "tokens_seen": 2411790336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718956870611836, + "loss": 1.3984, + "theoretical_loss": 3.406683360481596, + "tokens_seen": 2411855872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027187562688064194, + "loss": 1.2606, + "theoretical_loss": 3.4066759857642035, + "tokens_seen": 2411921408 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718555667001003, + "loss": 1.3034, + "theoretical_loss": 3.4066686113032976, + "tokens_seen": 2411986944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027183550651955867, + "loss": 1.4012, + "theoretical_loss": 3.4066612370988625, + "tokens_seen": 2412052480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718154463390171, + "loss": 1.3774, + "theoretical_loss": 3.4066538631508823, + "tokens_seen": 2412118016 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027179538615847544, + "loss": 1.3222, + "theoretical_loss": 3.406646489459341, + "tokens_seen": 2412183552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717753259779338, + "loss": 1.3702, + "theoretical_loss": 3.4066391160242224, + "tokens_seen": 2412249088 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027175526579739217, + "loss": 1.2503, + "theoretical_loss": 3.406631742845511, + "tokens_seen": 2412314624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027173520561685053, + "loss": 1.3344, + "theoretical_loss": 3.4066243699231906, + "tokens_seen": 2412380160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027171514543630895, + "loss": 1.3219, + "theoretical_loss": 3.4066169972572458, + "tokens_seen": 2412445696 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716950852557673, + "loss": 1.3941, + "theoretical_loss": 3.4066096248476607, + "tokens_seen": 2412511232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027167502507522567, + "loss": 1.3243, + "theoretical_loss": 3.4066022526944186, + "tokens_seen": 2412576768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027165496489468403, + "loss": 1.5133, + "theoretical_loss": 3.406594880797505, + "tokens_seen": 2412642304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027163490471414245, + "loss": 1.3319, + "theoretical_loss": 3.4065875091569024, + "tokens_seen": 2412707840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716148445336008, + "loss": 1.306, + "theoretical_loss": 3.4065801377725964, + "tokens_seen": 2412773376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715947843530592, + "loss": 1.4057, + "theoretical_loss": 3.40657276664457, + "tokens_seen": 2412838912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027157472417251754, + "loss": 1.286, + "theoretical_loss": 3.4065653957728084, + "tokens_seen": 2412904448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715546639919759, + "loss": 1.3028, + "theoretical_loss": 3.4065580251572944, + "tokens_seen": 2412969984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715346038114343, + "loss": 1.4019, + "theoretical_loss": 3.4065506547980138, + "tokens_seen": 2413035520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715145436308927, + "loss": 1.3433, + "theoretical_loss": 3.406543284694949, + "tokens_seen": 2413101056 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027149448345035104, + "loss": 1.3145, + "theoretical_loss": 3.406535914848085, + "tokens_seen": 2413166592 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714744232698094, + "loss": 1.2907, + "theoretical_loss": 3.4065285452574066, + "tokens_seen": 2413232128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714543630892678, + "loss": 1.4535, + "theoretical_loss": 3.406521175922897, + "tokens_seen": 2413297664 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714343029087262, + "loss": 1.2803, + "theoretical_loss": 3.40651380684454, + "tokens_seen": 2413363200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027141424272818454, + "loss": 1.4042, + "theoretical_loss": 3.406506438022321, + "tokens_seen": 2413428736 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 878709, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2260515689849854, + "objective/train/theoretical_loss": 3.406499069456223, + "objective/train/tokens_used": 783953376, + "theoretical_loss": 3.406499069456223, + "tokens_seen": 2413494272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713941825476429, + "loss": 1.2944, + "theoretical_loss": 3.406499069456223, + "tokens_seen": 2413494272 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027137412236710127, + "loss": 1.2526, + "theoretical_loss": 3.406491701146231, + "tokens_seen": 2413559808 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713540621865597, + "loss": 1.2666, + "theoretical_loss": 3.4064843330923282, + "tokens_seen": 2413625344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027133400200601805, + "loss": 1.3362, + "theoretical_loss": 3.4064769652944995, + "tokens_seen": 2413690880 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713139418254764, + "loss": 1.3257, + "theoretical_loss": 3.4064695977527286, + "tokens_seen": 2413756416 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027129388164493477, + "loss": 1.3463, + "theoretical_loss": 3.4064622304670005, + "tokens_seen": 2413821952 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712738214643932, + "loss": 1.313, + "theoretical_loss": 3.406454863437298, + "tokens_seen": 2413887488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027125376128385155, + "loss": 1.3596, + "theoretical_loss": 3.4064474966636067, + "tokens_seen": 2413953024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712337011033099, + "loss": 1.3633, + "theoretical_loss": 3.4064401301459095, + "tokens_seen": 2414018560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712136409227683, + "loss": 1.3177, + "theoretical_loss": 3.4064327638841916, + "tokens_seen": 2414084096 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711935807422267, + "loss": 1.4225, + "theoretical_loss": 3.4064253978784365, + "tokens_seen": 2414149632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027117352056168505, + "loss": 1.4033, + "theoretical_loss": 3.406418032128628, + "tokens_seen": 2414215168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711534603811434, + "loss": 1.3188, + "theoretical_loss": 3.406410666634751, + "tokens_seen": 2414280704 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711334002006018, + "loss": 1.3211, + "theoretical_loss": 3.40640330139679, + "tokens_seen": 2414346240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027111334002006014, + "loss": 1.3446, + "theoretical_loss": 3.406395936414728, + "tokens_seen": 2414411776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027109327983951856, + "loss": 1.3498, + "theoretical_loss": 3.4063885716885496, + "tokens_seen": 2414477312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710732196589769, + "loss": 1.3331, + "theoretical_loss": 3.4063812072182396, + "tokens_seen": 2414542848 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710531594784353, + "loss": 1.3824, + "theoretical_loss": 3.4063738430037813, + "tokens_seen": 2414608384 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027103309929789364, + "loss": 1.3836, + "theoretical_loss": 3.4063664790451598, + "tokens_seen": 2414673920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710130391173521, + "loss": 1.3344, + "theoretical_loss": 3.406359115342358, + "tokens_seen": 2414739456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709929789368105, + "loss": 1.3293, + "theoretical_loss": 3.4063517518953614, + "tokens_seen": 2414804992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027097291875626884, + "loss": 1.3345, + "theoretical_loss": 3.406344388704153, + "tokens_seen": 2414870528 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709528585757272, + "loss": 1.2308, + "theoretical_loss": 3.406337025768718, + "tokens_seen": 2414936064 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027093279839518556, + "loss": 1.3315, + "theoretical_loss": 3.40632966308904, + "tokens_seen": 2415001600 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270912738214644, + "loss": 1.3957, + "theoretical_loss": 3.4063223006651033, + "tokens_seen": 2415067136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027089267803410234, + "loss": 1.3345, + "theoretical_loss": 3.406314938496892, + "tokens_seen": 2415132672 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708726178535607, + "loss": 1.2748, + "theoretical_loss": 3.4063075765843904, + "tokens_seen": 2415198208 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027085255767301907, + "loss": 1.3211, + "theoretical_loss": 3.4063002149275823, + "tokens_seen": 2415263744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708324974924775, + "loss": 1.2757, + "theoretical_loss": 3.4062928535264527, + "tokens_seen": 2415329280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027081243731193584, + "loss": 1.3569, + "theoretical_loss": 3.406285492380985, + "tokens_seen": 2415394816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707923771313942, + "loss": 1.4019, + "theoretical_loss": 3.4062781314911637, + "tokens_seen": 2415460352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027077231695085257, + "loss": 1.2499, + "theoretical_loss": 3.406270770856973, + "tokens_seen": 2415525888 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027075225677031093, + "loss": 1.3974, + "theoretical_loss": 3.406263410478397, + "tokens_seen": 2415591424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027073219658976935, + "loss": 1.3, + "theoretical_loss": 3.40625605035542, + "tokens_seen": 2415656960 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707121364092277, + "loss": 1.4814, + "theoretical_loss": 3.406248690488026, + "tokens_seen": 2415722496 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027069207622868607, + "loss": 1.2349, + "theoretical_loss": 3.4062413308761994, + "tokens_seen": 2415788032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027067201604814443, + "loss": 1.3823, + "theoretical_loss": 3.4062339715199244, + "tokens_seen": 2415853568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027065195586760285, + "loss": 1.324, + "theoretical_loss": 3.406226612419185, + "tokens_seen": 2415919104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706318956870612, + "loss": 1.4081, + "theoretical_loss": 3.4062192535739655, + "tokens_seen": 2415984640 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706118355065196, + "loss": 1.3212, + "theoretical_loss": 3.4062118949842497, + "tokens_seen": 2416050176 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027059177532597794, + "loss": 1.291, + "theoretical_loss": 3.406204536650023, + "tokens_seen": 2416115712 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705717151454363, + "loss": 1.3856, + "theoretical_loss": 3.406197178571268, + "tokens_seen": 2416181248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705516549648947, + "loss": 1.4, + "theoretical_loss": 3.40618982074797, + "tokens_seen": 2416246784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705315947843531, + "loss": 1.3138, + "theoretical_loss": 3.406182463180113, + "tokens_seen": 2416312320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027051153460381144, + "loss": 1.2567, + "theoretical_loss": 3.406175105867681, + "tokens_seen": 2416377856 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704914744232698, + "loss": 1.3544, + "theoretical_loss": 3.4061677488106583, + "tokens_seen": 2416443392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704714142427282, + "loss": 1.437, + "theoretical_loss": 3.406160392009029, + "tokens_seen": 2416508928 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704513540621866, + "loss": 1.3461, + "theoretical_loss": 3.406153035462778, + "tokens_seen": 2416574464 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027043129388164494, + "loss": 1.3394, + "theoretical_loss": 3.406145679171888, + "tokens_seen": 2416640000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704112337011033, + "loss": 1.3404, + "theoretical_loss": 3.4061383231363447, + "tokens_seen": 2416705536 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 880009, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3705439567565918, + "objective/train/theoretical_loss": 3.4061309673561317, + "objective/train/tokens_used": 787230176, + "theoretical_loss": 3.4061309673561317, + "tokens_seen": 2416771072 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027039117352056167, + "loss": 1.3516, + "theoretical_loss": 3.4061309673561317, + "tokens_seen": 2416771072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703711133400201, + "loss": 1.4004, + "theoretical_loss": 3.4061236118312332, + "tokens_seen": 2416836608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027035105315947845, + "loss": 1.4456, + "theoretical_loss": 3.4061162565616336, + "tokens_seen": 2416902144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703309929789368, + "loss": 1.3281, + "theoretical_loss": 3.406108901547317, + "tokens_seen": 2416967680 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027031093279839517, + "loss": 1.3332, + "theoretical_loss": 3.4061015467882676, + "tokens_seen": 2417033216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002702908726178536, + "loss": 1.2566, + "theoretical_loss": 3.4060941922844696, + "tokens_seen": 2417098752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027027081243731195, + "loss": 1.4096, + "theoretical_loss": 3.406086838035907, + "tokens_seen": 2417164288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002702507522567703, + "loss": 1.4685, + "theoretical_loss": 3.4060794840425648, + "tokens_seen": 2417229824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702306920762287, + "loss": 1.5282, + "theoretical_loss": 3.4060721303044263, + "tokens_seen": 2417295360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702106318956871, + "loss": 1.3421, + "theoretical_loss": 3.4060647768214762, + "tokens_seen": 2417360896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027019057171514545, + "loss": 1.3677, + "theoretical_loss": 3.406057423593699, + "tokens_seen": 2417426432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701705115346038, + "loss": 1.3555, + "theoretical_loss": 3.4060500706210783, + "tokens_seen": 2417491968 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701504513540622, + "loss": 1.4035, + "theoretical_loss": 3.4060427179035986, + "tokens_seen": 2417557504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027013039117352054, + "loss": 1.2473, + "theoretical_loss": 3.406035365441244, + "tokens_seen": 2417623040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027011033099297896, + "loss": 1.3873, + "theoretical_loss": 3.406028013233999, + "tokens_seen": 2417688576 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700902708124373, + "loss": 1.3009, + "theoretical_loss": 3.4060206612818478, + "tokens_seen": 2417754112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700702106318957, + "loss": 1.3812, + "theoretical_loss": 3.4060133095847744, + "tokens_seen": 2417819648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027005015045135404, + "loss": 1.2852, + "theoretical_loss": 3.4060059581427633, + "tokens_seen": 2417885184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027003009027081246, + "loss": 1.2703, + "theoretical_loss": 3.4059986069557984, + "tokens_seen": 2417950720 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700100300902708, + "loss": 1.3213, + "theoretical_loss": 3.4059912560238645, + "tokens_seen": 2418016256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699899699097292, + "loss": 1.2318, + "theoretical_loss": 3.4059839053469454, + "tokens_seen": 2418081792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026996990972918755, + "loss": 1.3667, + "theoretical_loss": 3.4059765549250254, + "tokens_seen": 2418147328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699498495486459, + "loss": 1.2922, + "theoretical_loss": 3.4059692047580885, + "tokens_seen": 2418212864 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699297893681043, + "loss": 1.354, + "theoretical_loss": 3.4059618548461197, + "tokens_seen": 2418278400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699097291875627, + "loss": 1.3234, + "theoretical_loss": 3.4059545051891025, + "tokens_seen": 2418343936 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026988966900702105, + "loss": 1.2766, + "theoretical_loss": 3.4059471557870213, + "tokens_seen": 2418409472 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698696088264794, + "loss": 1.2844, + "theoretical_loss": 3.4059398066398607, + "tokens_seen": 2418475008 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026984954864593783, + "loss": 1.3467, + "theoretical_loss": 3.4059324577476047, + "tokens_seen": 2418540544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698294884653962, + "loss": 1.2876, + "theoretical_loss": 3.4059251091102376, + "tokens_seen": 2418606080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026980942828485455, + "loss": 1.3701, + "theoretical_loss": 3.4059177607277435, + "tokens_seen": 2418671616 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697893681043129, + "loss": 1.4601, + "theoretical_loss": 3.405910412600107, + "tokens_seen": 2418737152 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697693079237713, + "loss": 1.3751, + "theoretical_loss": 3.405903064727312, + "tokens_seen": 2418802688 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697492477432297, + "loss": 1.3724, + "theoretical_loss": 3.4058957171093427, + "tokens_seen": 2418868224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026972918756268806, + "loss": 1.2093, + "theoretical_loss": 3.405888369746184, + "tokens_seen": 2418933760 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697091273821464, + "loss": 1.359, + "theoretical_loss": 3.405881022637819, + "tokens_seen": 2418999296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696890672016048, + "loss": 1.3386, + "theoretical_loss": 3.405873675784233, + "tokens_seen": 2419064832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696690070210632, + "loss": 1.3968, + "theoretical_loss": 3.4058663291854105, + "tokens_seen": 2419130368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026964894684052156, + "loss": 1.3508, + "theoretical_loss": 3.405858982841335, + "tokens_seen": 2419195904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696288866599799, + "loss": 1.3091, + "theoretical_loss": 3.4058516367519904, + "tokens_seen": 2419261440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696088264794383, + "loss": 1.2846, + "theoretical_loss": 3.405844290917362, + "tokens_seen": 2419326976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026958876629889665, + "loss": 1.3162, + "theoretical_loss": 3.405836945337433, + "tokens_seen": 2419392512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026956870611835506, + "loss": 1.3486, + "theoretical_loss": 3.405829600012189, + "tokens_seen": 2419458048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695486459378134, + "loss": 1.3165, + "theoretical_loss": 3.4058222549416133, + "tokens_seen": 2419523584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695285857572718, + "loss": 1.3118, + "theoretical_loss": 3.4058149101256903, + "tokens_seen": 2419589120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695085255767302, + "loss": 1.3184, + "theoretical_loss": 3.4058075655644044, + "tokens_seen": 2419654656 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694884653961886, + "loss": 1.3961, + "theoretical_loss": 3.4058002212577403, + "tokens_seen": 2419720192 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269468405215647, + "loss": 1.3534, + "theoretical_loss": 3.4057928772056814, + "tokens_seen": 2419785728 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026944834503510534, + "loss": 1.399, + "theoretical_loss": 3.4057855334082126, + "tokens_seen": 2419851264 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694282848545637, + "loss": 1.2392, + "theoretical_loss": 3.405778189865318, + "tokens_seen": 2419916800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026940822467402207, + "loss": 1.3299, + "theoretical_loss": 3.4057708465769814, + "tokens_seen": 2419982336 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 881081, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4548755884170532, + "objective/train/theoretical_loss": 3.405763503543188, + "objective/train/tokens_used": 790506976, + "theoretical_loss": 3.405763503543188, + "tokens_seen": 2420047872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693881644934805, + "loss": 1.3281, + "theoretical_loss": 3.405763503543188, + "tokens_seen": 2420047872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026936810431293885, + "loss": 1.2841, + "theoretical_loss": 3.4057561607639215, + "tokens_seen": 2420113408 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693480441323972, + "loss": 1.3131, + "theoretical_loss": 3.4057488182391666, + "tokens_seen": 2420178944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026932798395185557, + "loss": 1.5235, + "theoretical_loss": 3.4057414759689073, + "tokens_seen": 2420244480 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269307923771314, + "loss": 1.3, + "theoretical_loss": 3.405734133953128, + "tokens_seen": 2420310016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026928786359077235, + "loss": 1.3561, + "theoretical_loss": 3.4057267921918126, + "tokens_seen": 2420375552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692678034102307, + "loss": 1.3014, + "theoretical_loss": 3.4057194506849457, + "tokens_seen": 2420441088 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692477432296891, + "loss": 1.368, + "theoretical_loss": 3.4057121094325113, + "tokens_seen": 2420506624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692276830491475, + "loss": 1.3419, + "theoretical_loss": 3.4057047684344948, + "tokens_seen": 2420572160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026920762286860585, + "loss": 1.31, + "theoretical_loss": 3.405697427690879, + "tokens_seen": 2420637696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691875626880642, + "loss": 1.2438, + "theoretical_loss": 3.4056900872016493, + "tokens_seen": 2420703232 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691675025075226, + "loss": 1.2784, + "theoretical_loss": 3.4056827469667894, + "tokens_seen": 2420768768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026914744232698094, + "loss": 1.3955, + "theoretical_loss": 3.4056754069862833, + "tokens_seen": 2420834304 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026912738214643936, + "loss": 1.2967, + "theoretical_loss": 3.405668067260116, + "tokens_seen": 2420899840 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691073219658977, + "loss": 1.351, + "theoretical_loss": 3.405660727788272, + "tokens_seen": 2420965376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690872617853561, + "loss": 1.3782, + "theoretical_loss": 3.4056533885707347, + "tokens_seen": 2421030912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026906720160481444, + "loss": 1.328, + "theoretical_loss": 3.4056460496074896, + "tokens_seen": 2421096448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026904714142427286, + "loss": 1.411, + "theoretical_loss": 3.4056387108985193, + "tokens_seen": 2421161984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690270812437312, + "loss": 1.3244, + "theoretical_loss": 3.4056313724438096, + "tokens_seen": 2421227520 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690070210631896, + "loss": 1.3849, + "theoretical_loss": 3.4056240342433446, + "tokens_seen": 2421293056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026898696088264795, + "loss": 1.2887, + "theoretical_loss": 3.4056166962971077, + "tokens_seen": 2421358592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689669007021063, + "loss": 1.4169, + "theoretical_loss": 3.4056093586050844, + "tokens_seen": 2421424128 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689468405215647, + "loss": 1.4382, + "theoretical_loss": 3.4056020211672577, + "tokens_seen": 2421489664 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689267803410231, + "loss": 1.2975, + "theoretical_loss": 3.4055946839836135, + "tokens_seen": 2421555200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026890672016048145, + "loss": 1.3015, + "theoretical_loss": 3.4055873470541345, + "tokens_seen": 2421620736 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688866599799398, + "loss": 1.3456, + "theoretical_loss": 3.4055800103788063, + "tokens_seen": 2421686272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026886659979939823, + "loss": 1.4122, + "theoretical_loss": 3.4055726739576127, + "tokens_seen": 2421751808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688465396188566, + "loss": 1.3985, + "theoretical_loss": 3.4055653377905375, + "tokens_seen": 2421817344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026882647943831495, + "loss": 1.2206, + "theoretical_loss": 3.4055580018775657, + "tokens_seen": 2421882880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688064192577733, + "loss": 1.4452, + "theoretical_loss": 3.405550666218682, + "tokens_seen": 2421948416 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687863590772317, + "loss": 1.2921, + "theoretical_loss": 3.40554333081387, + "tokens_seen": 2422013952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687662988966901, + "loss": 1.3464, + "theoretical_loss": 3.4055359956631137, + "tokens_seen": 2422079488 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026874623871614846, + "loss": 1.4432, + "theoretical_loss": 3.4055286607663984, + "tokens_seen": 2422145024 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687261785356068, + "loss": 1.1195, + "theoretical_loss": 3.405521326123708, + "tokens_seen": 2422210560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687061183550652, + "loss": 1.3454, + "theoretical_loss": 3.4055139917350266, + "tokens_seen": 2422276096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686860581745236, + "loss": 1.3041, + "theoretical_loss": 3.4055066576003385, + "tokens_seen": 2422341632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026866599799398196, + "loss": 1.368, + "theoretical_loss": 3.4054993237196287, + "tokens_seen": 2422407168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686459378134403, + "loss": 1.4555, + "theoretical_loss": 3.405491990092881, + "tokens_seen": 2422472704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686258776328987, + "loss": 1.3074, + "theoretical_loss": 3.4054846567200796, + "tokens_seen": 2422538240 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026860581745235705, + "loss": 1.4823, + "theoretical_loss": 3.4054773236012097, + "tokens_seen": 2422603776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026858575727181546, + "loss": 1.2772, + "theoretical_loss": 3.4054699907362544, + "tokens_seen": 2422669312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685656970912738, + "loss": 1.3256, + "theoretical_loss": 3.4054626581251988, + "tokens_seen": 2422734848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685456369107322, + "loss": 1.2827, + "theoretical_loss": 3.405455325768027, + "tokens_seen": 2422800384 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026852557673019055, + "loss": 1.2093, + "theoretical_loss": 3.4054479936647235, + "tokens_seen": 2422865920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026850551654964897, + "loss": 1.3149, + "theoretical_loss": 3.405440661815273, + "tokens_seen": 2422931456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026848545636910733, + "loss": 1.3921, + "theoretical_loss": 3.4054333302196587, + "tokens_seen": 2422996992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684653961885657, + "loss": 1.3477, + "theoretical_loss": 3.4054259988778663, + "tokens_seen": 2423062528 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026844533600802405, + "loss": 1.3062, + "theoretical_loss": 3.405418667789879, + "tokens_seen": 2423128064 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684252758274824, + "loss": 1.2946, + "theoretical_loss": 3.405411336955682, + "tokens_seen": 2423193600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026840521564694083, + "loss": 1.3284, + "theoretical_loss": 3.4054040063752593, + "tokens_seen": 2423259136 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 882232, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3485541343688965, + "objective/train/theoretical_loss": 3.405396676048595, + "objective/train/tokens_used": 793783776, + "theoretical_loss": 3.405396676048595, + "tokens_seen": 2423324672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683851554663992, + "loss": 1.3357, + "theoretical_loss": 3.405396676048595, + "tokens_seen": 2423324672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026836509528585756, + "loss": 1.3971, + "theoretical_loss": 3.405389345975674, + "tokens_seen": 2423390208 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683450351053159, + "loss": 1.2751, + "theoretical_loss": 3.40538201615648, + "tokens_seen": 2423455744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026832497492477433, + "loss": 1.4781, + "theoretical_loss": 3.4053746865909984, + "tokens_seen": 2423521280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683049147442327, + "loss": 1.3827, + "theoretical_loss": 3.4053673572792125, + "tokens_seen": 2423586816 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026828485456369106, + "loss": 1.4542, + "theoretical_loss": 3.405360028221107, + "tokens_seen": 2423652352 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682647943831494, + "loss": 1.4376, + "theoretical_loss": 3.4053526994166665, + "tokens_seen": 2423717888 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682447342026078, + "loss": 1.3843, + "theoretical_loss": 3.405345370865875, + "tokens_seen": 2423783424 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682246740220662, + "loss": 1.5332, + "theoretical_loss": 3.405338042568717, + "tokens_seen": 2423848960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026820461384152456, + "loss": 1.3222, + "theoretical_loss": 3.405330714525177, + "tokens_seen": 2423914496 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681845536609829, + "loss": 1.3028, + "theoretical_loss": 3.4053233867352395, + "tokens_seen": 2423980032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681644934804413, + "loss": 1.4225, + "theoretical_loss": 3.405316059198888, + "tokens_seen": 2424045568 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681444332998997, + "loss": 1.3292, + "theoretical_loss": 3.405308731916108, + "tokens_seen": 2424111104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026812437311935807, + "loss": 1.377, + "theoretical_loss": 3.4053014048868833, + "tokens_seen": 2424176640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026810431293881643, + "loss": 1.3468, + "theoretical_loss": 3.4052940781111984, + "tokens_seen": 2424242176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680842527582748, + "loss": 1.3393, + "theoretical_loss": 3.4052867515890375, + "tokens_seen": 2424307712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026806419257773315, + "loss": 1.2582, + "theoretical_loss": 3.405279425320385, + "tokens_seen": 2424373248 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026804413239719157, + "loss": 1.3182, + "theoretical_loss": 3.4052720993052255, + "tokens_seen": 2424438784 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026802407221664993, + "loss": 1.261, + "theoretical_loss": 3.4052647735435433, + "tokens_seen": 2424504320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026800401203610835, + "loss": 1.3059, + "theoretical_loss": 3.4052574480353224, + "tokens_seen": 2424569856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679839518555667, + "loss": 1.3245, + "theoretical_loss": 3.405250122780548, + "tokens_seen": 2424635392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679638916750251, + "loss": 1.2669, + "theoretical_loss": 3.405242797779204, + "tokens_seen": 2424700928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679438314944835, + "loss": 1.2908, + "theoretical_loss": 3.405235473031274, + "tokens_seen": 2424766464 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026792377131394185, + "loss": 1.2577, + "theoretical_loss": 3.405228148536744, + "tokens_seen": 2424832000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679037111334002, + "loss": 1.4439, + "theoretical_loss": 3.405220824295597, + "tokens_seen": 2424897536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026788365095285863, + "loss": 1.3717, + "theoretical_loss": 3.405213500307818, + "tokens_seen": 2424963072 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267863590772317, + "loss": 1.3757, + "theoretical_loss": 3.4052061765733916, + "tokens_seen": 2425028608 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026784353059177535, + "loss": 1.3104, + "theoretical_loss": 3.4051988530923016, + "tokens_seen": 2425094144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678234704112337, + "loss": 1.2968, + "theoretical_loss": 3.405191529864533, + "tokens_seen": 2425159680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678034102306921, + "loss": 1.3218, + "theoretical_loss": 3.4051842068900697, + "tokens_seen": 2425225216 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677833500501505, + "loss": 1.504, + "theoretical_loss": 3.4051768841688963, + "tokens_seen": 2425290752 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026776328986960886, + "loss": 1.2117, + "theoretical_loss": 3.4051695617009967, + "tokens_seen": 2425356288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677432296890672, + "loss": 1.3506, + "theoretical_loss": 3.4051622394863563, + "tokens_seen": 2425421824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677231695085256, + "loss": 1.4068, + "theoretical_loss": 3.405154917524959, + "tokens_seen": 2425487360 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267703109327984, + "loss": 1.2232, + "theoretical_loss": 3.405147595816789, + "tokens_seen": 2425552896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026768304914744236, + "loss": 1.3203, + "theoretical_loss": 3.405140274361831, + "tokens_seen": 2425618432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676629889669007, + "loss": 1.2788, + "theoretical_loss": 3.405132953160069, + "tokens_seen": 2425683968 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676429287863591, + "loss": 1.4393, + "theoretical_loss": 3.405125632211488, + "tokens_seen": 2425749504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026762286860581745, + "loss": 1.3712, + "theoretical_loss": 3.405118311516072, + "tokens_seen": 2425815040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026760280842527586, + "loss": 1.3375, + "theoretical_loss": 3.4051109910738053, + "tokens_seen": 2425880576 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675827482447342, + "loss": 1.4066, + "theoretical_loss": 3.4051036708846727, + "tokens_seen": 2425946112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675626880641926, + "loss": 1.3846, + "theoretical_loss": 3.405096350948658, + "tokens_seen": 2426011648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026754262788365095, + "loss": 1.3323, + "theoretical_loss": 3.4050890312657467, + "tokens_seen": 2426077184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026752256770310937, + "loss": 1.2681, + "theoretical_loss": 3.4050817118359222, + "tokens_seen": 2426142720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026750250752256773, + "loss": 1.3565, + "theoretical_loss": 3.405074392659169, + "tokens_seen": 2426208256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674824473420261, + "loss": 1.5088, + "theoretical_loss": 3.405067073735472, + "tokens_seen": 2426273792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026746238716148445, + "loss": 1.3013, + "theoretical_loss": 3.405059755064815, + "tokens_seen": 2426339328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674423269809428, + "loss": 1.2945, + "theoretical_loss": 3.4050524366471833, + "tokens_seen": 2426404864 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026742226680040123, + "loss": 1.3537, + "theoretical_loss": 3.4050451184825605, + "tokens_seen": 2426470400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674022066198596, + "loss": 1.2138, + "theoretical_loss": 3.4050378005709314, + "tokens_seen": 2426535936 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 883303, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3030461072921753, + "objective/train/theoretical_loss": 3.4050304829122804, + "objective/train/tokens_used": 797060576, + "theoretical_loss": 3.4050304829122804, + "tokens_seen": 2426601472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026738214643931796, + "loss": 1.258, + "theoretical_loss": 3.4050304829122804, + "tokens_seen": 2426601472 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673620862587763, + "loss": 1.4453, + "theoretical_loss": 3.405023165506592, + "tokens_seen": 2426667008 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026734202607823474, + "loss": 1.33, + "theoretical_loss": 3.40501584835385, + "tokens_seen": 2426732544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673219658976931, + "loss": 1.4086, + "theoretical_loss": 3.4050085314540395, + "tokens_seen": 2426798080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026730190571715146, + "loss": 1.2477, + "theoretical_loss": 3.405001214807145, + "tokens_seen": 2426863616 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672818455366098, + "loss": 1.2665, + "theoretical_loss": 3.4049938984131507, + "tokens_seen": 2426929152 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672617853560682, + "loss": 1.1864, + "theoretical_loss": 3.4049865822720404, + "tokens_seen": 2426994688 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672417251755266, + "loss": 1.2912, + "theoretical_loss": 3.4049792663838, + "tokens_seen": 2427060224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026722166499498496, + "loss": 1.2859, + "theoretical_loss": 3.404971950748412, + "tokens_seen": 2427125760 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672016048144433, + "loss": 1.3571, + "theoretical_loss": 3.4049646353658627, + "tokens_seen": 2427191296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671815446339017, + "loss": 1.2856, + "theoretical_loss": 3.404957320236136, + "tokens_seen": 2427256832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671614844533601, + "loss": 1.25, + "theoretical_loss": 3.404950005359215, + "tokens_seen": 2427322368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026714142427281847, + "loss": 1.258, + "theoretical_loss": 3.404942690735086, + "tokens_seen": 2427387904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026712136409227683, + "loss": 1.3684, + "theoretical_loss": 3.4049353763637327, + "tokens_seen": 2427453440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671013039117352, + "loss": 1.4446, + "theoretical_loss": 3.404928062245139, + "tokens_seen": 2427518976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026708124373119355, + "loss": 1.2308, + "theoretical_loss": 3.40492074837929, + "tokens_seen": 2427584512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026706118355065197, + "loss": 1.3159, + "theoretical_loss": 3.4049134347661703, + "tokens_seen": 2427650048 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026704112337011033, + "loss": 1.4214, + "theoretical_loss": 3.4049061214057637, + "tokens_seen": 2427715584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002670210631895687, + "loss": 1.4214, + "theoretical_loss": 3.404898808298055, + "tokens_seen": 2427781120 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026700100300902706, + "loss": 1.391, + "theoretical_loss": 3.4048914954430285, + "tokens_seen": 2427846656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026698094282848547, + "loss": 1.3988, + "theoretical_loss": 3.404884182840669, + "tokens_seen": 2427912192 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026696088264794383, + "loss": 1.2703, + "theoretical_loss": 3.4048768704909604, + "tokens_seen": 2427977728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669408224674022, + "loss": 1.3277, + "theoretical_loss": 3.4048695583938877, + "tokens_seen": 2428043264 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026692076228686056, + "loss": 1.3554, + "theoretical_loss": 3.404862246549435, + "tokens_seen": 2428108800 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266900702106319, + "loss": 1.5205, + "theoretical_loss": 3.404854934957587, + "tokens_seen": 2428174336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026688064192577734, + "loss": 1.3562, + "theoretical_loss": 3.404847623618328, + "tokens_seen": 2428239872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668605817452357, + "loss": 1.4315, + "theoretical_loss": 3.4048403125316424, + "tokens_seen": 2428305408 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026684052156469406, + "loss": 1.2801, + "theoretical_loss": 3.404833001697515, + "tokens_seen": 2428370944 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668204613841524, + "loss": 1.3293, + "theoretical_loss": 3.4048256911159296, + "tokens_seen": 2428436480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026680040120361084, + "loss": 1.3755, + "theoretical_loss": 3.404818380786871, + "tokens_seen": 2428502016 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667803410230692, + "loss": 1.2545, + "theoretical_loss": 3.4048110707103243, + "tokens_seen": 2428567552 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026676028084252757, + "loss": 1.3037, + "theoretical_loss": 3.404803760886273, + "tokens_seen": 2428633088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026674022066198593, + "loss": 1.3218, + "theoretical_loss": 3.4047964513147018, + "tokens_seen": 2428698624 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026672016048144434, + "loss": 1.3399, + "theoretical_loss": 3.404789141995596, + "tokens_seen": 2428764160 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667001003009027, + "loss": 1.1972, + "theoretical_loss": 3.4047818329289385, + "tokens_seen": 2428829696 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026668004012036107, + "loss": 1.3512, + "theoretical_loss": 3.404774524114715, + "tokens_seen": 2428895232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026665997993981943, + "loss": 1.2622, + "theoretical_loss": 3.40476721555291, + "tokens_seen": 2428960768 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666399197592778, + "loss": 1.3282, + "theoretical_loss": 3.404759907243507, + "tokens_seen": 2429026304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666198595787362, + "loss": 1.2539, + "theoretical_loss": 3.4047525991864913, + "tokens_seen": 2429091840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026659979939819457, + "loss": 1.4264, + "theoretical_loss": 3.4047452913818472, + "tokens_seen": 2429157376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026657973921765293, + "loss": 1.3425, + "theoretical_loss": 3.404737983829559, + "tokens_seen": 2429222912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665596790371113, + "loss": 1.2743, + "theoretical_loss": 3.4047306765296117, + "tokens_seen": 2429288448 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665396188565697, + "loss": 1.33, + "theoretical_loss": 3.404723369481989, + "tokens_seen": 2429353984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665195586760281, + "loss": 1.2016, + "theoretical_loss": 3.404716062686676, + "tokens_seen": 2429419520 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664994984954865, + "loss": 1.2349, + "theoretical_loss": 3.4047087561436564, + "tokens_seen": 2429485056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026647943831494485, + "loss": 1.3166, + "theoretical_loss": 3.404701449852916, + "tokens_seen": 2429550592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664593781344032, + "loss": 1.3174, + "theoretical_loss": 3.4046941438144382, + "tokens_seen": 2429616128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026643931795386163, + "loss": 1.3426, + "theoretical_loss": 3.4046868380282076, + "tokens_seen": 2429681664 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026641925777332, + "loss": 1.3944, + "theoretical_loss": 3.4046795324942094, + "tokens_seen": 2429747200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026639919759277836, + "loss": 1.2684, + "theoretical_loss": 3.404672227212427, + "tokens_seen": 2429812736 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 884501, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3901500701904297, + "objective/train/theoretical_loss": 3.4046649221828456, + "objective/train/tokens_used": 800337376, + "theoretical_loss": 3.4046649221828456, + "tokens_seen": 2429878272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663791374122367, + "loss": 1.3589, + "theoretical_loss": 3.4046649221828456, + "tokens_seen": 2429878272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026635907723169514, + "loss": 1.4137, + "theoretical_loss": 3.4046576174054497, + "tokens_seen": 2429943808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663390170511535, + "loss": 1.2823, + "theoretical_loss": 3.4046503128802237, + "tokens_seen": 2430009344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026631895687061186, + "loss": 1.3052, + "theoretical_loss": 3.404643008607152, + "tokens_seen": 2430074880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662988966900702, + "loss": 1.374, + "theoretical_loss": 3.404635704586219, + "tokens_seen": 2430140416 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662788365095286, + "loss": 1.3538, + "theoretical_loss": 3.4046284008174093, + "tokens_seen": 2430205952 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266258776328987, + "loss": 1.4029, + "theoretical_loss": 3.4046210973007076, + "tokens_seen": 2430271488 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026623871614844536, + "loss": 1.4687, + "theoretical_loss": 3.404613794036098, + "tokens_seen": 2430337024 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662186559679037, + "loss": 1.4055, + "theoretical_loss": 3.4046064910235656, + "tokens_seen": 2430402560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661985957873621, + "loss": 1.2183, + "theoretical_loss": 3.404599188263094, + "tokens_seen": 2430468096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661785356068205, + "loss": 1.2853, + "theoretical_loss": 3.4045918857546686, + "tokens_seen": 2430533632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026615847542627887, + "loss": 1.3001, + "theoretical_loss": 3.404584583498274, + "tokens_seen": 2430599168 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026613841524573723, + "loss": 1.4721, + "theoretical_loss": 3.4045772814938937, + "tokens_seen": 2430664704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661183550651956, + "loss": 1.1923, + "theoretical_loss": 3.4045699797415128, + "tokens_seen": 2430730240 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026609829488465395, + "loss": 1.3096, + "theoretical_loss": 3.404562678241116, + "tokens_seen": 2430795776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026607823470411237, + "loss": 1.2664, + "theoretical_loss": 3.404555376992687, + "tokens_seen": 2430861312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026605817452357073, + "loss": 1.2991, + "theoretical_loss": 3.4045480759962117, + "tokens_seen": 2430926848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660381143430291, + "loss": 1.3548, + "theoretical_loss": 3.4045407752516734, + "tokens_seen": 2430992384 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026601805416248746, + "loss": 1.2728, + "theoretical_loss": 3.4045334747590568, + "tokens_seen": 2431057920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026599799398194587, + "loss": 1.394, + "theoretical_loss": 3.404526174518347, + "tokens_seen": 2431123456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026597793380140423, + "loss": 1.3311, + "theoretical_loss": 3.4045188745295283, + "tokens_seen": 2431188992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659578736208626, + "loss": 1.2233, + "theoretical_loss": 3.4045115747925845, + "tokens_seen": 2431254528 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026593781344032096, + "loss": 1.5064, + "theoretical_loss": 3.404504275307501, + "tokens_seen": 2431320064 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659177532597794, + "loss": 1.2455, + "theoretical_loss": 3.4044969760742623, + "tokens_seen": 2431385600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026589769307923774, + "loss": 1.2831, + "theoretical_loss": 3.404489677092852, + "tokens_seen": 2431451136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658776328986961, + "loss": 1.3527, + "theoretical_loss": 3.404482378363256, + "tokens_seen": 2431516672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026585757271815446, + "loss": 1.306, + "theoretical_loss": 3.4044750798854575, + "tokens_seen": 2431582208 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658375125376128, + "loss": 1.3541, + "theoretical_loss": 3.4044677816594415, + "tokens_seen": 2431647744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026581745235707124, + "loss": 1.4123, + "theoretical_loss": 3.4044604836851935, + "tokens_seen": 2431713280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657973921765296, + "loss": 1.2521, + "theoretical_loss": 3.4044531859626965, + "tokens_seen": 2431778816 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026577733199598797, + "loss": 1.2421, + "theoretical_loss": 3.404445888491936, + "tokens_seen": 2431844352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026575727181544633, + "loss": 1.3793, + "theoretical_loss": 3.404438591272896, + "tokens_seen": 2431909888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026573721163490474, + "loss": 1.3008, + "theoretical_loss": 3.404431294305561, + "tokens_seen": 2431975424 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657171514543631, + "loss": 1.2833, + "theoretical_loss": 3.4044239975899164, + "tokens_seen": 2432040960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026569709127382147, + "loss": 1.4058, + "theoretical_loss": 3.404416701125946, + "tokens_seen": 2432106496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026567703109327983, + "loss": 1.3082, + "theoretical_loss": 3.4044094049136344, + "tokens_seen": 2432172032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656569709127382, + "loss": 1.3279, + "theoretical_loss": 3.4044021089529664, + "tokens_seen": 2432237568 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656369107321966, + "loss": 1.2609, + "theoretical_loss": 3.404394813243926, + "tokens_seen": 2432303104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026561685055165497, + "loss": 1.3395, + "theoretical_loss": 3.4043875177864984, + "tokens_seen": 2432368640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026559679037111333, + "loss": 1.2916, + "theoretical_loss": 3.4043802225806674, + "tokens_seen": 2432434176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655767301905717, + "loss": 1.3477, + "theoretical_loss": 3.4043729276264187, + "tokens_seen": 2432499712 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655566700100301, + "loss": 1.3896, + "theoretical_loss": 3.4043656329237355, + "tokens_seen": 2432565248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655366098294885, + "loss": 1.2429, + "theoretical_loss": 3.4043583384726035, + "tokens_seen": 2432630784 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026551654964894684, + "loss": 1.4478, + "theoretical_loss": 3.4043510442730067, + "tokens_seen": 2432696320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654964894684052, + "loss": 1.3163, + "theoretical_loss": 3.404343750324929, + "tokens_seen": 2432761856 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026547642928786356, + "loss": 1.2848, + "theoretical_loss": 3.4043364566283563, + "tokens_seen": 2432827392 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265456369107322, + "loss": 1.3382, + "theoretical_loss": 3.4043291631832724, + "tokens_seen": 2432892928 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026543630892678034, + "loss": 1.2891, + "theoretical_loss": 3.404321869989662, + "tokens_seen": 2432958464 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654162487462387, + "loss": 1.2454, + "theoretical_loss": 3.4043145770475096, + "tokens_seen": 2433024000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026539618856569707, + "loss": 1.3167, + "theoretical_loss": 3.4043072843567996, + "tokens_seen": 2433089536 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 885704, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2719991207122803, + "objective/train/theoretical_loss": 3.4042999919175165, + "objective/train/tokens_used": 803614176, + "theoretical_loss": 3.4042999919175165, + "tokens_seen": 2433155072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653761283851555, + "loss": 1.2637, + "theoretical_loss": 3.4042999919175165, + "tokens_seen": 2433155072 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026535606820461384, + "loss": 1.352, + "theoretical_loss": 3.4042926997296457, + "tokens_seen": 2433220608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653360080240722, + "loss": 1.3194, + "theoretical_loss": 3.4042854077931706, + "tokens_seen": 2433286144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026531594784353057, + "loss": 1.2921, + "theoretical_loss": 3.4042781161080766, + "tokens_seen": 2433351680 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026529588766298893, + "loss": 1.3071, + "theoretical_loss": 3.404270824674348, + "tokens_seen": 2433417216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026527582748244735, + "loss": 1.23, + "theoretical_loss": 3.404263533491969, + "tokens_seen": 2433482752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652557673019057, + "loss": 1.2463, + "theoretical_loss": 3.4042562425609244, + "tokens_seen": 2433548288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026523570712136407, + "loss": 1.405, + "theoretical_loss": 3.404248951881199, + "tokens_seen": 2433613824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026521564694082243, + "loss": 1.3834, + "theoretical_loss": 3.4042416614527777, + "tokens_seen": 2433679360 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026519558676028085, + "loss": 1.4448, + "theoretical_loss": 3.404234371275644, + "tokens_seen": 2433744896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651755265797392, + "loss": 1.3802, + "theoretical_loss": 3.4042270813497835, + "tokens_seen": 2433810432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651554663991976, + "loss": 1.3237, + "theoretical_loss": 3.4042197916751804, + "tokens_seen": 2433875968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026513540621865594, + "loss": 1.2549, + "theoretical_loss": 3.404212502251819, + "tokens_seen": 2433941504 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651153460381143, + "loss": 1.431, + "theoretical_loss": 3.404205213079684, + "tokens_seen": 2434007040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650952858575727, + "loss": 1.3434, + "theoretical_loss": 3.40419792415876, + "tokens_seen": 2434072576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650752256770311, + "loss": 1.4047, + "theoretical_loss": 3.404190635489032, + "tokens_seen": 2434138112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026505516549648944, + "loss": 1.2805, + "theoretical_loss": 3.404183347070484, + "tokens_seen": 2434203648 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650351053159478, + "loss": 1.2965, + "theoretical_loss": 3.404176058903101, + "tokens_seen": 2434269184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650150451354062, + "loss": 1.2121, + "theoretical_loss": 3.4041687709868675, + "tokens_seen": 2434334720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026499498495486464, + "loss": 1.3883, + "theoretical_loss": 3.4041614833217677, + "tokens_seen": 2434400256 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264974924774323, + "loss": 1.3495, + "theoretical_loss": 3.404154195907787, + "tokens_seen": 2434465792 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026495486459378136, + "loss": 1.3549, + "theoretical_loss": 3.404146908744909, + "tokens_seen": 2434531328 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002649348044132398, + "loss": 1.3597, + "theoretical_loss": 3.404139621833119, + "tokens_seen": 2434596864 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026491474423269814, + "loss": 1.2416, + "theoretical_loss": 3.404132335172401, + "tokens_seen": 2434662400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648946840521565, + "loss": 1.304, + "theoretical_loss": 3.40412504876274, + "tokens_seen": 2434727936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026487462387161486, + "loss": 1.307, + "theoretical_loss": 3.4041177626041206, + "tokens_seen": 2434793472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648545636910732, + "loss": 1.3008, + "theoretical_loss": 3.4041104766965273, + "tokens_seen": 2434859008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026483450351053164, + "loss": 1.2966, + "theoretical_loss": 3.404103191039945, + "tokens_seen": 2434924544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026481444332999, + "loss": 1.2517, + "theoretical_loss": 3.4040959056343576, + "tokens_seen": 2434990080 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026479438314944837, + "loss": 1.2835, + "theoretical_loss": 3.4040886204797505, + "tokens_seen": 2435055616 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026477432296890673, + "loss": 1.4008, + "theoretical_loss": 3.4040813355761075, + "tokens_seen": 2435121152 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026475426278836514, + "loss": 1.3413, + "theoretical_loss": 3.404074050923414, + "tokens_seen": 2435186688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647342026078235, + "loss": 1.3696, + "theoretical_loss": 3.404066766521654, + "tokens_seen": 2435252224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026471414242728187, + "loss": 1.3266, + "theoretical_loss": 3.4040594823708124, + "tokens_seen": 2435317760 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026469408224674023, + "loss": 1.2926, + "theoretical_loss": 3.4040521984708736, + "tokens_seen": 2435383296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646740220661986, + "loss": 1.2218, + "theoretical_loss": 3.404044914821822, + "tokens_seen": 2435448832 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264653961885657, + "loss": 1.2744, + "theoretical_loss": 3.404037631423643, + "tokens_seen": 2435514368 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026463390170511537, + "loss": 1.3833, + "theoretical_loss": 3.404030348276321, + "tokens_seen": 2435579904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026461384152457373, + "loss": 1.3537, + "theoretical_loss": 3.40402306537984, + "tokens_seen": 2435645440 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645937813440321, + "loss": 1.2694, + "theoretical_loss": 3.404015782734185, + "tokens_seen": 2435710976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645737211634905, + "loss": 1.2989, + "theoretical_loss": 3.4040085003393403, + "tokens_seen": 2435776512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645536609829489, + "loss": 1.3162, + "theoretical_loss": 3.4040012181952912, + "tokens_seen": 2435842048 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026453360080240724, + "loss": 1.3371, + "theoretical_loss": 3.4039939363020215, + "tokens_seen": 2435907584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645135406218656, + "loss": 1.273, + "theoretical_loss": 3.4039866546595166, + "tokens_seen": 2435973120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026449348044132396, + "loss": 1.236, + "theoretical_loss": 3.4039793732677603, + "tokens_seen": 2436038656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644734202607824, + "loss": 1.3863, + "theoretical_loss": 3.4039720921267382, + "tokens_seen": 2436104192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026445336008024074, + "loss": 1.3751, + "theoretical_loss": 3.4039648112364342, + "tokens_seen": 2436169728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644332998996991, + "loss": 1.4152, + "theoretical_loss": 3.4039575305968333, + "tokens_seen": 2436235264 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026441323971915747, + "loss": 1.3586, + "theoretical_loss": 3.4039502502079193, + "tokens_seen": 2436300800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643931795386159, + "loss": 1.2942, + "theoretical_loss": 3.403942970069678, + "tokens_seen": 2436366336 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 886917, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2300962209701538, + "objective/train/theoretical_loss": 3.4039356901820934, + "objective/train/tokens_used": 806890976, + "theoretical_loss": 3.4039356901820934, + "tokens_seen": 2436431872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026437311935807424, + "loss": 1.3366, + "theoretical_loss": 3.4039356901820934, + "tokens_seen": 2436431872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643530591775326, + "loss": 1.302, + "theoretical_loss": 3.4039284105451504, + "tokens_seen": 2436497408 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026433299899699097, + "loss": 1.335, + "theoretical_loss": 3.4039211311588327, + "tokens_seen": 2436562944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026431293881644933, + "loss": 1.3802, + "theoretical_loss": 3.4039138520231265, + "tokens_seen": 2436628480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026429287863590775, + "loss": 1.2518, + "theoretical_loss": 3.4039065731380154, + "tokens_seen": 2436694016 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642728184553661, + "loss": 1.3233, + "theoretical_loss": 3.403899294503484, + "tokens_seen": 2436759552 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026425275827482447, + "loss": 1.3496, + "theoretical_loss": 3.403892016119517, + "tokens_seen": 2436825088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026423269809428283, + "loss": 1.2334, + "theoretical_loss": 3.4038847379860995, + "tokens_seen": 2436890624 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026421263791374125, + "loss": 1.2668, + "theoretical_loss": 3.403877460103216, + "tokens_seen": 2436956160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641925777331996, + "loss": 1.2685, + "theoretical_loss": 3.403870182470851, + "tokens_seen": 2437021696 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264172517552658, + "loss": 1.3218, + "theoretical_loss": 3.403862905088989, + "tokens_seen": 2437087232 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026415245737211634, + "loss": 1.283, + "theoretical_loss": 3.4038556279576144, + "tokens_seen": 2437152768 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641323971915747, + "loss": 1.3254, + "theoretical_loss": 3.4038483510767126, + "tokens_seen": 2437218304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641123370110331, + "loss": 1.3425, + "theoretical_loss": 3.4038410744462677, + "tokens_seen": 2437283840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640922768304915, + "loss": 1.2866, + "theoretical_loss": 3.4038337980662643, + "tokens_seen": 2437349376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026407221664994984, + "loss": 1.3734, + "theoretical_loss": 3.403826521936687, + "tokens_seen": 2437414912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640521564694082, + "loss": 1.3671, + "theoretical_loss": 3.403819246057522, + "tokens_seen": 2437480448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640320962888666, + "loss": 1.3858, + "theoretical_loss": 3.4038119704287517, + "tokens_seen": 2437545984 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264012036108325, + "loss": 1.3578, + "theoretical_loss": 3.4038046950503618, + "tokens_seen": 2437611520 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026399197592778334, + "loss": 1.4024, + "theoretical_loss": 3.403797419922337, + "tokens_seen": 2437677056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639719157472417, + "loss": 1.2692, + "theoretical_loss": 3.4037901450446615, + "tokens_seen": 2437742592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026395185556670007, + "loss": 1.3677, + "theoretical_loss": 3.40378287041732, + "tokens_seen": 2437808128 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639317953861585, + "loss": 1.3466, + "theoretical_loss": 3.403775596040298, + "tokens_seen": 2437873664 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026391173520561685, + "loss": 1.2702, + "theoretical_loss": 3.4037683219135793, + "tokens_seen": 2437939200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638916750250752, + "loss": 1.38, + "theoretical_loss": 3.4037610480371487, + "tokens_seen": 2438004736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026387161484453357, + "loss": 1.4064, + "theoretical_loss": 3.4037537744109914, + "tokens_seen": 2438070272 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263851554663992, + "loss": 1.4466, + "theoretical_loss": 3.4037465010350916, + "tokens_seen": 2438135808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026383149448345035, + "loss": 1.3869, + "theoretical_loss": 3.403739227909434, + "tokens_seen": 2438201344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638114343029087, + "loss": 1.1736, + "theoretical_loss": 3.4037319550340026, + "tokens_seen": 2438266880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637913741223671, + "loss": 1.2752, + "theoretical_loss": 3.4037246824087837, + "tokens_seen": 2438332416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637713139418255, + "loss": 1.2928, + "theoretical_loss": 3.40371741003376, + "tokens_seen": 2438397952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026375125376128385, + "loss": 1.2996, + "theoretical_loss": 3.4037101379089183, + "tokens_seen": 2438463488 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637311935807422, + "loss": 1.3287, + "theoretical_loss": 3.4037028660342417, + "tokens_seen": 2438529024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637111334002006, + "loss": 1.2418, + "theoretical_loss": 3.403695594409715, + "tokens_seen": 2438594560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026369107321965894, + "loss": 1.4114, + "theoretical_loss": 3.403688323035323, + "tokens_seen": 2438660096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026367101303911736, + "loss": 1.3061, + "theoretical_loss": 3.403681051911051, + "tokens_seen": 2438725632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636509528585757, + "loss": 1.3155, + "theoretical_loss": 3.4036737810368836, + "tokens_seen": 2438791168 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636308926780341, + "loss": 1.31, + "theoretical_loss": 3.4036665104128048, + "tokens_seen": 2438856704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026361083249749244, + "loss": 1.3362, + "theoretical_loss": 3.4036592400387997, + "tokens_seen": 2438922240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026359077231695086, + "loss": 1.2689, + "theoretical_loss": 3.4036519699148524, + "tokens_seen": 2438987776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635707121364092, + "loss": 1.3707, + "theoretical_loss": 3.403644700040948, + "tokens_seen": 2439053312 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635506519558676, + "loss": 1.3808, + "theoretical_loss": 3.403637430417072, + "tokens_seen": 2439118848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026353059177532595, + "loss": 1.2902, + "theoretical_loss": 3.403630161043208, + "tokens_seen": 2439184384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026351053159478436, + "loss": 1.2855, + "theoretical_loss": 3.4036228919193405, + "tokens_seen": 2439249920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634904714142428, + "loss": 1.3485, + "theoretical_loss": 3.4036156230454555, + "tokens_seen": 2439315456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026347041123370114, + "loss": 1.2817, + "theoretical_loss": 3.4036083544215363, + "tokens_seen": 2439380992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634503510531595, + "loss": 1.3304, + "theoretical_loss": 3.4036010860475683, + "tokens_seen": 2439446528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026343029087261787, + "loss": 1.3429, + "theoretical_loss": 3.4035938179235363, + "tokens_seen": 2439512064 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634102306920763, + "loss": 1.2538, + "theoretical_loss": 3.4035865500494245, + "tokens_seen": 2439577600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026339017051153464, + "loss": 1.3226, + "theoretical_loss": 3.403579282425218, + "tokens_seen": 2439643136 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 888050, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.376977562904358, + "objective/train/theoretical_loss": 3.4035720150509015, + "objective/train/tokens_used": 810167776, + "theoretical_loss": 3.4035720150509015, + "tokens_seen": 2439708672 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263370110330993, + "loss": 1.432, + "theoretical_loss": 3.4035720150509015, + "tokens_seen": 2439708672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026335005015045137, + "loss": 1.2653, + "theoretical_loss": 3.403564747926459, + "tokens_seen": 2439774208 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026332998996990973, + "loss": 1.3982, + "theoretical_loss": 3.403557481051876, + "tokens_seen": 2439839744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026330992978936815, + "loss": 1.2869, + "theoretical_loss": 3.403550214427137, + "tokens_seen": 2439905280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632898696088265, + "loss": 1.3165, + "theoretical_loss": 3.4035429480522263, + "tokens_seen": 2439970816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026326980942828487, + "loss": 1.337, + "theoretical_loss": 3.4035356819271296, + "tokens_seen": 2440036352 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026324974924774323, + "loss": 1.3174, + "theoretical_loss": 3.4035284160518304, + "tokens_seen": 2440101888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026322968906720165, + "loss": 1.3245, + "theoretical_loss": 3.403521150426314, + "tokens_seen": 2440167424 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026320962888666, + "loss": 1.379, + "theoretical_loss": 3.403513885050565, + "tokens_seen": 2440232960 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631895687061184, + "loss": 1.2523, + "theoretical_loss": 3.4035066199245683, + "tokens_seen": 2440298496 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026316950852557674, + "loss": 1.2459, + "theoretical_loss": 3.403499355048308, + "tokens_seen": 2440364032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631494483450351, + "loss": 1.2831, + "theoretical_loss": 3.40349209042177, + "tokens_seen": 2440429568 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631293881644935, + "loss": 1.3923, + "theoretical_loss": 3.403484826044938, + "tokens_seen": 2440495104 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631093279839519, + "loss": 1.3163, + "theoretical_loss": 3.403477561917797, + "tokens_seen": 2440560640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026308926780341024, + "loss": 1.3579, + "theoretical_loss": 3.4034702980403315, + "tokens_seen": 2440626176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630692076228686, + "loss": 1.3234, + "theoretical_loss": 3.4034630344125265, + "tokens_seen": 2440691712 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263049147442327, + "loss": 1.3498, + "theoretical_loss": 3.4034557710343667, + "tokens_seen": 2440757248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630290872617854, + "loss": 1.3074, + "theoretical_loss": 3.4034485079058365, + "tokens_seen": 2440822784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026300902708124374, + "loss": 1.3297, + "theoretical_loss": 3.4034412450269214, + "tokens_seen": 2440888320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629889669007021, + "loss": 1.3327, + "theoretical_loss": 3.4034339823976048, + "tokens_seen": 2440953856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026296890672016047, + "loss": 1.3149, + "theoretical_loss": 3.403426720017873, + "tokens_seen": 2441019392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629488465396189, + "loss": 1.3262, + "theoretical_loss": 3.4034194578877095, + "tokens_seen": 2441084928 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026292878635907725, + "loss": 1.395, + "theoretical_loss": 3.4034121960070993, + "tokens_seen": 2441150464 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629087261785356, + "loss": 1.3374, + "theoretical_loss": 3.4034049343760278, + "tokens_seen": 2441216000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026288866599799397, + "loss": 1.266, + "theoretical_loss": 3.403397672994479, + "tokens_seen": 2441281536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628686058174524, + "loss": 1.2661, + "theoretical_loss": 3.4033904118624374, + "tokens_seen": 2441347072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026284854563691075, + "loss": 1.2804, + "theoretical_loss": 3.4033831509798884, + "tokens_seen": 2441412608 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628284854563691, + "loss": 1.4598, + "theoretical_loss": 3.4033758903468168, + "tokens_seen": 2441478144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628084252758275, + "loss": 1.3232, + "theoretical_loss": 3.4033686299632064, + "tokens_seen": 2441543680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627883650952859, + "loss": 1.414, + "theoretical_loss": 3.4033613698290432, + "tokens_seen": 2441609216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026276830491474425, + "loss": 1.4005, + "theoretical_loss": 3.4033541099443108, + "tokens_seen": 2441674752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627482447342026, + "loss": 1.2182, + "theoretical_loss": 3.4033468503089948, + "tokens_seen": 2441740288 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262728184553661, + "loss": 1.3543, + "theoretical_loss": 3.4033395909230792, + "tokens_seen": 2441805824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026270812437311934, + "loss": 1.2899, + "theoretical_loss": 3.4033323317865496, + "tokens_seen": 2441871360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026268806419257776, + "loss": 1.4146, + "theoretical_loss": 3.40332507289939, + "tokens_seen": 2441936896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626680040120361, + "loss": 1.4143, + "theoretical_loss": 3.403317814261585, + "tokens_seen": 2442002432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626479438314945, + "loss": 1.369, + "theoretical_loss": 3.4033105558731203, + "tokens_seen": 2442067968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026262788365095284, + "loss": 1.2887, + "theoretical_loss": 3.40330329773398, + "tokens_seen": 2442133504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026260782347041126, + "loss": 1.2332, + "theoretical_loss": 3.4032960398441485, + "tokens_seen": 2442199040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625877632898696, + "loss": 1.2901, + "theoretical_loss": 3.403288782203611, + "tokens_seen": 2442264576 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262567703109328, + "loss": 1.3541, + "theoretical_loss": 3.4032815248123525, + "tokens_seen": 2442330112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026254764292878635, + "loss": 1.356, + "theoretical_loss": 3.4032742676703576, + "tokens_seen": 2442395648 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625275827482447, + "loss": 1.3593, + "theoretical_loss": 3.4032670107776104, + "tokens_seen": 2442461184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625075225677031, + "loss": 1.3006, + "theoretical_loss": 3.4032597541340968, + "tokens_seen": 2442526720 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624874623871615, + "loss": 1.3696, + "theoretical_loss": 3.4032524977398, + "tokens_seen": 2442592256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026246740220661985, + "loss": 1.2597, + "theoretical_loss": 3.4032452415947065, + "tokens_seen": 2442657792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624473420260782, + "loss": 1.2731, + "theoretical_loss": 3.4032379856988, + "tokens_seen": 2442723328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026242728184553663, + "loss": 1.3704, + "theoretical_loss": 3.4032307300520657, + "tokens_seen": 2442788864 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262407221664995, + "loss": 1.3342, + "theoretical_loss": 3.403223474654488, + "tokens_seen": 2442854400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026238716148445335, + "loss": 1.2058, + "theoretical_loss": 3.403216219506051, + "tokens_seen": 2442919936 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 889343, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2826709747314453, + "objective/train/theoretical_loss": 3.4032089646067414, + "objective/train/tokens_used": 813444576, + "theoretical_loss": 3.4032089646067414, + "tokens_seen": 2442985472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623671013039117, + "loss": 1.3074, + "theoretical_loss": 3.4032089646067414, + "tokens_seen": 2442985472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623470411233701, + "loss": 1.3213, + "theoretical_loss": 3.403201709956542, + "tokens_seen": 2443051008 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623269809428285, + "loss": 1.4409, + "theoretical_loss": 3.4031944555554388, + "tokens_seen": 2443116544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026230692076228686, + "loss": 1.3941, + "theoretical_loss": 3.4031872014034166, + "tokens_seen": 2443182080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622868605817452, + "loss": 1.2495, + "theoretical_loss": 3.403179947500459, + "tokens_seen": 2443247616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622668004012036, + "loss": 1.3324, + "theoretical_loss": 3.4031726938465514, + "tokens_seen": 2443313152 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262246740220662, + "loss": 1.241, + "theoretical_loss": 3.403165440441679, + "tokens_seen": 2443378688 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026222668004012036, + "loss": 1.3866, + "theoretical_loss": 3.4031581872858263, + "tokens_seen": 2443444224 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622066198595787, + "loss": 1.4343, + "theoretical_loss": 3.403150934378978, + "tokens_seen": 2443509760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621865596790371, + "loss": 1.3575, + "theoretical_loss": 3.4031436817211187, + "tokens_seen": 2443575296 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026216649949849545, + "loss": 1.4157, + "theoretical_loss": 3.4031364293122337, + "tokens_seen": 2443640832 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026214643931795386, + "loss": 1.3242, + "theoretical_loss": 3.403129177152307, + "tokens_seen": 2443706368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621263791374122, + "loss": 1.3735, + "theoretical_loss": 3.403121925241324, + "tokens_seen": 2443771904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621063189568706, + "loss": 1.3958, + "theoretical_loss": 3.4031146735792692, + "tokens_seen": 2443837440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026208625877632895, + "loss": 1.2829, + "theoretical_loss": 3.4031074221661273, + "tokens_seen": 2443902976 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026206619859578737, + "loss": 1.2901, + "theoretical_loss": 3.4031001710018836, + "tokens_seen": 2443968512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026204613841524573, + "loss": 1.3508, + "theoretical_loss": 3.4030929200865225, + "tokens_seen": 2444034048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620260782347041, + "loss": 1.2354, + "theoretical_loss": 3.4030856694200287, + "tokens_seen": 2444099584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620060180541625, + "loss": 1.3049, + "theoretical_loss": 3.4030784190023873, + "tokens_seen": 2444165120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026198595787362087, + "loss": 1.3019, + "theoretical_loss": 3.4030711688335824, + "tokens_seen": 2444230656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619658976930793, + "loss": 1.3713, + "theoretical_loss": 3.4030639189135994, + "tokens_seen": 2444296192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026194583751253765, + "loss": 1.3814, + "theoretical_loss": 3.4030566692424236, + "tokens_seen": 2444361728 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261925777331996, + "loss": 1.2917, + "theoretical_loss": 3.4030494198200385, + "tokens_seen": 2444427264 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026190571715145437, + "loss": 1.3146, + "theoretical_loss": 3.4030421706464296, + "tokens_seen": 2444492800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618856569709128, + "loss": 1.2822, + "theoretical_loss": 3.4030349217215816, + "tokens_seen": 2444558336 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026186559679037115, + "loss": 1.3221, + "theoretical_loss": 3.4030276730454796, + "tokens_seen": 2444623872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618455366098295, + "loss": 1.4244, + "theoretical_loss": 3.403020424618108, + "tokens_seen": 2444689408 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618254764292879, + "loss": 1.2541, + "theoretical_loss": 3.403013176439452, + "tokens_seen": 2444754944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618054162487463, + "loss": 1.3652, + "theoretical_loss": 3.4030059285094953, + "tokens_seen": 2444820480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026178535606820465, + "loss": 1.3091, + "theoretical_loss": 3.402998680828224, + "tokens_seen": 2444886016 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261765295887663, + "loss": 1.2622, + "theoretical_loss": 3.4029914333956226, + "tokens_seen": 2444951552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617452357071214, + "loss": 1.327, + "theoretical_loss": 3.402984186211676, + "tokens_seen": 2445017088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026172517552657974, + "loss": 1.305, + "theoretical_loss": 3.4029769392763685, + "tokens_seen": 2445082624 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026170511534603816, + "loss": 1.34, + "theoretical_loss": 3.402969692589685, + "tokens_seen": 2445148160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616850551654965, + "loss": 1.4476, + "theoretical_loss": 3.4029624461516104, + "tokens_seen": 2445213696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616649949849549, + "loss": 1.3878, + "theoretical_loss": 3.4029551999621295, + "tokens_seen": 2445279232 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026164493480441324, + "loss": 1.3156, + "theoretical_loss": 3.4029479540212275, + "tokens_seen": 2445344768 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026162487462387166, + "loss": 1.5031, + "theoretical_loss": 3.4029407083288885, + "tokens_seen": 2445410304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026160481444333, + "loss": 1.2931, + "theoretical_loss": 3.402933462885098, + "tokens_seen": 2445475840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615847542627884, + "loss": 1.3988, + "theoretical_loss": 3.4029262176898403, + "tokens_seen": 2445541376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026156469408224675, + "loss": 1.2403, + "theoretical_loss": 3.4029189727431004, + "tokens_seen": 2445606912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615446339017051, + "loss": 1.2563, + "theoretical_loss": 3.402911728044863, + "tokens_seen": 2445672448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615245737211635, + "loss": 1.299, + "theoretical_loss": 3.4029044835951137, + "tokens_seen": 2445737984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615045135406219, + "loss": 1.3516, + "theoretical_loss": 3.402897239393836, + "tokens_seen": 2445803520 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026148445336008025, + "loss": 1.3638, + "theoretical_loss": 3.4028899954410154, + "tokens_seen": 2445869056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614643931795386, + "loss": 1.3364, + "theoretical_loss": 3.4028827517366373, + "tokens_seen": 2445934592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026144433299899703, + "loss": 1.3023, + "theoretical_loss": 3.4028755082806854, + "tokens_seen": 2446000128 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614242728184554, + "loss": 1.3301, + "theoretical_loss": 3.4028682650731454, + "tokens_seen": 2446065664 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026140421263791375, + "loss": 1.224, + "theoretical_loss": 3.4028610221140014, + "tokens_seen": 2446131200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613841524573721, + "loss": 1.258, + "theoretical_loss": 3.402853779403239, + "tokens_seen": 2446196736 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 890537, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.372076153755188, + "objective/train/theoretical_loss": 3.4028465369408423, + "objective/train/tokens_used": 816721376, + "theoretical_loss": 3.4028465369408423, + "tokens_seen": 2446262272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613640922768305, + "loss": 1.3616, + "theoretical_loss": 3.4028465369408423, + "tokens_seen": 2446262272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613440320962889, + "loss": 1.3643, + "theoretical_loss": 3.402839294726797, + "tokens_seen": 2446327808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026132397191574726, + "loss": 1.3351, + "theoretical_loss": 3.4028320527610867, + "tokens_seen": 2446393344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613039117352056, + "loss": 1.4195, + "theoretical_loss": 3.4028248110436974, + "tokens_seen": 2446458880 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261283851554664, + "loss": 1.286, + "theoretical_loss": 3.402817569574613, + "tokens_seen": 2446524416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612637913741224, + "loss": 1.217, + "theoretical_loss": 3.4028103283538194, + "tokens_seen": 2446589952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026124373119358076, + "loss": 1.2749, + "theoretical_loss": 3.4028030873813004, + "tokens_seen": 2446655488 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612236710130391, + "loss": 1.3649, + "theoretical_loss": 3.4027958466570416, + "tokens_seen": 2446721024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612036108324975, + "loss": 1.3661, + "theoretical_loss": 3.4027886061810273, + "tokens_seen": 2446786560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026118355065195585, + "loss": 1.3086, + "theoretical_loss": 3.402781365953243, + "tokens_seen": 2446852096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026116349047141426, + "loss": 1.2858, + "theoretical_loss": 3.4027741259736723, + "tokens_seen": 2446917632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611434302908726, + "loss": 1.3788, + "theoretical_loss": 3.4027668862423015, + "tokens_seen": 2446983168 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261123370110331, + "loss": 1.266, + "theoretical_loss": 3.4027596467591144, + "tokens_seen": 2447048704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026110330992978935, + "loss": 1.2482, + "theoretical_loss": 3.4027524075240967, + "tokens_seen": 2447114240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026108324974924777, + "loss": 1.3155, + "theoretical_loss": 3.402745168537232, + "tokens_seen": 2447179776 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026106318956870613, + "loss": 1.2567, + "theoretical_loss": 3.4027379297985068, + "tokens_seen": 2447245312 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610431293881645, + "loss": 1.3292, + "theoretical_loss": 3.4027306913079043, + "tokens_seen": 2447310848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026102306920762285, + "loss": 1.2774, + "theoretical_loss": 3.4027234530654105, + "tokens_seen": 2447376384 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610030090270812, + "loss": 1.2333, + "theoretical_loss": 3.4027162150710097, + "tokens_seen": 2447441920 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026098294884653963, + "loss": 1.2712, + "theoretical_loss": 3.4027089773246875, + "tokens_seen": 2447507456 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260962888665998, + "loss": 1.2864, + "theoretical_loss": 3.4027017398264277, + "tokens_seen": 2447572992 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026094282848545636, + "loss": 1.29, + "theoretical_loss": 3.4026945025762156, + "tokens_seen": 2447638528 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002609227683049147, + "loss": 1.4223, + "theoretical_loss": 3.4026872655740363, + "tokens_seen": 2447704064 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026090270812437313, + "loss": 1.2668, + "theoretical_loss": 3.402680028819874, + "tokens_seen": 2447769600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608826479438315, + "loss": 1.4167, + "theoretical_loss": 3.402672792313715, + "tokens_seen": 2447835136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026086258776328986, + "loss": 1.3056, + "theoretical_loss": 3.4026655560555423, + "tokens_seen": 2447900672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608425275827482, + "loss": 1.343, + "theoretical_loss": 3.402658320045342, + "tokens_seen": 2447966208 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608224674022066, + "loss": 1.1956, + "theoretical_loss": 3.4026510842830984, + "tokens_seen": 2448031744 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260802407221665, + "loss": 1.3718, + "theoretical_loss": 3.402643848768797, + "tokens_seen": 2448097280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026078234704112336, + "loss": 1.3934, + "theoretical_loss": 3.4026366135024215, + "tokens_seen": 2448162816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607622868605817, + "loss": 1.2082, + "theoretical_loss": 3.402629378483958, + "tokens_seen": 2448228352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607422266800401, + "loss": 1.2699, + "theoretical_loss": 3.402622143713391, + "tokens_seen": 2448293888 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607221664994985, + "loss": 1.4717, + "theoretical_loss": 3.4026149091907048, + "tokens_seen": 2448359424 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026070210631895687, + "loss": 1.3513, + "theoretical_loss": 3.402607674915885, + "tokens_seen": 2448424960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026068204613841523, + "loss": 1.3045, + "theoretical_loss": 3.402600440888916, + "tokens_seen": 2448490496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606619859578736, + "loss": 1.4123, + "theoretical_loss": 3.4025932071097835, + "tokens_seen": 2448556032 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026064192577733195, + "loss": 1.3672, + "theoretical_loss": 3.4025859735784714, + "tokens_seen": 2448621568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026062186559679037, + "loss": 1.2683, + "theoretical_loss": 3.4025787402949645, + "tokens_seen": 2448687104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026060180541624873, + "loss": 1.3746, + "theoretical_loss": 3.4025715072592484, + "tokens_seen": 2448752640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605817452357071, + "loss": 1.2209, + "theoretical_loss": 3.402564274471308, + "tokens_seen": 2448818176 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026056168505516546, + "loss": 1.3007, + "theoretical_loss": 3.4025570419311273, + "tokens_seen": 2448883712 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026054162487462387, + "loss": 1.2537, + "theoretical_loss": 3.402549809638692, + "tokens_seen": 2448949248 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026052156469408223, + "loss": 1.4302, + "theoretical_loss": 3.402542577593987, + "tokens_seen": 2449014784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026050150451354065, + "loss": 1.3112, + "theoretical_loss": 3.4025353457969967, + "tokens_seen": 2449080320 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260481444332999, + "loss": 1.3001, + "theoretical_loss": 3.4025281142477057, + "tokens_seen": 2449145856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026046138415245743, + "loss": 1.2979, + "theoretical_loss": 3.4025208829461002, + "tokens_seen": 2449211392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604413239719158, + "loss": 1.4334, + "theoretical_loss": 3.4025136518921637, + "tokens_seen": 2449276928 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026042126379137415, + "loss": 1.3403, + "theoretical_loss": 3.402506421085882, + "tokens_seen": 2449342464 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604012036108325, + "loss": 1.17, + "theoretical_loss": 3.40249919052724, + "tokens_seen": 2449408000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603811434302909, + "loss": 1.1721, + "theoretical_loss": 3.4024919602162216, + "tokens_seen": 2449473536 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 891543, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2751030921936035, + "objective/train/theoretical_loss": 3.4024847301528123, + "objective/train/tokens_used": 819998176, + "theoretical_loss": 3.4024847301528123, + "tokens_seen": 2449539072 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603610832497493, + "loss": 1.2485, + "theoretical_loss": 3.4024847301528123, + "tokens_seen": 2449539072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026034102306920766, + "loss": 1.226, + "theoretical_loss": 3.4024775003369974, + "tokens_seen": 2449604608 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260320962888666, + "loss": 1.2568, + "theoretical_loss": 3.4024702707687613, + "tokens_seen": 2449670144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603009027081244, + "loss": 1.3388, + "theoretical_loss": 3.4024630414480894, + "tokens_seen": 2449735680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602808425275828, + "loss": 1.2031, + "theoretical_loss": 3.4024558123749657, + "tokens_seen": 2449801216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026026078234704116, + "loss": 1.2208, + "theoretical_loss": 3.402448583549376, + "tokens_seen": 2449866752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602407221664995, + "loss": 1.2523, + "theoretical_loss": 3.4024413549713044, + "tokens_seen": 2449932288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602206619859579, + "loss": 1.2126, + "theoretical_loss": 3.402434126640737, + "tokens_seen": 2449997824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026020060180541625, + "loss": 1.2344, + "theoretical_loss": 3.4024268985576573, + "tokens_seen": 2450063360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026018054162487466, + "loss": 1.274, + "theoretical_loss": 3.4024196707220513, + "tokens_seen": 2450128896 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260160481444333, + "loss": 1.2667, + "theoretical_loss": 3.4024124431339033, + "tokens_seen": 2450194432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002601404212637914, + "loss": 1.1339, + "theoretical_loss": 3.4024052157931983, + "tokens_seen": 2450259968 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026012036108324975, + "loss": 1.2779, + "theoretical_loss": 3.4023979886999216, + "tokens_seen": 2450325504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026010030090270817, + "loss": 1.2613, + "theoretical_loss": 3.402390761854057, + "tokens_seen": 2450391040 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026008024072216653, + "loss": 1.127, + "theoretical_loss": 3.402383535255591, + "tokens_seen": 2450456576 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600601805416249, + "loss": 1.3083, + "theoretical_loss": 3.4023763089045076, + "tokens_seen": 2450522112 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026004012036108325, + "loss": 1.3698, + "theoretical_loss": 3.4023690828007918, + "tokens_seen": 2450587648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600200601805416, + "loss": 1.3242, + "theoretical_loss": 3.4023618569444283, + "tokens_seen": 2450653184 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026000000000000003, + "loss": 1.2993, + "theoretical_loss": 3.4023546313354025, + "tokens_seen": 2450718720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599799398194584, + "loss": 1.1712, + "theoretical_loss": 3.4023474059736993, + "tokens_seen": 2450784256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025995987963891676, + "loss": 1.3599, + "theoretical_loss": 3.4023401808593032, + "tokens_seen": 2450849792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599398194583751, + "loss": 1.3202, + "theoretical_loss": 3.4023329559921995, + "tokens_seen": 2450915328 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025991975927783353, + "loss": 1.2785, + "theoretical_loss": 3.4023257313723727, + "tokens_seen": 2450980864 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598996990972919, + "loss": 1.2373, + "theoretical_loss": 3.402318506999808, + "tokens_seen": 2451046400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025987963891675026, + "loss": 1.2973, + "theoretical_loss": 3.402311282874491, + "tokens_seen": 2451111936 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598595787362086, + "loss": 1.2893, + "theoretical_loss": 3.402304058996405, + "tokens_seen": 2451177472 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259839518555667, + "loss": 1.2794, + "theoretical_loss": 3.4022968353655365, + "tokens_seen": 2451243008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598194583751254, + "loss": 1.2109, + "theoretical_loss": 3.4022896119818697, + "tokens_seen": 2451308544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025979939819458376, + "loss": 1.2111, + "theoretical_loss": 3.4022823888453892, + "tokens_seen": 2451374080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597793380140421, + "loss": 1.2783, + "theoretical_loss": 3.402275165956081, + "tokens_seen": 2451439616 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597592778335005, + "loss": 1.2987, + "theoretical_loss": 3.4022679433139293, + "tokens_seen": 2451505152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597392176529589, + "loss": 1.2217, + "theoretical_loss": 3.402260720918919, + "tokens_seen": 2451570688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025971915747241727, + "loss": 1.2945, + "theoretical_loss": 3.402253498771035, + "tokens_seen": 2451636224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025969909729187563, + "loss": 1.3011, + "theoretical_loss": 3.4022462768702626, + "tokens_seen": 2451701760 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259679037111334, + "loss": 1.3744, + "theoretical_loss": 3.4022390552165866, + "tokens_seen": 2451767296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025965897693079235, + "loss": 1.3144, + "theoretical_loss": 3.4022318338099917, + "tokens_seen": 2451832832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025963891675025077, + "loss": 1.0988, + "theoretical_loss": 3.402224612650463, + "tokens_seen": 2451898368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025961885656970913, + "loss": 1.2753, + "theoretical_loss": 3.402217391737986, + "tokens_seen": 2451963904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595987963891675, + "loss": 1.2413, + "theoretical_loss": 3.4022101710725448, + "tokens_seen": 2452029440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025957873620862586, + "loss": 1.214, + "theoretical_loss": 3.4022029506541247, + "tokens_seen": 2452094976 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025955867602808427, + "loss": 1.25, + "theoretical_loss": 3.4021957304827106, + "tokens_seen": 2452160512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025953861584754263, + "loss": 1.1689, + "theoretical_loss": 3.4021885105582874, + "tokens_seen": 2452226048 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259518555667001, + "loss": 1.3016, + "theoretical_loss": 3.4021812908808404, + "tokens_seen": 2452291584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025949849548645936, + "loss": 1.2182, + "theoretical_loss": 3.402174071450354, + "tokens_seen": 2452357120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594784353059178, + "loss": 1.35, + "theoretical_loss": 3.4021668522668134, + "tokens_seen": 2452422656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025945837512537614, + "loss": 1.4057, + "theoretical_loss": 3.4021596333302035, + "tokens_seen": 2452488192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594383149448345, + "loss": 1.2432, + "theoretical_loss": 3.40215241464051, + "tokens_seen": 2452553728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025941825476429286, + "loss": 1.2764, + "theoretical_loss": 3.4021451961977167, + "tokens_seen": 2452619264 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593981945837512, + "loss": 1.1841, + "theoretical_loss": 3.402137978001809, + "tokens_seen": 2452684800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025937813440320964, + "loss": 1.2657, + "theoretical_loss": 3.402130760052772, + "tokens_seen": 2452750336 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 892743, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4029309749603271, + "objective/train/theoretical_loss": 3.4021235423505907, + "objective/train/tokens_used": 823274976, + "theoretical_loss": 3.4021235423505907, + "tokens_seen": 2452815872 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259358074222668, + "loss": 1.3487, + "theoretical_loss": 3.4021235423505907, + "tokens_seen": 2452815872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025933801404212636, + "loss": 1.3066, + "theoretical_loss": 3.4021163248952497, + "tokens_seen": 2452881408 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593179538615847, + "loss": 1.251, + "theoretical_loss": 3.4021091076867345, + "tokens_seen": 2452946944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025929789368104314, + "loss": 1.3149, + "theoretical_loss": 3.4021018907250298, + "tokens_seen": 2453012480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592778335005015, + "loss": 1.2697, + "theoretical_loss": 3.40209467401012, + "tokens_seen": 2453078016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025925777331995987, + "loss": 1.3618, + "theoretical_loss": 3.402087457541991, + "tokens_seen": 2453143552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025923771313941823, + "loss": 1.3523, + "theoretical_loss": 3.4020802413206273, + "tokens_seen": 2453209088 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592176529588766, + "loss": 1.312, + "theoretical_loss": 3.402073025346014, + "tokens_seen": 2453274624 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259197592778335, + "loss": 1.2164, + "theoretical_loss": 3.402065809618136, + "tokens_seen": 2453340160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025917753259779337, + "loss": 1.2917, + "theoretical_loss": 3.402058594136978, + "tokens_seen": 2453405696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025915747241725173, + "loss": 1.1973, + "theoretical_loss": 3.4020513789025255, + "tokens_seen": 2453471232 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591374122367101, + "loss": 1.3843, + "theoretical_loss": 3.402044163914763, + "tokens_seen": 2453536768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591173520561685, + "loss": 1.0975, + "theoretical_loss": 3.4020369491736764, + "tokens_seen": 2453602304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590972918756269, + "loss": 1.3724, + "theoretical_loss": 3.4020297346792496, + "tokens_seen": 2453667840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025907723169508524, + "loss": 1.3255, + "theoretical_loss": 3.4020225204314674, + "tokens_seen": 2453733376 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590571715145436, + "loss": 1.2759, + "theoretical_loss": 3.4020153064303162, + "tokens_seen": 2453798912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025903711133400196, + "loss": 1.1484, + "theoretical_loss": 3.4020080926757794, + "tokens_seen": 2453864448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590170511534604, + "loss": 1.2668, + "theoretical_loss": 3.402000879167843, + "tokens_seen": 2453929984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589969909729188, + "loss": 1.2611, + "theoretical_loss": 3.401993665906492, + "tokens_seen": 2453995520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025897693079237716, + "loss": 1.335, + "theoretical_loss": 3.4019864528917108, + "tokens_seen": 2454061056 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589568706118355, + "loss": 1.2145, + "theoretical_loss": 3.401979240123485, + "tokens_seen": 2454126592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025893681043129393, + "loss": 1.3004, + "theoretical_loss": 3.401972027601799, + "tokens_seen": 2454192128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589167502507523, + "loss": 1.4319, + "theoretical_loss": 3.4019648153266377, + "tokens_seen": 2454257664 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025889669007021066, + "loss": 1.1735, + "theoretical_loss": 3.401957603297987, + "tokens_seen": 2454323200 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258876629889669, + "loss": 1.2995, + "theoretical_loss": 3.401950391515831, + "tokens_seen": 2454388736 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588565697091274, + "loss": 1.3106, + "theoretical_loss": 3.4019431799801554, + "tokens_seen": 2454454272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588365095285858, + "loss": 1.2075, + "theoretical_loss": 3.4019359686909443, + "tokens_seen": 2454519808 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025881644934804416, + "loss": 1.3379, + "theoretical_loss": 3.4019287576481836, + "tokens_seen": 2454585344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587963891675025, + "loss": 1.3133, + "theoretical_loss": 3.4019215468518578, + "tokens_seen": 2454650880 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587763289869609, + "loss": 1.2299, + "theoretical_loss": 3.4019143363019517, + "tokens_seen": 2454716416 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587562688064193, + "loss": 1.4027, + "theoretical_loss": 3.401907125998451, + "tokens_seen": 2454781952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025873620862587767, + "loss": 1.3717, + "theoretical_loss": 3.4018999159413403, + "tokens_seen": 2454847488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025871614844533603, + "loss": 1.1951, + "theoretical_loss": 3.4018927061306043, + "tokens_seen": 2454913024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586960882647944, + "loss": 1.316, + "theoretical_loss": 3.4018854965662286, + "tokens_seen": 2454978560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025867602808425275, + "loss": 1.187, + "theoretical_loss": 3.401878287248198, + "tokens_seen": 2455044096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025865596790371117, + "loss": 1.3822, + "theoretical_loss": 3.401871078176497, + "tokens_seen": 2455109632 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025863590772316953, + "loss": 1.3052, + "theoretical_loss": 3.4018638693511116, + "tokens_seen": 2455175168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586158475426279, + "loss": 1.4023, + "theoretical_loss": 3.401856660772026, + "tokens_seen": 2455240704 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025859578736208626, + "loss": 1.2247, + "theoretical_loss": 3.4018494524392255, + "tokens_seen": 2455306240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025857572718154467, + "loss": 1.2596, + "theoretical_loss": 3.401842244352695, + "tokens_seen": 2455371776 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025855566700100303, + "loss": 1.3451, + "theoretical_loss": 3.40183503651242, + "tokens_seen": 2455437312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585356068204614, + "loss": 1.3343, + "theoretical_loss": 3.4018278289183845, + "tokens_seen": 2455502848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025851554663991976, + "loss": 1.2938, + "theoretical_loss": 3.4018206215705744, + "tokens_seen": 2455568384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584954864593782, + "loss": 1.2978, + "theoretical_loss": 3.4018134144689745, + "tokens_seen": 2455633920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025847542627883654, + "loss": 1.2582, + "theoretical_loss": 3.4018062076135696, + "tokens_seen": 2455699456 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584553660982949, + "loss": 1.1933, + "theoretical_loss": 3.4017990010043446, + "tokens_seen": 2455764992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025843530591775326, + "loss": 1.3183, + "theoretical_loss": 3.4017917946412854, + "tokens_seen": 2455830528 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584152457372116, + "loss": 1.2937, + "theoretical_loss": 3.401784588524376, + "tokens_seen": 2455896064 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025839518555667004, + "loss": 1.2139, + "theoretical_loss": 3.401777382653602, + "tokens_seen": 2455961600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583751253761284, + "loss": 1.2555, + "theoretical_loss": 3.4017701770289483, + "tokens_seen": 2456027136 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 893908, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1507830619812012, + "objective/train/theoretical_loss": 3.4017629716504, + "objective/train/tokens_used": 826551776, + "theoretical_loss": 3.4017629716504, + "tokens_seen": 2456092672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025835506519558677, + "loss": 1.1521, + "theoretical_loss": 3.4017629716504, + "tokens_seen": 2456092672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025833500501504513, + "loss": 1.3059, + "theoretical_loss": 3.4017557665179416, + "tokens_seen": 2456158208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025831494483450354, + "loss": 1.2003, + "theoretical_loss": 3.401748561631559, + "tokens_seen": 2456223744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582948846539619, + "loss": 1.3157, + "theoretical_loss": 3.4017413569912365, + "tokens_seen": 2456289280 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025827482447342027, + "loss": 1.1688, + "theoretical_loss": 3.4017341525969598, + "tokens_seen": 2456354816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025825476429287863, + "loss": 1.331, + "theoretical_loss": 3.401726948448713, + "tokens_seen": 2456420352 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258234704112337, + "loss": 1.2889, + "theoretical_loss": 3.401719744546482, + "tokens_seen": 2456485888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582146439317954, + "loss": 1.3216, + "theoretical_loss": 3.4017125408902515, + "tokens_seen": 2456551424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025819458375125377, + "loss": 1.3415, + "theoretical_loss": 3.4017053374800064, + "tokens_seen": 2456616960 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025817452357071213, + "loss": 1.3383, + "theoretical_loss": 3.4016981343157324, + "tokens_seen": 2456682496 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581544633901705, + "loss": 1.1546, + "theoretical_loss": 3.4016909313974133, + "tokens_seen": 2456748032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581344032096289, + "loss": 1.3471, + "theoretical_loss": 3.401683728725035, + "tokens_seen": 2456813568 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581143430290873, + "loss": 1.1808, + "theoretical_loss": 3.401676526298583, + "tokens_seen": 2456879104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025809428284854564, + "loss": 1.2624, + "theoretical_loss": 3.4016693241180413, + "tokens_seen": 2456944640 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258074222668004, + "loss": 1.3343, + "theoretical_loss": 3.401662122183396, + "tokens_seen": 2457010176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025805416248746236, + "loss": 1.2565, + "theoretical_loss": 3.4016549204946305, + "tokens_seen": 2457075712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002580341023069208, + "loss": 1.2243, + "theoretical_loss": 3.4016477190517316, + "tokens_seen": 2457141248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025801404212637914, + "loss": 1.3084, + "theoretical_loss": 3.4016405178546836, + "tokens_seen": 2457206784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579939819458375, + "loss": 1.326, + "theoretical_loss": 3.4016333169034714, + "tokens_seen": 2457272320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025797392176529586, + "loss": 1.3367, + "theoretical_loss": 3.4016261161980803, + "tokens_seen": 2457337856 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579538615847543, + "loss": 1.2767, + "theoretical_loss": 3.4016189157384953, + "tokens_seen": 2457403392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025793380140421264, + "loss": 1.394, + "theoretical_loss": 3.4016117155247017, + "tokens_seen": 2457468928 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257913741223671, + "loss": 1.3321, + "theoretical_loss": 3.4016045155566843, + "tokens_seen": 2457534464 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025789368104312937, + "loss": 1.2814, + "theoretical_loss": 3.4015973158344277, + "tokens_seen": 2457600000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025787362086258773, + "loss": 1.3176, + "theoretical_loss": 3.4015901163579176, + "tokens_seen": 2457665536 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025785356068204615, + "loss": 1.1297, + "theoretical_loss": 3.401582917127139, + "tokens_seen": 2457731072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578335005015045, + "loss": 1.3789, + "theoretical_loss": 3.401575718142077, + "tokens_seen": 2457796608 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025781344032096287, + "loss": 1.21, + "theoretical_loss": 3.4015685194027165, + "tokens_seen": 2457862144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025779338014042123, + "loss": 1.3223, + "theoretical_loss": 3.4015613209090425, + "tokens_seen": 2457927680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025777331995987965, + "loss": 1.417, + "theoretical_loss": 3.40155412266104, + "tokens_seen": 2457993216 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257753259779338, + "loss": 1.082, + "theoretical_loss": 3.4015469246586942, + "tokens_seen": 2458058752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577331995987964, + "loss": 1.2411, + "theoretical_loss": 3.4015397269019902, + "tokens_seen": 2458124288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025771313941825474, + "loss": 1.2932, + "theoretical_loss": 3.401532529390913, + "tokens_seen": 2458189824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576930792377131, + "loss": 1.1867, + "theoretical_loss": 3.401525332125448, + "tokens_seen": 2458255360 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576730190571715, + "loss": 1.3752, + "theoretical_loss": 3.4015181351055794, + "tokens_seen": 2458320896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576529588766299, + "loss": 1.2384, + "theoretical_loss": 3.4015109383312936, + "tokens_seen": 2458386432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025763289869608824, + "loss": 1.2866, + "theoretical_loss": 3.4015037418025744, + "tokens_seen": 2458451968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576128385155466, + "loss": 1.3302, + "theoretical_loss": 3.4014965455194073, + "tokens_seen": 2458517504 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257592778335005, + "loss": 1.2366, + "theoretical_loss": 3.401489349481778, + "tokens_seen": 2458583040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575727181544634, + "loss": 1.2519, + "theoretical_loss": 3.4014821536896704, + "tokens_seen": 2458648576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025755265797392174, + "loss": 1.2704, + "theoretical_loss": 3.401474958143071, + "tokens_seen": 2458714112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575325977933801, + "loss": 1.2832, + "theoretical_loss": 3.401467762841963, + "tokens_seen": 2458779648 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025751253761283847, + "loss": 1.2439, + "theoretical_loss": 3.4014605677863337, + "tokens_seen": 2458845184 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025749247743229694, + "loss": 1.2907, + "theoretical_loss": 3.4014533729761665, + "tokens_seen": 2458910720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574724172517553, + "loss": 1.2892, + "theoretical_loss": 3.4014461784114474, + "tokens_seen": 2458976256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025745235707121366, + "loss": 1.3071, + "theoretical_loss": 3.401438984092161, + "tokens_seen": 2459041792 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257432296890672, + "loss": 1.3161, + "theoretical_loss": 3.401431790018292, + "tokens_seen": 2459107328 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025741223671013044, + "loss": 1.3173, + "theoretical_loss": 3.401424596189827, + "tokens_seen": 2459172864 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573921765295888, + "loss": 1.3224, + "theoretical_loss": 3.4014174026067496, + "tokens_seen": 2459238400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025737211634904717, + "loss": 1.2555, + "theoretical_loss": 3.401410209269045, + "tokens_seen": 2459303936 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 895121, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0762553215026855, + "objective/train/theoretical_loss": 3.4014030161766993, + "objective/train/tokens_used": 829828576, + "theoretical_loss": 3.4014030161766993, + "tokens_seen": 2459369472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025735205616850553, + "loss": 1.1005, + "theoretical_loss": 3.4014030161766993, + "tokens_seen": 2459369472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025733199598796394, + "loss": 1.1974, + "theoretical_loss": 3.401395823329697, + "tokens_seen": 2459435008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573119358074223, + "loss": 1.3607, + "theoretical_loss": 3.401388630728023, + "tokens_seen": 2459500544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025729187562688067, + "loss": 1.2499, + "theoretical_loss": 3.4013814383716623, + "tokens_seen": 2459566080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025727181544633903, + "loss": 1.2714, + "theoretical_loss": 3.401374246260601, + "tokens_seen": 2459631616 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572517552657974, + "loss": 1.2088, + "theoretical_loss": 3.4013670543948225, + "tokens_seen": 2459697152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572316950852558, + "loss": 1.2267, + "theoretical_loss": 3.4013598627743136, + "tokens_seen": 2459762688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025721163490471417, + "loss": 1.2654, + "theoretical_loss": 3.4013526713990583, + "tokens_seen": 2459828224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025719157472417253, + "loss": 1.2365, + "theoretical_loss": 3.4013454802690424, + "tokens_seen": 2459893760 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571715145436309, + "loss": 1.2702, + "theoretical_loss": 3.4013382893842503, + "tokens_seen": 2459959296 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571514543630893, + "loss": 1.2643, + "theoretical_loss": 3.401331098744668, + "tokens_seen": 2460024832 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571313941825477, + "loss": 1.3124, + "theoretical_loss": 3.4013239083502795, + "tokens_seen": 2460090368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025711133400200604, + "loss": 1.3635, + "theoretical_loss": 3.4013167182010706, + "tokens_seen": 2460155904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570912738214644, + "loss": 1.1712, + "theoretical_loss": 3.4013095282970265, + "tokens_seen": 2460221440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025707121364092276, + "loss": 1.225, + "theoretical_loss": 3.401302338638132, + "tokens_seen": 2460286976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570511534603812, + "loss": 1.1791, + "theoretical_loss": 3.4012951492243726, + "tokens_seen": 2460352512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025703109327983954, + "loss": 1.245, + "theoretical_loss": 3.4012879600557326, + "tokens_seen": 2460418048 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570110330992979, + "loss": 1.3002, + "theoretical_loss": 3.4012807711321984, + "tokens_seen": 2460483584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025699097291875626, + "loss": 1.3345, + "theoretical_loss": 3.4012735824537534, + "tokens_seen": 2460549120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569709127382147, + "loss": 1.266, + "theoretical_loss": 3.401266394020385, + "tokens_seen": 2460614656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025695085255767304, + "loss": 1.1927, + "theoretical_loss": 3.401259205832076, + "tokens_seen": 2460680192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569307923771314, + "loss": 1.2874, + "theoretical_loss": 3.4012520178888126, + "tokens_seen": 2460745728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025691073219658977, + "loss": 1.2789, + "theoretical_loss": 3.40124483019058, + "tokens_seen": 2460811264 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025689067201604813, + "loss": 1.3014, + "theoretical_loss": 3.401237642737363, + "tokens_seen": 2460876800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025687061183550655, + "loss": 1.237, + "theoretical_loss": 3.4012304555291473, + "tokens_seen": 2460942336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568505516549649, + "loss": 1.3844, + "theoretical_loss": 3.4012232685659174, + "tokens_seen": 2461007872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025683049147442327, + "loss": 1.2603, + "theoretical_loss": 3.4012160818476587, + "tokens_seen": 2461073408 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025681043129388163, + "loss": 1.415, + "theoretical_loss": 3.401208895374356, + "tokens_seen": 2461138944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025679037111334005, + "loss": 1.2863, + "theoretical_loss": 3.4012017091459947, + "tokens_seen": 2461204480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567703109327984, + "loss": 1.2419, + "theoretical_loss": 3.40119452316256, + "tokens_seen": 2461270016 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567502507522568, + "loss": 1.2632, + "theoretical_loss": 3.4011873374240373, + "tokens_seen": 2461335552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025673019057171514, + "loss": 1.2436, + "theoretical_loss": 3.401180151930411, + "tokens_seen": 2461401088 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567101303911735, + "loss": 1.2178, + "theoretical_loss": 3.4011729666816666, + "tokens_seen": 2461466624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566900702106319, + "loss": 1.3521, + "theoretical_loss": 3.4011657816777894, + "tokens_seen": 2461532160 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566700100300903, + "loss": 1.2555, + "theoretical_loss": 3.4011585969187648, + "tokens_seen": 2461597696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025664994984954864, + "loss": 1.1631, + "theoretical_loss": 3.4011514124045767, + "tokens_seen": 2461663232 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256629889669007, + "loss": 1.3963, + "theoretical_loss": 3.4011442281352116, + "tokens_seen": 2461728768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566098294884654, + "loss": 1.2355, + "theoretical_loss": 3.401137044110654, + "tokens_seen": 2461794304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565897693079238, + "loss": 1.2411, + "theoretical_loss": 3.4011298603308893, + "tokens_seen": 2461859840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025656970912738214, + "loss": 1.2878, + "theoretical_loss": 3.401122676795902, + "tokens_seen": 2461925376 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565496489468405, + "loss": 1.2499, + "theoretical_loss": 3.401115493505678, + "tokens_seen": 2461990912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025652958876629887, + "loss": 1.382, + "theoretical_loss": 3.4011083104602022, + "tokens_seen": 2462056448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565095285857573, + "loss": 1.2319, + "theoretical_loss": 3.40110112765946, + "tokens_seen": 2462121984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025648946840521565, + "loss": 1.2203, + "theoretical_loss": 3.401093945103436, + "tokens_seen": 2462187520 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256469408224674, + "loss": 1.2867, + "theoretical_loss": 3.4010867627921155, + "tokens_seen": 2462253056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025644934804413237, + "loss": 1.2172, + "theoretical_loss": 3.4010795807254843, + "tokens_seen": 2462318592 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564292878635908, + "loss": 1.3653, + "theoretical_loss": 3.4010723989035263, + "tokens_seen": 2462384128 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025640922768304915, + "loss": 1.3665, + "theoretical_loss": 3.401065217326228, + "tokens_seen": 2462449664 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563891675025075, + "loss": 1.2652, + "theoretical_loss": 3.4010580359935734, + "tokens_seen": 2462515200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563691073219659, + "loss": 1.1988, + "theoretical_loss": 3.4010508549055487, + "tokens_seen": 2462580736 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 896263, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3929803371429443, + "objective/train/theoretical_loss": 3.401043674062138, + "objective/train/tokens_used": 833105376, + "theoretical_loss": 3.401043674062138, + "tokens_seen": 2462646272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025634904714142424, + "loss": 1.3371, + "theoretical_loss": 3.401043674062138, + "tokens_seen": 2462646272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025632898696088265, + "loss": 1.1339, + "theoretical_loss": 3.4010364934633275, + "tokens_seen": 2462711808 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256308926780341, + "loss": 1.267, + "theoretical_loss": 3.4010293131091016, + "tokens_seen": 2462777344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562888665997994, + "loss": 1.3505, + "theoretical_loss": 3.401022132999446, + "tokens_seen": 2462842880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025626880641925774, + "loss": 1.292, + "theoretical_loss": 3.401014953134345, + "tokens_seen": 2462908416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025624874623871616, + "loss": 1.2029, + "theoretical_loss": 3.401007773513785, + "tokens_seen": 2462973952 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562286860581745, + "loss": 1.1188, + "theoretical_loss": 3.4010005941377504, + "tokens_seen": 2463039488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562086258776329, + "loss": 1.3494, + "theoretical_loss": 3.400993415006226, + "tokens_seen": 2463105024 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025618856569709124, + "loss": 1.3765, + "theoretical_loss": 3.4009862361191976, + "tokens_seen": 2463170560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025616850551654966, + "loss": 1.2538, + "theoretical_loss": 3.4009790574766505, + "tokens_seen": 2463236096 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256148445336008, + "loss": 1.3168, + "theoretical_loss": 3.4009718790785692, + "tokens_seen": 2463301632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561283851554664, + "loss": 1.2821, + "theoretical_loss": 3.40096470092494, + "tokens_seen": 2463367168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025610832497492475, + "loss": 1.1803, + "theoretical_loss": 3.4009575230157467, + "tokens_seen": 2463432704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560882647943831, + "loss": 1.2587, + "theoretical_loss": 3.4009503453509753, + "tokens_seen": 2463498240 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560682046138415, + "loss": 1.2135, + "theoretical_loss": 3.4009431679306106, + "tokens_seen": 2463563776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560481444332999, + "loss": 1.2501, + "theoretical_loss": 3.400935990754638, + "tokens_seen": 2463629312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025602808425275825, + "loss": 1.3872, + "theoretical_loss": 3.4009288138230427, + "tokens_seen": 2463694848 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560080240722166, + "loss": 1.288, + "theoretical_loss": 3.4009216371358097, + "tokens_seen": 2463760384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559879638916751, + "loss": 1.1709, + "theoretical_loss": 3.4009144606929245, + "tokens_seen": 2463825920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025596790371113344, + "loss": 1.2113, + "theoretical_loss": 3.400907284494372, + "tokens_seen": 2463891456 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559478435305918, + "loss": 1.2224, + "theoretical_loss": 3.4009001085401374, + "tokens_seen": 2463956992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025592778335005017, + "loss": 1.2911, + "theoretical_loss": 3.4008929328302058, + "tokens_seen": 2464022528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025590772316950853, + "loss": 1.3853, + "theoretical_loss": 3.400885757364563, + "tokens_seen": 2464088064 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025588766298896695, + "loss": 1.2285, + "theoretical_loss": 3.4008785821431933, + "tokens_seen": 2464153600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558676028084253, + "loss": 1.2755, + "theoretical_loss": 3.400871407166082, + "tokens_seen": 2464219136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025584754262788367, + "loss": 1.2036, + "theoretical_loss": 3.4008642324332152, + "tokens_seen": 2464284672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025582748244734203, + "loss": 1.3086, + "theoretical_loss": 3.4008570579445774, + "tokens_seen": 2464350208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025580742226680045, + "loss": 1.2557, + "theoretical_loss": 3.4008498837001535, + "tokens_seen": 2464415744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557873620862588, + "loss": 1.3022, + "theoretical_loss": 3.4008427096999294, + "tokens_seen": 2464481280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557673019057172, + "loss": 1.2379, + "theoretical_loss": 3.40083553594389, + "tokens_seen": 2464546816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025574724172517554, + "loss": 1.3034, + "theoretical_loss": 3.40082836243202, + "tokens_seen": 2464612352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557271815446339, + "loss": 1.2848, + "theoretical_loss": 3.400821189164305, + "tokens_seen": 2464677888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557071213640923, + "loss": 1.2275, + "theoretical_loss": 3.400814016140731, + "tokens_seen": 2464743424 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556870611835507, + "loss": 1.2393, + "theoretical_loss": 3.4008068433612824, + "tokens_seen": 2464808960 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025566700100300904, + "loss": 1.2202, + "theoretical_loss": 3.400799670825944, + "tokens_seen": 2464874496 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556469408224674, + "loss": 1.3058, + "theoretical_loss": 3.4007924985347016, + "tokens_seen": 2464940032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556268806419258, + "loss": 1.3109, + "theoretical_loss": 3.40078532648754, + "tokens_seen": 2465005568 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556068204613842, + "loss": 1.2705, + "theoretical_loss": 3.4007781546844447, + "tokens_seen": 2465071104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025558676028084254, + "loss": 1.2203, + "theoretical_loss": 3.400770983125401, + "tokens_seen": 2465136640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555667001003009, + "loss": 1.4566, + "theoretical_loss": 3.400763811810394, + "tokens_seen": 2465202176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025554663991975927, + "loss": 1.1912, + "theoretical_loss": 3.400756640739409, + "tokens_seen": 2465267712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555265797392177, + "loss": 1.3859, + "theoretical_loss": 3.400749469912431, + "tokens_seen": 2465333248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025550651955867605, + "loss": 1.2354, + "theoretical_loss": 3.400742299329445, + "tokens_seen": 2465398784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554864593781344, + "loss": 1.2108, + "theoretical_loss": 3.400735128990437, + "tokens_seen": 2465464320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025546639919759277, + "loss": 1.1971, + "theoretical_loss": 3.4007279588953914, + "tokens_seen": 2465529856 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554463390170512, + "loss": 1.3361, + "theoretical_loss": 3.4007207890442936, + "tokens_seen": 2465595392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025542627883650955, + "loss": 1.2921, + "theoretical_loss": 3.4007136194371292, + "tokens_seen": 2465660928 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554062186559679, + "loss": 1.3289, + "theoretical_loss": 3.4007064500738835, + "tokens_seen": 2465726464 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553861584754263, + "loss": 1.288, + "theoretical_loss": 3.400699280954541, + "tokens_seen": 2465792000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025536609829488464, + "loss": 1.2986, + "theoretical_loss": 3.400692112079087, + "tokens_seen": 2465857536 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 897133, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2484326362609863, + "objective/train/theoretical_loss": 3.4006849434475077, + "objective/train/tokens_used": 836382176, + "theoretical_loss": 3.4006849434475077, + "tokens_seen": 2465923072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025534603811434305, + "loss": 1.2703, + "theoretical_loss": 3.4006849434475077, + "tokens_seen": 2465923072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553259779338014, + "loss": 1.2842, + "theoretical_loss": 3.4006777750597874, + "tokens_seen": 2465988608 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553059177532598, + "loss": 1.3129, + "theoretical_loss": 3.4006706069159116, + "tokens_seen": 2466054144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025528585757271814, + "loss": 1.3345, + "theoretical_loss": 3.400663439015865, + "tokens_seen": 2466119680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025526579739217656, + "loss": 1.245, + "theoretical_loss": 3.400656271359634, + "tokens_seen": 2466185216 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552457372116349, + "loss": 1.3945, + "theoretical_loss": 3.4006491039472033, + "tokens_seen": 2466250752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552256770310933, + "loss": 1.3552, + "theoretical_loss": 3.4006419367785576, + "tokens_seen": 2466316288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025520561685055164, + "loss": 1.1685, + "theoretical_loss": 3.4006347698536823, + "tokens_seen": 2466381824 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025518555667001006, + "loss": 1.2597, + "theoretical_loss": 3.4006276031725635, + "tokens_seen": 2466447360 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551654964894684, + "loss": 1.28, + "theoretical_loss": 3.4006204367351853, + "tokens_seen": 2466512896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551454363089268, + "loss": 1.2678, + "theoretical_loss": 3.4006132705415335, + "tokens_seen": 2466578432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025512537612838515, + "loss": 1.2468, + "theoretical_loss": 3.4006061045915934, + "tokens_seen": 2466643968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551053159478435, + "loss": 1.2012, + "theoretical_loss": 3.40059893888535, + "tokens_seen": 2466709504 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550852557673019, + "loss": 1.2066, + "theoretical_loss": 3.4005917734227884, + "tokens_seen": 2466775040 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550651955867603, + "loss": 1.2797, + "theoretical_loss": 3.4005846082038946, + "tokens_seen": 2466840576 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025504513540621865, + "loss": 1.3193, + "theoretical_loss": 3.400577443228653, + "tokens_seen": 2466906112 + }, + { + "epoch": 0.5, + "learning_rate": 0.000255025075225677, + "loss": 1.427, + "theoretical_loss": 3.400570278497049, + "tokens_seen": 2466971648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025500501504513543, + "loss": 1.2894, + "theoretical_loss": 3.400563114009068, + "tokens_seen": 2467037184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549849548645938, + "loss": 1.3065, + "theoretical_loss": 3.4005559497646956, + "tokens_seen": 2467102720 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025496489468405215, + "loss": 1.2431, + "theoretical_loss": 3.4005487857639167, + "tokens_seen": 2467168256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549448345035105, + "loss": 1.351, + "theoretical_loss": 3.400541622006716, + "tokens_seen": 2467233792 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549247743229689, + "loss": 1.3369, + "theoretical_loss": 3.40053445849308, + "tokens_seen": 2467299328 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549047141424273, + "loss": 1.1883, + "theoretical_loss": 3.4005272952229926, + "tokens_seen": 2467364864 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025488465396188566, + "loss": 1.2606, + "theoretical_loss": 3.40052013219644, + "tokens_seen": 2467430400 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254864593781344, + "loss": 1.1122, + "theoretical_loss": 3.4005129694134073, + "tokens_seen": 2467495936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548445336008024, + "loss": 1.2738, + "theoretical_loss": 3.400505806873879, + "tokens_seen": 2467561472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548244734202608, + "loss": 1.3744, + "theoretical_loss": 3.4004986445778416, + "tokens_seen": 2467627008 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025480441323971916, + "loss": 1.265, + "theoretical_loss": 3.400491482525279, + "tokens_seen": 2467692544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547843530591775, + "loss": 1.2596, + "theoretical_loss": 3.4004843207161777, + "tokens_seen": 2467758080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547642928786359, + "loss": 1.3174, + "theoretical_loss": 3.4004771591505225, + "tokens_seen": 2467823616 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025474423269809425, + "loss": 1.2611, + "theoretical_loss": 3.4004699978282984, + "tokens_seen": 2467889152 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025472417251755266, + "loss": 1.4096, + "theoretical_loss": 3.4004628367494907, + "tokens_seen": 2467954688 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254704112337011, + "loss": 1.3379, + "theoretical_loss": 3.400455675914085, + "tokens_seen": 2468020224 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546840521564694, + "loss": 1.2788, + "theoretical_loss": 3.4004485153220663, + "tokens_seen": 2468085760 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025466399197592775, + "loss": 1.2646, + "theoretical_loss": 3.40044135497342, + "tokens_seen": 2468151296 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025464393179538616, + "loss": 1.3432, + "theoretical_loss": 3.400434194868131, + "tokens_seen": 2468216832 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025462387161484453, + "loss": 1.2121, + "theoretical_loss": 3.400427035006185, + "tokens_seen": 2468282368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546038114343029, + "loss": 1.3065, + "theoretical_loss": 3.400419875387567, + "tokens_seen": 2468347904 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025458375125376125, + "loss": 1.3534, + "theoretical_loss": 3.400412716012263, + "tokens_seen": 2468413440 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545636910732196, + "loss": 1.299, + "theoretical_loss": 3.400405556880257, + "tokens_seen": 2468478976 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025454363089267803, + "loss": 1.2512, + "theoretical_loss": 3.4003983979915358, + "tokens_seen": 2468544512 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545235707121364, + "loss": 1.3126, + "theoretical_loss": 3.4003912393460833, + "tokens_seen": 2468610048 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545035105315948, + "loss": 1.2881, + "theoretical_loss": 3.4003840809438852, + "tokens_seen": 2468675584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025448345035105317, + "loss": 1.3258, + "theoretical_loss": 3.4003769227849268, + "tokens_seen": 2468741120 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544633901705116, + "loss": 1.3368, + "theoretical_loss": 3.4003697648691937, + "tokens_seen": 2468806656 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025444332998996995, + "loss": 1.2432, + "theoretical_loss": 3.4003626071966706, + "tokens_seen": 2468872192 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544232698094283, + "loss": 1.3649, + "theoretical_loss": 3.4003554497673436, + "tokens_seen": 2468937728 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544032096288867, + "loss": 1.2638, + "theoretical_loss": 3.400348292581197, + "tokens_seen": 2469003264 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025438314944834504, + "loss": 1.1754, + "theoretical_loss": 3.400341135638217, + "tokens_seen": 2469068800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025436308926780345, + "loss": 1.3857, + "theoretical_loss": 3.400333978938388, + "tokens_seen": 2469134336 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 898257, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.171244502067566, + "objective/train/theoretical_loss": 3.4003268224816963, + "objective/train/tokens_used": 839658976, + "theoretical_loss": 3.4003268224816963, + "tokens_seen": 2469199872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543430290872618, + "loss": 1.2643, + "theoretical_loss": 3.4003268224816963, + "tokens_seen": 2469199872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543229689067202, + "loss": 1.2239, + "theoretical_loss": 3.4003196662681265, + "tokens_seen": 2469265408 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025430290872617854, + "loss": 1.247, + "theoretical_loss": 3.4003125102976637, + "tokens_seen": 2469330944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025428284854563696, + "loss": 1.22, + "theoretical_loss": 3.400305354570294, + "tokens_seen": 2469396480 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542627883650953, + "loss": 1.3195, + "theoretical_loss": 3.4002981990860017, + "tokens_seen": 2469462016 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542427281845537, + "loss": 1.305, + "theoretical_loss": 3.400291043844773, + "tokens_seen": 2469527552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025422266800401204, + "loss": 1.3304, + "theoretical_loss": 3.4002838888465927, + "tokens_seen": 2469593088 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025420260782347046, + "loss": 1.3272, + "theoretical_loss": 3.4002767340914457, + "tokens_seen": 2469658624 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541825476429288, + "loss": 1.3424, + "theoretical_loss": 3.4002695795793185, + "tokens_seen": 2469724160 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541624874623872, + "loss": 1.1842, + "theoretical_loss": 3.400262425310195, + "tokens_seen": 2469789696 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025414242728184555, + "loss": 1.2724, + "theoretical_loss": 3.4002552712840615, + "tokens_seen": 2469855232 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541223671013039, + "loss": 1.2416, + "theoretical_loss": 3.400248117500903, + "tokens_seen": 2469920768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541023069207623, + "loss": 1.363, + "theoretical_loss": 3.400240963960705, + "tokens_seen": 2469986304 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540822467402207, + "loss": 1.3486, + "theoretical_loss": 3.4002338106634524, + "tokens_seen": 2470051840 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025406218655967905, + "loss": 1.447, + "theoretical_loss": 3.4002266576091307, + "tokens_seen": 2470117376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540421263791374, + "loss": 1.3055, + "theoretical_loss": 3.4002195047977253, + "tokens_seen": 2470182912 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025402206619859583, + "loss": 1.2536, + "theoretical_loss": 3.400212352229221, + "tokens_seen": 2470248448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540020060180542, + "loss": 1.4328, + "theoretical_loss": 3.4002051999036036, + "tokens_seen": 2470313984 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025398194583751255, + "loss": 1.4135, + "theoretical_loss": 3.400198047820859, + "tokens_seen": 2470379520 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539618856569709, + "loss": 1.3562, + "theoretical_loss": 3.400190895980971, + "tokens_seen": 2470445056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539418254764293, + "loss": 1.3296, + "theoretical_loss": 3.4001837443839262, + "tokens_seen": 2470510592 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539217652958877, + "loss": 1.3178, + "theoretical_loss": 3.400176593029709, + "tokens_seen": 2470576128 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025390170511534606, + "loss": 1.3027, + "theoretical_loss": 3.4001694419183055, + "tokens_seen": 2470641664 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538816449348044, + "loss": 1.3103, + "theoretical_loss": 3.4001622910497007, + "tokens_seen": 2470707200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538615847542628, + "loss": 1.4414, + "theoretical_loss": 3.4001551404238795, + "tokens_seen": 2470772736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538415245737212, + "loss": 1.2714, + "theoretical_loss": 3.400147990040828, + "tokens_seen": 2470838272 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025382146439317956, + "loss": 1.2991, + "theoretical_loss": 3.400140839900531, + "tokens_seen": 2470903808 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538014042126379, + "loss": 1.3553, + "theoretical_loss": 3.400133690002974, + "tokens_seen": 2470969344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537813440320963, + "loss": 1.3477, + "theoretical_loss": 3.4001265403481424, + "tokens_seen": 2471034880 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025376128385155465, + "loss": 1.285, + "theoretical_loss": 3.400119390936021, + "tokens_seen": 2471100416 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025374122367101306, + "loss": 1.2024, + "theoretical_loss": 3.4001122417665957, + "tokens_seen": 2471165952 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537211634904714, + "loss": 1.3761, + "theoretical_loss": 3.4001050928398517, + "tokens_seen": 2471231488 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537011033099298, + "loss": 1.3542, + "theoretical_loss": 3.400097944155774, + "tokens_seen": 2471297024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025368104312938815, + "loss": 1.1995, + "theoretical_loss": 3.4000907957143487, + "tokens_seen": 2471362560 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025366098294884657, + "loss": 1.3057, + "theoretical_loss": 3.4000836475155602, + "tokens_seen": 2471428096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025364092276830493, + "loss": 1.3117, + "theoretical_loss": 3.4000764995593946, + "tokens_seen": 2471493632 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536208625877633, + "loss": 1.2502, + "theoretical_loss": 3.4000693518458367, + "tokens_seen": 2471559168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025360080240722165, + "loss": 1.2898, + "theoretical_loss": 3.400062204374872, + "tokens_seen": 2471624704 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025358074222668, + "loss": 1.3149, + "theoretical_loss": 3.400055057146486, + "tokens_seen": 2471690240 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025356068204613843, + "loss": 1.3647, + "theoretical_loss": 3.4000479101606635, + "tokens_seen": 2471755776 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535406218655968, + "loss": 1.2987, + "theoretical_loss": 3.4000407634173904, + "tokens_seen": 2471821312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025352056168505516, + "loss": 1.2208, + "theoretical_loss": 3.400033616916652, + "tokens_seen": 2471886848 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535005015045135, + "loss": 1.2578, + "theoretical_loss": 3.4000264706584336, + "tokens_seen": 2471952384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025348044132397193, + "loss": 1.3675, + "theoretical_loss": 3.40001932464272, + "tokens_seen": 2472017920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534603811434303, + "loss": 1.3747, + "theoretical_loss": 3.4000121788694977, + "tokens_seen": 2472083456 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025344032096288866, + "loss": 1.1592, + "theoretical_loss": 3.4000050333387506, + "tokens_seen": 2472148992 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253420260782347, + "loss": 1.2827, + "theoretical_loss": 3.399997888050465, + "tokens_seen": 2472214528 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534002006018054, + "loss": 1.3685, + "theoretical_loss": 3.3999907430046266, + "tokens_seen": 2472280064 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533801404212638, + "loss": 1.4195, + "theoretical_loss": 3.3999835982012194, + "tokens_seen": 2472345600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025336008024072216, + "loss": 1.3339, + "theoretical_loss": 3.39997645364023, + "tokens_seen": 2472411136 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 899541, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3970298767089844, + "objective/train/theoretical_loss": 3.399969309321643, + "objective/train/tokens_used": 842935776, + "theoretical_loss": 3.399969309321643, + "tokens_seen": 2472476672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533400200601805, + "loss": 1.3013, + "theoretical_loss": 3.399969309321643, + "tokens_seen": 2472476672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533199598796389, + "loss": 1.1648, + "theoretical_loss": 3.399962165245444, + "tokens_seen": 2472542208 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532998996990973, + "loss": 1.1538, + "theoretical_loss": 3.3999550214116185, + "tokens_seen": 2472607744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025327983951855566, + "loss": 1.2646, + "theoretical_loss": 3.3999478778201517, + "tokens_seen": 2472673280 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253259779338014, + "loss": 1.4066, + "theoretical_loss": 3.399940734471029, + "tokens_seen": 2472738816 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532397191574724, + "loss": 1.4237, + "theoretical_loss": 3.3999335913642357, + "tokens_seen": 2472804352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025321965897693075, + "loss": 1.4168, + "theoretical_loss": 3.399926448499757, + "tokens_seen": 2472869888 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025319959879638917, + "loss": 1.3408, + "theoretical_loss": 3.3999193058775785, + "tokens_seen": 2472935424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025317953861584753, + "loss": 1.233, + "theoretical_loss": 3.3999121634976857, + "tokens_seen": 2473000960 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002531594784353059, + "loss": 1.2979, + "theoretical_loss": 3.3999050213600635, + "tokens_seen": 2473066496 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025313941825476425, + "loss": 1.347, + "theoretical_loss": 3.3998978794646977, + "tokens_seen": 2473132032 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025311935807422267, + "loss": 1.3366, + "theoretical_loss": 3.3998907378115737, + "tokens_seen": 2473197568 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025309929789368103, + "loss": 1.3087, + "theoretical_loss": 3.3998835964006764, + "tokens_seen": 2473263104 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530792377131394, + "loss": 1.2667, + "theoretical_loss": 3.3998764552319916, + "tokens_seen": 2473328640 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025305917753259776, + "loss": 1.307, + "theoretical_loss": 3.399869314305504, + "tokens_seen": 2473394176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530391173520562, + "loss": 1.2641, + "theoretical_loss": 3.3998621736212002, + "tokens_seen": 2473459712 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025301905717151454, + "loss": 1.3426, + "theoretical_loss": 3.3998550331790645, + "tokens_seen": 2473525248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025299899699097295, + "loss": 1.2919, + "theoretical_loss": 3.3998478929790825, + "tokens_seen": 2473590784 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529789368104313, + "loss": 1.2603, + "theoretical_loss": 3.39984075302124, + "tokens_seen": 2473656320 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529588766298897, + "loss": 1.3741, + "theoretical_loss": 3.3998336133055216, + "tokens_seen": 2473721856 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529388164493481, + "loss": 1.2717, + "theoretical_loss": 3.3998264738319133, + "tokens_seen": 2473787392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025291875626880646, + "loss": 1.3289, + "theoretical_loss": 3.3998193346004006, + "tokens_seen": 2473852928 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528986960882648, + "loss": 1.2296, + "theoretical_loss": 3.399812195610968, + "tokens_seen": 2473918464 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528786359077232, + "loss": 1.2522, + "theoretical_loss": 3.399805056863602, + "tokens_seen": 2473984000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528585757271816, + "loss": 1.2973, + "theoretical_loss": 3.399797918358287, + "tokens_seen": 2474049536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025283851554663996, + "loss": 1.2837, + "theoretical_loss": 3.399790780095009, + "tokens_seen": 2474115072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528184553660983, + "loss": 1.3227, + "theoretical_loss": 3.399783642073753, + "tokens_seen": 2474180608 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527983951855567, + "loss": 1.287, + "theoretical_loss": 3.399776504294505, + "tokens_seen": 2474246144 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025277833500501505, + "loss": 1.2835, + "theoretical_loss": 3.39976936675725, + "tokens_seen": 2474311680 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025275827482447346, + "loss": 1.3138, + "theoretical_loss": 3.399762229461973, + "tokens_seen": 2474377216 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527382146439318, + "loss": 1.2274, + "theoretical_loss": 3.3997550924086597, + "tokens_seen": 2474442752 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527181544633902, + "loss": 1.3367, + "theoretical_loss": 3.3997479555972956, + "tokens_seen": 2474508288 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025269809428284855, + "loss": 1.3539, + "theoretical_loss": 3.3997408190278664, + "tokens_seen": 2474573824 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025267803410230697, + "loss": 1.3539, + "theoretical_loss": 3.399733682700357, + "tokens_seen": 2474639360 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025265797392176533, + "loss": 1.2803, + "theoretical_loss": 3.3997265466147524, + "tokens_seen": 2474704896 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002526379137412237, + "loss": 1.1842, + "theoretical_loss": 3.399719410771039, + "tokens_seen": 2474770432 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025261785356068205, + "loss": 1.3009, + "theoretical_loss": 3.3997122751692013, + "tokens_seen": 2474835968 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002525977933801404, + "loss": 1.2362, + "theoretical_loss": 3.3997051398092255, + "tokens_seen": 2474901504 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025257773319959883, + "loss": 1.2856, + "theoretical_loss": 3.3996980046910963, + "tokens_seen": 2474967040 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002525576730190572, + "loss": 1.2006, + "theoretical_loss": 3.3996908698147994, + "tokens_seen": 2475032576 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025253761283851556, + "loss": 1.3353, + "theoretical_loss": 3.3996837351803206, + "tokens_seen": 2475098112 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002525175526579739, + "loss": 1.2818, + "theoretical_loss": 3.3996766007876444, + "tokens_seen": 2475163648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025249749247743233, + "loss": 1.4091, + "theoretical_loss": 3.399669466636757, + "tokens_seen": 2475229184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002524774322968907, + "loss": 1.3858, + "theoretical_loss": 3.3996623327276434, + "tokens_seen": 2475294720 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025245737211634906, + "loss": 1.296, + "theoretical_loss": 3.3996551990602892, + "tokens_seen": 2475360256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002524373119358074, + "loss": 1.4147, + "theoretical_loss": 3.3996480656346795, + "tokens_seen": 2475425792 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002524172517552658, + "loss": 1.262, + "theoretical_loss": 3.3996409324508, + "tokens_seen": 2475491328 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523971915747242, + "loss": 1.2315, + "theoretical_loss": 3.399633799508636, + "tokens_seen": 2475556864 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025237713139418256, + "loss": 1.3357, + "theoretical_loss": 3.399626666808173, + "tokens_seen": 2475622400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523570712136409, + "loss": 1.3354, + "theoretical_loss": 3.3996195343493962, + "tokens_seen": 2475687936 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 901124, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2519432306289673, + "objective/train/theoretical_loss": 3.3996124021322918, + "objective/train/tokens_used": 846212576, + "theoretical_loss": 3.3996124021322918, + "tokens_seen": 2475753472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523370110330993, + "loss": 1.3891, + "theoretical_loss": 3.3996124021322918, + "tokens_seen": 2475753472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523169508525577, + "loss": 1.3378, + "theoretical_loss": 3.3996052701568438, + "tokens_seen": 2475819008 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025229689067201606, + "loss": 1.2418, + "theoretical_loss": 3.399598138423039, + "tokens_seen": 2475884544 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025227683049147443, + "loss": 1.4269, + "theoretical_loss": 3.3995910069308617, + "tokens_seen": 2475950080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002522567703109328, + "loss": 1.3235, + "theoretical_loss": 3.399583875680298, + "tokens_seen": 2476015616 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025223671013039115, + "loss": 1.2395, + "theoretical_loss": 3.399576744671333, + "tokens_seen": 2476081152 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025221664994984957, + "loss": 1.3774, + "theoretical_loss": 3.3995696139039526, + "tokens_seen": 2476146688 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025219658976930793, + "loss": 1.2903, + "theoretical_loss": 3.3995624833781415, + "tokens_seen": 2476212224 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002521765295887663, + "loss": 1.3205, + "theoretical_loss": 3.399555353093886, + "tokens_seen": 2476277760 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025215646940822465, + "loss": 1.2984, + "theoretical_loss": 3.3995482230511707, + "tokens_seen": 2476343296 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025213640922768307, + "loss": 1.4112, + "theoretical_loss": 3.399541093249981, + "tokens_seen": 2476408832 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025211634904714143, + "loss": 1.3454, + "theoretical_loss": 3.3995339636903035, + "tokens_seen": 2476474368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002520962888665998, + "loss": 1.2686, + "theoretical_loss": 3.3995268343721223, + "tokens_seen": 2476539904 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025207622868605816, + "loss": 1.31, + "theoretical_loss": 3.3995197052954236, + "tokens_seen": 2476605440 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002520561685055166, + "loss": 1.2162, + "theoretical_loss": 3.3995125764601926, + "tokens_seen": 2476670976 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025203610832497494, + "loss": 1.3281, + "theoretical_loss": 3.3995054478664146, + "tokens_seen": 2476736512 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002520160481444333, + "loss": 1.3353, + "theoretical_loss": 3.399498319514075, + "tokens_seen": 2476802048 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025199598796389166, + "loss": 1.2379, + "theoretical_loss": 3.39949119140316, + "tokens_seen": 2476867584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025197592778335, + "loss": 1.2788, + "theoretical_loss": 3.3994840635336536, + "tokens_seen": 2476933120 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025195586760280844, + "loss": 1.2848, + "theoretical_loss": 3.3994769359055423, + "tokens_seen": 2476998656 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002519358074222668, + "loss": 1.2879, + "theoretical_loss": 3.3994698085188118, + "tokens_seen": 2477064192 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025191574724172516, + "loss": 1.2709, + "theoretical_loss": 3.3994626813734463, + "tokens_seen": 2477129728 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002518956870611835, + "loss": 1.3468, + "theoretical_loss": 3.3994555544694323, + "tokens_seen": 2477195264 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025187562688064194, + "loss": 1.3157, + "theoretical_loss": 3.399448427806755, + "tokens_seen": 2477260800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002518555667001003, + "loss": 1.271, + "theoretical_loss": 3.3994413013853997, + "tokens_seen": 2477326336 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025183550651955867, + "loss": 1.4645, + "theoretical_loss": 3.399434175205352, + "tokens_seen": 2477391872 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025181544633901703, + "loss": 1.3035, + "theoretical_loss": 3.3994270492665972, + "tokens_seen": 2477457408 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517953861584754, + "loss": 1.3667, + "theoretical_loss": 3.3994199235691203, + "tokens_seen": 2477522944 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517753259779338, + "loss": 1.3674, + "theoretical_loss": 3.399412798112908, + "tokens_seen": 2477588480 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025175526579739217, + "loss": 1.3269, + "theoretical_loss": 3.3994056728979443, + "tokens_seen": 2477654016 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025173520561685053, + "loss": 1.2677, + "theoretical_loss": 3.399398547924216, + "tokens_seen": 2477719552 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517151454363089, + "loss": 1.272, + "theoretical_loss": 3.3993914231917075, + "tokens_seen": 2477785088 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516950852557673, + "loss": 1.223, + "theoretical_loss": 3.399384298700405, + "tokens_seen": 2477850624 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516750250752257, + "loss": 1.382, + "theoretical_loss": 3.3993771744502927, + "tokens_seen": 2477916160 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025165496489468404, + "loss": 1.4047, + "theoretical_loss": 3.399370050441358, + "tokens_seen": 2477981696 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516349047141424, + "loss": 1.3301, + "theoretical_loss": 3.399362926673585, + "tokens_seen": 2478047232 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025161484453360076, + "loss": 1.3455, + "theoretical_loss": 3.3993558031469595, + "tokens_seen": 2478112768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515947843530592, + "loss": 1.2766, + "theoretical_loss": 3.3993486798614665, + "tokens_seen": 2478178304 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025157472417251754, + "loss": 1.3943, + "theoretical_loss": 3.3993415568170926, + "tokens_seen": 2478243840 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515546639919759, + "loss": 1.3411, + "theoretical_loss": 3.399334434013822, + "tokens_seen": 2478309376 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025153460381143426, + "loss": 1.2909, + "theoretical_loss": 3.399327311451641, + "tokens_seen": 2478374912 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515145436308927, + "loss": 1.3031, + "theoretical_loss": 3.3993201891305347, + "tokens_seen": 2478440448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514944834503511, + "loss": 1.3338, + "theoretical_loss": 3.3993130670504885, + "tokens_seen": 2478505984 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025147442326980946, + "loss": 1.4301, + "theoretical_loss": 3.399305945211488, + "tokens_seen": 2478571520 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514543630892678, + "loss": 1.336, + "theoretical_loss": 3.399298823613519, + "tokens_seen": 2478637056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514343029087262, + "loss": 1.34, + "theoretical_loss": 3.3992917022565665, + "tokens_seen": 2478702592 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514142427281846, + "loss": 1.3152, + "theoretical_loss": 3.3992845811406163, + "tokens_seen": 2478768128 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025139418254764296, + "loss": 1.2802, + "theoretical_loss": 3.3992774602656537, + "tokens_seen": 2478833664 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002513741223671013, + "loss": 1.5276, + "theoretical_loss": 3.399270339631664, + "tokens_seen": 2478899200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002513540621865597, + "loss": 1.2222, + "theoretical_loss": 3.399263219238633, + "tokens_seen": 2478964736 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 901918, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4798810482025146, + "objective/train/theoretical_loss": 3.399256099086546, + "objective/train/tokens_used": 849489376, + "theoretical_loss": 3.399256099086546, + "tokens_seen": 2479030272 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002513340020060181, + "loss": 1.3852, + "theoretical_loss": 3.399256099086546, + "tokens_seen": 2479030272 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025131394182547647, + "loss": 1.178, + "theoretical_loss": 3.3992489791753884, + "tokens_seen": 2479095808 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025129388164493483, + "loss": 1.3435, + "theoretical_loss": 3.3992418595051457, + "tokens_seen": 2479161344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002512738214643932, + "loss": 1.2602, + "theoretical_loss": 3.3992347400758036, + "tokens_seen": 2479226880 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025125376128385155, + "loss": 1.3079, + "theoretical_loss": 3.3992276208873475, + "tokens_seen": 2479292416 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025123370110330997, + "loss": 1.1964, + "theoretical_loss": 3.3992205019397628, + "tokens_seen": 2479357952 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025121364092276833, + "loss": 1.3512, + "theoretical_loss": 3.399213383233035, + "tokens_seen": 2479423488 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002511935807422267, + "loss": 1.3259, + "theoretical_loss": 3.3992062647671495, + "tokens_seen": 2479489024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025117352056168506, + "loss": 1.3042, + "theoretical_loss": 3.399199146542092, + "tokens_seen": 2479554560 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025115346038114347, + "loss": 1.3641, + "theoretical_loss": 3.399192028557848, + "tokens_seen": 2479620096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025113340020060183, + "loss": 1.1336, + "theoretical_loss": 3.3991849108144025, + "tokens_seen": 2479685632 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002511133400200602, + "loss": 1.3666, + "theoretical_loss": 3.3991777933117415, + "tokens_seen": 2479751168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025109327983951856, + "loss": 1.3548, + "theoretical_loss": 3.3991706760498506, + "tokens_seen": 2479816704 + }, + { + "epoch": 0.5, + "learning_rate": 0.000251073219658977, + "loss": 1.2685, + "theoretical_loss": 3.3991635590287146, + "tokens_seen": 2479882240 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025105315947843534, + "loss": 1.2976, + "theoretical_loss": 3.3991564422483203, + "tokens_seen": 2479947776 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002510330992978937, + "loss": 1.3874, + "theoretical_loss": 3.399149325708651, + "tokens_seen": 2480013312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025101303911735206, + "loss": 1.3662, + "theoretical_loss": 3.3991422094096944, + "tokens_seen": 2480078848 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002509929789368104, + "loss": 1.3587, + "theoretical_loss": 3.3991350933514353, + "tokens_seen": 2480144384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025097291875626884, + "loss": 1.2371, + "theoretical_loss": 3.3991279775338583, + "tokens_seen": 2480209920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002509528585757272, + "loss": 1.3304, + "theoretical_loss": 3.39912086195695, + "tokens_seen": 2480275456 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025093279839518556, + "loss": 1.3329, + "theoretical_loss": 3.3991137466206958, + "tokens_seen": 2480340992 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002509127382146439, + "loss": 1.3344, + "theoretical_loss": 3.3991066315250804, + "tokens_seen": 2480406528 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025089267803410234, + "loss": 1.1925, + "theoretical_loss": 3.3990995166700904, + "tokens_seen": 2480472064 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508726178535607, + "loss": 1.2723, + "theoretical_loss": 3.39909240205571, + "tokens_seen": 2480537600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025085255767301907, + "loss": 1.2843, + "theoretical_loss": 3.3990852876819257, + "tokens_seen": 2480603136 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025083249749247743, + "loss": 1.2822, + "theoretical_loss": 3.399078173548723, + "tokens_seen": 2480668672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508124373119358, + "loss": 1.2573, + "theoretical_loss": 3.399071059656087, + "tokens_seen": 2480734208 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507923771313942, + "loss": 1.3705, + "theoretical_loss": 3.3990639460040035, + "tokens_seen": 2480799744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025077231695085257, + "loss": 1.2406, + "theoretical_loss": 3.3990568325924575, + "tokens_seen": 2480865280 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025075225677031093, + "loss": 1.465, + "theoretical_loss": 3.399049719421435, + "tokens_seen": 2480930816 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507321965897693, + "loss": 1.3613, + "theoretical_loss": 3.399042606490922, + "tokens_seen": 2480996352 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507121364092277, + "loss": 1.3736, + "theoretical_loss": 3.399035493800903, + "tokens_seen": 2481061888 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002506920762286861, + "loss": 1.3005, + "theoretical_loss": 3.3990283813513633, + "tokens_seen": 2481127424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025067201604814444, + "loss": 1.2957, + "theoretical_loss": 3.39902126914229, + "tokens_seen": 2481192960 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002506519558676028, + "loss": 1.3358, + "theoretical_loss": 3.399014157173667, + "tokens_seen": 2481258496 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025063189568706116, + "loss": 1.2478, + "theoretical_loss": 3.3990070454454813, + "tokens_seen": 2481324032 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002506118355065196, + "loss": 1.2297, + "theoretical_loss": 3.398999933957717, + "tokens_seen": 2481389568 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025059177532597794, + "loss": 1.3491, + "theoretical_loss": 3.39899282271036, + "tokens_seen": 2481455104 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002505717151454363, + "loss": 1.2844, + "theoretical_loss": 3.3989857117033964, + "tokens_seen": 2481520640 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025055165496489466, + "loss": 1.376, + "theoretical_loss": 3.3989786009368115, + "tokens_seen": 2481586176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002505315947843531, + "loss": 1.3269, + "theoretical_loss": 3.3989714904105908, + "tokens_seen": 2481651712 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025051153460381144, + "loss": 1.3084, + "theoretical_loss": 3.3989643801247196, + "tokens_seen": 2481717248 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002504914744232698, + "loss": 1.2167, + "theoretical_loss": 3.3989572700791832, + "tokens_seen": 2481782784 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025047141424272817, + "loss": 1.3676, + "theoretical_loss": 3.3989501602739676, + "tokens_seen": 2481848320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025045135406218653, + "loss": 1.2946, + "theoretical_loss": 3.3989430507090588, + "tokens_seen": 2481913856 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025043129388164495, + "loss": 1.3691, + "theoretical_loss": 3.3989359413844413, + "tokens_seen": 2481979392 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002504112337011033, + "loss": 1.2622, + "theoretical_loss": 3.398928832300101, + "tokens_seen": 2482044928 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025039117352056167, + "loss": 1.2578, + "theoretical_loss": 3.398921723456024, + "tokens_seen": 2482110464 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025037111334002003, + "loss": 1.4824, + "theoretical_loss": 3.398914614852195, + "tokens_seen": 2482176000 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025035105315947845, + "loss": 1.3281, + "theoretical_loss": 3.3989075064886, + "tokens_seen": 2482241536 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 903113, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3541759252548218, + "objective/train/theoretical_loss": 3.3989003983652246, + "objective/train/tokens_used": 852766176, + "theoretical_loss": 3.3989003983652246, + "tokens_seen": 2482307072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002503309929789368, + "loss": 1.3556, + "theoretical_loss": 3.3989003983652246, + "tokens_seen": 2482307072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002503109327983952, + "loss": 1.2891, + "theoretical_loss": 3.3988932904820537, + "tokens_seen": 2482372608 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025029087261785354, + "loss": 1.2133, + "theoretical_loss": 3.3988861828390737, + "tokens_seen": 2482438144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002502708124373119, + "loss": 1.426, + "theoretical_loss": 3.3988790754362697, + "tokens_seen": 2482503680 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002502507522567703, + "loss": 1.3232, + "theoretical_loss": 3.3988719682736273, + "tokens_seen": 2482569216 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002502306920762287, + "loss": 1.3657, + "theoretical_loss": 3.3988648613511323, + "tokens_seen": 2482634752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025021063189568704, + "loss": 1.2924, + "theoretical_loss": 3.3988577546687697, + "tokens_seen": 2482700288 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501905717151454, + "loss": 1.2758, + "theoretical_loss": 3.3988506482265253, + "tokens_seen": 2482765824 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501705115346038, + "loss": 1.2373, + "theoretical_loss": 3.3988435420243848, + "tokens_seen": 2482831360 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501504513540622, + "loss": 1.2455, + "theoretical_loss": 3.3988364360623335, + "tokens_seen": 2482896896 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025013039117352054, + "loss": 1.263, + "theoretical_loss": 3.3988293303403574, + "tokens_seen": 2482962432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501103309929789, + "loss": 1.2571, + "theoretical_loss": 3.3988222248584417, + "tokens_seen": 2483027968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025009027081243727, + "loss": 1.3233, + "theoretical_loss": 3.398815119616572, + "tokens_seen": 2483093504 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002500702106318957, + "loss": 1.3388, + "theoretical_loss": 3.3988080146147337, + "tokens_seen": 2483159040 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025005015045135405, + "loss": 1.4493, + "theoretical_loss": 3.3988009098529126, + "tokens_seen": 2483224576 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002500300902708124, + "loss": 1.3901, + "theoretical_loss": 3.398793805331094, + "tokens_seen": 2483290112 + }, + { + "epoch": 0.51, + "learning_rate": 0.00025001003009027077, + "loss": 1.2823, + "theoretical_loss": 3.3987867010492643, + "tokens_seen": 2483355648 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002499899699097292, + "loss": 1.3034, + "theoretical_loss": 3.3987795970074077, + "tokens_seen": 2483421184 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024996990972918755, + "loss": 1.2635, + "theoretical_loss": 3.398772493205511, + "tokens_seen": 2483486720 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024994984954864597, + "loss": 1.3248, + "theoretical_loss": 3.398765389643559, + "tokens_seen": 2483552256 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024992978936810433, + "loss": 1.4511, + "theoretical_loss": 3.398758286321537, + "tokens_seen": 2483617792 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002499097291875627, + "loss": 1.3944, + "theoretical_loss": 3.398751183239432, + "tokens_seen": 2483683328 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024988966900702105, + "loss": 1.207, + "theoretical_loss": 3.3987440803972278, + "tokens_seen": 2483748864 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002498696088264794, + "loss": 1.3166, + "theoretical_loss": 3.3987369777949112, + "tokens_seen": 2483814400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024984954864593783, + "loss": 1.1807, + "theoretical_loss": 3.3987298754324673, + "tokens_seen": 2483879936 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002498294884653962, + "loss": 1.3815, + "theoretical_loss": 3.3987227733098817, + "tokens_seen": 2483945472 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024980942828485455, + "loss": 1.4362, + "theoretical_loss": 3.39871567142714, + "tokens_seen": 2484011008 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497893681043129, + "loss": 1.3367, + "theoretical_loss": 3.3987085697842274, + "tokens_seen": 2484076544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024976930792377133, + "loss": 1.2403, + "theoretical_loss": 3.3987014683811303, + "tokens_seen": 2484142080 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497492477432297, + "loss": 1.276, + "theoretical_loss": 3.398694367217834, + "tokens_seen": 2484207616 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024972918756268806, + "loss": 1.235, + "theoretical_loss": 3.3986872662943233, + "tokens_seen": 2484273152 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497091273821464, + "loss": 1.4189, + "theoretical_loss": 3.3986801656105845, + "tokens_seen": 2484338688 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496890672016048, + "loss": 1.4088, + "theoretical_loss": 3.3986730651666033, + "tokens_seen": 2484404224 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496690070210632, + "loss": 1.2772, + "theoretical_loss": 3.398665964962365, + "tokens_seen": 2484469760 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024964894684052156, + "loss": 1.3537, + "theoretical_loss": 3.3986588649978553, + "tokens_seen": 2484535296 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496288866599799, + "loss": 1.2367, + "theoretical_loss": 3.3986517652730597, + "tokens_seen": 2484600832 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024960882647943834, + "loss": 1.4176, + "theoretical_loss": 3.3986446657879634, + "tokens_seen": 2484666368 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495887662988967, + "loss": 1.4045, + "theoretical_loss": 3.398637566542553, + "tokens_seen": 2484731904 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024956870611835506, + "loss": 1.3051, + "theoretical_loss": 3.3986304675368126, + "tokens_seen": 2484797440 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495486459378135, + "loss": 1.3138, + "theoretical_loss": 3.3986233687707292, + "tokens_seen": 2484862976 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024952858575727184, + "loss": 1.2604, + "theoretical_loss": 3.398616270244288, + "tokens_seen": 2484928512 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495085255767302, + "loss": 1.1999, + "theoretical_loss": 3.398609171957474, + "tokens_seen": 2484994048 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024948846539618857, + "loss": 1.305, + "theoretical_loss": 3.3986020739102734, + "tokens_seen": 2485059584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024946840521564693, + "loss": 1.3133, + "theoretical_loss": 3.3985949761026717, + "tokens_seen": 2485125120 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024944834503510535, + "loss": 1.2485, + "theoretical_loss": 3.398587878534654, + "tokens_seen": 2485190656 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002494282848545637, + "loss": 1.2223, + "theoretical_loss": 3.3985807812062068, + "tokens_seen": 2485256192 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024940822467402207, + "loss": 1.3108, + "theoretical_loss": 3.3985736841173146, + "tokens_seen": 2485321728 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024938816449348043, + "loss": 1.2937, + "theoretical_loss": 3.3985665872679642, + "tokens_seen": 2485387264 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024936810431293885, + "loss": 1.2634, + "theoretical_loss": 3.3985594906581404, + "tokens_seen": 2485452800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002493480441323972, + "loss": 1.2663, + "theoretical_loss": 3.3985523942878286, + "tokens_seen": 2485518336 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 904288, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2782502174377441, + "objective/train/theoretical_loss": 3.3985452981570154, + "objective/train/tokens_used": 856042976, + "theoretical_loss": 3.3985452981570154, + "tokens_seen": 2485583872 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002493279839518556, + "loss": 1.252, + "theoretical_loss": 3.3985452981570154, + "tokens_seen": 2485583872 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024930792377131394, + "loss": 1.3401, + "theoretical_loss": 3.3985382022656854, + "tokens_seen": 2485649408 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492878635907723, + "loss": 1.4326, + "theoretical_loss": 3.3985311066138246, + "tokens_seen": 2485714944 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492678034102307, + "loss": 1.2876, + "theoretical_loss": 3.398524011201419, + "tokens_seen": 2485780480 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492477432296891, + "loss": 1.3128, + "theoretical_loss": 3.3985169160284534, + "tokens_seen": 2485846016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024922768304914744, + "loss": 1.24, + "theoretical_loss": 3.398509821094914, + "tokens_seen": 2485911552 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492076228686058, + "loss": 1.186, + "theoretical_loss": 3.3985027264007863, + "tokens_seen": 2485977088 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491875626880642, + "loss": 1.2708, + "theoretical_loss": 3.3984956319460555, + "tokens_seen": 2486042624 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491675025075226, + "loss": 1.366, + "theoretical_loss": 3.3984885377307075, + "tokens_seen": 2486108160 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024914744232698094, + "loss": 1.2984, + "theoretical_loss": 3.3984814437547284, + "tokens_seen": 2486173696 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491273821464393, + "loss": 1.3309, + "theoretical_loss": 3.398474350018103, + "tokens_seen": 2486239232 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024910732196589767, + "loss": 1.2778, + "theoretical_loss": 3.3984672565208176, + "tokens_seen": 2486304768 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490872617853561, + "loss": 1.2828, + "theoretical_loss": 3.3984601632628575, + "tokens_seen": 2486370304 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024906720160481445, + "loss": 1.2744, + "theoretical_loss": 3.3984530702442077, + "tokens_seen": 2486435840 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490471414242728, + "loss": 1.2582, + "theoretical_loss": 3.3984459774648554, + "tokens_seen": 2486501376 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024902708124373117, + "loss": 1.3138, + "theoretical_loss": 3.3984388849247846, + "tokens_seen": 2486566912 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490070210631896, + "loss": 1.2444, + "theoretical_loss": 3.398431792623982, + "tokens_seen": 2486632448 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024898696088264795, + "loss": 1.3583, + "theoretical_loss": 3.3984247005624324, + "tokens_seen": 2486697984 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002489669007021063, + "loss": 1.4093, + "theoretical_loss": 3.3984176087401217, + "tokens_seen": 2486763520 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002489468405215647, + "loss": 1.2117, + "theoretical_loss": 3.398410517157036, + "tokens_seen": 2486829056 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024892678034102304, + "loss": 1.3196, + "theoretical_loss": 3.398403425813161, + "tokens_seen": 2486894592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024890672016048145, + "loss": 1.3787, + "theoretical_loss": 3.3983963347084813, + "tokens_seen": 2486960128 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488866599799398, + "loss": 1.3075, + "theoretical_loss": 3.398389243842983, + "tokens_seen": 2487025664 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024886659979939823, + "loss": 1.3273, + "theoretical_loss": 3.398382153216652, + "tokens_seen": 2487091200 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488465396188566, + "loss": 1.3336, + "theoretical_loss": 3.398375062829474, + "tokens_seen": 2487156736 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024882647943831496, + "loss": 1.3448, + "theoretical_loss": 3.3983679726814344, + "tokens_seen": 2487222272 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488064192577733, + "loss": 1.2555, + "theoretical_loss": 3.398360882772519, + "tokens_seen": 2487287808 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024878635907723173, + "loss": 1.1584, + "theoretical_loss": 3.398353793102713, + "tokens_seen": 2487353344 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487662988966901, + "loss": 1.2994, + "theoretical_loss": 3.3983467036720025, + "tokens_seen": 2487418880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024874623871614846, + "loss": 1.4172, + "theoretical_loss": 3.398339614480373, + "tokens_seen": 2487484416 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487261785356068, + "loss": 1.2021, + "theoretical_loss": 3.3983325255278096, + "tokens_seen": 2487549952 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487061183550652, + "loss": 1.3081, + "theoretical_loss": 3.398325436814299, + "tokens_seen": 2487615488 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486860581745236, + "loss": 1.2459, + "theoretical_loss": 3.398318348339826, + "tokens_seen": 2487681024 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024866599799398196, + "loss": 1.4061, + "theoretical_loss": 3.398311260104377, + "tokens_seen": 2487746560 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486459378134403, + "loss": 1.2761, + "theoretical_loss": 3.398304172107937, + "tokens_seen": 2487812096 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486258776328987, + "loss": 1.4075, + "theoretical_loss": 3.3982970843504914, + "tokens_seen": 2487877632 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486058174523571, + "loss": 1.2773, + "theoretical_loss": 3.398289996832027, + "tokens_seen": 2487943168 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024858575727181546, + "loss": 1.3485, + "theoretical_loss": 3.398282909552528, + "tokens_seen": 2488008704 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002485656970912738, + "loss": 1.2861, + "theoretical_loss": 3.398275822511981, + "tokens_seen": 2488074240 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002485456369107322, + "loss": 1.2942, + "theoretical_loss": 3.3982687357103716, + "tokens_seen": 2488139776 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024852557673019055, + "loss": 1.2823, + "theoretical_loss": 3.398261649147685, + "tokens_seen": 2488205312 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024850551654964897, + "loss": 1.2922, + "theoretical_loss": 3.398254562823907, + "tokens_seen": 2488270848 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024848545636910733, + "loss": 1.3095, + "theoretical_loss": 3.398247476739024, + "tokens_seen": 2488336384 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002484653961885657, + "loss": 1.3002, + "theoretical_loss": 3.3982403908930205, + "tokens_seen": 2488401920 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024844533600802405, + "loss": 1.4044, + "theoretical_loss": 3.398233305285883, + "tokens_seen": 2488467456 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024842527582748247, + "loss": 1.3391, + "theoretical_loss": 3.3982262199175968, + "tokens_seen": 2488532992 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024840521564694083, + "loss": 1.3447, + "theoretical_loss": 3.3982191347881474, + "tokens_seen": 2488598528 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002483851554663992, + "loss": 1.3976, + "theoretical_loss": 3.398212049897521, + "tokens_seen": 2488664064 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024836509528585756, + "loss": 1.3185, + "theoretical_loss": 3.3982049652457023, + "tokens_seen": 2488729600 + }, + { + "epoch": 0.51, + "learning_rate": 0.000248345035105316, + "loss": 1.3527, + "theoretical_loss": 3.398197880832678, + "tokens_seen": 2488795136 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 905609, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2062811851501465, + "objective/train/theoretical_loss": 3.398190796658433, + "objective/train/tokens_used": 859319776, + "theoretical_loss": 3.398190796658433, + "tokens_seen": 2488860672 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024832497492477434, + "loss": 1.2004, + "theoretical_loss": 3.398190796658433, + "tokens_seen": 2488860672 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002483049147442327, + "loss": 1.2462, + "theoretical_loss": 3.398183712722954, + "tokens_seen": 2488926208 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024828485456369106, + "loss": 1.309, + "theoretical_loss": 3.3981766290262256, + "tokens_seen": 2488991744 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002482647943831494, + "loss": 1.3463, + "theoretical_loss": 3.3981695455682335, + "tokens_seen": 2489057280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024824473420260784, + "loss": 1.3427, + "theoretical_loss": 3.398162462348964, + "tokens_seen": 2489122816 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002482246740220662, + "loss": 1.4135, + "theoretical_loss": 3.398155379368403, + "tokens_seen": 2489188352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024820461384152456, + "loss": 1.3162, + "theoretical_loss": 3.398148296626535, + "tokens_seen": 2489253888 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481845536609829, + "loss": 1.3228, + "theoretical_loss": 3.3981412141233465, + "tokens_seen": 2489319424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024816449348044134, + "loss": 1.2983, + "theoretical_loss": 3.398134131858823, + "tokens_seen": 2489384960 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481444332998997, + "loss": 1.37, + "theoretical_loss": 3.39812704983295, + "tokens_seen": 2489450496 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024812437311935807, + "loss": 1.3446, + "theoretical_loss": 3.3981199680457137, + "tokens_seen": 2489516032 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481043129388165, + "loss": 1.2452, + "theoretical_loss": 3.398112886497099, + "tokens_seen": 2489581568 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024808425275827485, + "loss": 1.3364, + "theoretical_loss": 3.3981058051870923, + "tokens_seen": 2489647104 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002480641925777332, + "loss": 1.2841, + "theoretical_loss": 3.398098724115679, + "tokens_seen": 2489712640 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024804413239719157, + "loss": 1.2488, + "theoretical_loss": 3.3980916432828447, + "tokens_seen": 2489778176 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024802407221665, + "loss": 1.2844, + "theoretical_loss": 3.3980845626885756, + "tokens_seen": 2489843712 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024800401203610835, + "loss": 1.425, + "theoretical_loss": 3.398077482332856, + "tokens_seen": 2489909248 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002479839518555667, + "loss": 1.4244, + "theoretical_loss": 3.398070402215673, + "tokens_seen": 2489974784 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002479638916750251, + "loss": 1.3454, + "theoretical_loss": 3.398063322337012, + "tokens_seen": 2490040320 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024794383149448344, + "loss": 1.2685, + "theoretical_loss": 3.3980562426968586, + "tokens_seen": 2490105856 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024792377131394185, + "loss": 1.315, + "theoretical_loss": 3.3980491632951977, + "tokens_seen": 2490171392 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002479037111334002, + "loss": 1.3586, + "theoretical_loss": 3.3980420841320162, + "tokens_seen": 2490236928 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002478836509528586, + "loss": 1.2656, + "theoretical_loss": 3.398035005207299, + "tokens_seen": 2490302464 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024786359077231694, + "loss": 1.2893, + "theoretical_loss": 3.3980279265210322, + "tokens_seen": 2490368000 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024784353059177536, + "loss": 1.3092, + "theoretical_loss": 3.3980208480732013, + "tokens_seen": 2490433536 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002478234704112337, + "loss": 1.3621, + "theoretical_loss": 3.398013769863792, + "tokens_seen": 2490499072 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002478034102306921, + "loss": 1.3002, + "theoretical_loss": 3.39800669189279, + "tokens_seen": 2490564608 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024778335005015044, + "loss": 1.291, + "theoretical_loss": 3.397999614160181, + "tokens_seen": 2490630144 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024776328986960886, + "loss": 1.3411, + "theoretical_loss": 3.397992536665951, + "tokens_seen": 2490695680 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477432296890672, + "loss": 1.292, + "theoretical_loss": 3.3979854594100853, + "tokens_seen": 2490761216 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477231695085256, + "loss": 1.3295, + "theoretical_loss": 3.3979783823925693, + "tokens_seen": 2490826752 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024770310932798395, + "loss": 1.3149, + "theoretical_loss": 3.39797130561339, + "tokens_seen": 2490892288 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476830491474423, + "loss": 1.273, + "theoretical_loss": 3.3979642290725316, + "tokens_seen": 2490957824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476629889669007, + "loss": 1.4394, + "theoretical_loss": 3.3979571527699806, + "tokens_seen": 2491023360 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476429287863591, + "loss": 1.2769, + "theoretical_loss": 3.3979500767057225, + "tokens_seen": 2491088896 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024762286860581745, + "loss": 1.2908, + "theoretical_loss": 3.397943000879743, + "tokens_seen": 2491154432 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476028084252758, + "loss": 1.3369, + "theoretical_loss": 3.397935925292028, + "tokens_seen": 2491219968 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024758274824473423, + "loss": 1.3049, + "theoretical_loss": 3.397928849942563, + "tokens_seen": 2491285504 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475626880641926, + "loss": 1.3192, + "theoretical_loss": 3.3979217748313335, + "tokens_seen": 2491351040 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024754262788365095, + "loss": 1.2839, + "theoretical_loss": 3.397914699958326, + "tokens_seen": 2491416576 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475225677031093, + "loss": 1.3115, + "theoretical_loss": 3.3979076253235254, + "tokens_seen": 2491482112 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475025075225677, + "loss": 1.4467, + "theoretical_loss": 3.397900550926918, + "tokens_seen": 2491547648 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474824473420261, + "loss": 1.3443, + "theoretical_loss": 3.3978934767684885, + "tokens_seen": 2491613184 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024746238716148445, + "loss": 1.2875, + "theoretical_loss": 3.3978864028482243, + "tokens_seen": 2491678720 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474423269809428, + "loss": 1.3229, + "theoretical_loss": 3.3978793291661096, + "tokens_seen": 2491744256 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474222668004012, + "loss": 1.2385, + "theoretical_loss": 3.397872255722131, + "tokens_seen": 2491809792 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474022066198596, + "loss": 1.3834, + "theoretical_loss": 3.3978651825162736, + "tokens_seen": 2491875328 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024738214643931796, + "loss": 1.2925, + "theoretical_loss": 3.3978581095485234, + "tokens_seen": 2491940864 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002473620862587764, + "loss": 1.3056, + "theoretical_loss": 3.3978510368188664, + "tokens_seen": 2492006400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024734202607823474, + "loss": 1.285, + "theoretical_loss": 3.3978439643272877, + "tokens_seen": 2492071936 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 906987, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.148254156112671, + "objective/train/theoretical_loss": 3.397836892073774, + "objective/train/tokens_used": 862596576, + "theoretical_loss": 3.397836892073774, + "tokens_seen": 2492137472 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002473219658976931, + "loss": 1.1904, + "theoretical_loss": 3.397836892073774, + "tokens_seen": 2492137472 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024730190571715146, + "loss": 1.323, + "theoretical_loss": 3.39782982005831, + "tokens_seen": 2492203008 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472818455366098, + "loss": 1.5083, + "theoretical_loss": 3.397822748280882, + "tokens_seen": 2492268544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024726178535606824, + "loss": 1.2224, + "theoretical_loss": 3.3978156767414753, + "tokens_seen": 2492334080 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472417251755266, + "loss": 1.372, + "theoretical_loss": 3.3978086054400762, + "tokens_seen": 2492399616 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024722166499498496, + "loss": 1.2508, + "theoretical_loss": 3.3978015343766703, + "tokens_seen": 2492465152 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472016048144433, + "loss": 1.275, + "theoretical_loss": 3.397794463551243, + "tokens_seen": 2492530688 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024718154463390174, + "loss": 1.2373, + "theoretical_loss": 3.39778739296378, + "tokens_seen": 2492596224 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002471614844533601, + "loss": 1.2955, + "theoretical_loss": 3.3977803226142673, + "tokens_seen": 2492661760 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024714142427281847, + "loss": 1.2886, + "theoretical_loss": 3.3977732525026907, + "tokens_seen": 2492727296 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024712136409227683, + "loss": 1.2405, + "theoretical_loss": 3.397766182629036, + "tokens_seen": 2492792832 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002471013039117352, + "loss": 1.3852, + "theoretical_loss": 3.3977591129932883, + "tokens_seen": 2492858368 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002470812437311936, + "loss": 1.361, + "theoretical_loss": 3.3977520435954345, + "tokens_seen": 2492923904 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024706118355065197, + "loss": 1.1757, + "theoretical_loss": 3.397744974435459, + "tokens_seen": 2492989440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024704112337011033, + "loss": 1.4564, + "theoretical_loss": 3.3977379055133485, + "tokens_seen": 2493054976 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002470210631895687, + "loss": 1.3568, + "theoretical_loss": 3.397730836829088, + "tokens_seen": 2493120512 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002470010030090271, + "loss": 1.3232, + "theoretical_loss": 3.397723768382664, + "tokens_seen": 2493186048 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002469809428284855, + "loss": 1.3393, + "theoretical_loss": 3.3977167001740622, + "tokens_seen": 2493251584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024696088264794384, + "loss": 1.2738, + "theoretical_loss": 3.397709632203268, + "tokens_seen": 2493317120 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002469408224674022, + "loss": 1.2933, + "theoretical_loss": 3.3977025644702668, + "tokens_seen": 2493382656 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024692076228686056, + "loss": 1.2349, + "theoretical_loss": 3.397695496975045, + "tokens_seen": 2493448192 + }, + { + "epoch": 0.51, + "learning_rate": 0.000246900702106319, + "loss": 1.3842, + "theoretical_loss": 3.397688429717588, + "tokens_seen": 2493513728 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024688064192577734, + "loss": 1.161, + "theoretical_loss": 3.397681362697882, + "tokens_seen": 2493579264 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002468605817452357, + "loss": 1.1998, + "theoretical_loss": 3.3976742959159116, + "tokens_seen": 2493644800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024684052156469406, + "loss": 1.323, + "theoretical_loss": 3.397667229371664, + "tokens_seen": 2493710336 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002468204613841525, + "loss": 1.1944, + "theoretical_loss": 3.3976601630651246, + "tokens_seen": 2493775872 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024680040120361084, + "loss": 1.217, + "theoretical_loss": 3.3976530969962786, + "tokens_seen": 2493841408 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467803410230692, + "loss": 1.2876, + "theoretical_loss": 3.3976460311651118, + "tokens_seen": 2493906944 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024676028084252757, + "loss": 1.3305, + "theoretical_loss": 3.3976389655716104, + "tokens_seen": 2493972480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024674022066198593, + "loss": 1.2719, + "theoretical_loss": 3.3976319002157602, + "tokens_seen": 2494038016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024672016048144435, + "loss": 1.3139, + "theoretical_loss": 3.397624835097546, + "tokens_seen": 2494103552 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467001003009027, + "loss": 1.3514, + "theoretical_loss": 3.3976177702169554, + "tokens_seen": 2494169088 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024668004012036107, + "loss": 1.2691, + "theoretical_loss": 3.3976107055739724, + "tokens_seen": 2494234624 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024665997993981943, + "loss": 1.2836, + "theoretical_loss": 3.3976036411685833, + "tokens_seen": 2494300160 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024663991975927785, + "loss": 1.3656, + "theoretical_loss": 3.3975965770007743, + "tokens_seen": 2494365696 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002466198595787362, + "loss": 1.28, + "theoretical_loss": 3.397589513070531, + "tokens_seen": 2494431232 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024659979939819463, + "loss": 1.3324, + "theoretical_loss": 3.3975824493778384, + "tokens_seen": 2494496768 + }, + { + "epoch": 0.51, + "learning_rate": 0.000246579739217653, + "loss": 1.3077, + "theoretical_loss": 3.3975753859226834, + "tokens_seen": 2494562304 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024655967903711135, + "loss": 1.3581, + "theoretical_loss": 3.397568322705051, + "tokens_seen": 2494627840 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002465396188565697, + "loss": 1.3503, + "theoretical_loss": 3.3975612597249274, + "tokens_seen": 2494693376 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002465195586760281, + "loss": 1.2005, + "theoretical_loss": 3.3975541969822984, + "tokens_seen": 2494758912 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002464994984954865, + "loss": 1.3329, + "theoretical_loss": 3.3975471344771493, + "tokens_seen": 2494824448 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024647943831494486, + "loss": 1.1613, + "theoretical_loss": 3.397540072209466, + "tokens_seen": 2494889984 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002464593781344032, + "loss": 1.3635, + "theoretical_loss": 3.397533010179235, + "tokens_seen": 2494955520 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002464393179538616, + "loss": 1.3921, + "theoretical_loss": 3.3975259483864413, + "tokens_seen": 2495021056 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024641925777332, + "loss": 1.3385, + "theoretical_loss": 3.3975188868310706, + "tokens_seen": 2495086592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024639919759277836, + "loss": 1.2526, + "theoretical_loss": 3.397511825513109, + "tokens_seen": 2495152128 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002463791374122367, + "loss": 1.1815, + "theoretical_loss": 3.397504764432543, + "tokens_seen": 2495217664 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002463590772316951, + "loss": 1.2979, + "theoretical_loss": 3.397497703589357, + "tokens_seen": 2495283200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024633901705115345, + "loss": 1.3793, + "theoretical_loss": 3.397490642983538, + "tokens_seen": 2495348736 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 908214, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2217057943344116, + "objective/train/theoretical_loss": 3.3974835826150707, + "objective/train/tokens_used": 865873376, + "theoretical_loss": 3.3974835826150707, + "tokens_seen": 2495414272 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024631895687061186, + "loss": 1.2668, + "theoretical_loss": 3.3974835826150707, + "tokens_seen": 2495414272 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462988966900702, + "loss": 1.2666, + "theoretical_loss": 3.3974765224839416, + "tokens_seen": 2495479808 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462788365095286, + "loss": 1.2652, + "theoretical_loss": 3.397469462590136, + "tokens_seen": 2495545344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024625877632898695, + "loss": 1.2481, + "theoretical_loss": 3.3974624029336407, + "tokens_seen": 2495610880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024623871614844536, + "loss": 1.2666, + "theoretical_loss": 3.39745534351444, + "tokens_seen": 2495676416 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462186559679037, + "loss": 1.2957, + "theoretical_loss": 3.397448284332521, + "tokens_seen": 2495741952 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461985957873621, + "loss": 1.3204, + "theoretical_loss": 3.397441225387869, + "tokens_seen": 2495807488 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024617853560682045, + "loss": 1.4021, + "theoretical_loss": 3.3974341666804695, + "tokens_seen": 2495873024 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461584754262788, + "loss": 1.3076, + "theoretical_loss": 3.3974271082103087, + "tokens_seen": 2495938560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024613841524573723, + "loss": 1.3235, + "theoretical_loss": 3.397420049977372, + "tokens_seen": 2496004096 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461183550651956, + "loss": 1.2981, + "theoretical_loss": 3.397412991981646, + "tokens_seen": 2496069632 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024609829488465395, + "loss": 1.3519, + "theoretical_loss": 3.3974059342231158, + "tokens_seen": 2496135168 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002460782347041123, + "loss": 1.4067, + "theoretical_loss": 3.397398876701767, + "tokens_seen": 2496200704 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024605817452357073, + "loss": 1.3553, + "theoretical_loss": 3.397391819417586, + "tokens_seen": 2496266240 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002460381143430291, + "loss": 1.4182, + "theoretical_loss": 3.3973847623705584, + "tokens_seen": 2496331776 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024601805416248746, + "loss": 1.2988, + "theoretical_loss": 3.39737770556067, + "tokens_seen": 2496397312 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002459979939819458, + "loss": 1.2761, + "theoretical_loss": 3.3973706489879065, + "tokens_seen": 2496462848 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002459779338014042, + "loss": 1.2759, + "theoretical_loss": 3.397363592652254, + "tokens_seen": 2496528384 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002459578736208626, + "loss": 1.3149, + "theoretical_loss": 3.3973565365536977, + "tokens_seen": 2496593920 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024593781344032096, + "loss": 1.266, + "theoretical_loss": 3.397349480692224, + "tokens_seen": 2496659456 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002459177532597793, + "loss": 1.4351, + "theoretical_loss": 3.3973424250678184, + "tokens_seen": 2496724992 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458976930792377, + "loss": 1.2678, + "theoretical_loss": 3.397335369680467, + "tokens_seen": 2496790528 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458776328986961, + "loss": 1.2867, + "theoretical_loss": 3.3973283145301556, + "tokens_seen": 2496856064 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024585757271815446, + "loss": 1.359, + "theoretical_loss": 3.39732125961687, + "tokens_seen": 2496921600 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458375125376129, + "loss": 1.3574, + "theoretical_loss": 3.3973142049405953, + "tokens_seen": 2496987136 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024581745235707124, + "loss": 1.362, + "theoretical_loss": 3.397307150501318, + "tokens_seen": 2497052672 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457973921765296, + "loss": 1.2774, + "theoretical_loss": 3.3973000962990243, + "tokens_seen": 2497118208 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024577733199598797, + "loss": 1.257, + "theoretical_loss": 3.397293042333699, + "tokens_seen": 2497183744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024575727181544633, + "loss": 1.2144, + "theoretical_loss": 3.397285988605329, + "tokens_seen": 2497249280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024573721163490475, + "loss": 1.2676, + "theoretical_loss": 3.3972789351138992, + "tokens_seen": 2497314816 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457171514543631, + "loss": 1.4925, + "theoretical_loss": 3.397271881859396, + "tokens_seen": 2497380352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024569709127382147, + "loss": 1.4294, + "theoretical_loss": 3.397264828841805, + "tokens_seen": 2497445888 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024567703109327983, + "loss": 1.357, + "theoretical_loss": 3.3972577760611116, + "tokens_seen": 2497511424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024565697091273825, + "loss": 1.3072, + "theoretical_loss": 3.3972507235173026, + "tokens_seen": 2497576960 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002456369107321966, + "loss": 1.4013, + "theoretical_loss": 3.397243671210363, + "tokens_seen": 2497642496 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245616850551655, + "loss": 1.2713, + "theoretical_loss": 3.397236619140279, + "tokens_seen": 2497708032 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024559679037111334, + "loss": 1.2894, + "theoretical_loss": 3.3972295673070363, + "tokens_seen": 2497773568 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455767301905717, + "loss": 1.3014, + "theoretical_loss": 3.397222515710621, + "tokens_seen": 2497839104 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455566700100301, + "loss": 1.3756, + "theoretical_loss": 3.3972154643510186, + "tokens_seen": 2497904640 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455366098294885, + "loss": 1.303, + "theoretical_loss": 3.397208413228215, + "tokens_seen": 2497970176 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024551654964894684, + "loss": 1.2442, + "theoretical_loss": 3.397201362342196, + "tokens_seen": 2498035712 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454964894684052, + "loss": 1.2907, + "theoretical_loss": 3.397194311692948, + "tokens_seen": 2498101248 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454764292878636, + "loss": 1.3495, + "theoretical_loss": 3.3971872612804566, + "tokens_seen": 2498166784 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245456369107322, + "loss": 1.3344, + "theoretical_loss": 3.3971802111047067, + "tokens_seen": 2498232320 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024543630892678034, + "loss": 1.3165, + "theoretical_loss": 3.397173161165685, + "tokens_seen": 2498297856 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454162487462387, + "loss": 1.2856, + "theoretical_loss": 3.3971661114633775, + "tokens_seen": 2498363392 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024539618856569707, + "loss": 1.2515, + "theoretical_loss": 3.3971590619977694, + "tokens_seen": 2498428928 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002453761283851555, + "loss": 1.1653, + "theoretical_loss": 3.397152012768847, + "tokens_seen": 2498494464 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024535606820461385, + "loss": 1.3751, + "theoretical_loss": 3.3971449637765962, + "tokens_seen": 2498560000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002453360080240722, + "loss": 1.3467, + "theoretical_loss": 3.3971379150210024, + "tokens_seen": 2498625536 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 909311, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1775472164154053, + "objective/train/theoretical_loss": 3.397130866502052, + "objective/train/tokens_used": 869150176, + "theoretical_loss": 3.397130866502052, + "tokens_seen": 2498691072 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024531594784353057, + "loss": 1.243, + "theoretical_loss": 3.397130866502052, + "tokens_seen": 2498691072 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245295887662989, + "loss": 1.3678, + "theoretical_loss": 3.3971238182197303, + "tokens_seen": 2498756608 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024527582748244735, + "loss": 1.2951, + "theoretical_loss": 3.3971167701740237, + "tokens_seen": 2498822144 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002452557673019057, + "loss": 1.1966, + "theoretical_loss": 3.3971097223649176, + "tokens_seen": 2498887680 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002452357071213641, + "loss": 1.3145, + "theoretical_loss": 3.397102674792398, + "tokens_seen": 2498953216 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024521564694082244, + "loss": 1.3083, + "theoretical_loss": 3.3970956274564506, + "tokens_seen": 2499018752 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024519558676028085, + "loss": 1.4205, + "theoretical_loss": 3.3970885803570616, + "tokens_seen": 2499084288 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002451755265797392, + "loss": 1.1542, + "theoretical_loss": 3.397081533494217, + "tokens_seen": 2499149824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002451554663991976, + "loss": 1.44, + "theoretical_loss": 3.397074486867902, + "tokens_seen": 2499215360 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024513540621865594, + "loss": 1.4321, + "theoretical_loss": 3.3970674404781027, + "tokens_seen": 2499280896 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024511534603811435, + "loss": 1.3213, + "theoretical_loss": 3.3970603943248054, + "tokens_seen": 2499346432 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002450952858575727, + "loss": 1.2879, + "theoretical_loss": 3.3970533484079954, + "tokens_seen": 2499411968 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024507522567703113, + "loss": 1.3041, + "theoretical_loss": 3.397046302727659, + "tokens_seen": 2499477504 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002450551654964895, + "loss": 1.2256, + "theoretical_loss": 3.3970392572837818, + "tokens_seen": 2499543040 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024503510531594786, + "loss": 1.2722, + "theoretical_loss": 3.3970322120763496, + "tokens_seen": 2499608576 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002450150451354062, + "loss": 1.3751, + "theoretical_loss": 3.3970251671053484, + "tokens_seen": 2499674112 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002449949849548646, + "loss": 1.318, + "theoretical_loss": 3.3970181223707643, + "tokens_seen": 2499739648 + }, + { + "epoch": 0.52, + "learning_rate": 0.000244974924774323, + "loss": 1.2402, + "theoretical_loss": 3.3970110778725826, + "tokens_seen": 2499805184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024495486459378136, + "loss": 1.3678, + "theoretical_loss": 3.3970040336107896, + "tokens_seen": 2499870720 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002449348044132397, + "loss": 1.2922, + "theoretical_loss": 3.396996989585371, + "tokens_seen": 2499936256 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002449147442326981, + "loss": 1.2874, + "theoretical_loss": 3.396989945796313, + "tokens_seen": 2500001792 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448946840521565, + "loss": 1.2734, + "theoretical_loss": 3.3969829022436007, + "tokens_seen": 2500067328 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024487462387161486, + "loss": 1.189, + "theoretical_loss": 3.3969758589272208, + "tokens_seen": 2500132864 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448545636910732, + "loss": 1.2168, + "theoretical_loss": 3.396968815847159, + "tokens_seen": 2500198400 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448345035105316, + "loss": 1.3108, + "theoretical_loss": 3.396961773003401, + "tokens_seen": 2500263936 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024481444332998995, + "loss": 1.3047, + "theoretical_loss": 3.396954730395932, + "tokens_seen": 2500329472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024479438314944837, + "loss": 1.2748, + "theoretical_loss": 3.3969476880247393, + "tokens_seen": 2500395008 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024477432296890673, + "loss": 1.2455, + "theoretical_loss": 3.396940645889808, + "tokens_seen": 2500460544 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002447542627883651, + "loss": 1.3413, + "theoretical_loss": 3.3969336039911244, + "tokens_seen": 2500526080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024473420260782345, + "loss": 1.1791, + "theoretical_loss": 3.3969265623286735, + "tokens_seen": 2500591616 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024471414242728187, + "loss": 1.2259, + "theoretical_loss": 3.3969195209024416, + "tokens_seen": 2500657152 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024469408224674023, + "loss": 1.4091, + "theoretical_loss": 3.396912479712415, + "tokens_seen": 2500722688 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446740220661986, + "loss": 1.3666, + "theoretical_loss": 3.3969054387585795, + "tokens_seen": 2500788224 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024465396188565696, + "loss": 1.2633, + "theoretical_loss": 3.3968983980409204, + "tokens_seen": 2500853760 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446339017051154, + "loss": 1.1678, + "theoretical_loss": 3.396891357559424, + "tokens_seen": 2500919296 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024461384152457374, + "loss": 1.3439, + "theoretical_loss": 3.3968843173140764, + "tokens_seen": 2500984832 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445937813440321, + "loss": 1.3315, + "theoretical_loss": 3.3968772773048634, + "tokens_seen": 2501050368 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024457372116349046, + "loss": 1.3109, + "theoretical_loss": 3.39687023753177, + "tokens_seen": 2501115904 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445536609829488, + "loss": 1.3383, + "theoretical_loss": 3.396863197994784, + "tokens_seen": 2501181440 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024453360080240724, + "loss": 1.2633, + "theoretical_loss": 3.396856158693889, + "tokens_seen": 2501246976 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445135406218656, + "loss": 1.2642, + "theoretical_loss": 3.3968491196290724, + "tokens_seen": 2501312512 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024449348044132396, + "loss": 1.3262, + "theoretical_loss": 3.3968420808003197, + "tokens_seen": 2501378048 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002444734202607823, + "loss": 1.2824, + "theoretical_loss": 3.3968350422076172, + "tokens_seen": 2501443584 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024445336008024074, + "loss": 1.3248, + "theoretical_loss": 3.39682800385095, + "tokens_seen": 2501509120 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002444332998996991, + "loss": 1.2704, + "theoretical_loss": 3.3968209657303046, + "tokens_seen": 2501574656 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024441323971915747, + "loss": 1.3109, + "theoretical_loss": 3.396813927845667, + "tokens_seen": 2501640192 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024439317953861583, + "loss": 1.2614, + "theoretical_loss": 3.396806890197022, + "tokens_seen": 2501705728 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443731193580742, + "loss": 1.3496, + "theoretical_loss": 3.396799852784357, + "tokens_seen": 2501771264 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443530591775326, + "loss": 1.3203, + "theoretical_loss": 3.396792815607657, + "tokens_seen": 2501836800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024433299899699097, + "loss": 1.2492, + "theoretical_loss": 3.3967857786669082, + "tokens_seen": 2501902336 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 910572, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3356267213821411, + "objective/train/theoretical_loss": 3.3967787419620965, + "objective/train/tokens_used": 872426976, + "theoretical_loss": 3.3967787419620965, + "tokens_seen": 2501967872 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443129388164494, + "loss": 1.2816, + "theoretical_loss": 3.3967787419620965, + "tokens_seen": 2501967872 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024429287863590775, + "loss": 1.2678, + "theoretical_loss": 3.396771705493208, + "tokens_seen": 2502033408 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442728184553661, + "loss": 1.2394, + "theoretical_loss": 3.3967646692602282, + "tokens_seen": 2502098944 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442527582748245, + "loss": 1.2657, + "theoretical_loss": 3.3967576332631433, + "tokens_seen": 2502164480 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024423269809428284, + "loss": 1.1568, + "theoretical_loss": 3.3967505975019385, + "tokens_seen": 2502230016 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024421263791374125, + "loss": 1.3706, + "theoretical_loss": 3.396743561976601, + "tokens_seen": 2502295552 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002441925777331996, + "loss": 1.2704, + "theoretical_loss": 3.396736526687116, + "tokens_seen": 2502361088 + }, + { + "epoch": 0.52, + "learning_rate": 0.000244172517552658, + "loss": 1.2622, + "theoretical_loss": 3.396729491633469, + "tokens_seen": 2502426624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024415245737211634, + "loss": 1.2789, + "theoretical_loss": 3.396722456815647, + "tokens_seen": 2502492160 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024413239719157473, + "loss": 1.2568, + "theoretical_loss": 3.3967154222336347, + "tokens_seen": 2502557696 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024411233701103312, + "loss": 1.2607, + "theoretical_loss": 3.396708387887419, + "tokens_seen": 2502623232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024409227683049148, + "loss": 1.1742, + "theoretical_loss": 3.3967013537769852, + "tokens_seen": 2502688768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024407221664994987, + "loss": 1.2899, + "theoretical_loss": 3.3966943199023194, + "tokens_seen": 2502754304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024405215646940823, + "loss": 1.341, + "theoretical_loss": 3.3966872862634077, + "tokens_seen": 2502819840 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002440320962888666, + "loss": 1.1911, + "theoretical_loss": 3.396680252860236, + "tokens_seen": 2502885376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024401203610832498, + "loss": 1.4833, + "theoretical_loss": 3.39667321969279, + "tokens_seen": 2502950912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024399197592778335, + "loss": 1.245, + "theoretical_loss": 3.396666186761056, + "tokens_seen": 2503016448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024397191574724173, + "loss": 1.2568, + "theoretical_loss": 3.3966591540650195, + "tokens_seen": 2503081984 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002439518555667001, + "loss": 1.2373, + "theoretical_loss": 3.3966521216046663, + "tokens_seen": 2503147520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024393179538615849, + "loss": 1.3586, + "theoretical_loss": 3.396645089379983, + "tokens_seen": 2503213056 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024391173520561685, + "loss": 1.345, + "theoretical_loss": 3.3966380573909554, + "tokens_seen": 2503278592 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024389167502507524, + "loss": 1.304, + "theoretical_loss": 3.3966310256375687, + "tokens_seen": 2503344128 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002438716148445336, + "loss": 1.3118, + "theoretical_loss": 3.3966239941198095, + "tokens_seen": 2503409664 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024385155466399196, + "loss": 1.2612, + "theoretical_loss": 3.3966169628376637, + "tokens_seen": 2503475200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024383149448345035, + "loss": 1.1515, + "theoretical_loss": 3.3966099317911174, + "tokens_seen": 2503540736 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024381143430290871, + "loss": 1.3744, + "theoretical_loss": 3.396602900980156, + "tokens_seen": 2503606272 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002437913741223671, + "loss": 1.3877, + "theoretical_loss": 3.3965958704047656, + "tokens_seen": 2503671808 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024377131394182547, + "loss": 1.3577, + "theoretical_loss": 3.3965888400649322, + "tokens_seen": 2503737344 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024375125376128385, + "loss": 1.3073, + "theoretical_loss": 3.396581809960642, + "tokens_seen": 2503802880 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024373119358074222, + "loss": 1.182, + "theoretical_loss": 3.3965747800918806, + "tokens_seen": 2503868416 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002437111334002006, + "loss": 1.1259, + "theoretical_loss": 3.396567750458634, + "tokens_seen": 2503933952 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024369107321965897, + "loss": 1.2835, + "theoretical_loss": 3.3965607210608884, + "tokens_seen": 2503999488 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024367101303911736, + "loss": 1.3467, + "theoretical_loss": 3.3965536918986294, + "tokens_seen": 2504065024 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024365095285857572, + "loss": 1.2089, + "theoretical_loss": 3.3965466629718435, + "tokens_seen": 2504130560 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024363089267803408, + "loss": 1.3103, + "theoretical_loss": 3.396539634280516, + "tokens_seen": 2504196096 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002436108324974925, + "loss": 1.1838, + "theoretical_loss": 3.396532605824633, + "tokens_seen": 2504261632 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024359077231695086, + "loss": 1.3384, + "theoretical_loss": 3.3965255776041805, + "tokens_seen": 2504327168 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024357071213640925, + "loss": 1.3258, + "theoretical_loss": 3.3965185496191443, + "tokens_seen": 2504392704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002435506519558676, + "loss": 1.2457, + "theoretical_loss": 3.3965115218695114, + "tokens_seen": 2504458240 + }, + { + "epoch": 0.52, + "learning_rate": 0.000243530591775326, + "loss": 1.2626, + "theoretical_loss": 3.3965044943552662, + "tokens_seen": 2504523776 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024351053159478436, + "loss": 1.3249, + "theoretical_loss": 3.396497467076396, + "tokens_seen": 2504589312 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024349047141424275, + "loss": 1.2892, + "theoretical_loss": 3.3964904400328857, + "tokens_seen": 2504654848 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024347041123370112, + "loss": 1.258, + "theoretical_loss": 3.396483413224722, + "tokens_seen": 2504720384 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024345035105315948, + "loss": 1.223, + "theoretical_loss": 3.3964763866518903, + "tokens_seen": 2504785920 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024343029087261787, + "loss": 1.3423, + "theoretical_loss": 3.3964693603143767, + "tokens_seen": 2504851456 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024341023069207623, + "loss": 1.231, + "theoretical_loss": 3.396462334212168, + "tokens_seen": 2504916992 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024339017051153462, + "loss": 1.2587, + "theoretical_loss": 3.3964553083452484, + "tokens_seen": 2504982528 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024337011033099298, + "loss": 1.2877, + "theoretical_loss": 3.3964482827136058, + "tokens_seen": 2505048064 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024335005015045137, + "loss": 1.1973, + "theoretical_loss": 3.3964412573172247, + "tokens_seen": 2505113600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024332998996990973, + "loss": 1.189, + "theoretical_loss": 3.396434232156092, + "tokens_seen": 2505179136 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 911801, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2049455642700195, + "objective/train/theoretical_loss": 3.396427207230193, + "objective/train/tokens_used": 875703776, + "theoretical_loss": 3.396427207230193, + "tokens_seen": 2505244672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024330992978936812, + "loss": 1.255, + "theoretical_loss": 3.396427207230193, + "tokens_seen": 2505244672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024328986960882648, + "loss": 1.3085, + "theoretical_loss": 3.3964201825395146, + "tokens_seen": 2505310208 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024326980942828487, + "loss": 1.3258, + "theoretical_loss": 3.3964131580840418, + "tokens_seen": 2505375744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024324974924774324, + "loss": 1.2469, + "theoretical_loss": 3.396406133863761, + "tokens_seen": 2505441280 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002432296890672016, + "loss": 1.277, + "theoretical_loss": 3.396399109878658, + "tokens_seen": 2505506816 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024320962888666, + "loss": 1.2912, + "theoretical_loss": 3.396392086128719, + "tokens_seen": 2505572352 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024318956870611835, + "loss": 1.1993, + "theoretical_loss": 3.3963850626139296, + "tokens_seen": 2505637888 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024316950852557674, + "loss": 1.3634, + "theoretical_loss": 3.396378039334276, + "tokens_seen": 2505703424 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002431494483450351, + "loss": 1.3874, + "theoretical_loss": 3.3963710162897445, + "tokens_seen": 2505768960 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002431293881644935, + "loss": 1.3488, + "theoretical_loss": 3.3963639934803207, + "tokens_seen": 2505834496 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024310932798395185, + "loss": 1.3272, + "theoretical_loss": 3.3963569709059906, + "tokens_seen": 2505900032 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024308926780341024, + "loss": 1.3915, + "theoretical_loss": 3.3963499485667406, + "tokens_seen": 2505965568 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002430692076228686, + "loss": 1.2889, + "theoretical_loss": 3.396342926462556, + "tokens_seen": 2506031104 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024304914744232697, + "loss": 1.1886, + "theoretical_loss": 3.396335904593423, + "tokens_seen": 2506096640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024302908726178536, + "loss": 1.2166, + "theoretical_loss": 3.3963288829593283, + "tokens_seen": 2506162176 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024300902708124372, + "loss": 1.2082, + "theoretical_loss": 3.396321861560257, + "tokens_seen": 2506227712 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002429889669007021, + "loss": 1.3306, + "theoretical_loss": 3.396314840396195, + "tokens_seen": 2506293248 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024296890672016047, + "loss": 1.2791, + "theoretical_loss": 3.396307819467129, + "tokens_seen": 2506358784 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024294884653961886, + "loss": 1.3418, + "theoretical_loss": 3.3963007987730447, + "tokens_seen": 2506424320 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024292878635907722, + "loss": 1.2467, + "theoretical_loss": 3.396293778313928, + "tokens_seen": 2506489856 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002429087261785356, + "loss": 1.3428, + "theoretical_loss": 3.396286758089765, + "tokens_seen": 2506555392 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024288866599799397, + "loss": 1.1408, + "theoretical_loss": 3.3962797381005414, + "tokens_seen": 2506620928 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024286860581745234, + "loss": 1.3193, + "theoretical_loss": 3.396272718346244, + "tokens_seen": 2506686464 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024284854563691075, + "loss": 1.2052, + "theoretical_loss": 3.396265698826858, + "tokens_seen": 2506752000 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024282848545636911, + "loss": 1.2707, + "theoretical_loss": 3.3962586795423695, + "tokens_seen": 2506817536 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002428084252758275, + "loss": 1.3113, + "theoretical_loss": 3.3962516604927644, + "tokens_seen": 2506883072 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024278836509528587, + "loss": 1.2016, + "theoretical_loss": 3.396244641678029, + "tokens_seen": 2506948608 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024276830491474426, + "loss": 1.2756, + "theoretical_loss": 3.3962376230981497, + "tokens_seen": 2507014144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024274824473420262, + "loss": 1.2539, + "theoretical_loss": 3.3962306047531117, + "tokens_seen": 2507079680 + }, + { + "epoch": 0.52, + "learning_rate": 0.000242728184553661, + "loss": 1.3405, + "theoretical_loss": 3.3962235866429014, + "tokens_seen": 2507145216 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024270812437311937, + "loss": 1.2471, + "theoretical_loss": 3.3962165687675046, + "tokens_seen": 2507210752 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024268806419257776, + "loss": 1.3524, + "theoretical_loss": 3.3962095511269075, + "tokens_seen": 2507276288 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024266800401203612, + "loss": 1.1967, + "theoretical_loss": 3.3962025337210964, + "tokens_seen": 2507341824 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024264794383149448, + "loss": 1.4228, + "theoretical_loss": 3.3961955165500566, + "tokens_seen": 2507407360 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024262788365095287, + "loss": 1.2913, + "theoretical_loss": 3.396188499613775, + "tokens_seen": 2507472896 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024260782347041123, + "loss": 1.2601, + "theoretical_loss": 3.3961814829122363, + "tokens_seen": 2507538432 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024258776328986962, + "loss": 1.3025, + "theoretical_loss": 3.3961744664454274, + "tokens_seen": 2507603968 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024256770310932799, + "loss": 1.2971, + "theoretical_loss": 3.3961674502133348, + "tokens_seen": 2507669504 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024254764292878638, + "loss": 1.1873, + "theoretical_loss": 3.3961604342159437, + "tokens_seen": 2507735040 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024252758274824474, + "loss": 1.3039, + "theoretical_loss": 3.39615341845324, + "tokens_seen": 2507800576 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024250752256770313, + "loss": 1.2806, + "theoretical_loss": 3.3961464029252104, + "tokens_seen": 2507866112 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002424874623871615, + "loss": 1.4146, + "theoretical_loss": 3.3961393876318406, + "tokens_seen": 2507931648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024246740220661985, + "loss": 1.2456, + "theoretical_loss": 3.3961323725731165, + "tokens_seen": 2507997184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024244734202607824, + "loss": 1.2534, + "theoretical_loss": 3.3961253577490242, + "tokens_seen": 2508062720 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002424272818455366, + "loss": 1.2875, + "theoretical_loss": 3.3961183431595496, + "tokens_seen": 2508128256 + }, + { + "epoch": 0.52, + "learning_rate": 0.000242407221664995, + "loss": 1.267, + "theoretical_loss": 3.396111328804679, + "tokens_seen": 2508193792 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024238716148445335, + "loss": 1.241, + "theoretical_loss": 3.3961043146843983, + "tokens_seen": 2508259328 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024236710130391174, + "loss": 1.3231, + "theoretical_loss": 3.3960973007986937, + "tokens_seen": 2508324864 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002423470411233701, + "loss": 1.3257, + "theoretical_loss": 3.3960902871475507, + "tokens_seen": 2508390400 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002423269809428285, + "loss": 1.2854, + "theoretical_loss": 3.3960832737309556, + "tokens_seen": 2508455936 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 912960, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2665480375289917, + "objective/train/theoretical_loss": 3.3960762605488948, + "objective/train/tokens_used": 878980576, + "theoretical_loss": 3.3960762605488948, + "tokens_seen": 2508521472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024230692076228686, + "loss": 1.3118, + "theoretical_loss": 3.3960762605488948, + "tokens_seen": 2508521472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024228686058174522, + "loss": 1.2865, + "theoretical_loss": 3.396069247601354, + "tokens_seen": 2508587008 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002422668004012036, + "loss": 1.2653, + "theoretical_loss": 3.396062234888319, + "tokens_seen": 2508652544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024224674022066197, + "loss": 1.2423, + "theoretical_loss": 3.3960552224097764, + "tokens_seen": 2508718080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024222668004012036, + "loss": 1.2996, + "theoretical_loss": 3.396048210165712, + "tokens_seen": 2508783616 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024220661985957872, + "loss": 1.1542, + "theoretical_loss": 3.3960411981561114, + "tokens_seen": 2508849152 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002421865596790371, + "loss": 1.2604, + "theoretical_loss": 3.396034186380961, + "tokens_seen": 2508914688 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024216649949849547, + "loss": 1.2149, + "theoretical_loss": 3.3960271748402473, + "tokens_seen": 2508980224 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024214643931795386, + "loss": 1.3407, + "theoretical_loss": 3.3960201635339553, + "tokens_seen": 2509045760 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024212637913741223, + "loss": 1.2862, + "theoretical_loss": 3.396013152462072, + "tokens_seen": 2509111296 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024210631895687064, + "loss": 1.2362, + "theoretical_loss": 3.3960061416245826, + "tokens_seen": 2509176832 + }, + { + "epoch": 0.52, + "learning_rate": 0.000242086258776329, + "loss": 1.3402, + "theoretical_loss": 3.395999131021474, + "tokens_seen": 2509242368 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024206619859578737, + "loss": 1.2018, + "theoretical_loss": 3.3959921206527315, + "tokens_seen": 2509307904 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024204613841524576, + "loss": 1.4198, + "theoretical_loss": 3.3959851105183416, + "tokens_seen": 2509373440 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024202607823470412, + "loss": 1.3456, + "theoretical_loss": 3.3959781006182905, + "tokens_seen": 2509438976 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002420060180541625, + "loss": 1.2254, + "theoretical_loss": 3.395971090952564, + "tokens_seen": 2509504512 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024198595787362087, + "loss": 1.3242, + "theoretical_loss": 3.395964081521148, + "tokens_seen": 2509570048 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024196589769307926, + "loss": 1.258, + "theoretical_loss": 3.395957072324028, + "tokens_seen": 2509635584 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024194583751253762, + "loss": 1.2676, + "theoretical_loss": 3.3959500633611914, + "tokens_seen": 2509701120 + }, + { + "epoch": 0.52, + "learning_rate": 0.000241925777331996, + "loss": 1.2625, + "theoretical_loss": 3.3959430546326237, + "tokens_seen": 2509766656 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024190571715145437, + "loss": 1.3453, + "theoretical_loss": 3.3959360461383103, + "tokens_seen": 2509832192 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024188565697091274, + "loss": 1.1878, + "theoretical_loss": 3.395929037878238, + "tokens_seen": 2509897728 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024186559679037113, + "loss": 1.1101, + "theoretical_loss": 3.395922029852393, + "tokens_seen": 2509963264 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002418455366098295, + "loss": 1.265, + "theoretical_loss": 3.395915022060761, + "tokens_seen": 2510028800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024182547642928788, + "loss": 1.3557, + "theoretical_loss": 3.3959080145033274, + "tokens_seen": 2510094336 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024180541624874624, + "loss": 1.209, + "theoretical_loss": 3.395901007180079, + "tokens_seen": 2510159872 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024178535606820463, + "loss": 1.2192, + "theoretical_loss": 3.395894000091002, + "tokens_seen": 2510225408 + }, + { + "epoch": 0.52, + "learning_rate": 0.000241765295887663, + "loss": 1.3312, + "theoretical_loss": 3.3958869932360822, + "tokens_seen": 2510290944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024174523570712138, + "loss": 1.2973, + "theoretical_loss": 3.3958799866153058, + "tokens_seen": 2510356480 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024172517552657974, + "loss": 1.1978, + "theoretical_loss": 3.3958729802286585, + "tokens_seen": 2510422016 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024170511534603813, + "loss": 1.2592, + "theoretical_loss": 3.3958659740761266, + "tokens_seen": 2510487552 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002416850551654965, + "loss": 1.2362, + "theoretical_loss": 3.3958589681576963, + "tokens_seen": 2510553088 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024166499498495486, + "loss": 1.2761, + "theoretical_loss": 3.395851962473354, + "tokens_seen": 2510618624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024164493480441325, + "loss": 1.2353, + "theoretical_loss": 3.3958449570230846, + "tokens_seen": 2510684160 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002416248746238716, + "loss": 1.2603, + "theoretical_loss": 3.3958379518068753, + "tokens_seen": 2510749696 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024160481444333, + "loss": 1.2835, + "theoretical_loss": 3.3958309468247116, + "tokens_seen": 2510815232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024158475426278836, + "loss": 1.3098, + "theoretical_loss": 3.3958239420765794, + "tokens_seen": 2510880768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024156469408224675, + "loss": 1.2894, + "theoretical_loss": 3.395816937562466, + "tokens_seen": 2510946304 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002415446339017051, + "loss": 1.2669, + "theoretical_loss": 3.3958099332823557, + "tokens_seen": 2511011840 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002415245737211635, + "loss": 1.3402, + "theoretical_loss": 3.3958029292362357, + "tokens_seen": 2511077376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024150451354062186, + "loss": 1.2344, + "theoretical_loss": 3.395795925424092, + "tokens_seen": 2511142912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024148445336008022, + "loss": 1.2514, + "theoretical_loss": 3.3957889218459103, + "tokens_seen": 2511208448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024146439317953861, + "loss": 1.3256, + "theoretical_loss": 3.3957819185016773, + "tokens_seen": 2511273984 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024144433299899698, + "loss": 1.3806, + "theoretical_loss": 3.395774915391378, + "tokens_seen": 2511339520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024142427281845537, + "loss": 1.2892, + "theoretical_loss": 3.3957679125149998, + "tokens_seen": 2511405056 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024140421263791373, + "loss": 1.3458, + "theoretical_loss": 3.3957609098725277, + "tokens_seen": 2511470592 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024138415245737212, + "loss": 1.3419, + "theoretical_loss": 3.3957539074639485, + "tokens_seen": 2511536128 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002413640922768305, + "loss": 1.2786, + "theoretical_loss": 3.395746905289248, + "tokens_seen": 2511601664 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002413440320962889, + "loss": 1.2827, + "theoretical_loss": 3.395739903348412, + "tokens_seen": 2511667200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024132397191574726, + "loss": 1.2896, + "theoretical_loss": 3.395732901641427, + "tokens_seen": 2511732736 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 914176, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3319772481918335, + "objective/train/theoretical_loss": 3.395725900168279, + "objective/train/tokens_used": 882257376, + "theoretical_loss": 3.395725900168279, + "tokens_seen": 2511798272 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024130391173520562, + "loss": 1.3164, + "theoretical_loss": 3.395725900168279, + "tokens_seen": 2511798272 + }, + { + "epoch": 0.52, + "learning_rate": 0.000241283851554664, + "loss": 1.4194, + "theoretical_loss": 3.395718898928954, + "tokens_seen": 2511863808 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024126379137412237, + "loss": 1.3157, + "theoretical_loss": 3.395711897923438, + "tokens_seen": 2511929344 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024124373119358076, + "loss": 1.2739, + "theoretical_loss": 3.3957048971517176, + "tokens_seen": 2511994880 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024122367101303912, + "loss": 1.4492, + "theoretical_loss": 3.395697896613778, + "tokens_seen": 2512060416 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002412036108324975, + "loss": 1.3712, + "theoretical_loss": 3.395690896309606, + "tokens_seen": 2512125952 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024118355065195587, + "loss": 1.325, + "theoretical_loss": 3.3956838962391878, + "tokens_seen": 2512191488 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024116349047141426, + "loss": 1.2067, + "theoretical_loss": 3.395676896402509, + "tokens_seen": 2512257024 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024114343029087263, + "loss": 1.224, + "theoretical_loss": 3.395669896799556, + "tokens_seen": 2512322560 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024112337011033102, + "loss": 1.2227, + "theoretical_loss": 3.3956628974303147, + "tokens_seen": 2512388096 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024110330992978938, + "loss": 1.2728, + "theoretical_loss": 3.3956558982947715, + "tokens_seen": 2512453632 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024108324974924774, + "loss": 1.2426, + "theoretical_loss": 3.395648899392912, + "tokens_seen": 2512519168 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024106318956870613, + "loss": 1.2978, + "theoretical_loss": 3.395641900724723, + "tokens_seen": 2512584704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002410431293881645, + "loss": 1.257, + "theoretical_loss": 3.39563490229019, + "tokens_seen": 2512650240 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024102306920762288, + "loss": 1.3434, + "theoretical_loss": 3.3956279040892996, + "tokens_seen": 2512715776 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024100300902708124, + "loss": 1.4236, + "theoretical_loss": 3.395620906122037, + "tokens_seen": 2512781312 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024098294884653963, + "loss": 1.3639, + "theoretical_loss": 3.3956139083883894, + "tokens_seen": 2512846848 + }, + { + "epoch": 0.52, + "learning_rate": 0.000240962888665998, + "loss": 1.3421, + "theoretical_loss": 3.3956069108883424, + "tokens_seen": 2512912384 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024094282848545638, + "loss": 1.2332, + "theoretical_loss": 3.395599913621882, + "tokens_seen": 2512977920 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024092276830491475, + "loss": 1.1876, + "theoretical_loss": 3.3955929165889946, + "tokens_seen": 2513043456 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002409027081243731, + "loss": 1.3529, + "theoretical_loss": 3.3955859197896663, + "tokens_seen": 2513108992 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002408826479438315, + "loss": 1.362, + "theoretical_loss": 3.3955789232238827, + "tokens_seen": 2513174528 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024086258776328986, + "loss": 1.3722, + "theoretical_loss": 3.395571926891631, + "tokens_seen": 2513240064 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024084252758274825, + "loss": 1.2987, + "theoretical_loss": 3.395564930792896, + "tokens_seen": 2513305600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002408224674022066, + "loss": 1.3985, + "theoretical_loss": 3.395557934927665, + "tokens_seen": 2513371136 + }, + { + "epoch": 0.52, + "learning_rate": 0.000240802407221665, + "loss": 1.1562, + "theoretical_loss": 3.395550939295923, + "tokens_seen": 2513436672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024078234704112336, + "loss": 1.3234, + "theoretical_loss": 3.395543943897657, + "tokens_seen": 2513502208 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024076228686058175, + "loss": 1.3481, + "theoretical_loss": 3.395536948732852, + "tokens_seen": 2513567744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024074222668004012, + "loss": 1.2679, + "theoretical_loss": 3.395529953801496, + "tokens_seen": 2513633280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024072216649949848, + "loss": 1.3197, + "theoretical_loss": 3.3955229591035736, + "tokens_seen": 2513698816 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024070210631895687, + "loss": 1.1287, + "theoretical_loss": 3.3955159646390713, + "tokens_seen": 2513764352 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024068204613841523, + "loss": 1.3856, + "theoretical_loss": 3.3955089704079753, + "tokens_seen": 2513829888 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024066198595787362, + "loss": 1.3083, + "theoretical_loss": 3.395501976410272, + "tokens_seen": 2513895424 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024064192577733198, + "loss": 1.3458, + "theoretical_loss": 3.395494982645947, + "tokens_seen": 2513960960 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024062186559679037, + "loss": 1.2991, + "theoretical_loss": 3.3954879891149865, + "tokens_seen": 2514026496 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024060180541624876, + "loss": 1.3509, + "theoretical_loss": 3.395480995817377, + "tokens_seen": 2514092032 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024058174523570715, + "loss": 1.223, + "theoretical_loss": 3.3954740027531045, + "tokens_seen": 2514157568 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002405616850551655, + "loss": 1.2719, + "theoretical_loss": 3.395467009922155, + "tokens_seen": 2514223104 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002405416248746239, + "loss": 1.1867, + "theoretical_loss": 3.395460017324515, + "tokens_seen": 2514288640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024052156469408226, + "loss": 1.3767, + "theoretical_loss": 3.39545302496017, + "tokens_seen": 2514354176 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024050150451354062, + "loss": 1.2269, + "theoretical_loss": 3.3954460328291067, + "tokens_seen": 2514419712 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024048144433299901, + "loss": 1.2896, + "theoretical_loss": 3.395439040931311, + "tokens_seen": 2514485248 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024046138415245738, + "loss": 1.3091, + "theoretical_loss": 3.395432049266769, + "tokens_seen": 2514550784 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024044132397191577, + "loss": 1.3215, + "theoretical_loss": 3.3954250578354666, + "tokens_seen": 2514616320 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024042126379137413, + "loss": 1.2738, + "theoretical_loss": 3.3954180666373905, + "tokens_seen": 2514681856 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024040120361083252, + "loss": 1.2625, + "theoretical_loss": 3.395411075672526, + "tokens_seen": 2514747392 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024038114343029088, + "loss": 1.244, + "theoretical_loss": 3.3954040849408607, + "tokens_seen": 2514812928 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024036108324974927, + "loss": 1.2857, + "theoretical_loss": 3.3953970944423797, + "tokens_seen": 2514878464 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024034102306920763, + "loss": 1.2971, + "theoretical_loss": 3.395390104177069, + "tokens_seen": 2514944000 + }, + { + "epoch": 0.52, + "learning_rate": 0.000240320962888666, + "loss": 1.2655, + "theoretical_loss": 3.395383114144915, + "tokens_seen": 2515009536 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 915361, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.244124174118042, + "objective/train/theoretical_loss": 3.395376124345904, + "objective/train/tokens_used": 885534176, + "theoretical_loss": 3.395376124345904, + "tokens_seen": 2515075072 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024030090270812438, + "loss": 1.2465, + "theoretical_loss": 3.395376124345904, + "tokens_seen": 2515075072 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024028084252758274, + "loss": 1.2714, + "theoretical_loss": 3.3953691347800223, + "tokens_seen": 2515140608 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024026078234704113, + "loss": 1.249, + "theoretical_loss": 3.3953621454472556, + "tokens_seen": 2515206144 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002402407221664995, + "loss": 1.2827, + "theoretical_loss": 3.3953551563475903, + "tokens_seen": 2515271680 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024022066198595789, + "loss": 1.1938, + "theoretical_loss": 3.3953481674810124, + "tokens_seen": 2515337216 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024020060180541625, + "loss": 1.1646, + "theoretical_loss": 3.395341178847508, + "tokens_seen": 2515402752 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024018054162487464, + "loss": 1.3404, + "theoretical_loss": 3.3953341904470635, + "tokens_seen": 2515468288 + }, + { + "epoch": 0.52, + "learning_rate": 0.000240160481444333, + "loss": 1.1998, + "theoretical_loss": 3.3953272022796654, + "tokens_seen": 2515533824 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024014042126379136, + "loss": 1.3259, + "theoretical_loss": 3.395320214345299, + "tokens_seen": 2515599360 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024012036108324975, + "loss": 1.3914, + "theoretical_loss": 3.395313226643951, + "tokens_seen": 2515664896 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024010030090270811, + "loss": 1.3053, + "theoretical_loss": 3.3953062391756075, + "tokens_seen": 2515730432 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002400802407221665, + "loss": 1.3496, + "theoretical_loss": 3.3952992519402545, + "tokens_seen": 2515795968 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024006018054162487, + "loss": 1.2803, + "theoretical_loss": 3.3952922649378783, + "tokens_seen": 2515861504 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024004012036108325, + "loss": 1.3036, + "theoretical_loss": 3.395285278168465, + "tokens_seen": 2515927040 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024002006018054162, + "loss": 1.3084, + "theoretical_loss": 3.395278291632001, + "tokens_seen": 2515992576 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024, + "loss": 1.2416, + "theoretical_loss": 3.395271305328472, + "tokens_seen": 2516058112 + }, + { + "epoch": 0.52, + "learning_rate": 0.00023997993981945837, + "loss": 1.3358, + "theoretical_loss": 3.3952643192578646, + "tokens_seen": 2516123648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00023995987963891676, + "loss": 1.3276, + "theoretical_loss": 3.395257333420165, + "tokens_seen": 2516189184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023993981945837512, + "loss": 1.2792, + "theoretical_loss": 3.3952503478153586, + "tokens_seen": 2516254720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023991975927783348, + "loss": 1.3809, + "theoretical_loss": 3.3952433624434324, + "tokens_seen": 2516320256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023989969909729187, + "loss": 1.1999, + "theoretical_loss": 3.3952363773043723, + "tokens_seen": 2516385792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023987963891675023, + "loss": 1.1577, + "theoretical_loss": 3.3952293923981647, + "tokens_seen": 2516451328 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023985957873620865, + "loss": 1.3156, + "theoretical_loss": 3.3952224077247957, + "tokens_seen": 2516516864 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239839518555667, + "loss": 1.2288, + "theoretical_loss": 3.3952154232842506, + "tokens_seen": 2516582400 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002398194583751254, + "loss": 1.3285, + "theoretical_loss": 3.3952084390765167, + "tokens_seen": 2516647936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023979939819458376, + "loss": 1.3744, + "theoretical_loss": 3.3952014551015797, + "tokens_seen": 2516713472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023977933801404215, + "loss": 1.2413, + "theoretical_loss": 3.395194471359426, + "tokens_seen": 2516779008 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023975927783350052, + "loss": 1.2416, + "theoretical_loss": 3.3951874878500417, + "tokens_seen": 2516844544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023973921765295888, + "loss": 1.2554, + "theoretical_loss": 3.395180504573413, + "tokens_seen": 2516910080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023971915747241727, + "loss": 1.3108, + "theoretical_loss": 3.3951735215295256, + "tokens_seen": 2516975616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023969909729187563, + "loss": 1.2718, + "theoretical_loss": 3.3951665387183665, + "tokens_seen": 2517041152 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023967903711133402, + "loss": 1.2574, + "theoretical_loss": 3.3951595561399213, + "tokens_seen": 2517106688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023965897693079238, + "loss": 1.252, + "theoretical_loss": 3.395152573794176, + "tokens_seen": 2517172224 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023963891675025077, + "loss": 1.2349, + "theoretical_loss": 3.3951455916811177, + "tokens_seen": 2517237760 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023961885656970913, + "loss": 1.3255, + "theoretical_loss": 3.395138609800732, + "tokens_seen": 2517303296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023959879638916752, + "loss": 1.3004, + "theoretical_loss": 3.395131628153005, + "tokens_seen": 2517368832 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023957873620862588, + "loss": 1.305, + "theoretical_loss": 3.395124646737923, + "tokens_seen": 2517434368 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023955867602808427, + "loss": 1.2907, + "theoretical_loss": 3.3951176655554716, + "tokens_seen": 2517499904 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023953861584754264, + "loss": 1.4413, + "theoretical_loss": 3.3951106846056383, + "tokens_seen": 2517565440 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239518555667001, + "loss": 1.2821, + "theoretical_loss": 3.3951037038884087, + "tokens_seen": 2517630976 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002394984954864594, + "loss": 1.4294, + "theoretical_loss": 3.395096723403768, + "tokens_seen": 2517696512 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023947843530591775, + "loss": 1.2547, + "theoretical_loss": 3.395089743151704, + "tokens_seen": 2517762048 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023945837512537614, + "loss": 1.3577, + "theoretical_loss": 3.3950827631322023, + "tokens_seen": 2517827584 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002394383149448345, + "loss": 1.3869, + "theoretical_loss": 3.3950757833452485, + "tokens_seen": 2517893120 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002394182547642929, + "loss": 1.2721, + "theoretical_loss": 3.3950688037908296, + "tokens_seen": 2517958656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023939819458375125, + "loss": 1.3566, + "theoretical_loss": 3.3950618244689315, + "tokens_seen": 2518024192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023937813440320964, + "loss": 1.2574, + "theoretical_loss": 3.3950548453795406, + "tokens_seen": 2518089728 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239358074222668, + "loss": 1.2537, + "theoretical_loss": 3.3950478665226425, + "tokens_seen": 2518155264 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023933801404212637, + "loss": 1.2403, + "theoretical_loss": 3.3950408878982237, + "tokens_seen": 2518220800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023931795386158476, + "loss": 1.4048, + "theoretical_loss": 3.395033909506271, + "tokens_seen": 2518286336 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 916685, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2623939514160156, + "objective/train/theoretical_loss": 3.395026931346769, + "objective/train/tokens_used": 888810976, + "theoretical_loss": 3.395026931346769, + "tokens_seen": 2518351872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023929789368104312, + "loss": 1.2771, + "theoretical_loss": 3.395026931346769, + "tokens_seen": 2518351872 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002392778335005015, + "loss": 1.2298, + "theoretical_loss": 3.3950199534197063, + "tokens_seen": 2518417408 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023925777331995987, + "loss": 1.1146, + "theoretical_loss": 3.3950129757250673, + "tokens_seen": 2518482944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023923771313941826, + "loss": 1.283, + "theoretical_loss": 3.3950059982628384, + "tokens_seen": 2518548480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023921765295887662, + "loss": 1.2702, + "theoretical_loss": 3.3949990210330068, + "tokens_seen": 2518614016 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239197592778335, + "loss": 1.2369, + "theoretical_loss": 3.3949920440355577, + "tokens_seen": 2518679552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023917753259779337, + "loss": 1.229, + "theoretical_loss": 3.3949850672704773, + "tokens_seen": 2518745088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023915747241725174, + "loss": 1.3067, + "theoretical_loss": 3.394978090737753, + "tokens_seen": 2518810624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023913741223671012, + "loss": 1.359, + "theoretical_loss": 3.3949711144373698, + "tokens_seen": 2518876160 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002391173520561685, + "loss": 1.3256, + "theoretical_loss": 3.394964138369314, + "tokens_seen": 2518941696 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002390972918756269, + "loss": 1.2956, + "theoretical_loss": 3.3949571625335726, + "tokens_seen": 2519007232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023907723169508527, + "loss": 1.2952, + "theoretical_loss": 3.394950186930131, + "tokens_seen": 2519072768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023905717151454365, + "loss": 1.2274, + "theoretical_loss": 3.394943211558976, + "tokens_seen": 2519138304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023903711133400202, + "loss": 1.2334, + "theoretical_loss": 3.3949362364200937, + "tokens_seen": 2519203840 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002390170511534604, + "loss": 1.2963, + "theoretical_loss": 3.39492926151347, + "tokens_seen": 2519269376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023899699097291877, + "loss": 1.3311, + "theoretical_loss": 3.3949222868390914, + "tokens_seen": 2519334912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023897693079237716, + "loss": 1.3855, + "theoretical_loss": 3.394915312396944, + "tokens_seen": 2519400448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023895687061183552, + "loss": 1.3365, + "theoretical_loss": 3.3949083381870144, + "tokens_seen": 2519465984 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023893681043129388, + "loss": 1.2198, + "theoretical_loss": 3.394901364209288, + "tokens_seen": 2519531520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023891675025075227, + "loss": 1.3557, + "theoretical_loss": 3.394894390463752, + "tokens_seen": 2519597056 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023889669007021063, + "loss": 1.3104, + "theoretical_loss": 3.394887416950392, + "tokens_seen": 2519662592 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023887662988966902, + "loss": 1.2074, + "theoretical_loss": 3.3948804436691944, + "tokens_seen": 2519728128 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023885656970912739, + "loss": 1.4014, + "theoretical_loss": 3.3948734706201456, + "tokens_seen": 2519793664 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023883650952858577, + "loss": 1.3765, + "theoretical_loss": 3.394866497803232, + "tokens_seen": 2519859200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023881644934804414, + "loss": 1.2059, + "theoretical_loss": 3.394859525218439, + "tokens_seen": 2519924736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023879638916750253, + "loss": 1.2375, + "theoretical_loss": 3.3948525528657534, + "tokens_seen": 2519990272 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002387763289869609, + "loss": 1.189, + "theoretical_loss": 3.3948455807451614, + "tokens_seen": 2520055808 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023875626880641925, + "loss": 1.3316, + "theoretical_loss": 3.3948386088566496, + "tokens_seen": 2520121344 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023873620862587764, + "loss": 1.2478, + "theoretical_loss": 3.3948316372002036, + "tokens_seen": 2520186880 + }, + { + "epoch": 0.53, + "learning_rate": 0.000238716148445336, + "loss": 1.2374, + "theoretical_loss": 3.3948246657758094, + "tokens_seen": 2520252416 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002386960882647944, + "loss": 1.3467, + "theoretical_loss": 3.3948176945834545, + "tokens_seen": 2520317952 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023867602808425275, + "loss": 1.4208, + "theoretical_loss": 3.394810723623124, + "tokens_seen": 2520383488 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023865596790371114, + "loss": 1.1307, + "theoretical_loss": 3.3948037528948047, + "tokens_seen": 2520449024 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002386359077231695, + "loss": 1.3776, + "theoretical_loss": 3.394796782398483, + "tokens_seen": 2520514560 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002386158475426279, + "loss": 1.2455, + "theoretical_loss": 3.3947898121341447, + "tokens_seen": 2520580096 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023859578736208626, + "loss": 1.3522, + "theoretical_loss": 3.3947828421017756, + "tokens_seen": 2520645632 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023857572718154462, + "loss": 1.3456, + "theoretical_loss": 3.3947758723013632, + "tokens_seen": 2520711168 + }, + { + "epoch": 0.53, + "learning_rate": 0.000238555667001003, + "loss": 1.3175, + "theoretical_loss": 3.394768902732893, + "tokens_seen": 2520776704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023853560682046137, + "loss": 1.2439, + "theoretical_loss": 3.394761933396351, + "tokens_seen": 2520842240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023851554663991976, + "loss": 1.3561, + "theoretical_loss": 3.394754964291724, + "tokens_seen": 2520907776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023849548645937812, + "loss": 1.2372, + "theoretical_loss": 3.3947479954189976, + "tokens_seen": 2520973312 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002384754262788365, + "loss": 1.2974, + "theoretical_loss": 3.394741026778159, + "tokens_seen": 2521038848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023845536609829487, + "loss": 1.3473, + "theoretical_loss": 3.394734058369194, + "tokens_seen": 2521104384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023843530591775326, + "loss": 1.2912, + "theoretical_loss": 3.394727090192089, + "tokens_seen": 2521169920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023841524573721163, + "loss": 1.368, + "theoretical_loss": 3.3947201222468295, + "tokens_seen": 2521235456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023839518555667002, + "loss": 1.2389, + "theoretical_loss": 3.3947131545334024, + "tokens_seen": 2521300992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023837512537612838, + "loss": 1.2624, + "theoretical_loss": 3.3947061870517943, + "tokens_seen": 2521366528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023835506519558677, + "loss": 1.4093, + "theoretical_loss": 3.394699219801991, + "tokens_seen": 2521432064 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023833500501504516, + "loss": 1.2918, + "theoretical_loss": 3.3946922527839782, + "tokens_seen": 2521497600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023831494483450352, + "loss": 1.3125, + "theoretical_loss": 3.3946852859977437, + "tokens_seen": 2521563136 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 917863, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2736694812774658, + "objective/train/theoretical_loss": 3.394678319443272, + "objective/train/tokens_used": 892087776, + "theoretical_loss": 3.394678319443272, + "tokens_seen": 2521628672 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002382948846539619, + "loss": 1.2443, + "theoretical_loss": 3.394678319443272, + "tokens_seen": 2521628672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023827482447342027, + "loss": 1.3449, + "theoretical_loss": 3.3946713531205504, + "tokens_seen": 2521694208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023825476429287866, + "loss": 1.2789, + "theoretical_loss": 3.3946643870295654, + "tokens_seen": 2521759744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023823470411233702, + "loss": 1.2904, + "theoretical_loss": 3.3946574211703027, + "tokens_seen": 2521825280 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002382146439317954, + "loss": 1.3102, + "theoretical_loss": 3.3946504555427484, + "tokens_seen": 2521890816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023819458375125377, + "loss": 1.3501, + "theoretical_loss": 3.3946434901468896, + "tokens_seen": 2521956352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023817452357071214, + "loss": 1.2194, + "theoretical_loss": 3.3946365249827117, + "tokens_seen": 2522021888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023815446339017052, + "loss": 1.3184, + "theoretical_loss": 3.3946295600502014, + "tokens_seen": 2522087424 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002381344032096289, + "loss": 1.2719, + "theoretical_loss": 3.394622595349345, + "tokens_seen": 2522152960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023811434302908728, + "loss": 1.3278, + "theoretical_loss": 3.3946156308801285, + "tokens_seen": 2522218496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023809428284854564, + "loss": 1.3833, + "theoretical_loss": 3.3946086666425384, + "tokens_seen": 2522284032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023807422266800403, + "loss": 1.2255, + "theoretical_loss": 3.394601702636561, + "tokens_seen": 2522349568 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002380541624874624, + "loss": 1.2968, + "theoretical_loss": 3.3945947388621827, + "tokens_seen": 2522415104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023803410230692078, + "loss": 1.3707, + "theoretical_loss": 3.394587775319389, + "tokens_seen": 2522480640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023801404212637914, + "loss": 1.3019, + "theoretical_loss": 3.3945808120081677, + "tokens_seen": 2522546176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023799398194583753, + "loss": 1.2885, + "theoretical_loss": 3.394573848928504, + "tokens_seen": 2522611712 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002379739217652959, + "loss": 1.3273, + "theoretical_loss": 3.3945668860803835, + "tokens_seen": 2522677248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023795386158475426, + "loss": 1.3853, + "theoretical_loss": 3.394559923463794, + "tokens_seen": 2522742784 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023793380140421265, + "loss": 1.3099, + "theoretical_loss": 3.3945529610787215, + "tokens_seen": 2522808320 + }, + { + "epoch": 0.53, + "learning_rate": 0.000237913741223671, + "loss": 1.3246, + "theoretical_loss": 3.3945459989251514, + "tokens_seen": 2522873856 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378936810431294, + "loss": 1.1867, + "theoretical_loss": 3.3945390370030704, + "tokens_seen": 2522939392 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023787362086258776, + "loss": 1.2945, + "theoretical_loss": 3.394532075312465, + "tokens_seen": 2523004928 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023785356068204615, + "loss": 1.3469, + "theoretical_loss": 3.3945251138533212, + "tokens_seen": 2523070464 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378335005015045, + "loss": 1.258, + "theoretical_loss": 3.394518152625626, + "tokens_seen": 2523136000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378134403209629, + "loss": 1.2601, + "theoretical_loss": 3.394511191629365, + "tokens_seen": 2523201536 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023779338014042126, + "loss": 1.2274, + "theoretical_loss": 3.3945042308645244, + "tokens_seen": 2523267072 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023777331995987962, + "loss": 1.3227, + "theoretical_loss": 3.394497270331091, + "tokens_seen": 2523332608 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023775325977933801, + "loss": 1.246, + "theoretical_loss": 3.3944903100290507, + "tokens_seen": 2523398144 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023773319959879638, + "loss": 1.3277, + "theoretical_loss": 3.39448334995839, + "tokens_seen": 2523463680 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023771313941825477, + "loss": 1.2785, + "theoretical_loss": 3.3944763901190953, + "tokens_seen": 2523529216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023769307923771313, + "loss": 1.4128, + "theoretical_loss": 3.394469430511153, + "tokens_seen": 2523594752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023767301905717152, + "loss": 1.3937, + "theoretical_loss": 3.3944624711345486, + "tokens_seen": 2523660288 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023765295887662988, + "loss": 1.2275, + "theoretical_loss": 3.394455511989269, + "tokens_seen": 2523725824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023763289869608827, + "loss": 1.3531, + "theoretical_loss": 3.394448553075301, + "tokens_seen": 2523791360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023761283851554663, + "loss": 1.2761, + "theoretical_loss": 3.39444159439263, + "tokens_seen": 2523856896 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023759277833500502, + "loss": 1.3607, + "theoretical_loss": 3.3944346359412423, + "tokens_seen": 2523922432 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002375727181544634, + "loss": 1.3944, + "theoretical_loss": 3.3944276777211253, + "tokens_seen": 2523987968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023755265797392177, + "loss": 1.2724, + "theoretical_loss": 3.3944207197322642, + "tokens_seen": 2524053504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023753259779338016, + "loss": 1.3558, + "theoretical_loss": 3.3944137619746457, + "tokens_seen": 2524119040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023751253761283852, + "loss": 1.2184, + "theoretical_loss": 3.3944068044482565, + "tokens_seen": 2524184576 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002374924774322969, + "loss": 1.3499, + "theoretical_loss": 3.394399847153082, + "tokens_seen": 2524250112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023747241725175527, + "loss": 1.3423, + "theoretical_loss": 3.3943928900891094, + "tokens_seen": 2524315648 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023745235707121366, + "loss": 1.342, + "theoretical_loss": 3.3943859332563244, + "tokens_seen": 2524381184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023743229689067203, + "loss": 1.3008, + "theoretical_loss": 3.394378976654714, + "tokens_seen": 2524446720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023741223671013042, + "loss": 1.3382, + "theoretical_loss": 3.3943720202842638, + "tokens_seen": 2524512256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023739217652958878, + "loss": 1.2082, + "theoretical_loss": 3.39436506414496, + "tokens_seen": 2524577792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023737211634904714, + "loss": 1.3622, + "theoretical_loss": 3.3943581082367897, + "tokens_seen": 2524643328 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023735205616850553, + "loss": 1.539, + "theoretical_loss": 3.3943511525597394, + "tokens_seen": 2524708864 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002373319959879639, + "loss": 1.3451, + "theoretical_loss": 3.394344197113794, + "tokens_seen": 2524774400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023731193580742228, + "loss": 1.3228, + "theoretical_loss": 3.394337241898941, + "tokens_seen": 2524839936 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 918983, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3001699447631836, + "objective/train/theoretical_loss": 3.394330286915167, + "objective/train/tokens_used": 895364576, + "theoretical_loss": 3.394330286915167, + "tokens_seen": 2524905472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023729187562688064, + "loss": 1.3305, + "theoretical_loss": 3.394330286915167, + "tokens_seen": 2524905472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023727181544633903, + "loss": 1.3807, + "theoretical_loss": 3.394323332162457, + "tokens_seen": 2524971008 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002372517552657974, + "loss": 1.3349, + "theoretical_loss": 3.3943163776407985, + "tokens_seen": 2525036544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023723169508525578, + "loss": 1.3456, + "theoretical_loss": 3.394309423350177, + "tokens_seen": 2525102080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023721163490471415, + "loss": 1.3051, + "theoretical_loss": 3.3943024692905794, + "tokens_seen": 2525167616 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002371915747241725, + "loss": 1.4065, + "theoretical_loss": 3.394295515461992, + "tokens_seen": 2525233152 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002371715145436309, + "loss": 1.3745, + "theoretical_loss": 3.3942885618644008, + "tokens_seen": 2525298688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023715145436308926, + "loss": 1.3283, + "theoretical_loss": 3.3942816084977925, + "tokens_seen": 2525364224 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023713139418254765, + "loss": 1.3107, + "theoretical_loss": 3.394274655362153, + "tokens_seen": 2525429760 + }, + { + "epoch": 0.53, + "learning_rate": 0.000237111334002006, + "loss": 1.314, + "theoretical_loss": 3.3942677024574692, + "tokens_seen": 2525495296 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002370912738214644, + "loss": 1.3154, + "theoretical_loss": 3.3942607497837267, + "tokens_seen": 2525560832 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023707121364092276, + "loss": 1.2463, + "theoretical_loss": 3.3942537973409124, + "tokens_seen": 2525626368 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023705115346038115, + "loss": 1.2468, + "theoretical_loss": 3.394246845129013, + "tokens_seen": 2525691904 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023703109327983952, + "loss": 1.3303, + "theoretical_loss": 3.3942398931480136, + "tokens_seen": 2525757440 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023701103309929788, + "loss": 1.387, + "theoretical_loss": 3.3942329413979015, + "tokens_seen": 2525822976 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023699097291875627, + "loss": 1.325, + "theoretical_loss": 3.394225989878663, + "tokens_seen": 2525888512 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023697091273821463, + "loss": 1.3172, + "theoretical_loss": 3.3942190385902844, + "tokens_seen": 2525954048 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023695085255767302, + "loss": 1.2369, + "theoretical_loss": 3.3942120875327513, + "tokens_seen": 2526019584 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023693079237713138, + "loss": 1.457, + "theoretical_loss": 3.3942051367060513, + "tokens_seen": 2526085120 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023691073219658977, + "loss": 1.2633, + "theoretical_loss": 3.3941981861101698, + "tokens_seen": 2526150656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023689067201604813, + "loss": 1.246, + "theoretical_loss": 3.3941912357450934, + "tokens_seen": 2526216192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023687061183550652, + "loss": 1.2951, + "theoretical_loss": 3.394184285610809, + "tokens_seen": 2526281728 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002368505516549649, + "loss": 1.3114, + "theoretical_loss": 3.3941773357073015, + "tokens_seen": 2526347264 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002368304914744233, + "loss": 1.2443, + "theoretical_loss": 3.394170386034559, + "tokens_seen": 2526412800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023681043129388166, + "loss": 1.306, + "theoretical_loss": 3.3941634365925664, + "tokens_seen": 2526478336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023679037111334002, + "loss": 1.2768, + "theoretical_loss": 3.394156487381311, + "tokens_seen": 2526543872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023677031093279841, + "loss": 1.3769, + "theoretical_loss": 3.394149538400779, + "tokens_seen": 2526609408 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023675025075225678, + "loss": 1.27, + "theoretical_loss": 3.3941425896509565, + "tokens_seen": 2526674944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023673019057171517, + "loss": 1.341, + "theoretical_loss": 3.3941356411318298, + "tokens_seen": 2526740480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023671013039117353, + "loss": 1.3168, + "theoretical_loss": 3.3941286928433856, + "tokens_seen": 2526806016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023669007021063192, + "loss": 1.3324, + "theoretical_loss": 3.3941217447856102, + "tokens_seen": 2526871552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023667001003009028, + "loss": 1.2687, + "theoretical_loss": 3.39411479695849, + "tokens_seen": 2526937088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023664994984954867, + "loss": 1.292, + "theoretical_loss": 3.3941078493620105, + "tokens_seen": 2527002624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023662988966900703, + "loss": 1.2878, + "theoretical_loss": 3.3941009019961594, + "tokens_seen": 2527068160 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002366098294884654, + "loss": 1.3061, + "theoretical_loss": 3.394093954860922, + "tokens_seen": 2527133696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023658976930792378, + "loss": 1.3248, + "theoretical_loss": 3.3940870079562853, + "tokens_seen": 2527199232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023656970912738214, + "loss": 1.3349, + "theoretical_loss": 3.3940800612822355, + "tokens_seen": 2527264768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023654964894684053, + "loss": 1.1914, + "theoretical_loss": 3.3940731148387586, + "tokens_seen": 2527330304 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002365295887662989, + "loss": 1.3029, + "theoretical_loss": 3.3940661686258418, + "tokens_seen": 2527395840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023650952858575729, + "loss": 1.3017, + "theoretical_loss": 3.3940592226434707, + "tokens_seen": 2527461376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023648946840521565, + "loss": 1.2516, + "theoretical_loss": 3.3940522768916317, + "tokens_seen": 2527526912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023646940822467404, + "loss": 1.3855, + "theoretical_loss": 3.394045331370312, + "tokens_seen": 2527592448 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002364493480441324, + "loss": 1.1429, + "theoretical_loss": 3.394038386079497, + "tokens_seen": 2527657984 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023642928786359076, + "loss": 1.2563, + "theoretical_loss": 3.394031441019173, + "tokens_seen": 2527723520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023640922768304915, + "loss": 1.3341, + "theoretical_loss": 3.3940244961893278, + "tokens_seen": 2527789056 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002363891675025075, + "loss": 1.2451, + "theoretical_loss": 3.394017551589946, + "tokens_seen": 2527854592 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002363691073219659, + "loss": 1.3222, + "theoretical_loss": 3.3940106072210154, + "tokens_seen": 2527920128 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023634904714142426, + "loss": 1.2938, + "theoretical_loss": 3.394003663082521, + "tokens_seen": 2527985664 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023632898696088265, + "loss": 1.2194, + "theoretical_loss": 3.3939967191744507, + "tokens_seen": 2528051200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023630892678034102, + "loss": 1.3052, + "theoretical_loss": 3.39398977549679, + "tokens_seen": 2528116736 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 920165, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1944116353988647, + "objective/train/theoretical_loss": 3.3939828320495247, + "objective/train/tokens_used": 898641376, + "theoretical_loss": 3.3939828320495247, + "tokens_seen": 2528182272 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002362888665997994, + "loss": 1.19, + "theoretical_loss": 3.3939828320495247, + "tokens_seen": 2528182272 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023626880641925777, + "loss": 1.2717, + "theoretical_loss": 3.3939758888326423, + "tokens_seen": 2528247808 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023624874623871616, + "loss": 1.4383, + "theoretical_loss": 3.393968945846129, + "tokens_seen": 2528313344 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023622868605817452, + "loss": 1.3235, + "theoretical_loss": 3.3939620030899706, + "tokens_seen": 2528378880 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023620862587763288, + "loss": 1.1821, + "theoretical_loss": 3.393955060564154, + "tokens_seen": 2528444416 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023618856569709127, + "loss": 1.3457, + "theoretical_loss": 3.393948118268666, + "tokens_seen": 2528509952 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023616850551654963, + "loss": 1.2929, + "theoretical_loss": 3.3939411762034917, + "tokens_seen": 2528575488 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023614844533600802, + "loss": 1.292, + "theoretical_loss": 3.393934234368618, + "tokens_seen": 2528641024 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023612838515546639, + "loss": 1.3213, + "theoretical_loss": 3.3939272927640323, + "tokens_seen": 2528706560 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002361083249749248, + "loss": 1.235, + "theoretical_loss": 3.3939203513897196, + "tokens_seen": 2528772096 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023608826479438316, + "loss": 1.3484, + "theoretical_loss": 3.393913410245667, + "tokens_seen": 2528837632 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023606820461384155, + "loss": 1.2588, + "theoretical_loss": 3.393906469331861, + "tokens_seen": 2528903168 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023604814443329992, + "loss": 1.3215, + "theoretical_loss": 3.393899528648287, + "tokens_seen": 2528968704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023602808425275828, + "loss": 1.3271, + "theoretical_loss": 3.3938925881949333, + "tokens_seen": 2529034240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023600802407221667, + "loss": 1.1908, + "theoretical_loss": 3.3938856479717847, + "tokens_seen": 2529099776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023598796389167503, + "loss": 1.2718, + "theoretical_loss": 3.393878707978828, + "tokens_seen": 2529165312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023596790371113342, + "loss": 1.302, + "theoretical_loss": 3.3938717682160497, + "tokens_seen": 2529230848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023594784353059178, + "loss": 1.3865, + "theoretical_loss": 3.393864828683436, + "tokens_seen": 2529296384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023592778335005017, + "loss": 1.3541, + "theoretical_loss": 3.393857889380974, + "tokens_seen": 2529361920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023590772316950853, + "loss": 1.3214, + "theoretical_loss": 3.393850950308649, + "tokens_seen": 2529427456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023588766298896692, + "loss": 1.3339, + "theoretical_loss": 3.393844011466448, + "tokens_seen": 2529492992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023586760280842528, + "loss": 1.3494, + "theoretical_loss": 3.3938370728543577, + "tokens_seen": 2529558528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023584754262788367, + "loss": 1.2793, + "theoretical_loss": 3.3938301344723643, + "tokens_seen": 2529624064 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023582748244734204, + "loss": 1.2974, + "theoretical_loss": 3.3938231963204535, + "tokens_seen": 2529689600 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002358074222668004, + "loss": 1.3017, + "theoretical_loss": 3.393816258398613, + "tokens_seen": 2529755136 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357873620862588, + "loss": 1.3097, + "theoretical_loss": 3.3938093207068283, + "tokens_seen": 2529820672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023576730190571715, + "loss": 1.229, + "theoretical_loss": 3.3938023832450854, + "tokens_seen": 2529886208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023574724172517554, + "loss": 1.3584, + "theoretical_loss": 3.3937954460133724, + "tokens_seen": 2529951744 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357271815446339, + "loss": 1.2374, + "theoretical_loss": 3.393788509011674, + "tokens_seen": 2530017280 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357071213640923, + "loss": 1.3936, + "theoretical_loss": 3.393781572239978, + "tokens_seen": 2530082816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023568706118355065, + "loss": 1.2647, + "theoretical_loss": 3.393774635698269, + "tokens_seen": 2530148352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023566700100300904, + "loss": 1.3391, + "theoretical_loss": 3.3937676993865353, + "tokens_seen": 2530213888 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002356469408224674, + "loss": 1.29, + "theoretical_loss": 3.3937607633047624, + "tokens_seen": 2530279424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023562688064192577, + "loss": 1.2838, + "theoretical_loss": 3.3937538274529366, + "tokens_seen": 2530344960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023560682046138416, + "loss": 1.2893, + "theoretical_loss": 3.3937468918310447, + "tokens_seen": 2530410496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023558676028084252, + "loss": 1.2798, + "theoretical_loss": 3.3937399564390733, + "tokens_seen": 2530476032 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002355667001003009, + "loss": 1.2985, + "theoretical_loss": 3.3937330212770083, + "tokens_seen": 2530541568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023554663991975927, + "loss": 1.316, + "theoretical_loss": 3.393726086344836, + "tokens_seen": 2530607104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023552657973921766, + "loss": 1.3255, + "theoretical_loss": 3.393719151642544, + "tokens_seen": 2530672640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023550651955867602, + "loss": 1.2738, + "theoretical_loss": 3.3937122171701173, + "tokens_seen": 2530738176 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002354864593781344, + "loss": 1.3738, + "theoretical_loss": 3.393705282927543, + "tokens_seen": 2530803712 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023546639919759277, + "loss": 1.3518, + "theoretical_loss": 3.3936983489148074, + "tokens_seen": 2530869248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023544633901705113, + "loss": 1.254, + "theoretical_loss": 3.3936914151318973, + "tokens_seen": 2530934784 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023542627883650952, + "loss": 1.4147, + "theoretical_loss": 3.3936844815787985, + "tokens_seen": 2531000320 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002354062186559679, + "loss": 1.3344, + "theoretical_loss": 3.3936775482554977, + "tokens_seen": 2531065856 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023538615847542628, + "loss": 1.3206, + "theoretical_loss": 3.3936706151619815, + "tokens_seen": 2531131392 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023536609829488464, + "loss": 1.2576, + "theoretical_loss": 3.3936636822982367, + "tokens_seen": 2531196928 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023534603811434305, + "loss": 1.3308, + "theoretical_loss": 3.3936567496642485, + "tokens_seen": 2531262464 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023532597793380142, + "loss": 1.2024, + "theoretical_loss": 3.3936498172600045, + "tokens_seen": 2531328000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002353059177532598, + "loss": 1.3267, + "theoretical_loss": 3.393642885085491, + "tokens_seen": 2531393536 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 921348, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2266067266464233, + "objective/train/theoretical_loss": 3.3936359531406937, + "objective/train/tokens_used": 901918176, + "theoretical_loss": 3.3936359531406937, + "tokens_seen": 2531459072 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023528585757271817, + "loss": 1.2252, + "theoretical_loss": 3.3936359531406937, + "tokens_seen": 2531459072 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023526579739217656, + "loss": 1.2849, + "theoretical_loss": 3.3936290214255997, + "tokens_seen": 2531524608 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023524573721163492, + "loss": 1.421, + "theoretical_loss": 3.3936220899401954, + "tokens_seen": 2531590144 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023522567703109328, + "loss": 1.2169, + "theoretical_loss": 3.393615158684467, + "tokens_seen": 2531655680 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023520561685055167, + "loss": 1.3707, + "theoretical_loss": 3.393608227658401, + "tokens_seen": 2531721216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023518555667001003, + "loss": 1.4081, + "theoretical_loss": 3.3936012968619838, + "tokens_seen": 2531786752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023516549648946842, + "loss": 1.32, + "theoretical_loss": 3.393594366295202, + "tokens_seen": 2531852288 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023514543630892679, + "loss": 1.3821, + "theoretical_loss": 3.393587435958042, + "tokens_seen": 2531917824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023512537612838517, + "loss": 1.3061, + "theoretical_loss": 3.39358050585049, + "tokens_seen": 2531983360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023510531594784354, + "loss": 1.3634, + "theoretical_loss": 3.393573575972533, + "tokens_seen": 2532048896 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023508525576730193, + "loss": 1.2712, + "theoretical_loss": 3.393566646324157, + "tokens_seen": 2532114432 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002350651955867603, + "loss": 1.3174, + "theoretical_loss": 3.393559716905349, + "tokens_seen": 2532179968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023504513540621865, + "loss": 1.3451, + "theoretical_loss": 3.3935527877160947, + "tokens_seen": 2532245504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023502507522567704, + "loss": 1.377, + "theoretical_loss": 3.3935458587563807, + "tokens_seen": 2532311040 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002350050150451354, + "loss": 1.3501, + "theoretical_loss": 3.3935389300261942, + "tokens_seen": 2532376576 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002349849548645938, + "loss": 1.247, + "theoretical_loss": 3.3935320015255206, + "tokens_seen": 2532442112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023496489468405215, + "loss": 1.2696, + "theoretical_loss": 3.393525073254347, + "tokens_seen": 2532507648 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023494483450351054, + "loss": 1.2861, + "theoretical_loss": 3.39351814521266, + "tokens_seen": 2532573184 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002349247743229689, + "loss": 1.3149, + "theoretical_loss": 3.3935112174004454, + "tokens_seen": 2532638720 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002349047141424273, + "loss": 1.2406, + "theoretical_loss": 3.39350428981769, + "tokens_seen": 2532704256 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023488465396188566, + "loss": 1.3341, + "theoretical_loss": 3.3934973624643807, + "tokens_seen": 2532769792 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023486459378134402, + "loss": 1.306, + "theoretical_loss": 3.393490435340503, + "tokens_seen": 2532835328 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002348445336008024, + "loss": 1.239, + "theoretical_loss": 3.3934835084460446, + "tokens_seen": 2532900864 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023482447342026077, + "loss": 1.444, + "theoretical_loss": 3.393476581780991, + "tokens_seen": 2532966400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023480441323971916, + "loss": 1.3266, + "theoretical_loss": 3.393469655345329, + "tokens_seen": 2533031936 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023478435305917752, + "loss": 1.2563, + "theoretical_loss": 3.3934627291390456, + "tokens_seen": 2533097472 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002347642928786359, + "loss": 1.2682, + "theoretical_loss": 3.393455803162126, + "tokens_seen": 2533163008 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023474423269809427, + "loss": 1.2032, + "theoretical_loss": 3.3934488774145577, + "tokens_seen": 2533228544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023472417251755266, + "loss": 1.2747, + "theoretical_loss": 3.393441951896327, + "tokens_seen": 2533294080 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023470411233701103, + "loss": 1.3323, + "theoretical_loss": 3.3934350266074196, + "tokens_seen": 2533359616 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023468405215646942, + "loss": 1.2968, + "theoretical_loss": 3.3934281015478227, + "tokens_seen": 2533425152 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023466399197592778, + "loss": 1.305, + "theoretical_loss": 3.3934211767175233, + "tokens_seen": 2533490688 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023464393179538614, + "loss": 1.3011, + "theoretical_loss": 3.3934142521165067, + "tokens_seen": 2533556224 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023462387161484453, + "loss": 1.3367, + "theoretical_loss": 3.3934073277447605, + "tokens_seen": 2533621760 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023460381143430292, + "loss": 1.319, + "theoretical_loss": 3.39340040360227, + "tokens_seen": 2533687296 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002345837512537613, + "loss": 1.3371, + "theoretical_loss": 3.393393479689023, + "tokens_seen": 2533752832 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023456369107321967, + "loss": 1.3563, + "theoretical_loss": 3.3933865560050043, + "tokens_seen": 2533818368 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023454363089267806, + "loss": 1.3537, + "theoretical_loss": 3.393379632550202, + "tokens_seen": 2533883904 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023452357071213642, + "loss": 1.2425, + "theoretical_loss": 3.393372709324602, + "tokens_seen": 2533949440 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002345035105315948, + "loss": 1.2524, + "theoretical_loss": 3.3933657863281903, + "tokens_seen": 2534014976 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023448345035105317, + "loss": 1.3222, + "theoretical_loss": 3.3933588635609544, + "tokens_seen": 2534080512 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023446339017051154, + "loss": 1.3306, + "theoretical_loss": 3.39335194102288, + "tokens_seen": 2534146048 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023444332998996992, + "loss": 1.2474, + "theoretical_loss": 3.3933450187139536, + "tokens_seen": 2534211584 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002344232698094283, + "loss": 1.2812, + "theoretical_loss": 3.393338096634162, + "tokens_seen": 2534277120 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023440320962888668, + "loss": 1.2894, + "theoretical_loss": 3.3933311747834916, + "tokens_seen": 2534342656 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023438314944834504, + "loss": 1.3491, + "theoretical_loss": 3.3933242531619285, + "tokens_seen": 2534408192 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023436308926780343, + "loss": 1.3092, + "theoretical_loss": 3.3933173317694596, + "tokens_seen": 2534473728 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002343430290872618, + "loss": 1.2819, + "theoretical_loss": 3.393310410606072, + "tokens_seen": 2534539264 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023432296890672018, + "loss": 1.2693, + "theoretical_loss": 3.3933034896717507, + "tokens_seen": 2534604800 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023430290872617854, + "loss": 1.3747, + "theoretical_loss": 3.3932965689664836, + "tokens_seen": 2534670336 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 922493, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4441323280334473, + "objective/train/theoretical_loss": 3.3932896484902564, + "objective/train/tokens_used": 905194976, + "theoretical_loss": 3.3932896484902564, + "tokens_seen": 2534735872 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002342828485456369, + "loss": 1.3382, + "theoretical_loss": 3.3932896484902564, + "tokens_seen": 2534735872 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002342627883650953, + "loss": 1.2646, + "theoretical_loss": 3.393282728243056, + "tokens_seen": 2534801408 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023424272818455366, + "loss": 1.3002, + "theoretical_loss": 3.3932758082248684, + "tokens_seen": 2534866944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023422266800401204, + "loss": 1.2672, + "theoretical_loss": 3.393268888435681, + "tokens_seen": 2534932480 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002342026078234704, + "loss": 1.2629, + "theoretical_loss": 3.3932619688754793, + "tokens_seen": 2534998016 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002341825476429288, + "loss": 1.2561, + "theoretical_loss": 3.3932550495442504, + "tokens_seen": 2535063552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023416248746238716, + "loss": 1.3855, + "theoretical_loss": 3.3932481304419806, + "tokens_seen": 2535129088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023414242728184555, + "loss": 1.2565, + "theoretical_loss": 3.393241211568656, + "tokens_seen": 2535194624 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002341223671013039, + "loss": 1.2771, + "theoretical_loss": 3.393234292924264, + "tokens_seen": 2535260160 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002341023069207623, + "loss": 1.3158, + "theoretical_loss": 3.3932273745087906, + "tokens_seen": 2535325696 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023408224674022066, + "loss": 1.3222, + "theoretical_loss": 3.3932204563222226, + "tokens_seen": 2535391232 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023406218655967902, + "loss": 1.2322, + "theoretical_loss": 3.393213538364546, + "tokens_seen": 2535456768 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002340421263791374, + "loss": 1.3096, + "theoretical_loss": 3.393206620635748, + "tokens_seen": 2535522304 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023402206619859578, + "loss": 1.3569, + "theoretical_loss": 3.393199703135814, + "tokens_seen": 2535587840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023400200601805416, + "loss": 1.2447, + "theoretical_loss": 3.3931927858647315, + "tokens_seen": 2535653376 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023398194583751253, + "loss": 1.1389, + "theoretical_loss": 3.3931858688224867, + "tokens_seen": 2535718912 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023396188565697092, + "loss": 1.2994, + "theoretical_loss": 3.3931789520090665, + "tokens_seen": 2535784448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023394182547642928, + "loss": 1.268, + "theoretical_loss": 3.393172035424457, + "tokens_seen": 2535849984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023392176529588767, + "loss": 1.2489, + "theoretical_loss": 3.3931651190686445, + "tokens_seen": 2535915520 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023390170511534603, + "loss": 1.2966, + "theoretical_loss": 3.393158202941616, + "tokens_seen": 2535981056 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002338816449348044, + "loss": 1.2845, + "theoretical_loss": 3.3931512870433576, + "tokens_seen": 2536046592 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023386158475426278, + "loss": 1.2369, + "theoretical_loss": 3.393144371373856, + "tokens_seen": 2536112128 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023384152457372117, + "loss": 1.3497, + "theoretical_loss": 3.393137455933098, + "tokens_seen": 2536177664 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023382146439317956, + "loss": 1.3591, + "theoretical_loss": 3.39313054072107, + "tokens_seen": 2536243200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023380140421263792, + "loss": 1.3728, + "theoretical_loss": 3.3931236257377577, + "tokens_seen": 2536308736 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002337813440320963, + "loss": 1.3139, + "theoretical_loss": 3.393116710983149, + "tokens_seen": 2536374272 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023376128385155467, + "loss": 1.245, + "theoretical_loss": 3.3931097964572294, + "tokens_seen": 2536439808 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023374122367101306, + "loss": 1.3315, + "theoretical_loss": 3.393102882159986, + "tokens_seen": 2536505344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023372116349047143, + "loss": 1.2661, + "theoretical_loss": 3.393095968091405, + "tokens_seen": 2536570880 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023370110330992982, + "loss": 1.1864, + "theoretical_loss": 3.3930890542514733, + "tokens_seen": 2536636416 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023368104312938818, + "loss": 1.2417, + "theoretical_loss": 3.393082140640177, + "tokens_seen": 2536701952 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023366098294884654, + "loss": 1.2916, + "theoretical_loss": 3.393075227257503, + "tokens_seen": 2536767488 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023364092276830493, + "loss": 1.308, + "theoretical_loss": 3.3930683141034375, + "tokens_seen": 2536833024 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002336208625877633, + "loss": 1.377, + "theoretical_loss": 3.3930614011779667, + "tokens_seen": 2536898560 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023360080240722168, + "loss": 1.2584, + "theoretical_loss": 3.3930544884810785, + "tokens_seen": 2536964096 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023358074222668004, + "loss": 1.1891, + "theoretical_loss": 3.393047576012758, + "tokens_seen": 2537029632 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023356068204613843, + "loss": 1.2358, + "theoretical_loss": 3.393040663772992, + "tokens_seen": 2537095168 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002335406218655968, + "loss": 1.2922, + "theoretical_loss": 3.3930337517617684, + "tokens_seen": 2537160704 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023352056168505518, + "loss": 1.2685, + "theoretical_loss": 3.393026839979072, + "tokens_seen": 2537226240 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023350050150451355, + "loss": 1.2198, + "theoretical_loss": 3.39301992842489, + "tokens_seen": 2537291776 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002334804413239719, + "loss": 1.2906, + "theoretical_loss": 3.393013017099209, + "tokens_seen": 2537357312 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002334603811434303, + "loss": 1.3042, + "theoretical_loss": 3.3930061060020154, + "tokens_seen": 2537422848 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023344032096288866, + "loss": 1.2958, + "theoretical_loss": 3.392999195133296, + "tokens_seen": 2537488384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023342026078234705, + "loss": 1.2147, + "theoretical_loss": 3.392992284493037, + "tokens_seen": 2537553920 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002334002006018054, + "loss": 1.244, + "theoretical_loss": 3.3929853740812255, + "tokens_seen": 2537619456 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002333801404212638, + "loss": 1.2696, + "theoretical_loss": 3.3929784638978475, + "tokens_seen": 2537684992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023336008024072216, + "loss": 1.3701, + "theoretical_loss": 3.39297155394289, + "tokens_seen": 2537750528 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023334002006018055, + "loss": 1.4499, + "theoretical_loss": 3.3929646442163386, + "tokens_seen": 2537816064 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023331995987963891, + "loss": 1.2674, + "theoretical_loss": 3.392957734718181, + "tokens_seen": 2537881600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023329989969909728, + "loss": 1.1724, + "theoretical_loss": 3.3929508254484033, + "tokens_seen": 2537947136 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 923645, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3264087438583374, + "objective/train/theoretical_loss": 3.392943916406992, + "objective/train/tokens_used": 908471776, + "theoretical_loss": 3.392943916406992, + "tokens_seen": 2538012672 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023327983951855567, + "loss": 1.3633, + "theoretical_loss": 3.392943916406992, + "tokens_seen": 2538012672 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023325977933801403, + "loss": 1.2765, + "theoretical_loss": 3.392937007593934, + "tokens_seen": 2538078208 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023323971915747242, + "loss": 1.1971, + "theoretical_loss": 3.392930099009215, + "tokens_seen": 2538143744 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023321965897693078, + "loss": 1.2715, + "theoretical_loss": 3.392923190652822, + "tokens_seen": 2538209280 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023319959879638917, + "loss": 1.2952, + "theoretical_loss": 3.3929162825247423, + "tokens_seen": 2538274816 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023317953861584753, + "loss": 1.1082, + "theoretical_loss": 3.3929093746249617, + "tokens_seen": 2538340352 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023315947843530592, + "loss": 1.3841, + "theoretical_loss": 3.392902466953467, + "tokens_seen": 2538405888 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023313941825476428, + "loss": 1.1833, + "theoretical_loss": 3.3928955595102446, + "tokens_seen": 2538471424 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023311935807422267, + "loss": 1.2684, + "theoretical_loss": 3.392888652295281, + "tokens_seen": 2538536960 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023309929789368106, + "loss": 1.1557, + "theoretical_loss": 3.3928817453085625, + "tokens_seen": 2538602496 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023307923771313942, + "loss": 1.2416, + "theoretical_loss": 3.3928748385500764, + "tokens_seen": 2538668032 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023305917753259781, + "loss": 1.3206, + "theoretical_loss": 3.392867932019809, + "tokens_seen": 2538733568 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023303911735205618, + "loss": 1.2026, + "theoretical_loss": 3.392861025717747, + "tokens_seen": 2538799104 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023301905717151457, + "loss": 1.2755, + "theoretical_loss": 3.3928541196438764, + "tokens_seen": 2538864640 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023299899699097293, + "loss": 1.306, + "theoretical_loss": 3.392847213798184, + "tokens_seen": 2538930176 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023297893681043132, + "loss": 1.3124, + "theoretical_loss": 3.392840308180657, + "tokens_seen": 2538995712 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023295887662988968, + "loss": 1.2694, + "theoretical_loss": 3.3928334027912816, + "tokens_seen": 2539061248 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023293881644934807, + "loss": 1.3038, + "theoretical_loss": 3.3928264976300437, + "tokens_seen": 2539126784 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023291875626880643, + "loss": 1.3387, + "theoretical_loss": 3.3928195926969305, + "tokens_seen": 2539192320 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002328986960882648, + "loss": 1.2893, + "theoretical_loss": 3.3928126879919285, + "tokens_seen": 2539257856 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023287863590772318, + "loss": 1.4551, + "theoretical_loss": 3.3928057835150245, + "tokens_seen": 2539323392 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023285857572718154, + "loss": 1.3237, + "theoretical_loss": 3.3927988792662047, + "tokens_seen": 2539388928 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023283851554663993, + "loss": 1.2466, + "theoretical_loss": 3.392791975245456, + "tokens_seen": 2539454464 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002328184553660983, + "loss": 1.2544, + "theoretical_loss": 3.3927850714527645, + "tokens_seen": 2539520000 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023279839518555669, + "loss": 1.3134, + "theoretical_loss": 3.3927781678881175, + "tokens_seen": 2539585536 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023277833500501505, + "loss": 1.1559, + "theoretical_loss": 3.3927712645515005, + "tokens_seen": 2539651072 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023275827482447344, + "loss": 1.2282, + "theoretical_loss": 3.3927643614429015, + "tokens_seen": 2539716608 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002327382146439318, + "loss": 1.3168, + "theoretical_loss": 3.392757458562306, + "tokens_seen": 2539782144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023271815446339016, + "loss": 1.243, + "theoretical_loss": 3.392750555909701, + "tokens_seen": 2539847680 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023269809428284855, + "loss": 1.3382, + "theoretical_loss": 3.392743653485073, + "tokens_seen": 2539913216 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002326780341023069, + "loss": 1.3677, + "theoretical_loss": 3.3927367512884086, + "tokens_seen": 2539978752 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002326579739217653, + "loss": 1.2787, + "theoretical_loss": 3.3927298493196947, + "tokens_seen": 2540044288 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023263791374122366, + "loss": 1.1864, + "theoretical_loss": 3.392722947578917, + "tokens_seen": 2540109824 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023261785356068205, + "loss": 1.363, + "theoretical_loss": 3.392716046066063, + "tokens_seen": 2540175360 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023259779338014042, + "loss": 1.4196, + "theoretical_loss": 3.392709144781119, + "tokens_seen": 2540240896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002325777331995988, + "loss": 1.2948, + "theoretical_loss": 3.3927022437240715, + "tokens_seen": 2540306432 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023255767301905717, + "loss": 1.2873, + "theoretical_loss": 3.3926953428949074, + "tokens_seen": 2540371968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023253761283851556, + "loss": 1.337, + "theoretical_loss": 3.3926884422936125, + "tokens_seen": 2540437504 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023251755265797392, + "loss": 1.3602, + "theoretical_loss": 3.3926815419201746, + "tokens_seen": 2540503040 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023249749247743228, + "loss": 1.1958, + "theoretical_loss": 3.3926746417745792, + "tokens_seen": 2540568576 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023247743229689067, + "loss": 1.3665, + "theoretical_loss": 3.3926677418568136, + "tokens_seen": 2540634112 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023245737211634903, + "loss": 1.237, + "theoretical_loss": 3.392660842166864, + "tokens_seen": 2540699648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023243731193580742, + "loss": 1.2613, + "theoretical_loss": 3.3926539427047175, + "tokens_seen": 2540765184 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023241725175526578, + "loss": 1.2628, + "theoretical_loss": 3.39264704347036, + "tokens_seen": 2540830720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023239719157472417, + "loss": 1.2362, + "theoretical_loss": 3.3926401444637784, + "tokens_seen": 2540896256 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023237713139418254, + "loss": 1.2816, + "theoretical_loss": 3.3926332456849595, + "tokens_seen": 2540961792 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023235707121364095, + "loss": 1.2716, + "theoretical_loss": 3.39262634713389, + "tokens_seen": 2541027328 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023233701103309932, + "loss": 1.2402, + "theoretical_loss": 3.392619448810556, + "tokens_seen": 2541092864 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023231695085255768, + "loss": 1.4142, + "theoretical_loss": 3.3926125507149445, + "tokens_seen": 2541158400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023229689067201607, + "loss": 1.388, + "theoretical_loss": 3.3926056528470423, + "tokens_seen": 2541223936 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 924855, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3667056560516357, + "objective/train/theoretical_loss": 3.3925987552068353, + "objective/train/tokens_used": 911748576, + "theoretical_loss": 3.3925987552068353, + "tokens_seen": 2541289472 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023227683049147443, + "loss": 1.2384, + "theoretical_loss": 3.3925987552068353, + "tokens_seen": 2541289472 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023225677031093282, + "loss": 1.2457, + "theoretical_loss": 3.3925918577943106, + "tokens_seen": 2541355008 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023223671013039118, + "loss": 1.3912, + "theoretical_loss": 3.392584960609455, + "tokens_seen": 2541420544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023221664994984957, + "loss": 1.2465, + "theoretical_loss": 3.3925780636522544, + "tokens_seen": 2541486080 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023219658976930793, + "loss": 1.1891, + "theoretical_loss": 3.3925711669226963, + "tokens_seen": 2541551616 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023217652958876632, + "loss": 1.4094, + "theoretical_loss": 3.3925642704207664, + "tokens_seen": 2541617152 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023215646940822468, + "loss": 1.3169, + "theoretical_loss": 3.392557374146452, + "tokens_seen": 2541682688 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023213640922768307, + "loss": 1.2887, + "theoretical_loss": 3.3925504780997398, + "tokens_seen": 2541748224 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023211634904714144, + "loss": 1.3069, + "theoretical_loss": 3.392543582280616, + "tokens_seen": 2541813760 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002320962888665998, + "loss": 1.2283, + "theoretical_loss": 3.392536686689067, + "tokens_seen": 2541879296 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002320762286860582, + "loss": 1.41, + "theoretical_loss": 3.3925297913250807, + "tokens_seen": 2541944832 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023205616850551655, + "loss": 1.2219, + "theoretical_loss": 3.392522896188642, + "tokens_seen": 2542010368 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023203610832497494, + "loss": 1.348, + "theoretical_loss": 3.3925160012797386, + "tokens_seen": 2542075904 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002320160481444333, + "loss": 1.1912, + "theoretical_loss": 3.392509106598357, + "tokens_seen": 2542141440 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002319959879638917, + "loss": 1.2898, + "theoretical_loss": 3.392502212144483, + "tokens_seen": 2542206976 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023197592778335005, + "loss": 1.2949, + "theoretical_loss": 3.392495317918105, + "tokens_seen": 2542272512 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023195586760280844, + "loss": 1.376, + "theoretical_loss": 3.392488423919208, + "tokens_seen": 2542338048 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002319358074222668, + "loss": 1.3114, + "theoretical_loss": 3.392481530147779, + "tokens_seen": 2542403584 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023191574724172517, + "loss": 1.3126, + "theoretical_loss": 3.392474636603805, + "tokens_seen": 2542469120 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023189568706118356, + "loss": 1.3893, + "theoretical_loss": 3.3924677432872725, + "tokens_seen": 2542534656 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023187562688064192, + "loss": 1.2242, + "theoretical_loss": 3.392460850198168, + "tokens_seen": 2542600192 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002318555667001003, + "loss": 1.3638, + "theoretical_loss": 3.3924539573364783, + "tokens_seen": 2542665728 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023183550651955867, + "loss": 1.2766, + "theoretical_loss": 3.3924470647021896, + "tokens_seen": 2542731264 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023181544633901706, + "loss": 1.3009, + "theoretical_loss": 3.3924401722952893, + "tokens_seen": 2542796800 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023179538615847542, + "loss": 1.366, + "theoretical_loss": 3.3924332801157635, + "tokens_seen": 2542862336 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002317753259779338, + "loss": 1.3684, + "theoretical_loss": 3.392426388163599, + "tokens_seen": 2542927872 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023175526579739217, + "loss": 1.3311, + "theoretical_loss": 3.392419496438782, + "tokens_seen": 2542993408 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023173520561685053, + "loss": 1.3037, + "theoretical_loss": 3.3924126049413, + "tokens_seen": 2543058944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023171514543630892, + "loss": 1.2468, + "theoretical_loss": 3.392405713671139, + "tokens_seen": 2543124480 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023169508525576729, + "loss": 1.2259, + "theoretical_loss": 3.392398822628286, + "tokens_seen": 2543190016 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023167502507522568, + "loss": 1.2246, + "theoretical_loss": 3.3923919318127274, + "tokens_seen": 2543255552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023165496489468404, + "loss": 1.2855, + "theoretical_loss": 3.3923850412244496, + "tokens_seen": 2543321088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023163490471414243, + "loss": 1.2819, + "theoretical_loss": 3.3923781508634403, + "tokens_seen": 2543386624 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002316148445336008, + "loss": 1.3187, + "theoretical_loss": 3.3923712607296848, + "tokens_seen": 2543452160 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002315947843530592, + "loss": 1.3124, + "theoretical_loss": 3.3923643708231705, + "tokens_seen": 2543517696 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023157472417251757, + "loss": 1.253, + "theoretical_loss": 3.392357481143884, + "tokens_seen": 2543583232 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023155466399197596, + "loss": 1.4152, + "theoretical_loss": 3.3923505916918115, + "tokens_seen": 2543648768 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023153460381143432, + "loss": 1.404, + "theoretical_loss": 3.3923437024669405, + "tokens_seen": 2543714304 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023151454363089268, + "loss": 1.2665, + "theoretical_loss": 3.392336813469257, + "tokens_seen": 2543779840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023149448345035107, + "loss": 1.2368, + "theoretical_loss": 3.392329924698748, + "tokens_seen": 2543845376 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023147442326980943, + "loss": 1.3588, + "theoretical_loss": 3.3923230361553998, + "tokens_seen": 2543910912 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023145436308926782, + "loss": 1.2957, + "theoretical_loss": 3.392316147839199, + "tokens_seen": 2543976448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023143430290872619, + "loss": 1.3076, + "theoretical_loss": 3.392309259750133, + "tokens_seen": 2544041984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023141424272818457, + "loss": 1.2901, + "theoretical_loss": 3.3923023718881877, + "tokens_seen": 2544107520 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023139418254764294, + "loss": 1.2834, + "theoretical_loss": 3.39229548425335, + "tokens_seen": 2544173056 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023137412236710133, + "loss": 1.2669, + "theoretical_loss": 3.392288596845607, + "tokens_seen": 2544238592 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002313540621865597, + "loss": 1.2922, + "theoretical_loss": 3.3922817096649442, + "tokens_seen": 2544304128 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023133400200601805, + "loss": 1.3144, + "theoretical_loss": 3.3922748227113497, + "tokens_seen": 2544369664 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023131394182547644, + "loss": 1.2929, + "theoretical_loss": 3.392267935984809, + "tokens_seen": 2544435200 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002312938816449348, + "loss": 1.3303, + "theoretical_loss": 3.3922610494853096, + "tokens_seen": 2544500736 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 925876, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3360546827316284, + "objective/train/theoretical_loss": 3.3922541632128373, + "objective/train/tokens_used": 915025376, + "theoretical_loss": 3.3922541632128373, + "tokens_seen": 2544566272 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002312738214643932, + "loss": 1.3073, + "theoretical_loss": 3.3922541632128373, + "tokens_seen": 2544566272 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023125376128385155, + "loss": 1.2513, + "theoretical_loss": 3.39224727716738, + "tokens_seen": 2544631808 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023123370110330994, + "loss": 1.2782, + "theoretical_loss": 3.392240391348923, + "tokens_seen": 2544697344 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002312136409227683, + "loss": 1.2373, + "theoretical_loss": 3.392233505757454, + "tokens_seen": 2544762880 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002311935807422267, + "loss": 1.2918, + "theoretical_loss": 3.392226620392959, + "tokens_seen": 2544828416 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023117352056168506, + "loss": 1.3595, + "theoretical_loss": 3.3922197352554253, + "tokens_seen": 2544893952 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023115346038114342, + "loss": 1.3242, + "theoretical_loss": 3.392212850344839, + "tokens_seen": 2544959488 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002311334002006018, + "loss": 1.2397, + "theoretical_loss": 3.392205965661187, + "tokens_seen": 2545025024 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023111334002006017, + "loss": 1.2491, + "theoretical_loss": 3.3921990812044562, + "tokens_seen": 2545090560 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023109327983951856, + "loss": 1.1877, + "theoretical_loss": 3.392192196974633, + "tokens_seen": 2545156096 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023107321965897692, + "loss": 1.2053, + "theoretical_loss": 3.392185312971704, + "tokens_seen": 2545221632 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002310531594784353, + "loss": 1.187, + "theoretical_loss": 3.3921784291956563, + "tokens_seen": 2545287168 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023103309929789367, + "loss": 1.1851, + "theoretical_loss": 3.3921715456464763, + "tokens_seen": 2545352704 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023101303911735206, + "loss": 1.3078, + "theoretical_loss": 3.392164662324151, + "tokens_seen": 2545418240 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023099297893681043, + "loss": 1.2729, + "theoretical_loss": 3.392157779228666, + "tokens_seen": 2545483776 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023097291875626881, + "loss": 1.3199, + "theoretical_loss": 3.3921508963600093, + "tokens_seen": 2545549312 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023095285857572718, + "loss": 1.3278, + "theoretical_loss": 3.3921440137181667, + "tokens_seen": 2545614848 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023093279839518554, + "loss": 1.1528, + "theoretical_loss": 3.392137131303125, + "tokens_seen": 2545680384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023091273821464393, + "loss": 1.323, + "theoretical_loss": 3.3921302491148717, + "tokens_seen": 2545745920 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002308926780341023, + "loss": 1.2429, + "theoretical_loss": 3.392123367153393, + "tokens_seen": 2545811456 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023087261785356068, + "loss": 1.2823, + "theoretical_loss": 3.392116485418675, + "tokens_seen": 2545876992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023085255767301907, + "loss": 1.1817, + "theoretical_loss": 3.392109603910705, + "tokens_seen": 2545942528 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023083249749247746, + "loss": 1.3439, + "theoretical_loss": 3.3921027226294695, + "tokens_seen": 2546008064 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023081243731193582, + "loss": 1.3047, + "theoretical_loss": 3.392095841574956, + "tokens_seen": 2546073600 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002307923771313942, + "loss": 1.2534, + "theoretical_loss": 3.3920889607471496, + "tokens_seen": 2546139136 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023077231695085257, + "loss": 1.3664, + "theoretical_loss": 3.3920820801460385, + "tokens_seen": 2546204672 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023075225677031094, + "loss": 1.3099, + "theoretical_loss": 3.3920751997716083, + "tokens_seen": 2546270208 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023073219658976932, + "loss": 1.1967, + "theoretical_loss": 3.392068319623846, + "tokens_seen": 2546335744 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002307121364092277, + "loss": 1.271, + "theoretical_loss": 3.392061439702739, + "tokens_seen": 2546401280 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023069207622868608, + "loss": 1.3284, + "theoretical_loss": 3.3920545600082734, + "tokens_seen": 2546466816 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023067201604814444, + "loss": 1.2488, + "theoretical_loss": 3.392047680540436, + "tokens_seen": 2546532352 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023065195586760283, + "loss": 1.3052, + "theoretical_loss": 3.392040801299213, + "tokens_seen": 2546597888 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002306318956870612, + "loss": 1.4035, + "theoretical_loss": 3.392033922284592, + "tokens_seen": 2546663424 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023061183550651958, + "loss": 1.3251, + "theoretical_loss": 3.392027043496559, + "tokens_seen": 2546728960 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023059177532597794, + "loss": 1.2543, + "theoretical_loss": 3.392020164935101, + "tokens_seen": 2546794496 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002305717151454363, + "loss": 1.2776, + "theoretical_loss": 3.392013286600205, + "tokens_seen": 2546860032 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002305516549648947, + "loss": 1.1164, + "theoretical_loss": 3.392006408491857, + "tokens_seen": 2546925568 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023053159478435306, + "loss": 1.3665, + "theoretical_loss": 3.391999530610044, + "tokens_seen": 2546991104 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023051153460381144, + "loss": 1.276, + "theoretical_loss": 3.3919926529547535, + "tokens_seen": 2547056640 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002304914744232698, + "loss": 1.3972, + "theoretical_loss": 3.391985775525971, + "tokens_seen": 2547122176 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002304714142427282, + "loss": 1.2194, + "theoretical_loss": 3.391978898323684, + "tokens_seen": 2547187712 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023045135406218656, + "loss": 1.321, + "theoretical_loss": 3.3919720213478786, + "tokens_seen": 2547253248 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023043129388164495, + "loss": 1.2579, + "theoretical_loss": 3.391965144598542, + "tokens_seen": 2547318784 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002304112337011033, + "loss": 1.3014, + "theoretical_loss": 3.391958268075661, + "tokens_seen": 2547384320 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002303911735205617, + "loss": 1.2547, + "theoretical_loss": 3.391951391779222, + "tokens_seen": 2547449856 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023037111334002006, + "loss": 1.2699, + "theoretical_loss": 3.391944515709212, + "tokens_seen": 2547515392 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023035105315947842, + "loss": 1.3736, + "theoretical_loss": 3.391937639865617, + "tokens_seen": 2547580928 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002303309929789368, + "loss": 1.2853, + "theoretical_loss": 3.3919307642484244, + "tokens_seen": 2547646464 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023031093279839518, + "loss": 1.1486, + "theoretical_loss": 3.391923888857621, + "tokens_seen": 2547712000 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023029087261785356, + "loss": 1.2565, + "theoretical_loss": 3.3919170136931935, + "tokens_seen": 2547777536 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 927107, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2384496927261353, + "objective/train/theoretical_loss": 3.391910138755128, + "objective/train/tokens_used": 918302176, + "theoretical_loss": 3.391910138755128, + "tokens_seen": 2547843072 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023027081243731193, + "loss": 1.2026, + "theoretical_loss": 3.391910138755128, + "tokens_seen": 2547843072 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023025075225677032, + "loss": 1.2926, + "theoretical_loss": 3.3919032640434117, + "tokens_seen": 2547908608 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023023069207622868, + "loss": 1.2563, + "theoretical_loss": 3.391896389558031, + "tokens_seen": 2547974144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023021063189568707, + "loss": 1.1939, + "theoretical_loss": 3.3918895152989736, + "tokens_seen": 2548039680 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023019057171514543, + "loss": 1.201, + "theoretical_loss": 3.3918826412662253, + "tokens_seen": 2548105216 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002301705115346038, + "loss": 1.2752, + "theoretical_loss": 3.391875767459773, + "tokens_seen": 2548170752 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023015045135406218, + "loss": 1.2772, + "theoretical_loss": 3.3918688938796033, + "tokens_seen": 2548236288 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023013039117352054, + "loss": 1.3354, + "theoretical_loss": 3.3918620205257035, + "tokens_seen": 2548301824 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023011033099297893, + "loss": 1.2105, + "theoretical_loss": 3.3918551473980596, + "tokens_seen": 2548367360 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023009027081243732, + "loss": 1.2032, + "theoretical_loss": 3.3918482744966587, + "tokens_seen": 2548432896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002300702106318957, + "loss": 1.3504, + "theoretical_loss": 3.391841401821488, + "tokens_seen": 2548498432 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023005015045135407, + "loss": 1.2992, + "theoretical_loss": 3.391834529372533, + "tokens_seen": 2548563968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023003009027081246, + "loss": 1.3862, + "theoretical_loss": 3.391827657149782, + "tokens_seen": 2548629504 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023001003009027083, + "loss": 1.2772, + "theoretical_loss": 3.3918207851532203, + "tokens_seen": 2548695040 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022998996990972922, + "loss": 1.3028, + "theoretical_loss": 3.3918139133828356, + "tokens_seen": 2548760576 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022996990972918758, + "loss": 1.4012, + "theoretical_loss": 3.3918070418386144, + "tokens_seen": 2548826112 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022994984954864594, + "loss": 1.275, + "theoretical_loss": 3.391800170520543, + "tokens_seen": 2548891648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022992978936810433, + "loss": 1.3267, + "theoretical_loss": 3.391793299428609, + "tokens_seen": 2548957184 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002299097291875627, + "loss": 1.2164, + "theoretical_loss": 3.391786428562798, + "tokens_seen": 2549022720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022988966900702108, + "loss": 1.2215, + "theoretical_loss": 3.3917795579230976, + "tokens_seen": 2549088256 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022986960882647944, + "loss": 1.2221, + "theoretical_loss": 3.3917726875094947, + "tokens_seen": 2549153792 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022984954864593783, + "loss": 1.342, + "theoretical_loss": 3.3917658173219754, + "tokens_seen": 2549219328 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002298294884653962, + "loss": 1.1806, + "theoretical_loss": 3.3917589473605267, + "tokens_seen": 2549284864 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022980942828485458, + "loss": 1.3192, + "theoretical_loss": 3.3917520776251355, + "tokens_seen": 2549350400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022978936810431295, + "loss": 1.3126, + "theoretical_loss": 3.3917452081157884, + "tokens_seen": 2549415936 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002297693079237713, + "loss": 1.2251, + "theoretical_loss": 3.391738338832472, + "tokens_seen": 2549481472 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002297492477432297, + "loss": 1.3276, + "theoretical_loss": 3.3917314697751735, + "tokens_seen": 2549547008 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022972918756268806, + "loss": 1.2749, + "theoretical_loss": 3.391724600943879, + "tokens_seen": 2549612544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022970912738214645, + "loss": 1.2326, + "theoretical_loss": 3.391717732338576, + "tokens_seen": 2549678080 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002296890672016048, + "loss": 1.2217, + "theoretical_loss": 3.3917108639592506, + "tokens_seen": 2549743616 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002296690070210632, + "loss": 1.3089, + "theoretical_loss": 3.3917039958058903, + "tokens_seen": 2549809152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022964894684052156, + "loss": 1.2547, + "theoretical_loss": 3.3916971278784813, + "tokens_seen": 2549874688 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022962888665997995, + "loss": 1.3146, + "theoretical_loss": 3.3916902601770103, + "tokens_seen": 2549940224 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022960882647943831, + "loss": 1.2868, + "theoretical_loss": 3.3916833927014642, + "tokens_seen": 2550005760 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022958876629889668, + "loss": 1.2445, + "theoretical_loss": 3.39167652545183, + "tokens_seen": 2550071296 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022956870611835507, + "loss": 1.2995, + "theoretical_loss": 3.391669658428094, + "tokens_seen": 2550136832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022954864593781343, + "loss": 1.331, + "theoretical_loss": 3.3916627916302433, + "tokens_seen": 2550202368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022952858575727182, + "loss": 1.2807, + "theoretical_loss": 3.3916559250582647, + "tokens_seen": 2550267904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022950852557673018, + "loss": 1.3521, + "theoretical_loss": 3.391649058712145, + "tokens_seen": 2550333440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022948846539618857, + "loss": 1.2518, + "theoretical_loss": 3.3916421925918705, + "tokens_seen": 2550398976 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022946840521564693, + "loss": 1.2686, + "theoretical_loss": 3.3916353266974286, + "tokens_seen": 2550464512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022944834503510532, + "loss": 1.2378, + "theoretical_loss": 3.391628461028805, + "tokens_seen": 2550530048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022942828485456368, + "loss": 1.2267, + "theoretical_loss": 3.3916215955859883, + "tokens_seen": 2550595584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022940822467402207, + "loss": 1.2759, + "theoretical_loss": 3.3916147303689637, + "tokens_seen": 2550661120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022938816449348043, + "loss": 1.3356, + "theoretical_loss": 3.3916078653777184, + "tokens_seen": 2550726656 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002293681043129388, + "loss": 1.2648, + "theoretical_loss": 3.3916010006122397, + "tokens_seen": 2550792192 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002293480441323972, + "loss": 1.1256, + "theoretical_loss": 3.3915941360725133, + "tokens_seen": 2550857728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022932798395185558, + "loss": 1.2613, + "theoretical_loss": 3.3915872717585267, + "tokens_seen": 2550923264 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022930792377131397, + "loss": 1.3427, + "theoretical_loss": 3.3915804076702667, + "tokens_seen": 2550988800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022928786359077233, + "loss": 1.2914, + "theoretical_loss": 3.39157354380772, + "tokens_seen": 2551054336 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 928186, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2692723274230957, + "objective/train/theoretical_loss": 3.3915666801708735, + "objective/train/tokens_used": 921578976, + "theoretical_loss": 3.3915666801708735, + "tokens_seen": 2551119872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022926780341023072, + "loss": 1.3094, + "theoretical_loss": 3.3915666801708735, + "tokens_seen": 2551119872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022924774322968908, + "loss": 1.2983, + "theoretical_loss": 3.3915598167597136, + "tokens_seen": 2551185408 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022922768304914747, + "loss": 1.3001, + "theoretical_loss": 3.3915529535742275, + "tokens_seen": 2551250944 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022920762286860583, + "loss": 1.2585, + "theoretical_loss": 3.3915460906144017, + "tokens_seen": 2551316480 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002291875626880642, + "loss": 1.2536, + "theoretical_loss": 3.391539227880223, + "tokens_seen": 2551382016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022916750250752258, + "loss": 1.353, + "theoretical_loss": 3.391532365371678, + "tokens_seen": 2551447552 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022914744232698094, + "loss": 1.1998, + "theoretical_loss": 3.3915255030887543, + "tokens_seen": 2551513088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022912738214643933, + "loss": 1.2663, + "theoretical_loss": 3.3915186410314377, + "tokens_seen": 2551578624 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002291073219658977, + "loss": 1.3219, + "theoretical_loss": 3.3915117791997154, + "tokens_seen": 2551644160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022908726178535609, + "loss": 1.2352, + "theoretical_loss": 3.3915049175935743, + "tokens_seen": 2551709696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022906720160481445, + "loss": 1.337, + "theoretical_loss": 3.3914980562130013, + "tokens_seen": 2551775232 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022904714142427284, + "loss": 1.2546, + "theoretical_loss": 3.3914911950579825, + "tokens_seen": 2551840768 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002290270812437312, + "loss": 1.2496, + "theoretical_loss": 3.3914843341285055, + "tokens_seen": 2551906304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022900702106318956, + "loss": 1.1115, + "theoretical_loss": 3.391477473424557, + "tokens_seen": 2551971840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022898696088264795, + "loss": 1.2667, + "theoretical_loss": 3.391470612946123, + "tokens_seen": 2552037376 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002289669007021063, + "loss": 1.122, + "theoretical_loss": 3.3914637526931917, + "tokens_seen": 2552102912 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002289468405215647, + "loss": 1.2336, + "theoretical_loss": 3.3914568926657482, + "tokens_seen": 2552168448 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022892678034102306, + "loss": 1.2311, + "theoretical_loss": 3.391450032863781, + "tokens_seen": 2552233984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022890672016048145, + "loss": 1.3124, + "theoretical_loss": 3.3914431732872754, + "tokens_seen": 2552299520 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022888665997993982, + "loss": 1.3253, + "theoretical_loss": 3.391436313936219, + "tokens_seen": 2552365056 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002288665997993982, + "loss": 1.2424, + "theoretical_loss": 3.3914294548105985, + "tokens_seen": 2552430592 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022884653961885657, + "loss": 1.2629, + "theoretical_loss": 3.3914225959104005, + "tokens_seen": 2552496128 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022882647943831496, + "loss": 1.2983, + "theoretical_loss": 3.3914157372356124, + "tokens_seen": 2552561664 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022880641925777332, + "loss": 1.3037, + "theoretical_loss": 3.39140887878622, + "tokens_seen": 2552627200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022878635907723168, + "loss": 1.2505, + "theoretical_loss": 3.391402020562211, + "tokens_seen": 2552692736 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022876629889669007, + "loss": 1.308, + "theoretical_loss": 3.3913951625635717, + "tokens_seen": 2552758272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022874623871614843, + "loss": 1.2239, + "theoretical_loss": 3.3913883047902895, + "tokens_seen": 2552823808 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022872617853560682, + "loss": 1.2598, + "theoretical_loss": 3.3913814472423507, + "tokens_seen": 2552889344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022870611835506518, + "loss": 1.2808, + "theoretical_loss": 3.3913745899197423, + "tokens_seen": 2552954880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022868605817452357, + "loss": 1.2019, + "theoretical_loss": 3.3913677328224505, + "tokens_seen": 2553020416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022866599799398194, + "loss": 1.2037, + "theoretical_loss": 3.391360875950463, + "tokens_seen": 2553085952 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022864593781344033, + "loss": 1.1416, + "theoretical_loss": 3.3913540193037663, + "tokens_seen": 2553151488 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002286258776328987, + "loss": 1.2984, + "theoretical_loss": 3.391347162882347, + "tokens_seen": 2553217024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022860581745235705, + "loss": 1.233, + "theoretical_loss": 3.391340306686192, + "tokens_seen": 2553282560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022858575727181547, + "loss": 1.194, + "theoretical_loss": 3.3913334507152886, + "tokens_seen": 2553348096 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022856569709127383, + "loss": 1.2404, + "theoretical_loss": 3.3913265949696227, + "tokens_seen": 2553413632 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022854563691073222, + "loss": 1.2232, + "theoretical_loss": 3.391319739449182, + "tokens_seen": 2553479168 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022852557673019058, + "loss": 1.2146, + "theoretical_loss": 3.391312884153953, + "tokens_seen": 2553544704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022850551654964897, + "loss": 1.1606, + "theoretical_loss": 3.3913060290839225, + "tokens_seen": 2553610240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022848545636910733, + "loss": 1.3204, + "theoretical_loss": 3.391299174239077, + "tokens_seen": 2553675776 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022846539618856572, + "loss": 1.2898, + "theoretical_loss": 3.391292319619404, + "tokens_seen": 2553741312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022844533600802408, + "loss": 1.3152, + "theoretical_loss": 3.3912854652248896, + "tokens_seen": 2553806848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022842527582748247, + "loss": 1.2375, + "theoretical_loss": 3.391278611055521, + "tokens_seen": 2553872384 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022840521564694084, + "loss": 1.2807, + "theoretical_loss": 3.3912717571112854, + "tokens_seen": 2553937920 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002283851554663992, + "loss": 1.2345, + "theoretical_loss": 3.391264903392169, + "tokens_seen": 2554003456 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002283650952858576, + "loss": 1.2174, + "theoretical_loss": 3.391258049898159, + "tokens_seen": 2554068992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022834503510531595, + "loss": 1.4115, + "theoretical_loss": 3.3912511966292422, + "tokens_seen": 2554134528 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022832497492477434, + "loss": 1.2741, + "theoretical_loss": 3.391244343585405, + "tokens_seen": 2554200064 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002283049147442327, + "loss": 1.2154, + "theoretical_loss": 3.3912374907666347, + "tokens_seen": 2554265600 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002282848545636911, + "loss": 1.1808, + "theoretical_loss": 3.391230638172918, + "tokens_seen": 2554331136 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 929369, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2488348484039307, + "objective/train/theoretical_loss": 3.3912237858042413, + "objective/train/tokens_used": 924855776, + "theoretical_loss": 3.3912237858042413, + "tokens_seen": 2554396672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022826479438314945, + "loss": 1.2962, + "theoretical_loss": 3.3912237858042413, + "tokens_seen": 2554396672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022824473420260784, + "loss": 1.2765, + "theoretical_loss": 3.3912169336605924, + "tokens_seen": 2554462208 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002282246740220662, + "loss": 1.2384, + "theoretical_loss": 3.3912100817419577, + "tokens_seen": 2554527744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022820461384152457, + "loss": 1.1954, + "theoretical_loss": 3.3912032300483235, + "tokens_seen": 2554593280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022818455366098296, + "loss": 1.3625, + "theoretical_loss": 3.3911963785796777, + "tokens_seen": 2554658816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022816449348044132, + "loss": 1.2302, + "theoretical_loss": 3.3911895273360058, + "tokens_seen": 2554724352 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002281444332998997, + "loss": 1.294, + "theoretical_loss": 3.391182676317296, + "tokens_seen": 2554789888 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022812437311935807, + "loss": 1.1749, + "theoretical_loss": 3.391175825523534, + "tokens_seen": 2554855424 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022810431293881646, + "loss": 1.2554, + "theoretical_loss": 3.3911689749547076, + "tokens_seen": 2554920960 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022808425275827482, + "loss": 1.281, + "theoretical_loss": 3.391162124610803, + "tokens_seen": 2554986496 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002280641925777332, + "loss": 1.3678, + "theoretical_loss": 3.391155274491807, + "tokens_seen": 2555052032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022804413239719157, + "loss": 1.2437, + "theoretical_loss": 3.391148424597707, + "tokens_seen": 2555117568 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022802407221664993, + "loss": 1.2758, + "theoretical_loss": 3.3911415749284894, + "tokens_seen": 2555183104 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022800401203610832, + "loss": 1.2632, + "theoretical_loss": 3.3911347254841413, + "tokens_seen": 2555248640 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022798395185556669, + "loss": 1.2471, + "theoretical_loss": 3.3911278762646493, + "tokens_seen": 2555314176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022796389167502508, + "loss": 1.2772, + "theoretical_loss": 3.3911210272700005, + "tokens_seen": 2555379712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022794383149448344, + "loss": 1.2512, + "theoretical_loss": 3.3911141785001817, + "tokens_seen": 2555445248 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022792377131394183, + "loss": 1.2253, + "theoretical_loss": 3.3911073299551795, + "tokens_seen": 2555510784 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002279037111334002, + "loss": 1.2573, + "theoretical_loss": 3.391100481634981, + "tokens_seen": 2555576320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022788365095285858, + "loss": 1.2862, + "theoretical_loss": 3.3910936335395734, + "tokens_seen": 2555641856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022786359077231694, + "loss": 1.2627, + "theoretical_loss": 3.391086785668943, + "tokens_seen": 2555707392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022784353059177536, + "loss": 1.2816, + "theoretical_loss": 3.3910799380230765, + "tokens_seen": 2555772928 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022782347041123372, + "loss": 1.2736, + "theoretical_loss": 3.391073090601961, + "tokens_seen": 2555838464 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022780341023069208, + "loss": 1.3276, + "theoretical_loss": 3.391066243405584, + "tokens_seen": 2555904000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022778335005015047, + "loss": 1.3233, + "theoretical_loss": 3.3910593964339313, + "tokens_seen": 2555969536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022776328986960883, + "loss": 1.2602, + "theoretical_loss": 3.3910525496869908, + "tokens_seen": 2556035072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022774322968906722, + "loss": 1.244, + "theoretical_loss": 3.3910457031647483, + "tokens_seen": 2556100608 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022772316950852558, + "loss": 1.3074, + "theoretical_loss": 3.3910388568671914, + "tokens_seen": 2556166144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022770310932798397, + "loss": 1.1998, + "theoretical_loss": 3.391032010794307, + "tokens_seen": 2556231680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022768304914744234, + "loss": 1.2459, + "theoretical_loss": 3.3910251649460816, + "tokens_seen": 2556297216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022766298896690073, + "loss": 1.276, + "theoretical_loss": 3.391018319322502, + "tokens_seen": 2556362752 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002276429287863591, + "loss": 1.3326, + "theoretical_loss": 3.3910114739235553, + "tokens_seen": 2556428288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022762286860581745, + "loss": 1.2504, + "theoretical_loss": 3.391004628749229, + "tokens_seen": 2556493824 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022760280842527584, + "loss": 1.3782, + "theoretical_loss": 3.3909977837995084, + "tokens_seen": 2556559360 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002275827482447342, + "loss": 1.2602, + "theoretical_loss": 3.3909909390743818, + "tokens_seen": 2556624896 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002275626880641926, + "loss": 1.258, + "theoretical_loss": 3.3909840945738354, + "tokens_seen": 2556690432 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022754262788365095, + "loss": 1.3264, + "theoretical_loss": 3.390977250297856, + "tokens_seen": 2556755968 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022752256770310934, + "loss": 1.2632, + "theoretical_loss": 3.390970406246431, + "tokens_seen": 2556821504 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002275025075225677, + "loss": 1.3073, + "theoretical_loss": 3.390963562419547, + "tokens_seen": 2556887040 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002274824473420261, + "loss": 1.2205, + "theoretical_loss": 3.3909567188171907, + "tokens_seen": 2556952576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022746238716148446, + "loss": 1.2598, + "theoretical_loss": 3.3909498754393494, + "tokens_seen": 2557018112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022744232698094282, + "loss": 1.2794, + "theoretical_loss": 3.3909430322860095, + "tokens_seen": 2557083648 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002274222668004012, + "loss": 1.2823, + "theoretical_loss": 3.3909361893571583, + "tokens_seen": 2557149184 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022740220661985957, + "loss": 1.3034, + "theoretical_loss": 3.3909293466527823, + "tokens_seen": 2557214720 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022738214643931796, + "loss": 1.2941, + "theoretical_loss": 3.3909225041728686, + "tokens_seen": 2557280256 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022736208625877632, + "loss": 1.2401, + "theoretical_loss": 3.390915661917404, + "tokens_seen": 2557345792 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002273420260782347, + "loss": 1.5283, + "theoretical_loss": 3.3909088198863757, + "tokens_seen": 2557411328 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022732196589769307, + "loss": 1.2034, + "theoretical_loss": 3.3909019780797705, + "tokens_seen": 2557476864 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022730190571715146, + "loss": 1.3338, + "theoretical_loss": 3.3908951364975746, + "tokens_seen": 2557542400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022728184553660983, + "loss": 1.288, + "theoretical_loss": 3.3908882951397756, + "tokens_seen": 2557607936 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 930481, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1277267932891846, + "objective/train/theoretical_loss": 3.3908814540063603, + "objective/train/tokens_used": 928132576, + "theoretical_loss": 3.3908814540063603, + "tokens_seen": 2557673472 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022726178535606821, + "loss": 1.1303, + "theoretical_loss": 3.3908814540063603, + "tokens_seen": 2557673472 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022724172517552658, + "loss": 1.3335, + "theoretical_loss": 3.3908746130973153, + "tokens_seen": 2557739008 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022722166499498494, + "loss": 1.192, + "theoretical_loss": 3.390867772412628, + "tokens_seen": 2557804544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022720160481444333, + "loss": 1.2036, + "theoretical_loss": 3.3908609319522847, + "tokens_seen": 2557870080 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002271815446339017, + "loss": 1.1932, + "theoretical_loss": 3.390854091716273, + "tokens_seen": 2557935616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022716148445336008, + "loss": 1.2301, + "theoretical_loss": 3.390847251704579, + "tokens_seen": 2558001152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022714142427281844, + "loss": 1.3051, + "theoretical_loss": 3.3908404119171895, + "tokens_seen": 2558066688 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022712136409227683, + "loss": 1.2399, + "theoretical_loss": 3.3908335723540928, + "tokens_seen": 2558132224 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022710130391173522, + "loss": 1.3998, + "theoretical_loss": 3.3908267330152744, + "tokens_seen": 2558197760 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002270812437311936, + "loss": 1.2031, + "theoretical_loss": 3.3908198939007215, + "tokens_seen": 2558263296 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022706118355065197, + "loss": 1.1693, + "theoretical_loss": 3.390813055010421, + "tokens_seen": 2558328832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022704112337011033, + "loss": 1.3285, + "theoretical_loss": 3.3908062163443606, + "tokens_seen": 2558394368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022702106318956872, + "loss": 1.2584, + "theoretical_loss": 3.3907993779025265, + "tokens_seen": 2558459904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022700100300902709, + "loss": 1.2561, + "theoretical_loss": 3.390792539684905, + "tokens_seen": 2558525440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022698094282848548, + "loss": 1.3101, + "theoretical_loss": 3.3907857016914846, + "tokens_seen": 2558590976 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022696088264794384, + "loss": 1.226, + "theoretical_loss": 3.3907788639222507, + "tokens_seen": 2558656512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022694082246740223, + "loss": 1.2375, + "theoretical_loss": 3.390772026377191, + "tokens_seen": 2558722048 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002269207622868606, + "loss": 1.2791, + "theoretical_loss": 3.390765189056292, + "tokens_seen": 2558787584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022690070210631898, + "loss": 1.253, + "theoretical_loss": 3.390758351959541, + "tokens_seen": 2558853120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022688064192577734, + "loss": 1.3176, + "theoretical_loss": 3.3907515150869245, + "tokens_seen": 2558918656 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002268605817452357, + "loss": 1.2282, + "theoretical_loss": 3.39074467843843, + "tokens_seen": 2558984192 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002268405215646941, + "loss": 1.2043, + "theoretical_loss": 3.390737842014044, + "tokens_seen": 2559049728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022682046138415245, + "loss": 1.2071, + "theoretical_loss": 3.390731005813753, + "tokens_seen": 2559115264 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022680040120361084, + "loss": 1.2949, + "theoretical_loss": 3.390724169837545, + "tokens_seen": 2559180800 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002267803410230692, + "loss": 1.2831, + "theoretical_loss": 3.390717334085406, + "tokens_seen": 2559246336 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002267602808425276, + "loss": 1.2761, + "theoretical_loss": 3.390710498557323, + "tokens_seen": 2559311872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022674022066198596, + "loss": 1.2964, + "theoretical_loss": 3.3907036632532837, + "tokens_seen": 2559377408 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022672016048144435, + "loss": 1.2564, + "theoretical_loss": 3.390696828173274, + "tokens_seen": 2559442944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002267001003009027, + "loss": 1.2203, + "theoretical_loss": 3.3906899933172814, + "tokens_seen": 2559508480 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002266800401203611, + "loss": 1.1786, + "theoretical_loss": 3.3906831586852926, + "tokens_seen": 2559574016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022665997993981946, + "loss": 1.3037, + "theoretical_loss": 3.390676324277295, + "tokens_seen": 2559639552 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022663991975927782, + "loss": 1.2932, + "theoretical_loss": 3.3906694900932743, + "tokens_seen": 2559705088 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002266198595787362, + "loss": 1.2071, + "theoretical_loss": 3.390662656133219, + "tokens_seen": 2559770624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022659979939819458, + "loss": 1.2184, + "theoretical_loss": 3.390655822397115, + "tokens_seen": 2559836160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022657973921765296, + "loss": 1.2582, + "theoretical_loss": 3.3906489888849496, + "tokens_seen": 2559901696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022655967903711133, + "loss": 1.31, + "theoretical_loss": 3.39064215559671, + "tokens_seen": 2559967232 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022653961885656972, + "loss": 1.2104, + "theoretical_loss": 3.3906353225323826, + "tokens_seen": 2560032768 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022651955867602808, + "loss": 1.3441, + "theoretical_loss": 3.3906284896919545, + "tokens_seen": 2560098304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022649949849548647, + "loss": 1.2178, + "theoretical_loss": 3.390621657075412, + "tokens_seen": 2560163840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022647943831494483, + "loss": 1.3156, + "theoretical_loss": 3.3906148246827437, + "tokens_seen": 2560229376 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002264593781344032, + "loss": 1.1912, + "theoretical_loss": 3.390607992513935, + "tokens_seen": 2560294912 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022643931795386158, + "loss": 1.312, + "theoretical_loss": 3.3906011605689734, + "tokens_seen": 2560360448 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022641925777331994, + "loss": 1.2481, + "theoretical_loss": 3.390594328847846, + "tokens_seen": 2560425984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022639919759277833, + "loss": 1.3445, + "theoretical_loss": 3.3905874973505394, + "tokens_seen": 2560491520 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002263791374122367, + "loss": 1.19, + "theoretical_loss": 3.3905806660770406, + "tokens_seen": 2560557056 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022635907723169508, + "loss": 1.3432, + "theoretical_loss": 3.3905738350273364, + "tokens_seen": 2560622592 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022633901705115347, + "loss": 1.2233, + "theoretical_loss": 3.390567004201414, + "tokens_seen": 2560688128 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022631895687061186, + "loss": 1.2607, + "theoretical_loss": 3.3905601735992605, + "tokens_seen": 2560753664 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022629889669007023, + "loss": 1.2545, + "theoretical_loss": 3.390553343220863, + "tokens_seen": 2560819200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022627883650952861, + "loss": 1.268, + "theoretical_loss": 3.3905465130662074, + "tokens_seen": 2560884736 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 931622, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0067193508148193, + "objective/train/theoretical_loss": 3.3905396831352816, + "objective/train/tokens_used": 931409376, + "theoretical_loss": 3.3905396831352816, + "tokens_seen": 2560950272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022625877632898698, + "loss": 1.0906, + "theoretical_loss": 3.3905396831352816, + "tokens_seen": 2560950272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022623871614844534, + "loss": 1.2708, + "theoretical_loss": 3.3905328534280725, + "tokens_seen": 2561015808 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022621865596790373, + "loss": 1.2504, + "theoretical_loss": 3.3905260239445663, + "tokens_seen": 2561081344 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002261985957873621, + "loss": 1.1916, + "theoretical_loss": 3.390519194684751, + "tokens_seen": 2561146880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022617853560682048, + "loss": 1.1895, + "theoretical_loss": 3.3905123656486125, + "tokens_seen": 2561212416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022615847542627884, + "loss": 1.2994, + "theoretical_loss": 3.3905055368361383, + "tokens_seen": 2561277952 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022613841524573723, + "loss": 1.2573, + "theoretical_loss": 3.390498708247316, + "tokens_seen": 2561343488 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002261183550651956, + "loss": 1.4366, + "theoretical_loss": 3.390491879882131, + "tokens_seen": 2561409024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022609829488465398, + "loss": 1.1878, + "theoretical_loss": 3.390485051740572, + "tokens_seen": 2561474560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022607823470411235, + "loss": 1.2546, + "theoretical_loss": 3.3904782238226243, + "tokens_seen": 2561540096 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002260581745235707, + "loss": 1.3899, + "theoretical_loss": 3.3904713961282757, + "tokens_seen": 2561605632 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002260381143430291, + "loss": 1.1874, + "theoretical_loss": 3.3904645686575137, + "tokens_seen": 2561671168 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022601805416248746, + "loss": 1.1859, + "theoretical_loss": 3.390457741410324, + "tokens_seen": 2561736704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022599799398194585, + "loss": 1.2505, + "theoretical_loss": 3.3904509143866948, + "tokens_seen": 2561802240 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002259779338014042, + "loss": 1.2346, + "theoretical_loss": 3.390444087586612, + "tokens_seen": 2561867776 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002259578736208626, + "loss": 1.2056, + "theoretical_loss": 3.3904372610100633, + "tokens_seen": 2561933312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022593781344032096, + "loss": 1.3845, + "theoretical_loss": 3.390430434657035, + "tokens_seen": 2561998848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022591775325977935, + "loss": 1.2934, + "theoretical_loss": 3.390423608527515, + "tokens_seen": 2562064384 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022589769307923771, + "loss": 1.2505, + "theoretical_loss": 3.3904167826214895, + "tokens_seen": 2562129920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022587763289869608, + "loss": 1.267, + "theoretical_loss": 3.3904099569389454, + "tokens_seen": 2562195456 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022585757271815447, + "loss": 1.1387, + "theoretical_loss": 3.3904031314798706, + "tokens_seen": 2562260992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022583751253761283, + "loss": 1.2045, + "theoretical_loss": 3.390396306244251, + "tokens_seen": 2562326528 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022581745235707122, + "loss": 1.3322, + "theoretical_loss": 3.3903894812320736, + "tokens_seen": 2562392064 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022579739217652958, + "loss": 1.253, + "theoretical_loss": 3.390382656443326, + "tokens_seen": 2562457600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022577733199598797, + "loss": 1.3081, + "theoretical_loss": 3.3903758318779955, + "tokens_seen": 2562523136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022575727181544633, + "loss": 1.37, + "theoretical_loss": 3.3903690075360684, + "tokens_seen": 2562588672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022573721163490472, + "loss": 1.3448, + "theoretical_loss": 3.3903621834175315, + "tokens_seen": 2562654208 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022571715145436308, + "loss": 1.2017, + "theoretical_loss": 3.390355359522372, + "tokens_seen": 2562719744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022569709127382147, + "loss": 1.2806, + "theoretical_loss": 3.3903485358505767, + "tokens_seen": 2562785280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022567703109327983, + "loss": 1.2277, + "theoretical_loss": 3.390341712402133, + "tokens_seen": 2562850816 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002256569709127382, + "loss": 1.1669, + "theoretical_loss": 3.390334889177028, + "tokens_seen": 2562916352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022563691073219659, + "loss": 1.3417, + "theoretical_loss": 3.3903280661752486, + "tokens_seen": 2562981888 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022561685055165495, + "loss": 1.2812, + "theoretical_loss": 3.390321243396781, + "tokens_seen": 2563047424 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022559679037111336, + "loss": 1.4431, + "theoretical_loss": 3.3903144208416127, + "tokens_seen": 2563112960 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022557673019057173, + "loss": 1.301, + "theoretical_loss": 3.390307598509731, + "tokens_seen": 2563178496 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022555667001003012, + "loss": 1.3115, + "theoretical_loss": 3.390300776401123, + "tokens_seen": 2563244032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022553660982948848, + "loss": 1.3287, + "theoretical_loss": 3.390293954515774, + "tokens_seen": 2563309568 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022551654964894687, + "loss": 1.2775, + "theoretical_loss": 3.3902871328536732, + "tokens_seen": 2563375104 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022549648946840523, + "loss": 1.2908, + "theoretical_loss": 3.3902803114148066, + "tokens_seen": 2563440640 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002254764292878636, + "loss": 1.278, + "theoretical_loss": 3.390273490199161, + "tokens_seen": 2563506176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022545636910732198, + "loss": 1.3345, + "theoretical_loss": 3.390266669206724, + "tokens_seen": 2563571712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022543630892678034, + "loss": 1.2854, + "theoretical_loss": 3.390259848437482, + "tokens_seen": 2563637248 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022541624874623873, + "loss": 1.3501, + "theoretical_loss": 3.390253027891422, + "tokens_seen": 2563702784 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002253961885656971, + "loss": 1.2909, + "theoretical_loss": 3.3902462075685316, + "tokens_seen": 2563768320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022537612838515548, + "loss": 1.2359, + "theoretical_loss": 3.390239387468797, + "tokens_seen": 2563833856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022535606820461385, + "loss": 1.1691, + "theoretical_loss": 3.390232567592206, + "tokens_seen": 2563899392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022533600802407224, + "loss": 1.3987, + "theoretical_loss": 3.3902257479387448, + "tokens_seen": 2563964928 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002253159478435306, + "loss": 1.2887, + "theoretical_loss": 3.390218928508401, + "tokens_seen": 2564030464 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022529588766298896, + "loss": 1.1763, + "theoretical_loss": 3.3902121093011615, + "tokens_seen": 2564096000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022527582748244735, + "loss": 1.2393, + "theoretical_loss": 3.390205290317013, + "tokens_seen": 2564161536 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 932851, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3499561548233032, + "objective/train/theoretical_loss": 3.3901984715559426, + "objective/train/tokens_used": 934686176, + "theoretical_loss": 3.3901984715559426, + "tokens_seen": 2564227072 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002252557673019057, + "loss": 1.3266, + "theoretical_loss": 3.3901984715559426, + "tokens_seen": 2564227072 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002252357071213641, + "loss": 1.2986, + "theoretical_loss": 3.3901916530179372, + "tokens_seen": 2564292608 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022521564694082246, + "loss": 1.2773, + "theoretical_loss": 3.3901848347029846, + "tokens_seen": 2564358144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022519558676028085, + "loss": 1.3602, + "theoretical_loss": 3.390178016611071, + "tokens_seen": 2564423680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022517552657973922, + "loss": 1.4323, + "theoretical_loss": 3.390171198742183, + "tokens_seen": 2564489216 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002251554663991976, + "loss": 1.2046, + "theoretical_loss": 3.390164381096309, + "tokens_seen": 2564554752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022513540621865597, + "loss": 1.3094, + "theoretical_loss": 3.390157563673435, + "tokens_seen": 2564620288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022511534603811436, + "loss": 1.2662, + "theoretical_loss": 3.390150746473548, + "tokens_seen": 2564685824 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022509528585757272, + "loss": 1.3273, + "theoretical_loss": 3.3901439294966353, + "tokens_seen": 2564751360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022507522567703108, + "loss": 1.2659, + "theoretical_loss": 3.390137112742684, + "tokens_seen": 2564816896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022505516549648947, + "loss": 1.1546, + "theoretical_loss": 3.3901302962116806, + "tokens_seen": 2564882432 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022503510531594783, + "loss": 1.3612, + "theoretical_loss": 3.3901234799036124, + "tokens_seen": 2564947968 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022501504513540622, + "loss": 1.2759, + "theoretical_loss": 3.3901166638184668, + "tokens_seen": 2565013504 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022499498495486458, + "loss": 1.3883, + "theoretical_loss": 3.3901098479562304, + "tokens_seen": 2565079040 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022497492477432297, + "loss": 1.3977, + "theoretical_loss": 3.3901030323168904, + "tokens_seen": 2565144576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022495486459378134, + "loss": 1.287, + "theoretical_loss": 3.3900962169004334, + "tokens_seen": 2565210112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022493480441323973, + "loss": 1.3046, + "theoretical_loss": 3.390089401706847, + "tokens_seen": 2565275648 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002249147442326981, + "loss": 1.2899, + "theoretical_loss": 3.390082586736118, + "tokens_seen": 2565341184 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022489468405215645, + "loss": 1.3483, + "theoretical_loss": 3.390075771988233, + "tokens_seen": 2565406720 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022487462387161484, + "loss": 1.311, + "theoretical_loss": 3.3900689574631797, + "tokens_seen": 2565472256 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002248545636910732, + "loss": 1.3224, + "theoretical_loss": 3.390062143160945, + "tokens_seen": 2565537792 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022483450351053162, + "loss": 1.3757, + "theoretical_loss": 3.390055329081515, + "tokens_seen": 2565603328 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022481444332998998, + "loss": 1.2855, + "theoretical_loss": 3.390048515224878, + "tokens_seen": 2565668864 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022479438314944837, + "loss": 1.3388, + "theoretical_loss": 3.3900417015910205, + "tokens_seen": 2565734400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022477432296890673, + "loss": 1.2142, + "theoretical_loss": 3.390034888179929, + "tokens_seen": 2565799936 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022475426278836512, + "loss": 1.2702, + "theoretical_loss": 3.3900280749915916, + "tokens_seen": 2565865472 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022473420260782348, + "loss": 1.1059, + "theoretical_loss": 3.3900212620259946, + "tokens_seen": 2565931008 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022471414242728187, + "loss": 1.2501, + "theoretical_loss": 3.3900144492831252, + "tokens_seen": 2565996544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022469408224674023, + "loss": 1.3234, + "theoretical_loss": 3.3900076367629706, + "tokens_seen": 2566062080 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002246740220661986, + "loss": 1.2152, + "theoretical_loss": 3.3900008244655173, + "tokens_seen": 2566127616 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022465396188565699, + "loss": 1.2538, + "theoretical_loss": 3.389994012390753, + "tokens_seen": 2566193152 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022463390170511535, + "loss": 1.4163, + "theoretical_loss": 3.3899872005386644, + "tokens_seen": 2566258688 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022461384152457374, + "loss": 1.2178, + "theoretical_loss": 3.389980388909238, + "tokens_seen": 2566324224 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002245937813440321, + "loss": 1.282, + "theoretical_loss": 3.389973577502462, + "tokens_seen": 2566389760 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002245737211634905, + "loss": 1.3096, + "theoretical_loss": 3.389966766318323, + "tokens_seen": 2566455296 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022455366098294885, + "loss": 1.2495, + "theoretical_loss": 3.389959955356807, + "tokens_seen": 2566520832 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022453360080240724, + "loss": 1.2295, + "theoretical_loss": 3.3899531446179028, + "tokens_seen": 2566586368 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002245135406218656, + "loss": 1.3284, + "theoretical_loss": 3.389946334101596, + "tokens_seen": 2566651904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022449348044132397, + "loss": 1.2621, + "theoretical_loss": 3.3899395238078744, + "tokens_seen": 2566717440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022447342026078236, + "loss": 1.3465, + "theoretical_loss": 3.389932713736725, + "tokens_seen": 2566782976 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022445336008024072, + "loss": 1.3138, + "theoretical_loss": 3.389925903888134, + "tokens_seen": 2566848512 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002244332998996991, + "loss": 1.3155, + "theoretical_loss": 3.3899190942620896, + "tokens_seen": 2566914048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022441323971915747, + "loss": 1.4059, + "theoretical_loss": 3.3899122848585783, + "tokens_seen": 2566979584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022439317953861586, + "loss": 1.4369, + "theoretical_loss": 3.3899054756775873, + "tokens_seen": 2567045120 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022437311935807422, + "loss": 1.1655, + "theoretical_loss": 3.3898986667191036, + "tokens_seen": 2567110656 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002243530591775326, + "loss": 1.347, + "theoretical_loss": 3.389891857983114, + "tokens_seen": 2567176192 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022433299899699097, + "loss": 1.3014, + "theoretical_loss": 3.389885049469606, + "tokens_seen": 2567241728 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022431293881644933, + "loss": 1.2648, + "theoretical_loss": 3.3898782411785664, + "tokens_seen": 2567307264 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022429287863590772, + "loss": 1.2783, + "theoretical_loss": 3.389871433109982, + "tokens_seen": 2567372800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022427281845536609, + "loss": 1.2455, + "theoretical_loss": 3.38986462526384, + "tokens_seen": 2567438336 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 933971, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1539313793182373, + "objective/train/theoretical_loss": 3.389857817640128, + "objective/train/tokens_used": 937962976, + "theoretical_loss": 3.389857817640128, + "tokens_seen": 2567503872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022425275827482448, + "loss": 1.2347, + "theoretical_loss": 3.389857817640128, + "tokens_seen": 2567503872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022423269809428284, + "loss": 1.3264, + "theoretical_loss": 3.389851010238832, + "tokens_seen": 2567569408 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022421263791374123, + "loss": 1.4469, + "theoretical_loss": 3.3898442030599405, + "tokens_seen": 2567634944 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002241925777331996, + "loss": 1.3354, + "theoretical_loss": 3.3898373961034394, + "tokens_seen": 2567700480 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022417251755265798, + "loss": 1.323, + "theoretical_loss": 3.389830589369316, + "tokens_seen": 2567766016 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022415245737211634, + "loss": 1.3827, + "theoretical_loss": 3.3898237828575573, + "tokens_seen": 2567831552 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002241323971915747, + "loss": 1.3502, + "theoretical_loss": 3.389816976568151, + "tokens_seen": 2567897088 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002241123370110331, + "loss": 1.2593, + "theoretical_loss": 3.3898101705010832, + "tokens_seen": 2567962624 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022409227683049148, + "loss": 1.224, + "theoretical_loss": 3.3898033646563417, + "tokens_seen": 2568028160 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022407221664994987, + "loss": 1.308, + "theoretical_loss": 3.389796559033913, + "tokens_seen": 2568093696 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022405215646940823, + "loss": 1.2603, + "theoretical_loss": 3.389789753633785, + "tokens_seen": 2568159232 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022403209628886662, + "loss": 1.3496, + "theoretical_loss": 3.389782948455944, + "tokens_seen": 2568224768 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022401203610832498, + "loss": 1.2856, + "theoretical_loss": 3.3897761435003773, + "tokens_seen": 2568290304 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022399197592778337, + "loss": 1.3056, + "theoretical_loss": 3.3897693387670715, + "tokens_seen": 2568355840 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022397191574724174, + "loss": 1.4114, + "theoretical_loss": 3.3897625342560147, + "tokens_seen": 2568421376 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022395185556670013, + "loss": 1.2533, + "theoretical_loss": 3.3897557299671934, + "tokens_seen": 2568486912 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002239317953861585, + "loss": 1.3902, + "theoretical_loss": 3.3897489259005944, + "tokens_seen": 2568552448 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022391173520561685, + "loss": 1.3716, + "theoretical_loss": 3.389742122056205, + "tokens_seen": 2568617984 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022389167502507524, + "loss": 1.3404, + "theoretical_loss": 3.389735318434013, + "tokens_seen": 2568683520 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002238716148445336, + "loss": 1.2121, + "theoretical_loss": 3.389728515034004, + "tokens_seen": 2568749056 + }, + { + "epoch": 0.56, + "learning_rate": 0.000223851554663992, + "loss": 1.3619, + "theoretical_loss": 3.389721711856166, + "tokens_seen": 2568814592 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022383149448345035, + "loss": 1.3159, + "theoretical_loss": 3.3897149089004865, + "tokens_seen": 2568880128 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022381143430290874, + "loss": 1.3131, + "theoretical_loss": 3.3897081061669514, + "tokens_seen": 2568945664 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002237913741223671, + "loss": 1.2708, + "theoretical_loss": 3.389701303655549, + "tokens_seen": 2569011200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002237713139418255, + "loss": 1.3967, + "theoretical_loss": 3.3896945013662654, + "tokens_seen": 2569076736 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022375125376128386, + "loss": 1.2901, + "theoretical_loss": 3.3896876992990883, + "tokens_seen": 2569142272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022373119358074222, + "loss": 1.3518, + "theoretical_loss": 3.3896808974540042, + "tokens_seen": 2569207808 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002237111334002006, + "loss": 1.2018, + "theoretical_loss": 3.389674095831001, + "tokens_seen": 2569273344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022369107321965897, + "loss": 1.3063, + "theoretical_loss": 3.389667294430065, + "tokens_seen": 2569338880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022367101303911736, + "loss": 1.2333, + "theoretical_loss": 3.389660493251184, + "tokens_seen": 2569404416 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022365095285857572, + "loss": 1.2359, + "theoretical_loss": 3.389653692294344, + "tokens_seen": 2569469952 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002236308926780341, + "loss": 1.4002, + "theoretical_loss": 3.3896468915595337, + "tokens_seen": 2569535488 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022361083249749247, + "loss": 1.2422, + "theoretical_loss": 3.3896400910467386, + "tokens_seen": 2569601024 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022359077231695086, + "loss": 1.3985, + "theoretical_loss": 3.3896332907559468, + "tokens_seen": 2569666560 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022357071213640923, + "loss": 1.2322, + "theoretical_loss": 3.389626490687145, + "tokens_seen": 2569732096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022355065195586761, + "loss": 1.4224, + "theoretical_loss": 3.3896196908403198, + "tokens_seen": 2569797632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022353059177532598, + "loss": 1.399, + "theoretical_loss": 3.3896128912154593, + "tokens_seen": 2569863168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022351053159478434, + "loss": 1.3241, + "theoretical_loss": 3.38960609181255, + "tokens_seen": 2569928704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022349047141424273, + "loss": 1.3168, + "theoretical_loss": 3.38959929263158, + "tokens_seen": 2569994240 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002234704112337011, + "loss": 1.3395, + "theoretical_loss": 3.3895924936725343, + "tokens_seen": 2570059776 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022345035105315948, + "loss": 1.3385, + "theoretical_loss": 3.389585694935402, + "tokens_seen": 2570125312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022343029087261784, + "loss": 1.2579, + "theoretical_loss": 3.389578896420169, + "tokens_seen": 2570190848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022341023069207623, + "loss": 1.3115, + "theoretical_loss": 3.389572098126823, + "tokens_seen": 2570256384 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002233901705115346, + "loss": 1.3606, + "theoretical_loss": 3.389565300055351, + "tokens_seen": 2570321920 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022337011033099298, + "loss": 1.3374, + "theoretical_loss": 3.3895585022057397, + "tokens_seen": 2570387456 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022335005015045135, + "loss": 1.4263, + "theoretical_loss": 3.389551704577977, + "tokens_seen": 2570452992 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022332998996990973, + "loss": 1.2502, + "theoretical_loss": 3.3895449071720494, + "tokens_seen": 2570518528 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022330992978936812, + "loss": 1.299, + "theoretical_loss": 3.389538109987944, + "tokens_seen": 2570584064 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022328986960882649, + "loss": 1.2784, + "theoretical_loss": 3.389531313025648, + "tokens_seen": 2570649600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022326980942828488, + "loss": 1.2771, + "theoretical_loss": 3.3895245162851486, + "tokens_seen": 2570715136 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 935107, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1545178890228271, + "objective/train/theoretical_loss": 3.389517719766433, + "objective/train/tokens_used": 941239776, + "theoretical_loss": 3.389517719766433, + "tokens_seen": 2570780672 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022324974924774324, + "loss": 1.2435, + "theoretical_loss": 3.389517719766433, + "tokens_seen": 2570780672 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022322968906720163, + "loss": 1.4146, + "theoretical_loss": 3.389510923469488, + "tokens_seen": 2570846208 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022320962888666, + "loss": 1.4585, + "theoretical_loss": 3.389504127394301, + "tokens_seen": 2570911744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022318956870611838, + "loss": 1.2648, + "theoretical_loss": 3.3894973315408587, + "tokens_seen": 2570977280 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022316950852557674, + "loss": 1.2951, + "theoretical_loss": 3.389490535909149, + "tokens_seen": 2571042816 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002231494483450351, + "loss": 1.26, + "theoretical_loss": 3.3894837404991582, + "tokens_seen": 2571108352 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002231293881644935, + "loss": 1.3512, + "theoretical_loss": 3.3894769453108737, + "tokens_seen": 2571173888 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022310932798395185, + "loss": 1.234, + "theoretical_loss": 3.3894701503442826, + "tokens_seen": 2571239424 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022308926780341024, + "loss": 1.2663, + "theoretical_loss": 3.3894633555993723, + "tokens_seen": 2571304960 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002230692076228686, + "loss": 1.3575, + "theoretical_loss": 3.3894565610761296, + "tokens_seen": 2571370496 + }, + { + "epoch": 0.56, + "learning_rate": 0.000223049147442327, + "loss": 1.3399, + "theoretical_loss": 3.3894497667745416, + "tokens_seen": 2571436032 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022302908726178536, + "loss": 1.3957, + "theoretical_loss": 3.389442972694596, + "tokens_seen": 2571501568 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022300902708124375, + "loss": 1.2917, + "theoretical_loss": 3.389436178836279, + "tokens_seen": 2571567104 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002229889669007021, + "loss": 1.4657, + "theoretical_loss": 3.3894293851995783, + "tokens_seen": 2571632640 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002229689067201605, + "loss": 1.2593, + "theoretical_loss": 3.3894225917844807, + "tokens_seen": 2571698176 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022294884653961886, + "loss": 1.3117, + "theoretical_loss": 3.3894157985909734, + "tokens_seen": 2571763712 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022292878635907722, + "loss": 1.2708, + "theoretical_loss": 3.389409005619044, + "tokens_seen": 2571829248 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002229087261785356, + "loss": 1.2876, + "theoretical_loss": 3.389402212868679, + "tokens_seen": 2571894784 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022288866599799397, + "loss": 1.4493, + "theoretical_loss": 3.3893954203398664, + "tokens_seen": 2571960320 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022286860581745236, + "loss": 1.2858, + "theoretical_loss": 3.3893886280325924, + "tokens_seen": 2572025856 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022284854563691073, + "loss": 1.4089, + "theoretical_loss": 3.3893818359468444, + "tokens_seen": 2572091392 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022282848545636912, + "loss": 1.3053, + "theoretical_loss": 3.389375044082609, + "tokens_seen": 2572156928 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022280842527582748, + "loss": 1.288, + "theoretical_loss": 3.3893682524398745, + "tokens_seen": 2572222464 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022278836509528587, + "loss": 1.3242, + "theoretical_loss": 3.3893614610186273, + "tokens_seen": 2572288000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022276830491474423, + "loss": 1.2921, + "theoretical_loss": 3.389354669818855, + "tokens_seen": 2572353536 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002227482447342026, + "loss": 1.3193, + "theoretical_loss": 3.3893478788405442, + "tokens_seen": 2572419072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022272818455366098, + "loss": 1.3499, + "theoretical_loss": 3.3893410880836825, + "tokens_seen": 2572484608 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022270812437311934, + "loss": 1.3765, + "theoretical_loss": 3.389334297548256, + "tokens_seen": 2572550144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022268806419257773, + "loss": 1.292, + "theoretical_loss": 3.3893275072342535, + "tokens_seen": 2572615680 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002226680040120361, + "loss": 1.2545, + "theoretical_loss": 3.389320717141661, + "tokens_seen": 2572681216 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022264794383149448, + "loss": 1.3835, + "theoretical_loss": 3.3893139272704658, + "tokens_seen": 2572746752 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022262788365095285, + "loss": 1.2129, + "theoretical_loss": 3.389307137620655, + "tokens_seen": 2572812288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022260782347041124, + "loss": 1.3797, + "theoretical_loss": 3.3893003481922164, + "tokens_seen": 2572877824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022258776328986963, + "loss": 1.2878, + "theoretical_loss": 3.3892935589851363, + "tokens_seen": 2572943360 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022256770310932801, + "loss": 1.1948, + "theoretical_loss": 3.3892867699994023, + "tokens_seen": 2573008896 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022254764292878638, + "loss": 1.3066, + "theoretical_loss": 3.389279981235001, + "tokens_seen": 2573074432 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022252758274824474, + "loss": 1.3384, + "theoretical_loss": 3.3892731926919204, + "tokens_seen": 2573139968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022250752256770313, + "loss": 1.3614, + "theoretical_loss": 3.3892664043701473, + "tokens_seen": 2573205504 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002224874623871615, + "loss": 1.3016, + "theoretical_loss": 3.3892596162696687, + "tokens_seen": 2573271040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022246740220661988, + "loss": 1.2924, + "theoretical_loss": 3.3892528283904717, + "tokens_seen": 2573336576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022244734202607824, + "loss": 1.3467, + "theoretical_loss": 3.3892460407325435, + "tokens_seen": 2573402112 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022242728184553663, + "loss": 1.1972, + "theoretical_loss": 3.3892392532958717, + "tokens_seen": 2573467648 + }, + { + "epoch": 0.56, + "learning_rate": 0.000222407221664995, + "loss": 1.3852, + "theoretical_loss": 3.389232466080443, + "tokens_seen": 2573533184 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022238716148445338, + "loss": 1.2848, + "theoretical_loss": 3.3892256790862447, + "tokens_seen": 2573598720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022236710130391175, + "loss": 1.3283, + "theoretical_loss": 3.3892188923132633, + "tokens_seen": 2573664256 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002223470411233701, + "loss": 1.2935, + "theoretical_loss": 3.3892121057614872, + "tokens_seen": 2573729792 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002223269809428285, + "loss": 1.3555, + "theoretical_loss": 3.3892053194309026, + "tokens_seen": 2573795328 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022230692076228686, + "loss": 1.3757, + "theoretical_loss": 3.389198533321497, + "tokens_seen": 2573860864 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022228686058174525, + "loss": 1.3584, + "theoretical_loss": 3.389191747433258, + "tokens_seen": 2573926400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002222668004012036, + "loss": 1.2843, + "theoretical_loss": 3.3891849617661713, + "tokens_seen": 2573991936 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 936262, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2910391092300415, + "objective/train/theoretical_loss": 3.3891781763202258, + "objective/train/tokens_used": 944516576, + "theoretical_loss": 3.3891781763202258, + "tokens_seen": 2574057472 + }, + { + "epoch": 0.56, + "learning_rate": 0.000222246740220662, + "loss": 1.2646, + "theoretical_loss": 3.3891781763202258, + "tokens_seen": 2574057472 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022222668004012036, + "loss": 1.3274, + "theoretical_loss": 3.3891713910954078, + "tokens_seen": 2574123008 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022220661985957875, + "loss": 1.1979, + "theoretical_loss": 3.3891646060917044, + "tokens_seen": 2574188544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022218655967903711, + "loss": 1.4173, + "theoretical_loss": 3.389157821309103, + "tokens_seen": 2574254080 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022216649949849548, + "loss": 1.3303, + "theoretical_loss": 3.3891510367475908, + "tokens_seen": 2574319616 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022214643931795387, + "loss": 1.3349, + "theoretical_loss": 3.3891442524071547, + "tokens_seen": 2574385152 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022212637913741223, + "loss": 1.2275, + "theoretical_loss": 3.389137468287782, + "tokens_seen": 2574450688 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022210631895687062, + "loss": 1.24, + "theoretical_loss": 3.3891306843894604, + "tokens_seen": 2574516224 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022208625877632898, + "loss": 1.2977, + "theoretical_loss": 3.389123900712176, + "tokens_seen": 2574581760 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022206619859578737, + "loss": 1.2669, + "theoretical_loss": 3.3891171172559167, + "tokens_seen": 2574647296 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022204613841524573, + "loss": 1.2546, + "theoretical_loss": 3.38911033402067, + "tokens_seen": 2574712832 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022202607823470412, + "loss": 1.254, + "theoretical_loss": 3.3891035510064222, + "tokens_seen": 2574778368 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022200601805416248, + "loss": 1.4988, + "theoretical_loss": 3.389096768213161, + "tokens_seen": 2574843904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022198595787362084, + "loss": 1.2963, + "theoretical_loss": 3.389089985640873, + "tokens_seen": 2574909440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022196589769307923, + "loss": 1.3037, + "theoretical_loss": 3.3890832032895464, + "tokens_seen": 2574974976 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002219458375125376, + "loss": 1.3525, + "theoretical_loss": 3.3890764211591677, + "tokens_seen": 2575040512 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022192577733199599, + "loss": 1.3097, + "theoretical_loss": 3.3890696392497244, + "tokens_seen": 2575106048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022190571715145435, + "loss": 1.2034, + "theoretical_loss": 3.389062857561203, + "tokens_seen": 2575171584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022188565697091274, + "loss": 1.3004, + "theoretical_loss": 3.3890560760935915, + "tokens_seen": 2575237120 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002218655967903711, + "loss": 1.3135, + "theoretical_loss": 3.3890492948468767, + "tokens_seen": 2575302656 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022184553660982952, + "loss": 1.2574, + "theoretical_loss": 3.3890425138210456, + "tokens_seen": 2575368192 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022182547642928788, + "loss": 1.3913, + "theoretical_loss": 3.3890357330160854, + "tokens_seen": 2575433728 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022180541624874627, + "loss": 1.3158, + "theoretical_loss": 3.3890289524319837, + "tokens_seen": 2575499264 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022178535606820463, + "loss": 1.4259, + "theoretical_loss": 3.389022172068728, + "tokens_seen": 2575564800 + }, + { + "epoch": 0.56, + "learning_rate": 0.000221765295887663, + "loss": 1.302, + "theoretical_loss": 3.3890153919263044, + "tokens_seen": 2575630336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022174523570712138, + "loss": 1.2542, + "theoretical_loss": 3.3890086120047007, + "tokens_seen": 2575695872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022172517552657974, + "loss": 1.3244, + "theoretical_loss": 3.389001832303904, + "tokens_seen": 2575761408 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022170511534603813, + "loss": 1.2714, + "theoretical_loss": 3.3889950528239017, + "tokens_seen": 2575826944 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002216850551654965, + "loss": 1.3623, + "theoretical_loss": 3.3889882735646806, + "tokens_seen": 2575892480 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022166499498495488, + "loss": 1.2513, + "theoretical_loss": 3.3889814945262287, + "tokens_seen": 2575958016 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022164493480441325, + "loss": 1.2617, + "theoretical_loss": 3.388974715708532, + "tokens_seen": 2576023552 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022162487462387164, + "loss": 1.3275, + "theoretical_loss": 3.3889679371115786, + "tokens_seen": 2576089088 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022160481444333, + "loss": 1.2478, + "theoretical_loss": 3.3889611587353556, + "tokens_seen": 2576154624 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022158475426278836, + "loss": 1.3191, + "theoretical_loss": 3.3889543805798494, + "tokens_seen": 2576220160 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022156469408224675, + "loss": 1.2841, + "theoretical_loss": 3.388947602645048, + "tokens_seen": 2576285696 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002215446339017051, + "loss": 1.2245, + "theoretical_loss": 3.3889408249309385, + "tokens_seen": 2576351232 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002215245737211635, + "loss": 1.3988, + "theoretical_loss": 3.388934047437508, + "tokens_seen": 2576416768 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022150451354062186, + "loss": 1.4035, + "theoretical_loss": 3.3889272701647437, + "tokens_seen": 2576482304 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022148445336008025, + "loss": 1.2229, + "theoretical_loss": 3.388920493112633, + "tokens_seen": 2576547840 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022146439317953862, + "loss": 1.2585, + "theoretical_loss": 3.388913716281163, + "tokens_seen": 2576613376 + }, + { + "epoch": 0.56, + "learning_rate": 0.000221444332998997, + "loss": 1.3534, + "theoretical_loss": 3.3889069396703206, + "tokens_seen": 2576678912 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022142427281845537, + "loss": 1.408, + "theoretical_loss": 3.388900163280093, + "tokens_seen": 2576744448 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022140421263791376, + "loss": 1.2418, + "theoretical_loss": 3.3888933871104676, + "tokens_seen": 2576809984 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022138415245737212, + "loss": 1.3517, + "theoretical_loss": 3.388886611161432, + "tokens_seen": 2576875520 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022136409227683048, + "loss": 1.3343, + "theoretical_loss": 3.3888798354329728, + "tokens_seen": 2576941056 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022134403209628887, + "loss": 1.2922, + "theoretical_loss": 3.3888730599250776, + "tokens_seen": 2577006592 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022132397191574723, + "loss": 1.365, + "theoretical_loss": 3.388866284637733, + "tokens_seen": 2577072128 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022130391173520562, + "loss": 1.213, + "theoretical_loss": 3.3888595095709273, + "tokens_seen": 2577137664 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022128385155466398, + "loss": 1.3841, + "theoretical_loss": 3.388852734724647, + "tokens_seen": 2577203200 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022126379137412237, + "loss": 1.3248, + "theoretical_loss": 3.388845960098879, + "tokens_seen": 2577268736 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 937411, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2590935230255127, + "objective/train/theoretical_loss": 3.3888391856936115, + "objective/train/tokens_used": 947793376, + "theoretical_loss": 3.3888391856936115, + "tokens_seen": 2577334272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022124373119358074, + "loss": 1.3489, + "theoretical_loss": 3.3888391856936115, + "tokens_seen": 2577334272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022122367101303913, + "loss": 1.1843, + "theoretical_loss": 3.3888324115088304, + "tokens_seen": 2577399808 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002212036108324975, + "loss": 1.1495, + "theoretical_loss": 3.3888256375445245, + "tokens_seen": 2577465344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022118355065195585, + "loss": 1.3111, + "theoretical_loss": 3.3888188638006795, + "tokens_seen": 2577530880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022116349047141424, + "loss": 1.1546, + "theoretical_loss": 3.3888120902772836, + "tokens_seen": 2577596416 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002211434302908726, + "loss": 1.2159, + "theoretical_loss": 3.3888053169743237, + "tokens_seen": 2577661952 + }, + { + "epoch": 0.56, + "learning_rate": 0.000221123370110331, + "loss": 1.3333, + "theoretical_loss": 3.388798543891787, + "tokens_seen": 2577727488 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022110330992978935, + "loss": 1.4019, + "theoretical_loss": 3.3887917710296604, + "tokens_seen": 2577793024 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022108324974924777, + "loss": 1.3249, + "theoretical_loss": 3.388784998387932, + "tokens_seen": 2577858560 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022106318956870613, + "loss": 1.3419, + "theoretical_loss": 3.3887782259665884, + "tokens_seen": 2577924096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022104312938816452, + "loss": 1.2986, + "theoretical_loss": 3.388771453765617, + "tokens_seen": 2577989632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022102306920762288, + "loss": 1.3086, + "theoretical_loss": 3.3887646817850046, + "tokens_seen": 2578055168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022100300902708125, + "loss": 1.3599, + "theoretical_loss": 3.388757910024739, + "tokens_seen": 2578120704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022098294884653963, + "loss": 1.3771, + "theoretical_loss": 3.3887511384848072, + "tokens_seen": 2578186240 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220962888665998, + "loss": 1.3966, + "theoretical_loss": 3.388744367165197, + "tokens_seen": 2578251776 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022094282848545639, + "loss": 1.3356, + "theoretical_loss": 3.3887375960658943, + "tokens_seen": 2578317312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022092276830491475, + "loss": 1.3992, + "theoretical_loss": 3.388730825186887, + "tokens_seen": 2578382848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022090270812437314, + "loss": 1.3421, + "theoretical_loss": 3.388724054528163, + "tokens_seen": 2578448384 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002208826479438315, + "loss": 1.3526, + "theoretical_loss": 3.388717284089709, + "tokens_seen": 2578513920 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002208625877632899, + "loss": 1.3457, + "theoretical_loss": 3.388710513871512, + "tokens_seen": 2578579456 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022084252758274825, + "loss": 1.2566, + "theoretical_loss": 3.3887037438735597, + "tokens_seen": 2578644992 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022082246740220664, + "loss": 1.2289, + "theoretical_loss": 3.3886969740958386, + "tokens_seen": 2578710528 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220802407221665, + "loss": 1.4146, + "theoretical_loss": 3.388690204538337, + "tokens_seen": 2578776064 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022078234704112337, + "loss": 1.2102, + "theoretical_loss": 3.388683435201041, + "tokens_seen": 2578841600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022076228686058175, + "loss": 1.3504, + "theoretical_loss": 3.3886766660839385, + "tokens_seen": 2578907136 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022074222668004012, + "loss": 1.1813, + "theoretical_loss": 3.3886698971870173, + "tokens_seen": 2578972672 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002207221664994985, + "loss": 1.323, + "theoretical_loss": 3.388663128510263, + "tokens_seen": 2579038208 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022070210631895687, + "loss": 1.2812, + "theoretical_loss": 3.3886563600536648, + "tokens_seen": 2579103744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022068204613841526, + "loss": 1.2407, + "theoretical_loss": 3.3886495918172086, + "tokens_seen": 2579169280 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022066198595787362, + "loss": 1.3465, + "theoretical_loss": 3.3886428238008817, + "tokens_seen": 2579234816 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220641925777332, + "loss": 1.2961, + "theoretical_loss": 3.3886360560046724, + "tokens_seen": 2579300352 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022062186559679037, + "loss": 1.2614, + "theoretical_loss": 3.3886292884285667, + "tokens_seen": 2579365888 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022060180541624873, + "loss": 1.3342, + "theoretical_loss": 3.3886225210725525, + "tokens_seen": 2579431424 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022058174523570712, + "loss": 1.3126, + "theoretical_loss": 3.388615753936617, + "tokens_seen": 2579496960 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022056168505516549, + "loss": 1.3641, + "theoretical_loss": 3.3886089870207474, + "tokens_seen": 2579562496 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022054162487462387, + "loss": 1.2825, + "theoretical_loss": 3.3886022203249304, + "tokens_seen": 2579628032 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022052156469408224, + "loss": 1.2858, + "theoretical_loss": 3.3885954538491543, + "tokens_seen": 2579693568 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022050150451354063, + "loss": 1.026, + "theoretical_loss": 3.388588687593406, + "tokens_seen": 2579759104 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220481444332999, + "loss": 1.475, + "theoretical_loss": 3.3885819215576722, + "tokens_seen": 2579824640 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022046138415245738, + "loss": 1.2341, + "theoretical_loss": 3.388575155741941, + "tokens_seen": 2579890176 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022044132397191574, + "loss": 1.225, + "theoretical_loss": 3.388568390146199, + "tokens_seen": 2579955712 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002204212637913741, + "loss": 1.2431, + "theoretical_loss": 3.3885616247704338, + "tokens_seen": 2580021248 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002204012036108325, + "loss": 1.1438, + "theoretical_loss": 3.388554859614632, + "tokens_seen": 2580086784 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022038114343029085, + "loss": 1.323, + "theoretical_loss": 3.3885480946787823, + "tokens_seen": 2580152320 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022036108324974924, + "loss": 1.3085, + "theoretical_loss": 3.3885413299628704, + "tokens_seen": 2580217856 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022034102306920763, + "loss": 1.2261, + "theoretical_loss": 3.3885345654668844, + "tokens_seen": 2580283392 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022032096288866602, + "loss": 1.2305, + "theoretical_loss": 3.3885278011908113, + "tokens_seen": 2580348928 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022030090270812438, + "loss": 1.4236, + "theoretical_loss": 3.3885210371346384, + "tokens_seen": 2580414464 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022028084252758277, + "loss": 1.371, + "theoretical_loss": 3.388514273298353, + "tokens_seen": 2580480000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022026078234704114, + "loss": 1.238, + "theoretical_loss": 3.3885075096819426, + "tokens_seen": 2580545536 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 938550, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.151209831237793, + "objective/train/theoretical_loss": 3.388500746285394, + "objective/train/tokens_used": 951070176, + "theoretical_loss": 3.388500746285394, + "tokens_seen": 2580611072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022024072216649953, + "loss": 1.2161, + "theoretical_loss": 3.388500746285394, + "tokens_seen": 2580611072 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002202206619859579, + "loss": 1.3549, + "theoretical_loss": 3.388493983108695, + "tokens_seen": 2580676608 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022020060180541625, + "loss": 1.2652, + "theoretical_loss": 3.3884872201518323, + "tokens_seen": 2580742144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022018054162487464, + "loss": 1.3827, + "theoretical_loss": 3.388480457414794, + "tokens_seen": 2580807680 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220160481444333, + "loss": 1.2145, + "theoretical_loss": 3.3884736948975664, + "tokens_seen": 2580873216 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002201404212637914, + "loss": 1.1846, + "theoretical_loss": 3.3884669326001373, + "tokens_seen": 2580938752 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022012036108324975, + "loss": 1.4169, + "theoretical_loss": 3.3884601705224937, + "tokens_seen": 2581004288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022010030090270814, + "loss": 1.3475, + "theoretical_loss": 3.388453408664623, + "tokens_seen": 2581069824 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002200802407221665, + "loss": 1.2245, + "theoretical_loss": 3.3884466470265133, + "tokens_seen": 2581135360 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002200601805416249, + "loss": 1.2955, + "theoretical_loss": 3.3884398856081503, + "tokens_seen": 2581200896 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022004012036108326, + "loss": 1.2953, + "theoretical_loss": 3.3884331244095227, + "tokens_seen": 2581266432 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022002006018054162, + "loss": 1.3739, + "theoretical_loss": 3.388426363430617, + "tokens_seen": 2581331968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022, + "loss": 1.3167, + "theoretical_loss": 3.38841960267142, + "tokens_seen": 2581397504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021997993981945837, + "loss": 1.0896, + "theoretical_loss": 3.388412842131921, + "tokens_seen": 2581463040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021995987963891676, + "loss": 1.3279, + "theoretical_loss": 3.388406081812105, + "tokens_seen": 2581528576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021993981945837512, + "loss": 1.3452, + "theoretical_loss": 3.3883993217119603, + "tokens_seen": 2581594112 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002199197592778335, + "loss": 1.3415, + "theoretical_loss": 3.3883925618314743, + "tokens_seen": 2581659648 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021989969909729187, + "loss": 1.498, + "theoretical_loss": 3.3883858021706335, + "tokens_seen": 2581725184 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021987963891675026, + "loss": 1.3083, + "theoretical_loss": 3.3883790427294267, + "tokens_seen": 2581790720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021985957873620862, + "loss": 1.3676, + "theoretical_loss": 3.38837228350784, + "tokens_seen": 2581856256 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021983951855566701, + "loss": 1.3168, + "theoretical_loss": 3.3883655245058604, + "tokens_seen": 2581921792 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021981945837512538, + "loss": 1.4092, + "theoretical_loss": 3.388358765723476, + "tokens_seen": 2581987328 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021979939819458374, + "loss": 1.2599, + "theoretical_loss": 3.388352007160674, + "tokens_seen": 2582052864 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021977933801404213, + "loss": 1.3216, + "theoretical_loss": 3.3883452488174415, + "tokens_seen": 2582118400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002197592778335005, + "loss": 1.2602, + "theoretical_loss": 3.3883384906937657, + "tokens_seen": 2582183936 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021973921765295888, + "loss": 1.3284, + "theoretical_loss": 3.388331732789634, + "tokens_seen": 2582249472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021971915747241724, + "loss": 1.3153, + "theoretical_loss": 3.388324975105034, + "tokens_seen": 2582315008 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021969909729187563, + "loss": 1.2298, + "theoretical_loss": 3.3883182176399522, + "tokens_seen": 2582380544 + }, + { + "epoch": 0.57, + "learning_rate": 0.000219679037111334, + "loss": 1.2799, + "theoretical_loss": 3.3883114603943767, + "tokens_seen": 2582446080 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021965897693079238, + "loss": 1.4192, + "theoretical_loss": 3.3883047033682945, + "tokens_seen": 2582511616 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021963891675025075, + "loss": 1.2732, + "theoretical_loss": 3.388297946561693, + "tokens_seen": 2582577152 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002196188565697091, + "loss": 1.3011, + "theoretical_loss": 3.3882911899745594, + "tokens_seen": 2582642688 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002195987963891675, + "loss": 1.2516, + "theoretical_loss": 3.388284433606881, + "tokens_seen": 2582708224 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021957873620862589, + "loss": 1.3317, + "theoretical_loss": 3.388277677458645, + "tokens_seen": 2582773760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021955867602808428, + "loss": 1.2456, + "theoretical_loss": 3.388270921529839, + "tokens_seen": 2582839296 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021953861584754264, + "loss": 1.278, + "theoretical_loss": 3.38826416582045, + "tokens_seen": 2582904832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021951855566700103, + "loss": 1.1486, + "theoretical_loss": 3.3882574103304655, + "tokens_seen": 2582970368 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002194984954864594, + "loss": 1.2977, + "theoretical_loss": 3.3882506550598728, + "tokens_seen": 2583035904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021947843530591778, + "loss": 1.1829, + "theoretical_loss": 3.388243900008659, + "tokens_seen": 2583101440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021945837512537614, + "loss": 1.3225, + "theoretical_loss": 3.3882371451768116, + "tokens_seen": 2583166976 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002194383149448345, + "loss": 1.2473, + "theoretical_loss": 3.388230390564318, + "tokens_seen": 2583232512 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002194182547642929, + "loss": 1.2673, + "theoretical_loss": 3.388223636171165, + "tokens_seen": 2583298048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021939819458375125, + "loss": 1.2482, + "theoretical_loss": 3.388216881997341, + "tokens_seen": 2583363584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021937813440320964, + "loss": 1.2795, + "theoretical_loss": 3.388210128042832, + "tokens_seen": 2583429120 + }, + { + "epoch": 0.57, + "learning_rate": 0.000219358074222668, + "loss": 1.2556, + "theoretical_loss": 3.388203374307626, + "tokens_seen": 2583494656 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002193380140421264, + "loss": 1.2754, + "theoretical_loss": 3.3881966207917102, + "tokens_seen": 2583560192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021931795386158476, + "loss": 1.2542, + "theoretical_loss": 3.3881898674950723, + "tokens_seen": 2583625728 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021929789368104315, + "loss": 1.2379, + "theoretical_loss": 3.388183114417699, + "tokens_seen": 2583691264 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002192778335005015, + "loss": 1.335, + "theoretical_loss": 3.3881763615595784, + "tokens_seen": 2583756800 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002192577733199599, + "loss": 1.2097, + "theoretical_loss": 3.3881696089206965, + "tokens_seen": 2583822336 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 939673, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.261291742324829, + "objective/train/theoretical_loss": 3.388162856501042, + "objective/train/tokens_used": 954346976, + "theoretical_loss": 3.388162856501042, + "tokens_seen": 2583887872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021923771313941826, + "loss": 1.2778, + "theoretical_loss": 3.388162856501042, + "tokens_seen": 2583887872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021921765295887662, + "loss": 1.4465, + "theoretical_loss": 3.3881561043006014, + "tokens_seen": 2583953408 + }, + { + "epoch": 0.57, + "learning_rate": 0.000219197592778335, + "loss": 1.2542, + "theoretical_loss": 3.3881493523193624, + "tokens_seen": 2584018944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021917753259779337, + "loss": 1.3014, + "theoretical_loss": 3.3881426005573125, + "tokens_seen": 2584084480 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021915747241725176, + "loss": 1.2977, + "theoretical_loss": 3.3881358490144384, + "tokens_seen": 2584150016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021913741223671013, + "loss": 1.3086, + "theoretical_loss": 3.3881290976907277, + "tokens_seen": 2584215552 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021911735205616852, + "loss": 1.2667, + "theoretical_loss": 3.3881223465861683, + "tokens_seen": 2584281088 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021909729187562688, + "loss": 1.3341, + "theoretical_loss": 3.3881155957007465, + "tokens_seen": 2584346624 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021907723169508527, + "loss": 1.2774, + "theoretical_loss": 3.3881088450344503, + "tokens_seen": 2584412160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021905717151454363, + "loss": 1.3352, + "theoretical_loss": 3.388102094587267, + "tokens_seen": 2584477696 + }, + { + "epoch": 0.57, + "learning_rate": 0.000219037111334002, + "loss": 1.2903, + "theoretical_loss": 3.3880953443591837, + "tokens_seen": 2584543232 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021901705115346038, + "loss": 1.2157, + "theoretical_loss": 3.3880885943501875, + "tokens_seen": 2584608768 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021899699097291874, + "loss": 1.2359, + "theoretical_loss": 3.3880818445602667, + "tokens_seen": 2584674304 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021897693079237713, + "loss": 1.3053, + "theoretical_loss": 3.388075094989408, + "tokens_seen": 2584739840 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002189568706118355, + "loss": 1.142, + "theoretical_loss": 3.388068345637598, + "tokens_seen": 2584805376 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021893681043129388, + "loss": 1.3428, + "theoretical_loss": 3.3880615965048255, + "tokens_seen": 2584870912 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021891675025075225, + "loss": 1.3832, + "theoretical_loss": 3.388054847591077, + "tokens_seen": 2584936448 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021889669007021064, + "loss": 1.2131, + "theoretical_loss": 3.38804809889634, + "tokens_seen": 2585001984 + }, + { + "epoch": 0.57, + "learning_rate": 0.000218876629889669, + "loss": 1.2205, + "theoretical_loss": 3.3880413504206017, + "tokens_seen": 2585067520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021885656970912736, + "loss": 1.2751, + "theoretical_loss": 3.3880346021638497, + "tokens_seen": 2585133056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021883650952858578, + "loss": 1.2977, + "theoretical_loss": 3.388027854126071, + "tokens_seen": 2585198592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021881644934804414, + "loss": 1.3599, + "theoretical_loss": 3.388021106307253, + "tokens_seen": 2585264128 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021879638916750253, + "loss": 1.254, + "theoretical_loss": 3.3880143587073834, + "tokens_seen": 2585329664 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002187763289869609, + "loss": 1.423, + "theoretical_loss": 3.3880076113264495, + "tokens_seen": 2585395200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021875626880641928, + "loss": 1.3074, + "theoretical_loss": 3.3880008641644386, + "tokens_seen": 2585460736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021873620862587764, + "loss": 1.4021, + "theoretical_loss": 3.3879941172213375, + "tokens_seen": 2585526272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021871614844533603, + "loss": 1.2713, + "theoretical_loss": 3.3879873704971337, + "tokens_seen": 2585591808 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002186960882647944, + "loss": 1.2147, + "theoretical_loss": 3.3879806239918153, + "tokens_seen": 2585657344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021867602808425278, + "loss": 1.3216, + "theoretical_loss": 3.3879738777053694, + "tokens_seen": 2585722880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021865596790371115, + "loss": 1.3081, + "theoretical_loss": 3.3879671316377826, + "tokens_seen": 2585788416 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002186359077231695, + "loss": 1.1993, + "theoretical_loss": 3.387960385789043, + "tokens_seen": 2585853952 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002186158475426279, + "loss": 1.2671, + "theoretical_loss": 3.3879536401591377, + "tokens_seen": 2585919488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021859578736208626, + "loss": 1.224, + "theoretical_loss": 3.387946894748054, + "tokens_seen": 2585985024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021857572718154465, + "loss": 1.3833, + "theoretical_loss": 3.3879401495557797, + "tokens_seen": 2586050560 + }, + { + "epoch": 0.57, + "learning_rate": 0.000218555667001003, + "loss": 1.2145, + "theoretical_loss": 3.3879334045823017, + "tokens_seen": 2586116096 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002185356068204614, + "loss": 1.3177, + "theoretical_loss": 3.387926659827607, + "tokens_seen": 2586181632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021851554663991976, + "loss": 1.2118, + "theoretical_loss": 3.387919915291684, + "tokens_seen": 2586247168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021849548645937815, + "loss": 1.3505, + "theoretical_loss": 3.387913170974519, + "tokens_seen": 2586312704 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021847542627883651, + "loss": 1.2221, + "theoretical_loss": 3.3879064268761003, + "tokens_seen": 2586378240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021845536609829488, + "loss": 1.2083, + "theoretical_loss": 3.3878996829964145, + "tokens_seen": 2586443776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021843530591775327, + "loss": 1.2276, + "theoretical_loss": 3.3878929393354493, + "tokens_seen": 2586509312 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021841524573721163, + "loss": 1.3386, + "theoretical_loss": 3.3878861958931923, + "tokens_seen": 2586574848 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021839518555667002, + "loss": 1.2346, + "theoretical_loss": 3.3878794526696305, + "tokens_seen": 2586640384 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021837512537612838, + "loss": 1.401, + "theoretical_loss": 3.387872709664751, + "tokens_seen": 2586705920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021835506519558677, + "loss": 1.4459, + "theoretical_loss": 3.387865966878542, + "tokens_seen": 2586771456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021833500501504513, + "loss": 1.3222, + "theoretical_loss": 3.38785922431099, + "tokens_seen": 2586836992 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021831494483450352, + "loss": 1.2416, + "theoretical_loss": 3.387852481962083, + "tokens_seen": 2586902528 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021829488465396188, + "loss": 1.3538, + "theoretical_loss": 3.3878457398318083, + "tokens_seen": 2586968064 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021827482447342024, + "loss": 1.3868, + "theoretical_loss": 3.387838997920153, + "tokens_seen": 2587033600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021825476429287863, + "loss": 1.2339, + "theoretical_loss": 3.3878322562271044, + "tokens_seen": 2587099136 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 940829, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2509891986846924, + "objective/train/theoretical_loss": 3.38782551475265, + "objective/train/tokens_used": 957623776, + "theoretical_loss": 3.38782551475265, + "tokens_seen": 2587164672 + }, + { + "epoch": 0.57, + "learning_rate": 0.000218234704112337, + "loss": 1.3083, + "theoretical_loss": 3.38782551475265, + "tokens_seen": 2587164672 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021821464393179539, + "loss": 1.3246, + "theoretical_loss": 3.3878187734967775, + "tokens_seen": 2587230208 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021819458375125375, + "loss": 1.3046, + "theoretical_loss": 3.387812032459474, + "tokens_seen": 2587295744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021817452357071214, + "loss": 1.2092, + "theoretical_loss": 3.387805291640727, + "tokens_seen": 2587361280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002181544633901705, + "loss": 1.3138, + "theoretical_loss": 3.3877985510405235, + "tokens_seen": 2587426816 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002181344032096289, + "loss": 1.2732, + "theoretical_loss": 3.387791810658851, + "tokens_seen": 2587492352 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021811434302908725, + "loss": 1.239, + "theoretical_loss": 3.3877850704956978, + "tokens_seen": 2587557888 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021809428284854564, + "loss": 1.3111, + "theoretical_loss": 3.3877783305510496, + "tokens_seen": 2587623424 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021807422266800403, + "loss": 1.2669, + "theoretical_loss": 3.3877715908248955, + "tokens_seen": 2587688960 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002180541624874624, + "loss": 1.234, + "theoretical_loss": 3.387764851317222, + "tokens_seen": 2587754496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021803410230692078, + "loss": 1.3953, + "theoretical_loss": 3.387758112028016, + "tokens_seen": 2587820032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021801404212637914, + "loss": 1.2365, + "theoretical_loss": 3.3877513729572657, + "tokens_seen": 2587885568 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021799398194583753, + "loss": 1.2418, + "theoretical_loss": 3.3877446341049584, + "tokens_seen": 2587951104 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002179739217652959, + "loss": 1.3406, + "theoretical_loss": 3.387737895471081, + "tokens_seen": 2588016640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021795386158475428, + "loss": 1.2822, + "theoretical_loss": 3.3877311570556214, + "tokens_seen": 2588082176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021793380140421265, + "loss": 1.2864, + "theoretical_loss": 3.387724418858567, + "tokens_seen": 2588147712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021791374122367104, + "loss": 1.2476, + "theoretical_loss": 3.3877176808799043, + "tokens_seen": 2588213248 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002178936810431294, + "loss": 1.2869, + "theoretical_loss": 3.3877109431196217, + "tokens_seen": 2588278784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021787362086258776, + "loss": 1.3003, + "theoretical_loss": 3.3877042055777067, + "tokens_seen": 2588344320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021785356068204615, + "loss": 1.2826, + "theoretical_loss": 3.387697468254146, + "tokens_seen": 2588409856 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002178335005015045, + "loss": 1.3873, + "theoretical_loss": 3.3876907311489273, + "tokens_seen": 2588475392 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002178134403209629, + "loss": 1.2663, + "theoretical_loss": 3.387683994262038, + "tokens_seen": 2588540928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021779338014042126, + "loss": 1.3139, + "theoretical_loss": 3.3876772575934657, + "tokens_seen": 2588606464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021777331995987965, + "loss": 1.236, + "theoretical_loss": 3.387670521143197, + "tokens_seen": 2588672000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021775325977933802, + "loss": 1.1984, + "theoretical_loss": 3.3876637849112203, + "tokens_seen": 2588737536 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002177331995987964, + "loss": 1.3467, + "theoretical_loss": 3.3876570488975224, + "tokens_seen": 2588803072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021771313941825477, + "loss": 1.4377, + "theoretical_loss": 3.3876503131020907, + "tokens_seen": 2588868608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021769307923771316, + "loss": 1.2978, + "theoretical_loss": 3.387643577524913, + "tokens_seen": 2588934144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021767301905717152, + "loss": 1.3231, + "theoretical_loss": 3.3876368421659766, + "tokens_seen": 2588999680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021765295887662988, + "loss": 1.3015, + "theoretical_loss": 3.3876301070252683, + "tokens_seen": 2589065216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021763289869608827, + "loss": 1.3072, + "theoretical_loss": 3.3876233721027766, + "tokens_seen": 2589130752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021761283851554663, + "loss": 1.31, + "theoretical_loss": 3.3876166373984877, + "tokens_seen": 2589196288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021759277833500502, + "loss": 1.2507, + "theoretical_loss": 3.3876099029123896, + "tokens_seen": 2589261824 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021757271815446338, + "loss": 1.4041, + "theoretical_loss": 3.3876031686444703, + "tokens_seen": 2589327360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021755265797392177, + "loss": 1.2979, + "theoretical_loss": 3.387596434594716, + "tokens_seen": 2589392896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021753259779338014, + "loss": 1.273, + "theoretical_loss": 3.3875897007631153, + "tokens_seen": 2589458432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021751253761283852, + "loss": 1.3271, + "theoretical_loss": 3.3875829671496547, + "tokens_seen": 2589523968 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002174924774322969, + "loss": 1.4244, + "theoretical_loss": 3.387576233754322, + "tokens_seen": 2589589504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021747241725175525, + "loss": 1.2764, + "theoretical_loss": 3.3875695005771043, + "tokens_seen": 2589655040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021745235707121364, + "loss": 1.3825, + "theoretical_loss": 3.3875627676179896, + "tokens_seen": 2589720576 + }, + { + "epoch": 0.57, + "learning_rate": 0.000217432296890672, + "loss": 1.3443, + "theoretical_loss": 3.387556034876965, + "tokens_seen": 2589786112 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002174122367101304, + "loss": 1.3369, + "theoretical_loss": 3.3875493023540173, + "tokens_seen": 2589851648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021739217652958875, + "loss": 1.2365, + "theoretical_loss": 3.3875425700491353, + "tokens_seen": 2589917184 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021737211634904714, + "loss": 1.2822, + "theoretical_loss": 3.3875358379623055, + "tokens_seen": 2589982720 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002173520561685055, + "loss": 1.3167, + "theoretical_loss": 3.387529106093515, + "tokens_seen": 2590048256 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021733199598796392, + "loss": 1.3383, + "theoretical_loss": 3.387522374442752, + "tokens_seen": 2590113792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021731193580742228, + "loss": 1.3321, + "theoretical_loss": 3.387515643010004, + "tokens_seen": 2590179328 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021729187562688065, + "loss": 1.2522, + "theoretical_loss": 3.3875089117952575, + "tokens_seen": 2590244864 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021727181544633903, + "loss": 1.3387, + "theoretical_loss": 3.3875021807985006, + "tokens_seen": 2590310400 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002172517552657974, + "loss": 1.2404, + "theoretical_loss": 3.3874954500197205, + "tokens_seen": 2590375936 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 942074, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3940274715423584, + "objective/train/theoretical_loss": 3.387488719458905, + "objective/train/tokens_used": 960900576, + "theoretical_loss": 3.387488719458905, + "tokens_seen": 2590441472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021723169508525579, + "loss": 1.3043, + "theoretical_loss": 3.387488719458905, + "tokens_seen": 2590441472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021721163490471415, + "loss": 1.1847, + "theoretical_loss": 3.387481989116041, + "tokens_seen": 2590507008 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021719157472417254, + "loss": 1.3461, + "theoretical_loss": 3.387475258991116, + "tokens_seen": 2590572544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002171715145436309, + "loss": 1.2616, + "theoretical_loss": 3.387468529084118, + "tokens_seen": 2590638080 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002171514543630893, + "loss": 1.2405, + "theoretical_loss": 3.387461799395034, + "tokens_seen": 2590703616 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021713139418254765, + "loss": 1.2641, + "theoretical_loss": 3.3874550699238513, + "tokens_seen": 2590769152 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021711133400200604, + "loss": 1.2447, + "theoretical_loss": 3.3874483406705576, + "tokens_seen": 2590834688 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002170912738214644, + "loss": 1.27, + "theoretical_loss": 3.38744161163514, + "tokens_seen": 2590900224 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021707121364092277, + "loss": 1.3115, + "theoretical_loss": 3.387434882817586, + "tokens_seen": 2590965760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021705115346038115, + "loss": 1.1958, + "theoretical_loss": 3.3874281542178837, + "tokens_seen": 2591031296 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021703109327983952, + "loss": 1.3434, + "theoretical_loss": 3.3874214258360196, + "tokens_seen": 2591096832 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002170110330992979, + "loss": 1.2896, + "theoretical_loss": 3.387414697671982, + "tokens_seen": 2591162368 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021699097291875627, + "loss": 1.2482, + "theoretical_loss": 3.3874079697257575, + "tokens_seen": 2591227904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021697091273821466, + "loss": 1.2597, + "theoretical_loss": 3.3874012419973343, + "tokens_seen": 2591293440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021695085255767302, + "loss": 1.2256, + "theoretical_loss": 3.387394514486699, + "tokens_seen": 2591358976 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002169307923771314, + "loss": 1.4062, + "theoretical_loss": 3.3873877871938403, + "tokens_seen": 2591424512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021691073219658977, + "loss": 1.2218, + "theoretical_loss": 3.387381060118744, + "tokens_seen": 2591490048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021689067201604813, + "loss": 1.3944, + "theoretical_loss": 3.3873743332613993, + "tokens_seen": 2591555584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021687061183550652, + "loss": 1.3162, + "theoretical_loss": 3.387367606621792, + "tokens_seen": 2591621120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021685055165496489, + "loss": 1.2974, + "theoretical_loss": 3.387360880199911, + "tokens_seen": 2591686656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021683049147442327, + "loss": 1.2311, + "theoretical_loss": 3.3873541539957426, + "tokens_seen": 2591752192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021681043129388164, + "loss": 1.2581, + "theoretical_loss": 3.387347428009275, + "tokens_seen": 2591817728 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021679037111334003, + "loss": 1.2783, + "theoretical_loss": 3.387340702240495, + "tokens_seen": 2591883264 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002167703109327984, + "loss": 1.3354, + "theoretical_loss": 3.3873339766893906, + "tokens_seen": 2591948800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021675025075225678, + "loss": 1.3031, + "theoretical_loss": 3.387327251355949, + "tokens_seen": 2592014336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021673019057171514, + "loss": 1.2351, + "theoretical_loss": 3.387320526240158, + "tokens_seen": 2592079872 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002167101303911735, + "loss": 1.3426, + "theoretical_loss": 3.387313801342004, + "tokens_seen": 2592145408 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002166900702106319, + "loss": 1.3081, + "theoretical_loss": 3.387307076661476, + "tokens_seen": 2592210944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021667001003009025, + "loss": 1.3367, + "theoretical_loss": 3.3873003521985603, + "tokens_seen": 2592276480 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021664994984954864, + "loss": 1.3211, + "theoretical_loss": 3.387293627953245, + "tokens_seen": 2592342016 + }, + { + "epoch": 0.57, + "learning_rate": 0.000216629889669007, + "loss": 1.282, + "theoretical_loss": 3.387286903925517, + "tokens_seen": 2592407552 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002166098294884654, + "loss": 1.2634, + "theoretical_loss": 3.3872801801153645, + "tokens_seen": 2592473088 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021658976930792378, + "loss": 1.1624, + "theoretical_loss": 3.387273456522774, + "tokens_seen": 2592538624 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021656970912738217, + "loss": 1.2401, + "theoretical_loss": 3.3872667331477335, + "tokens_seen": 2592604160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021654964894684054, + "loss": 1.3269, + "theoretical_loss": 3.387260009990231, + "tokens_seen": 2592669696 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021652958876629893, + "loss": 1.3005, + "theoretical_loss": 3.3872532870502527, + "tokens_seen": 2592735232 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002165095285857573, + "loss": 1.2359, + "theoretical_loss": 3.387246564327787, + "tokens_seen": 2592800768 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021648946840521565, + "loss": 1.3499, + "theoretical_loss": 3.3872398418228213, + "tokens_seen": 2592866304 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021646940822467404, + "loss": 1.219, + "theoretical_loss": 3.3872331195353427, + "tokens_seen": 2592931840 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002164493480441324, + "loss": 1.3621, + "theoretical_loss": 3.387226397465339, + "tokens_seen": 2592997376 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002164292878635908, + "loss": 1.3074, + "theoretical_loss": 3.3872196756127972, + "tokens_seen": 2593062912 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021640922768304915, + "loss": 1.3631, + "theoretical_loss": 3.3872129539777056, + "tokens_seen": 2593128448 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021638916750250754, + "loss": 1.2258, + "theoretical_loss": 3.387206232560051, + "tokens_seen": 2593193984 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002163691073219659, + "loss": 1.2506, + "theoretical_loss": 3.387199511359821, + "tokens_seen": 2593259520 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002163490471414243, + "loss": 1.309, + "theoretical_loss": 3.3871927903770027, + "tokens_seen": 2593325056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021632898696088266, + "loss": 1.2013, + "theoretical_loss": 3.3871860696115847, + "tokens_seen": 2593390592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021630892678034102, + "loss": 1.2941, + "theoretical_loss": 3.3871793490635533, + "tokens_seen": 2593456128 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002162888665997994, + "loss": 1.2226, + "theoretical_loss": 3.3871726287328965, + "tokens_seen": 2593521664 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021626880641925777, + "loss": 1.3222, + "theoretical_loss": 3.3871659086196018, + "tokens_seen": 2593587200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021624874623871616, + "loss": 1.2064, + "theoretical_loss": 3.3871591887236567, + "tokens_seen": 2593652736 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 943233, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2437530755996704, + "objective/train/theoretical_loss": 3.387152469045048, + "objective/train/tokens_used": 964177376, + "theoretical_loss": 3.387152469045048, + "tokens_seen": 2593718272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021622868605817452, + "loss": 1.271, + "theoretical_loss": 3.387152469045048, + "tokens_seen": 2593718272 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002162086258776329, + "loss": 1.2822, + "theoretical_loss": 3.3871457495837642, + "tokens_seen": 2593783808 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021618856569709127, + "loss": 1.3069, + "theoretical_loss": 3.3871390303397924, + "tokens_seen": 2593849344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021616850551654966, + "loss": 1.211, + "theoretical_loss": 3.38713231131312, + "tokens_seen": 2593914880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021614844533600802, + "loss": 1.3634, + "theoretical_loss": 3.3871255925037342, + "tokens_seen": 2593980416 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021612838515546641, + "loss": 1.349, + "theoretical_loss": 3.387118873911623, + "tokens_seen": 2594045952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021610832497492478, + "loss": 1.3906, + "theoretical_loss": 3.3871121555367734, + "tokens_seen": 2594111488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021608826479438314, + "loss": 1.2689, + "theoretical_loss": 3.3871054373791734, + "tokens_seen": 2594177024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021606820461384153, + "loss": 1.2012, + "theoretical_loss": 3.3870987194388102, + "tokens_seen": 2594242560 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002160481444332999, + "loss": 1.2212, + "theoretical_loss": 3.387092001715671, + "tokens_seen": 2594308096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021602808425275828, + "loss": 1.4563, + "theoretical_loss": 3.387085284209744, + "tokens_seen": 2594373632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021600802407221664, + "loss": 1.3245, + "theoretical_loss": 3.3870785669210166, + "tokens_seen": 2594439168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021598796389167503, + "loss": 1.3423, + "theoretical_loss": 3.3870718498494754, + "tokens_seen": 2594504704 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002159679037111334, + "loss": 1.3317, + "theoretical_loss": 3.387065132995109, + "tokens_seen": 2594570240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021594784353059178, + "loss": 1.2311, + "theoretical_loss": 3.3870584163579034, + "tokens_seen": 2594635776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021592778335005014, + "loss": 1.2042, + "theoretical_loss": 3.3870516999378477, + "tokens_seen": 2594701312 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002159077231695085, + "loss": 1.1755, + "theoretical_loss": 3.387044983734929, + "tokens_seen": 2594766848 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002158876629889669, + "loss": 1.204, + "theoretical_loss": 3.3870382677491344, + "tokens_seen": 2594832384 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021586760280842526, + "loss": 1.2663, + "theoretical_loss": 3.3870315519804515, + "tokens_seen": 2594897920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021584754262788365, + "loss": 1.2467, + "theoretical_loss": 3.3870248364288678, + "tokens_seen": 2594963456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021582748244734204, + "loss": 1.2131, + "theoretical_loss": 3.387018121094371, + "tokens_seen": 2595028992 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021580742226680043, + "loss": 1.4148, + "theoretical_loss": 3.3870114059769483, + "tokens_seen": 2595094528 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002157873620862588, + "loss": 1.2572, + "theoretical_loss": 3.387004691076587, + "tokens_seen": 2595160064 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021576730190571718, + "loss": 1.2429, + "theoretical_loss": 3.3869979763932756, + "tokens_seen": 2595225600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021574724172517554, + "loss": 1.3457, + "theoretical_loss": 3.3869912619270006, + "tokens_seen": 2595291136 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002157271815446339, + "loss": 1.4355, + "theoretical_loss": 3.3869845476777503, + "tokens_seen": 2595356672 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002157071213640923, + "loss": 1.296, + "theoretical_loss": 3.386977833645511, + "tokens_seen": 2595422208 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021568706118355065, + "loss": 1.2851, + "theoretical_loss": 3.386971119830272, + "tokens_seen": 2595487744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021566700100300904, + "loss": 1.3013, + "theoretical_loss": 3.386964406232019, + "tokens_seen": 2595553280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002156469408224674, + "loss": 1.1833, + "theoretical_loss": 3.38695769285074, + "tokens_seen": 2595618816 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002156268806419258, + "loss": 1.2655, + "theoretical_loss": 3.3869509796864237, + "tokens_seen": 2595684352 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021560682046138416, + "loss": 1.1923, + "theoretical_loss": 3.3869442667390564, + "tokens_seen": 2595749888 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021558676028084255, + "loss": 1.3299, + "theoretical_loss": 3.3869375540086257, + "tokens_seen": 2595815424 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002155667001003009, + "loss": 1.3791, + "theoretical_loss": 3.3869308414951194, + "tokens_seen": 2595880960 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002155466399197593, + "loss": 1.2975, + "theoretical_loss": 3.3869241291985253, + "tokens_seen": 2595946496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021552657973921766, + "loss": 1.4468, + "theoretical_loss": 3.38691741711883, + "tokens_seen": 2596012032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021550651955867602, + "loss": 1.3449, + "theoretical_loss": 3.386910705256022, + "tokens_seen": 2596077568 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002154864593781344, + "loss": 1.2333, + "theoretical_loss": 3.386903993610088, + "tokens_seen": 2596143104 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021546639919759277, + "loss": 1.2764, + "theoretical_loss": 3.3868972821810166, + "tokens_seen": 2596208640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021544633901705116, + "loss": 1.3744, + "theoretical_loss": 3.386890570968794, + "tokens_seen": 2596274176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021542627883650953, + "loss": 1.2479, + "theoretical_loss": 3.3868838599734086, + "tokens_seen": 2596339712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021540621865596792, + "loss": 1.1589, + "theoretical_loss": 3.3868771491948477, + "tokens_seen": 2596405248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021538615847542628, + "loss": 1.2463, + "theoretical_loss": 3.386870438633099, + "tokens_seen": 2596470784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021536609829488467, + "loss": 1.2987, + "theoretical_loss": 3.3868637282881493, + "tokens_seen": 2596536320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021534603811434303, + "loss": 1.2384, + "theoretical_loss": 3.3868570181599873, + "tokens_seen": 2596601856 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002153259779338014, + "loss": 1.4199, + "theoretical_loss": 3.3868503082485995, + "tokens_seen": 2596667392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021530591775325978, + "loss": 1.3321, + "theoretical_loss": 3.3868435985539738, + "tokens_seen": 2596732928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021528585757271814, + "loss": 1.3804, + "theoretical_loss": 3.386836889076098, + "tokens_seen": 2596798464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021526579739217653, + "loss": 1.3499, + "theoretical_loss": 3.3868301798149587, + "tokens_seen": 2596864000 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002152457372116349, + "loss": 1.3372, + "theoretical_loss": 3.3868234707705445, + "tokens_seen": 2596929536 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 944531, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2842936515808105, + "objective/train/theoretical_loss": 3.3868167619428426, + "objective/train/tokens_used": 967454176, + "theoretical_loss": 3.3868167619428426, + "tokens_seen": 2596995072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021522567703109328, + "loss": 1.3485, + "theoretical_loss": 3.3868167619428426, + "tokens_seen": 2596995072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021520561685055165, + "loss": 1.2758, + "theoretical_loss": 3.38681005333184, + "tokens_seen": 2597060608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021518555667001004, + "loss": 1.2756, + "theoretical_loss": 3.386803344937525, + "tokens_seen": 2597126144 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002151654964894684, + "loss": 1.3194, + "theoretical_loss": 3.386796636759885, + "tokens_seen": 2597191680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021514543630892676, + "loss": 1.3259, + "theoretical_loss": 3.3867899287989074, + "tokens_seen": 2597257216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021512537612838515, + "loss": 1.2657, + "theoretical_loss": 3.386783221054579, + "tokens_seen": 2597322752 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002151053159478435, + "loss": 1.2777, + "theoretical_loss": 3.386776513526889, + "tokens_seen": 2597388288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021508525576730193, + "loss": 1.2804, + "theoretical_loss": 3.386769806215823, + "tokens_seen": 2597453824 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002150651955867603, + "loss": 1.5169, + "theoretical_loss": 3.38676309912137, + "tokens_seen": 2597519360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021504513540621868, + "loss": 1.3961, + "theoretical_loss": 3.386756392243517, + "tokens_seen": 2597584896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021502507522567704, + "loss": 1.3238, + "theoretical_loss": 3.3867496855822514, + "tokens_seen": 2597650432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021500501504513543, + "loss": 1.4555, + "theoretical_loss": 3.386742979137561, + "tokens_seen": 2597715968 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002149849548645938, + "loss": 1.2589, + "theoretical_loss": 3.3867362729094332, + "tokens_seen": 2597781504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021496489468405218, + "loss": 1.3459, + "theoretical_loss": 3.3867295668978556, + "tokens_seen": 2597847040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021494483450351055, + "loss": 1.2854, + "theoretical_loss": 3.386722861102816, + "tokens_seen": 2597912576 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002149247743229689, + "loss": 1.3886, + "theoretical_loss": 3.3867161555243017, + "tokens_seen": 2597978112 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002149047141424273, + "loss": 1.3515, + "theoretical_loss": 3.3867094501623, + "tokens_seen": 2598043648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021488465396188566, + "loss": 1.3191, + "theoretical_loss": 3.3867027450167986, + "tokens_seen": 2598109184 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021486459378134405, + "loss": 1.3681, + "theoretical_loss": 3.3866960400877852, + "tokens_seen": 2598174720 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002148445336008024, + "loss": 1.329, + "theoretical_loss": 3.3866893353752476, + "tokens_seen": 2598240256 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002148244734202608, + "loss": 1.4251, + "theoretical_loss": 3.3866826308791724, + "tokens_seen": 2598305792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021480441323971916, + "loss": 1.2626, + "theoretical_loss": 3.3866759265995485, + "tokens_seen": 2598371328 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021478435305917755, + "loss": 1.2658, + "theoretical_loss": 3.386669222536362, + "tokens_seen": 2598436864 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021476429287863591, + "loss": 1.4198, + "theoretical_loss": 3.3866625186896018, + "tokens_seen": 2598502400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021474423269809428, + "loss": 1.3032, + "theoretical_loss": 3.386655815059255, + "tokens_seen": 2598567936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021472417251755267, + "loss": 1.326, + "theoretical_loss": 3.3866491116453084, + "tokens_seen": 2598633472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021470411233701103, + "loss": 1.2853, + "theoretical_loss": 3.3866424084477504, + "tokens_seen": 2598699008 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021468405215646942, + "loss": 1.2904, + "theoretical_loss": 3.386635705466569, + "tokens_seen": 2598764544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021466399197592778, + "loss": 1.244, + "theoretical_loss": 3.38662900270175, + "tokens_seen": 2598830080 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021464393179538617, + "loss": 1.2915, + "theoretical_loss": 3.3866223001532827, + "tokens_seen": 2598895616 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021462387161484453, + "loss": 1.2506, + "theoretical_loss": 3.3866155978211534, + "tokens_seen": 2598961152 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021460381143430292, + "loss": 1.2992, + "theoretical_loss": 3.3866088957053506, + "tokens_seen": 2599026688 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021458375125376128, + "loss": 1.326, + "theoretical_loss": 3.386602193805862, + "tokens_seen": 2599092224 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021456369107321964, + "loss": 1.2076, + "theoretical_loss": 3.386595492122674, + "tokens_seen": 2599157760 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021454363089267803, + "loss": 1.2627, + "theoretical_loss": 3.3865887906557752, + "tokens_seen": 2599223296 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002145235707121364, + "loss": 1.3047, + "theoretical_loss": 3.386582089405153, + "tokens_seen": 2599288832 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021450351053159479, + "loss": 1.2666, + "theoretical_loss": 3.3865753883707943, + "tokens_seen": 2599354368 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021448345035105315, + "loss": 1.247, + "theoretical_loss": 3.3865686875526873, + "tokens_seen": 2599419904 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021446339017051154, + "loss": 1.3109, + "theoretical_loss": 3.3865619869508197, + "tokens_seen": 2599485440 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002144433299899699, + "loss": 1.2982, + "theoretical_loss": 3.3865552865651782, + "tokens_seen": 2599550976 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002144232698094283, + "loss": 1.225, + "theoretical_loss": 3.386548586395752, + "tokens_seen": 2599616512 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021440320962888665, + "loss": 1.3705, + "theoretical_loss": 3.3865418864425267, + "tokens_seen": 2599682048 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021438314944834504, + "loss": 1.2968, + "theoretical_loss": 3.3865351867054914, + "tokens_seen": 2599747584 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002143630892678034, + "loss": 1.2971, + "theoretical_loss": 3.3865284871846324, + "tokens_seen": 2599813120 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021434302908726176, + "loss": 1.2057, + "theoretical_loss": 3.3865217878799387, + "tokens_seen": 2599878656 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021432296890672018, + "loss": 1.3999, + "theoretical_loss": 3.3865150887913966, + "tokens_seen": 2599944192 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021430290872617854, + "loss": 1.2995, + "theoretical_loss": 3.386508389918995, + "tokens_seen": 2600009728 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021428284854563693, + "loss": 1.3812, + "theoretical_loss": 3.38650169126272, + "tokens_seen": 2600075264 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002142627883650953, + "loss": 1.348, + "theoretical_loss": 3.38649499282256, + "tokens_seen": 2600140800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021424272818455368, + "loss": 1.2649, + "theoretical_loss": 3.3864882945985024, + "tokens_seen": 2600206336 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 945762, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.239673137664795, + "objective/train/theoretical_loss": 3.386481596590535, + "objective/train/tokens_used": 970730976, + "theoretical_loss": 3.386481596590535, + "tokens_seen": 2600271872 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021422266800401205, + "loss": 1.2688, + "theoretical_loss": 3.386481596590535, + "tokens_seen": 2600271872 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021420260782347044, + "loss": 1.3786, + "theoretical_loss": 3.386474898798645, + "tokens_seen": 2600337408 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002141825476429288, + "loss": 1.2528, + "theoretical_loss": 3.3864682012228204, + "tokens_seen": 2600402944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021416248746238716, + "loss": 1.2738, + "theoretical_loss": 3.386461503863049, + "tokens_seen": 2600468480 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021414242728184555, + "loss": 1.4186, + "theoretical_loss": 3.386454806719317, + "tokens_seen": 2600534016 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002141223671013039, + "loss": 1.3249, + "theoretical_loss": 3.3864481097916137, + "tokens_seen": 2600599552 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002141023069207623, + "loss": 1.4794, + "theoretical_loss": 3.386441413079926, + "tokens_seen": 2600665088 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021408224674022066, + "loss": 1.3045, + "theoretical_loss": 3.3864347165842412, + "tokens_seen": 2600730624 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021406218655967905, + "loss": 1.2861, + "theoretical_loss": 3.386428020304547, + "tokens_seen": 2600796160 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021404212637913742, + "loss": 1.2858, + "theoretical_loss": 3.386421324240831, + "tokens_seen": 2600861696 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002140220661985958, + "loss": 1.2522, + "theoretical_loss": 3.3864146283930814, + "tokens_seen": 2600927232 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021400200601805417, + "loss": 1.3613, + "theoretical_loss": 3.3864079327612853, + "tokens_seen": 2600992768 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021398194583751256, + "loss": 1.315, + "theoretical_loss": 3.38640123734543, + "tokens_seen": 2601058304 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021396188565697092, + "loss": 1.3365, + "theoretical_loss": 3.3863945421455037, + "tokens_seen": 2601123840 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021394182547642928, + "loss": 1.2409, + "theoretical_loss": 3.3863878471614934, + "tokens_seen": 2601189376 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021392176529588767, + "loss": 1.4198, + "theoretical_loss": 3.386381152393387, + "tokens_seen": 2601254912 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021390170511534603, + "loss": 1.3346, + "theoretical_loss": 3.386374457841172, + "tokens_seen": 2601320448 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021388164493480442, + "loss": 1.3286, + "theoretical_loss": 3.3863677635048366, + "tokens_seen": 2601385984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021386158475426278, + "loss": 1.3035, + "theoretical_loss": 3.3863610693843675, + "tokens_seen": 2601451520 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021384152457372117, + "loss": 1.2868, + "theoretical_loss": 3.3863543754797525, + "tokens_seen": 2601517056 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021382146439317954, + "loss": 1.178, + "theoretical_loss": 3.38634768179098, + "tokens_seen": 2601582592 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021380140421263792, + "loss": 1.4094, + "theoretical_loss": 3.3863409883180364, + "tokens_seen": 2601648128 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002137813440320963, + "loss": 1.367, + "theoretical_loss": 3.38633429506091, + "tokens_seen": 2601713664 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021376128385155465, + "loss": 1.248, + "theoretical_loss": 3.3863276020195885, + "tokens_seen": 2601779200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021374122367101304, + "loss": 1.3276, + "theoretical_loss": 3.386320909194059, + "tokens_seen": 2601844736 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002137211634904714, + "loss": 1.2629, + "theoretical_loss": 3.38631421658431, + "tokens_seen": 2601910272 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002137011033099298, + "loss": 1.3289, + "theoretical_loss": 3.386307524190328, + "tokens_seen": 2601975808 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021368104312938815, + "loss": 1.2458, + "theoretical_loss": 3.3863008320121013, + "tokens_seen": 2602041344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021366098294884654, + "loss": 1.3434, + "theoretical_loss": 3.3862941400496176, + "tokens_seen": 2602106880 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002136409227683049, + "loss": 1.281, + "theoretical_loss": 3.386287448302864, + "tokens_seen": 2602172416 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002136208625877633, + "loss": 1.3291, + "theoretical_loss": 3.3862807567718285, + "tokens_seen": 2602237952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021360080240722166, + "loss": 1.259, + "theoretical_loss": 3.3862740654564987, + "tokens_seen": 2602303488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021358074222668004, + "loss": 1.1932, + "theoretical_loss": 3.3862673743568616, + "tokens_seen": 2602369024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021356068204613843, + "loss": 1.2586, + "theoretical_loss": 3.386260683472906, + "tokens_seen": 2602434560 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002135406218655968, + "loss": 1.3246, + "theoretical_loss": 3.386253992804618, + "tokens_seen": 2602500096 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021352056168505519, + "loss": 1.4207, + "theoretical_loss": 3.3862473023519866, + "tokens_seen": 2602565632 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021350050150451355, + "loss": 1.2571, + "theoretical_loss": 3.3862406121149986, + "tokens_seen": 2602631168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021348044132397194, + "loss": 1.4282, + "theoretical_loss": 3.386233922093642, + "tokens_seen": 2602696704 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002134603811434303, + "loss": 1.2704, + "theoretical_loss": 3.386227232287905, + "tokens_seen": 2602762240 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002134403209628887, + "loss": 1.2008, + "theoretical_loss": 3.3862205426977736, + "tokens_seen": 2602827776 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021342026078234705, + "loss": 1.3949, + "theoretical_loss": 3.3862138533232367, + "tokens_seen": 2602893312 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021340020060180544, + "loss": 1.2713, + "theoretical_loss": 3.3862071641642815, + "tokens_seen": 2602958848 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002133801404212638, + "loss": 1.3066, + "theoretical_loss": 3.3862004752208956, + "tokens_seen": 2603024384 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021336008024072216, + "loss": 1.2632, + "theoretical_loss": 3.3861937864930667, + "tokens_seen": 2603089920 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021334002006018055, + "loss": 1.3674, + "theoretical_loss": 3.386187097980783, + "tokens_seen": 2603155456 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021331995987963892, + "loss": 1.4546, + "theoretical_loss": 3.386180409684031, + "tokens_seen": 2603220992 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002132998996990973, + "loss": 1.2932, + "theoretical_loss": 3.386173721602799, + "tokens_seen": 2603286528 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021327983951855567, + "loss": 1.3861, + "theoretical_loss": 3.386167033737075, + "tokens_seen": 2603352064 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021325977933801406, + "loss": 1.2028, + "theoretical_loss": 3.3861603460868457, + "tokens_seen": 2603417600 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021323971915747242, + "loss": 1.3224, + "theoretical_loss": 3.3861536586520993, + "tokens_seen": 2603483136 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 946919, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1864168643951416, + "objective/train/theoretical_loss": 3.386146971432823, + "objective/train/tokens_used": 974007776, + "theoretical_loss": 3.386146971432823, + "tokens_seen": 2603548672 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002132196589769308, + "loss": 1.3414, + "theoretical_loss": 3.386146971432823, + "tokens_seen": 2603548672 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021319959879638917, + "loss": 1.4402, + "theoretical_loss": 3.3861402844290054, + "tokens_seen": 2603614208 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021317953861584753, + "loss": 1.3805, + "theoretical_loss": 3.386133597640633, + "tokens_seen": 2603679744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021315947843530592, + "loss": 1.3066, + "theoretical_loss": 3.386126911067694, + "tokens_seen": 2603745280 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021313941825476429, + "loss": 1.4219, + "theoretical_loss": 3.3861202247101767, + "tokens_seen": 2603810816 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021311935807422267, + "loss": 1.2963, + "theoretical_loss": 3.3861135385680674, + "tokens_seen": 2603876352 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021309929789368104, + "loss": 1.1878, + "theoretical_loss": 3.3861068526413542, + "tokens_seen": 2603941888 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021307923771313943, + "loss": 1.4805, + "theoretical_loss": 3.3861001669300252, + "tokens_seen": 2604007424 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002130591775325978, + "loss": 1.5368, + "theoretical_loss": 3.3860934814340675, + "tokens_seen": 2604072960 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021303911735205618, + "loss": 1.2183, + "theoretical_loss": 3.3860867961534695, + "tokens_seen": 2604138496 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021301905717151454, + "loss": 1.3409, + "theoretical_loss": 3.386080111088218, + "tokens_seen": 2604204032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002129989969909729, + "loss": 1.3847, + "theoretical_loss": 3.3860734262383008, + "tokens_seen": 2604269568 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002129789368104313, + "loss": 1.2489, + "theoretical_loss": 3.386066741603706, + "tokens_seen": 2604335104 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021295887662988965, + "loss": 1.3172, + "theoretical_loss": 3.3860600571844204, + "tokens_seen": 2604400640 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021293881644934804, + "loss": 1.1577, + "theoretical_loss": 3.3860533729804327, + "tokens_seen": 2604466176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002129187562688064, + "loss": 1.3699, + "theoretical_loss": 3.38604668899173, + "tokens_seen": 2604531712 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002128986960882648, + "loss": 1.3584, + "theoretical_loss": 3.3860400052183, + "tokens_seen": 2604597248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021287863590772316, + "loss": 1.323, + "theoretical_loss": 3.3860333216601304, + "tokens_seen": 2604662784 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021285857572718155, + "loss": 1.3588, + "theoretical_loss": 3.3860266383172086, + "tokens_seen": 2604728320 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002128385155466399, + "loss": 1.3598, + "theoretical_loss": 3.3860199551895227, + "tokens_seen": 2604793856 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021281845536609832, + "loss": 1.216, + "theoretical_loss": 3.3860132722770597, + "tokens_seen": 2604859392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002127983951855567, + "loss": 1.2507, + "theoretical_loss": 3.386006589579808, + "tokens_seen": 2604924928 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021277833500501505, + "loss": 1.2601, + "theoretical_loss": 3.385999907097755, + "tokens_seen": 2604990464 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021275827482447344, + "loss": 1.3663, + "theoretical_loss": 3.385993224830888, + "tokens_seen": 2605056000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002127382146439318, + "loss": 1.4498, + "theoretical_loss": 3.3859865427791953, + "tokens_seen": 2605121536 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002127181544633902, + "loss": 1.184, + "theoretical_loss": 3.385979860942664, + "tokens_seen": 2605187072 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021269809428284855, + "loss": 1.38, + "theoretical_loss": 3.385973179321282, + "tokens_seen": 2605252608 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021267803410230694, + "loss": 1.3054, + "theoretical_loss": 3.3859664979150366, + "tokens_seen": 2605318144 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002126579739217653, + "loss": 1.2271, + "theoretical_loss": 3.385959816723916, + "tokens_seen": 2605383680 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002126379137412237, + "loss": 1.2764, + "theoretical_loss": 3.385953135747908, + "tokens_seen": 2605449216 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021261785356068206, + "loss": 1.3442, + "theoretical_loss": 3.3859464549869993, + "tokens_seen": 2605514752 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021259779338014042, + "loss": 1.3302, + "theoretical_loss": 3.3859397744411783, + "tokens_seen": 2605580288 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002125777331995988, + "loss": 1.3114, + "theoretical_loss": 3.385933094110433, + "tokens_seen": 2605645824 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021255767301905717, + "loss": 1.2694, + "theoretical_loss": 3.38592641399475, + "tokens_seen": 2605711360 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021253761283851556, + "loss": 1.2228, + "theoretical_loss": 3.385919734094118, + "tokens_seen": 2605776896 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021251755265797392, + "loss": 1.2205, + "theoretical_loss": 3.385913054408524, + "tokens_seen": 2605842432 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002124974924774323, + "loss": 1.234, + "theoretical_loss": 3.3859063749379557, + "tokens_seen": 2605907968 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021247743229689067, + "loss": 1.4278, + "theoretical_loss": 3.3858996956824012, + "tokens_seen": 2605973504 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021245737211634906, + "loss": 1.2267, + "theoretical_loss": 3.385893016641848, + "tokens_seen": 2606039040 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021243731193580742, + "loss": 1.3179, + "theoretical_loss": 3.3858863378162836, + "tokens_seen": 2606104576 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002124172517552658, + "loss": 1.3058, + "theoretical_loss": 3.385879659205696, + "tokens_seen": 2606170112 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021239719157472418, + "loss": 1.394, + "theoretical_loss": 3.3858729808100727, + "tokens_seen": 2606235648 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021237713139418254, + "loss": 1.361, + "theoretical_loss": 3.385866302629401, + "tokens_seen": 2606301184 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021235707121364093, + "loss": 1.3683, + "theoretical_loss": 3.385859624663669, + "tokens_seen": 2606366720 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002123370110330993, + "loss": 1.3794, + "theoretical_loss": 3.3858529469128644, + "tokens_seen": 2606432256 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021231695085255768, + "loss": 1.3559, + "theoretical_loss": 3.385846269376975, + "tokens_seen": 2606497792 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021229689067201604, + "loss": 1.3473, + "theoretical_loss": 3.385839592055988, + "tokens_seen": 2606563328 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021227683049147443, + "loss": 1.4184, + "theoretical_loss": 3.385832914949891, + "tokens_seen": 2606628864 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002122567703109328, + "loss": 1.2923, + "theoretical_loss": 3.3858262380586726, + "tokens_seen": 2606694400 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021223671013039118, + "loss": 1.3233, + "theoretical_loss": 3.38581956138232, + "tokens_seen": 2606759936 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 948007, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.389406442642212, + "objective/train/theoretical_loss": 3.38581288492082, + "objective/train/tokens_used": 977284576, + "theoretical_loss": 3.38581288492082, + "tokens_seen": 2606825472 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021221664994984954, + "loss": 1.3685, + "theoretical_loss": 3.38581288492082, + "tokens_seen": 2606825472 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002121965897693079, + "loss": 1.2768, + "theoretical_loss": 3.3858062086741616, + "tokens_seen": 2606891008 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002121765295887663, + "loss": 1.2539, + "theoretical_loss": 3.385799532642332, + "tokens_seen": 2606956544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021215646940822466, + "loss": 1.2671, + "theoretical_loss": 3.3857928568253186, + "tokens_seen": 2607022080 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021213640922768305, + "loss": 1.2328, + "theoretical_loss": 3.38578618122311, + "tokens_seen": 2607087616 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002121163490471414, + "loss": 1.3718, + "theoretical_loss": 3.3857795058356923, + "tokens_seen": 2607153152 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002120962888665998, + "loss": 1.3025, + "theoretical_loss": 3.3857728306630546, + "tokens_seen": 2607218688 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002120762286860582, + "loss": 1.2705, + "theoretical_loss": 3.385766155705184, + "tokens_seen": 2607284224 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021205616850551658, + "loss": 1.4094, + "theoretical_loss": 3.385759480962068, + "tokens_seen": 2607349760 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021203610832497494, + "loss": 1.2648, + "theoretical_loss": 3.385752806433695, + "tokens_seen": 2607415296 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002120160481444333, + "loss": 1.1654, + "theoretical_loss": 3.385746132120052, + "tokens_seen": 2607480832 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002119959879638917, + "loss": 1.3795, + "theoretical_loss": 3.3857394580211273, + "tokens_seen": 2607546368 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021197592778335005, + "loss": 1.3531, + "theoretical_loss": 3.3857327841369083, + "tokens_seen": 2607611904 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021195586760280844, + "loss": 1.1575, + "theoretical_loss": 3.3857261104673824, + "tokens_seen": 2607677440 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002119358074222668, + "loss": 1.3679, + "theoretical_loss": 3.3857194370125376, + "tokens_seen": 2607742976 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002119157472417252, + "loss": 1.4051, + "theoretical_loss": 3.3857127637723616, + "tokens_seen": 2607808512 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021189568706118356, + "loss": 1.3877, + "theoretical_loss": 3.385706090746842, + "tokens_seen": 2607874048 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021187562688064195, + "loss": 1.2889, + "theoretical_loss": 3.3856994179359665, + "tokens_seen": 2607939584 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002118555667001003, + "loss": 1.3065, + "theoretical_loss": 3.385692745339723, + "tokens_seen": 2608005120 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002118355065195587, + "loss": 1.2586, + "theoretical_loss": 3.3856860729580993, + "tokens_seen": 2608070656 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021181544633901706, + "loss": 1.2676, + "theoretical_loss": 3.3856794007910826, + "tokens_seen": 2608136192 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021179538615847542, + "loss": 1.4326, + "theoretical_loss": 3.385672728838661, + "tokens_seen": 2608201728 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002117753259779338, + "loss": 1.2722, + "theoretical_loss": 3.385666057100822, + "tokens_seen": 2608267264 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021175526579739217, + "loss": 1.2817, + "theoretical_loss": 3.3856593855775534, + "tokens_seen": 2608332800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021173520561685056, + "loss": 1.2932, + "theoretical_loss": 3.3856527142688426, + "tokens_seen": 2608398336 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021171514543630893, + "loss": 1.2759, + "theoretical_loss": 3.385646043174678, + "tokens_seen": 2608463872 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021169508525576732, + "loss": 1.3388, + "theoretical_loss": 3.3856393722950466, + "tokens_seen": 2608529408 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021167502507522568, + "loss": 1.2351, + "theoretical_loss": 3.385632701629937, + "tokens_seen": 2608594944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021165496489468407, + "loss": 1.2411, + "theoretical_loss": 3.385626031179336, + "tokens_seen": 2608660480 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021163490471414243, + "loss": 1.277, + "theoretical_loss": 3.3856193609432315, + "tokens_seen": 2608726016 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002116148445336008, + "loss": 1.339, + "theoretical_loss": 3.3856126909216115, + "tokens_seen": 2608791552 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021159478435305918, + "loss": 1.2675, + "theoretical_loss": 3.3856060211144636, + "tokens_seen": 2608857088 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021157472417251754, + "loss": 1.3877, + "theoretical_loss": 3.3855993515217757, + "tokens_seen": 2608922624 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021155466399197593, + "loss": 1.2376, + "theoretical_loss": 3.385592682143535, + "tokens_seen": 2608988160 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002115346038114343, + "loss": 1.2086, + "theoretical_loss": 3.3855860129797297, + "tokens_seen": 2609053696 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021151454363089268, + "loss": 1.2228, + "theoretical_loss": 3.385579344030347, + "tokens_seen": 2609119232 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021149448345035105, + "loss": 1.3658, + "theoretical_loss": 3.385572675295376, + "tokens_seen": 2609184768 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021147442326980944, + "loss": 1.3255, + "theoretical_loss": 3.385566006774803, + "tokens_seen": 2609250304 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002114543630892678, + "loss": 1.2569, + "theoretical_loss": 3.3855593384686156, + "tokens_seen": 2609315840 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021143430290872616, + "loss": 1.3271, + "theoretical_loss": 3.385552670376802, + "tokens_seen": 2609381376 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021141424272818455, + "loss": 1.276, + "theoretical_loss": 3.3855460024993502, + "tokens_seen": 2609446912 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002113941825476429, + "loss": 1.3329, + "theoretical_loss": 3.385539334836248, + "tokens_seen": 2609512448 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002113741223671013, + "loss": 1.3814, + "theoretical_loss": 3.3855326673874826, + "tokens_seen": 2609577984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021135406218655966, + "loss": 1.1943, + "theoretical_loss": 3.385526000153042, + "tokens_seen": 2609643520 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021133400200601808, + "loss": 1.3316, + "theoretical_loss": 3.3855193331329136, + "tokens_seen": 2609709056 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021131394182547644, + "loss": 1.307, + "theoretical_loss": 3.385512666327086, + "tokens_seen": 2609774592 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021129388164493483, + "loss": 1.3388, + "theoretical_loss": 3.3855059997355457, + "tokens_seen": 2609840128 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002112738214643932, + "loss": 1.3299, + "theoretical_loss": 3.3854993333582817, + "tokens_seen": 2609905664 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021125376128385158, + "loss": 1.3542, + "theoretical_loss": 3.3854926671952805, + "tokens_seen": 2609971200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021123370110330994, + "loss": 1.4304, + "theoretical_loss": 3.3854860012465307, + "tokens_seen": 2610036736 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 949200, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3941702842712402, + "objective/train/theoretical_loss": 3.3854793355120196, + "objective/train/tokens_used": 980561376, + "theoretical_loss": 3.3854793355120196, + "tokens_seen": 2610102272 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002112136409227683, + "loss": 1.3241, + "theoretical_loss": 3.3854793355120196, + "tokens_seen": 2610102272 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002111935807422267, + "loss": 1.2624, + "theoretical_loss": 3.3854726699917355, + "tokens_seen": 2610167808 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021117352056168506, + "loss": 1.3665, + "theoretical_loss": 3.3854660046856653, + "tokens_seen": 2610233344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021115346038114345, + "loss": 1.1832, + "theoretical_loss": 3.3854593395937975, + "tokens_seen": 2610298880 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002111334002006018, + "loss": 1.3294, + "theoretical_loss": 3.3854526747161193, + "tokens_seen": 2610364416 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002111133400200602, + "loss": 1.2467, + "theoretical_loss": 3.3854460100526187, + "tokens_seen": 2610429952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021109327983951856, + "loss": 1.3404, + "theoretical_loss": 3.3854393456032836, + "tokens_seen": 2610495488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021107321965897695, + "loss": 1.255, + "theoretical_loss": 3.3854326813681013, + "tokens_seen": 2610561024 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002110531594784353, + "loss": 1.2404, + "theoretical_loss": 3.38542601734706, + "tokens_seen": 2610626560 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021103309929789368, + "loss": 1.3178, + "theoretical_loss": 3.385419353540147, + "tokens_seen": 2610692096 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021101303911735207, + "loss": 1.3363, + "theoretical_loss": 3.3854126899473505, + "tokens_seen": 2610757632 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021099297893681043, + "loss": 1.4226, + "theoretical_loss": 3.385406026568658, + "tokens_seen": 2610823168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021097291875626882, + "loss": 1.1809, + "theoretical_loss": 3.385399363404057, + "tokens_seen": 2610888704 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021095285857572718, + "loss": 1.1952, + "theoretical_loss": 3.3853927004535356, + "tokens_seen": 2610954240 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021093279839518557, + "loss": 1.3801, + "theoretical_loss": 3.3853860377170815, + "tokens_seen": 2611019776 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021091273821464393, + "loss": 1.2187, + "theoretical_loss": 3.385379375194683, + "tokens_seen": 2611085312 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021089267803410232, + "loss": 1.2584, + "theoretical_loss": 3.3853727128863262, + "tokens_seen": 2611150848 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021087261785356068, + "loss": 1.4076, + "theoretical_loss": 3.3853660507920007, + "tokens_seen": 2611216384 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021085255767301904, + "loss": 1.2887, + "theoretical_loss": 3.3853593889116933, + "tokens_seen": 2611281920 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021083249749247743, + "loss": 1.3198, + "theoretical_loss": 3.3853527272453916, + "tokens_seen": 2611347456 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002108124373119358, + "loss": 1.3302, + "theoretical_loss": 3.385346065793084, + "tokens_seen": 2611412992 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021079237713139419, + "loss": 1.2385, + "theoretical_loss": 3.385339404554758, + "tokens_seen": 2611478528 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021077231695085255, + "loss": 1.2978, + "theoretical_loss": 3.385332743530401, + "tokens_seen": 2611544064 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021075225677031094, + "loss": 1.2065, + "theoretical_loss": 3.385326082720001, + "tokens_seen": 2611609600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002107321965897693, + "loss": 1.2002, + "theoretical_loss": 3.385319422123546, + "tokens_seen": 2611675136 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002107121364092277, + "loss": 1.3562, + "theoretical_loss": 3.3853127617410235, + "tokens_seen": 2611740672 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021069207622868605, + "loss": 1.3017, + "theoretical_loss": 3.385306101572421, + "tokens_seen": 2611806208 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021067201604814444, + "loss": 1.3705, + "theoretical_loss": 3.385299441617727, + "tokens_seen": 2611871744 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002106519558676028, + "loss": 1.2761, + "theoretical_loss": 3.385292781876929, + "tokens_seen": 2611937280 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021063189568706116, + "loss": 1.3827, + "theoretical_loss": 3.3852861223500144, + "tokens_seen": 2612002816 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021061183550651955, + "loss": 1.2891, + "theoretical_loss": 3.385279463036971, + "tokens_seen": 2612068352 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021059177532597792, + "loss": 1.3492, + "theoretical_loss": 3.385272803937787, + "tokens_seen": 2612133888 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021057171514543633, + "loss": 1.2769, + "theoretical_loss": 3.3852661450524497, + "tokens_seen": 2612199424 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002105516549648947, + "loss": 1.2566, + "theoretical_loss": 3.3852594863809475, + "tokens_seen": 2612264960 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021053159478435308, + "loss": 1.2464, + "theoretical_loss": 3.385252827923267, + "tokens_seen": 2612330496 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021051153460381145, + "loss": 1.1797, + "theoretical_loss": 3.385246169679397, + "tokens_seen": 2612396032 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021049147442326984, + "loss": 1.2945, + "theoretical_loss": 3.3852395116493255, + "tokens_seen": 2612461568 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002104714142427282, + "loss": 1.273, + "theoretical_loss": 3.385232853833039, + "tokens_seen": 2612527104 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021045135406218656, + "loss": 1.3013, + "theoretical_loss": 3.385226196230527, + "tokens_seen": 2612592640 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021043129388164495, + "loss": 1.3807, + "theoretical_loss": 3.385219538841776, + "tokens_seen": 2612658176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002104112337011033, + "loss": 1.3754, + "theoretical_loss": 3.3852128816667735, + "tokens_seen": 2612723712 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002103911735205617, + "loss": 1.2452, + "theoretical_loss": 3.385206224705508, + "tokens_seen": 2612789248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021037111334002006, + "loss": 1.3381, + "theoretical_loss": 3.3851995679579674, + "tokens_seen": 2612854784 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021035105315947845, + "loss": 1.3398, + "theoretical_loss": 3.385192911424139, + "tokens_seen": 2612920320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021033099297893681, + "loss": 1.3751, + "theoretical_loss": 3.385186255104011, + "tokens_seen": 2612985856 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002103109327983952, + "loss": 1.3993, + "theoretical_loss": 3.3851795989975706, + "tokens_seen": 2613051392 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021029087261785357, + "loss": 1.3228, + "theoretical_loss": 3.3851729431048065, + "tokens_seen": 2613116928 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021027081243731196, + "loss": 1.2678, + "theoretical_loss": 3.3851662874257054, + "tokens_seen": 2613182464 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021025075225677032, + "loss": 1.2178, + "theoretical_loss": 3.385159631960256, + "tokens_seen": 2613248000 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021023069207622868, + "loss": 1.2701, + "theoretical_loss": 3.3851529767084454, + "tokens_seen": 2613313536 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 950332, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1345751285552979, + "objective/train/theoretical_loss": 3.3851463216702617, + "objective/train/tokens_used": 983838176, + "theoretical_loss": 3.3851463216702617, + "tokens_seen": 2613379072 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021021063189568707, + "loss": 1.2405, + "theoretical_loss": 3.3851463216702617, + "tokens_seen": 2613379072 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021019057171514543, + "loss": 1.2675, + "theoretical_loss": 3.385139666845693, + "tokens_seen": 2613444608 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021017051153460382, + "loss": 1.3132, + "theoretical_loss": 3.3851330122347267, + "tokens_seen": 2613510144 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021015045135406218, + "loss": 1.2989, + "theoretical_loss": 3.3851263578373505, + "tokens_seen": 2613575680 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021013039117352057, + "loss": 1.352, + "theoretical_loss": 3.3851197036535523, + "tokens_seen": 2613641216 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021011033099297894, + "loss": 1.2469, + "theoretical_loss": 3.38511304968332, + "tokens_seen": 2613706752 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021009027081243732, + "loss": 1.3733, + "theoretical_loss": 3.385106395926641, + "tokens_seen": 2613772288 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002100702106318957, + "loss": 1.3916, + "theoretical_loss": 3.385099742383504, + "tokens_seen": 2613837824 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021005015045135405, + "loss": 1.2952, + "theoretical_loss": 3.3850930890538957, + "tokens_seen": 2613903360 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021003009027081244, + "loss": 1.3203, + "theoretical_loss": 3.385086435937805, + "tokens_seen": 2613968896 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002100100300902708, + "loss": 1.3856, + "theoretical_loss": 3.385079783035218, + "tokens_seen": 2614034432 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002099899699097292, + "loss": 1.4235, + "theoretical_loss": 3.3850731303461243, + "tokens_seen": 2614099968 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020996990972918755, + "loss": 1.3422, + "theoretical_loss": 3.3850664778705113, + "tokens_seen": 2614165504 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020994984954864594, + "loss": 1.2666, + "theoretical_loss": 3.3850598256083657, + "tokens_seen": 2614231040 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002099297893681043, + "loss": 1.2404, + "theoretical_loss": 3.385053173559677, + "tokens_seen": 2614296576 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002099097291875627, + "loss": 1.3187, + "theoretical_loss": 3.385046521724431, + "tokens_seen": 2614362112 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020988966900702106, + "loss": 1.2882, + "theoretical_loss": 3.3850398701026174, + "tokens_seen": 2614427648 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020986960882647942, + "loss": 1.2034, + "theoretical_loss": 3.385033218694223, + "tokens_seen": 2614493184 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002098495486459378, + "loss": 1.3275, + "theoretical_loss": 3.385026567499236, + "tokens_seen": 2614558720 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002098294884653962, + "loss": 1.3267, + "theoretical_loss": 3.3850199165176433, + "tokens_seen": 2614624256 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020980942828485459, + "loss": 1.2717, + "theoretical_loss": 3.385013265749434, + "tokens_seen": 2614689792 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020978936810431295, + "loss": 1.3211, + "theoretical_loss": 3.385006615194595, + "tokens_seen": 2614755328 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020976930792377134, + "loss": 1.2694, + "theoretical_loss": 3.3849999648531144, + "tokens_seen": 2614820864 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002097492477432297, + "loss": 1.4034, + "theoretical_loss": 3.38499331472498, + "tokens_seen": 2614886400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002097291875626881, + "loss": 1.2451, + "theoretical_loss": 3.38498666481018, + "tokens_seen": 2614951936 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020970912738214645, + "loss": 1.3335, + "theoretical_loss": 3.3849800151087015, + "tokens_seen": 2615017472 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020968906720160484, + "loss": 1.3094, + "theoretical_loss": 3.3849733656205325, + "tokens_seen": 2615083008 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002096690070210632, + "loss": 1.3608, + "theoretical_loss": 3.3849667163456614, + "tokens_seen": 2615148544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020964894684052156, + "loss": 1.2232, + "theoretical_loss": 3.384960067284075, + "tokens_seen": 2615214080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020962888665997995, + "loss": 1.3217, + "theoretical_loss": 3.384953418435762, + "tokens_seen": 2615279616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020960882647943832, + "loss": 1.2801, + "theoretical_loss": 3.38494676980071, + "tokens_seen": 2615345152 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002095887662988967, + "loss": 1.1749, + "theoretical_loss": 3.3849401213789068, + "tokens_seen": 2615410688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020956870611835507, + "loss": 1.2152, + "theoretical_loss": 3.3849334731703395, + "tokens_seen": 2615476224 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020954864593781346, + "loss": 1.3174, + "theoretical_loss": 3.3849268251749973, + "tokens_seen": 2615541760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020952858575727182, + "loss": 1.2834, + "theoretical_loss": 3.3849201773928668, + "tokens_seen": 2615607296 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002095085255767302, + "loss": 1.3839, + "theoretical_loss": 3.384913529823937, + "tokens_seen": 2615672832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020948846539618857, + "loss": 1.2381, + "theoretical_loss": 3.384906882468194, + "tokens_seen": 2615738368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020946840521564693, + "loss": 1.2972, + "theoretical_loss": 3.384900235325627, + "tokens_seen": 2615803904 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020944834503510532, + "loss": 1.3334, + "theoretical_loss": 3.3848935883962237, + "tokens_seen": 2615869440 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020942828485456368, + "loss": 1.2936, + "theoretical_loss": 3.384886941679971, + "tokens_seen": 2615934976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020940822467402207, + "loss": 1.2695, + "theoretical_loss": 3.3848802951768584, + "tokens_seen": 2616000512 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020938816449348044, + "loss": 1.2069, + "theoretical_loss": 3.384873648886872, + "tokens_seen": 2616066048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020936810431293883, + "loss": 1.3178, + "theoretical_loss": 3.384867002810001, + "tokens_seen": 2616131584 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002093480441323972, + "loss": 1.2583, + "theoretical_loss": 3.384860356946232, + "tokens_seen": 2616197120 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020932798395185558, + "loss": 1.3277, + "theoretical_loss": 3.3848537112955537, + "tokens_seen": 2616262656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020930792377131394, + "loss": 1.3963, + "theoretical_loss": 3.3848470658579535, + "tokens_seen": 2616328192 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002092878635907723, + "loss": 1.2654, + "theoretical_loss": 3.3848404206334193, + "tokens_seen": 2616393728 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002092678034102307, + "loss": 1.2415, + "theoretical_loss": 3.384833775621939, + "tokens_seen": 2616459264 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020924774322968905, + "loss": 1.4234, + "theoretical_loss": 3.3848271308235005, + "tokens_seen": 2616524800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020922768304914744, + "loss": 1.2251, + "theoretical_loss": 3.3848204862380915, + "tokens_seen": 2616590336 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 951607, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1770057678222656, + "objective/train/theoretical_loss": 3.3848138418657, + "objective/train/tokens_used": 987114976, + "theoretical_loss": 3.3848138418657, + "tokens_seen": 2616655872 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002092076228686058, + "loss": 1.2223, + "theoretical_loss": 3.3848138418657, + "tokens_seen": 2616655872 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002091875626880642, + "loss": 1.2837, + "theoretical_loss": 3.384807197706314, + "tokens_seen": 2616721408 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020916750250752256, + "loss": 1.2457, + "theoretical_loss": 3.384800553759921, + "tokens_seen": 2616786944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020914744232698095, + "loss": 1.2019, + "theoretical_loss": 3.3847939100265085, + "tokens_seen": 2616852480 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002091273821464393, + "loss": 1.376, + "theoretical_loss": 3.384787266506065, + "tokens_seen": 2616918016 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002091073219658977, + "loss": 1.3133, + "theoretical_loss": 3.384780623198578, + "tokens_seen": 2616983552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020908726178535606, + "loss": 1.284, + "theoretical_loss": 3.3847739801040357, + "tokens_seen": 2617049088 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020906720160481445, + "loss": 1.3725, + "theoretical_loss": 3.3847673372224256, + "tokens_seen": 2617114624 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020904714142427284, + "loss": 1.2273, + "theoretical_loss": 3.3847606945537354, + "tokens_seen": 2617180160 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002090270812437312, + "loss": 1.2256, + "theoretical_loss": 3.3847540520979535, + "tokens_seen": 2617245696 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002090070210631896, + "loss": 1.176, + "theoretical_loss": 3.384747409855067, + "tokens_seen": 2617311232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020898696088264795, + "loss": 1.3427, + "theoretical_loss": 3.3847407678250647, + "tokens_seen": 2617376768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020896690070210634, + "loss": 1.3989, + "theoretical_loss": 3.3847341260079338, + "tokens_seen": 2617442304 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002089468405215647, + "loss": 1.3529, + "theoretical_loss": 3.384727484403662, + "tokens_seen": 2617507840 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002089267803410231, + "loss": 1.2614, + "theoretical_loss": 3.3847208430122375, + "tokens_seen": 2617573376 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020890672016048146, + "loss": 1.2949, + "theoretical_loss": 3.384714201833648, + "tokens_seen": 2617638912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020888665997993982, + "loss": 1.1856, + "theoretical_loss": 3.384707560867881, + "tokens_seen": 2617704448 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002088665997993982, + "loss": 1.2664, + "theoretical_loss": 3.384700920114925, + "tokens_seen": 2617769984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020884653961885657, + "loss": 1.2858, + "theoretical_loss": 3.3846942795747683, + "tokens_seen": 2617835520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020882647943831496, + "loss": 1.3635, + "theoretical_loss": 3.3846876392473972, + "tokens_seen": 2617901056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020880641925777332, + "loss": 1.2692, + "theoretical_loss": 3.384680999132801, + "tokens_seen": 2617966592 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002087863590772317, + "loss": 1.3224, + "theoretical_loss": 3.3846743592309667, + "tokens_seen": 2618032128 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020876629889669007, + "loss": 1.2806, + "theoretical_loss": 3.3846677195418824, + "tokens_seen": 2618097664 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020874623871614846, + "loss": 1.2066, + "theoretical_loss": 3.384661080065536, + "tokens_seen": 2618163200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020872617853560682, + "loss": 1.2574, + "theoretical_loss": 3.3846544408019152, + "tokens_seen": 2618228736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020870611835506519, + "loss": 1.2838, + "theoretical_loss": 3.3846478017510084, + "tokens_seen": 2618294272 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020868605817452358, + "loss": 1.4147, + "theoretical_loss": 3.384641162912803, + "tokens_seen": 2618359808 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020866599799398194, + "loss": 1.2744, + "theoretical_loss": 3.3846345242872866, + "tokens_seen": 2618425344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020864593781344033, + "loss": 1.175, + "theoretical_loss": 3.3846278858744476, + "tokens_seen": 2618490880 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002086258776328987, + "loss": 1.462, + "theoretical_loss": 3.3846212476742736, + "tokens_seen": 2618556416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020860581745235708, + "loss": 1.2659, + "theoretical_loss": 3.3846146096867527, + "tokens_seen": 2618621952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020858575727181544, + "loss": 1.3922, + "theoretical_loss": 3.3846079719118727, + "tokens_seen": 2618687488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020856569709127383, + "loss": 1.2167, + "theoretical_loss": 3.384601334349621, + "tokens_seen": 2618753024 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002085456369107322, + "loss": 1.2984, + "theoretical_loss": 3.3845946969999856, + "tokens_seen": 2618818560 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020852557673019058, + "loss": 1.3219, + "theoretical_loss": 3.384588059862955, + "tokens_seen": 2618884096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020850551654964894, + "loss": 1.3638, + "theoretical_loss": 3.3845814229385165, + "tokens_seen": 2618949632 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002084854563691073, + "loss": 1.3519, + "theoretical_loss": 3.384574786226658, + "tokens_seen": 2619015168 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002084653961885657, + "loss": 1.3442, + "theoretical_loss": 3.384568149727368, + "tokens_seen": 2619080704 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020844533600802406, + "loss": 1.3298, + "theoretical_loss": 3.3845615134406337, + "tokens_seen": 2619146240 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020842527582748245, + "loss": 1.3213, + "theoretical_loss": 3.384554877366443, + "tokens_seen": 2619211776 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002084052156469408, + "loss": 1.3295, + "theoretical_loss": 3.384548241504784, + "tokens_seen": 2619277312 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002083851554663992, + "loss": 1.3272, + "theoretical_loss": 3.3845416058556443, + "tokens_seen": 2619342848 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020836509528585756, + "loss": 1.394, + "theoretical_loss": 3.384534970419012, + "tokens_seen": 2619408384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020834503510531595, + "loss": 1.2891, + "theoretical_loss": 3.3845283351948754, + "tokens_seen": 2619473920 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020832497492477434, + "loss": 1.3231, + "theoretical_loss": 3.3845217001832215, + "tokens_seen": 2619539456 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002083049147442327, + "loss": 1.3214, + "theoretical_loss": 3.384515065384039, + "tokens_seen": 2619604992 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002082848545636911, + "loss": 1.2981, + "theoretical_loss": 3.384508430797315, + "tokens_seen": 2619670528 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020826479438314945, + "loss": 1.3241, + "theoretical_loss": 3.384501796423038, + "tokens_seen": 2619736064 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020824473420260784, + "loss": 1.3119, + "theoretical_loss": 3.3844951622611954, + "tokens_seen": 2619801600 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002082246740220662, + "loss": 1.3339, + "theoretical_loss": 3.384488528311776, + "tokens_seen": 2619867136 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 952663, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2578439712524414, + "objective/train/theoretical_loss": 3.3844818945747663, + "objective/train/tokens_used": 990391776, + "theoretical_loss": 3.3844818945747663, + "tokens_seen": 2619932672 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002082046138415246, + "loss": 1.2398, + "theoretical_loss": 3.3844818945747663, + "tokens_seen": 2619932672 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020818455366098296, + "loss": 1.2191, + "theoretical_loss": 3.384475261050155, + "tokens_seen": 2619998208 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020816449348044135, + "loss": 1.2984, + "theoretical_loss": 3.38446862773793, + "tokens_seen": 2620063744 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002081444332998997, + "loss": 1.287, + "theoretical_loss": 3.3844619946380794, + "tokens_seen": 2620129280 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002081243731193581, + "loss": 1.355, + "theoretical_loss": 3.3844553617505904, + "tokens_seen": 2620194816 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020810431293881646, + "loss": 1.4031, + "theoretical_loss": 3.3844487290754515, + "tokens_seen": 2620260352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020808425275827482, + "loss": 1.3084, + "theoretical_loss": 3.3844420966126503, + "tokens_seen": 2620325888 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002080641925777332, + "loss": 1.1663, + "theoretical_loss": 3.384435464362175, + "tokens_seen": 2620391424 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020804413239719157, + "loss": 1.2969, + "theoretical_loss": 3.3844288323240126, + "tokens_seen": 2620456960 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020802407221664996, + "loss": 1.2917, + "theoretical_loss": 3.384422200498152, + "tokens_seen": 2620522496 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020800401203610833, + "loss": 1.2512, + "theoretical_loss": 3.384415568884581, + "tokens_seen": 2620588032 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020798395185556671, + "loss": 1.3212, + "theoretical_loss": 3.384408937483287, + "tokens_seen": 2620653568 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020796389167502508, + "loss": 1.4728, + "theoretical_loss": 3.384402306294258, + "tokens_seen": 2620719104 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020794383149448347, + "loss": 1.3355, + "theoretical_loss": 3.384395675317482, + "tokens_seen": 2620784640 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020792377131394183, + "loss": 1.2694, + "theoretical_loss": 3.384389044552947, + "tokens_seen": 2620850176 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002079037111334002, + "loss": 1.3449, + "theoretical_loss": 3.384382414000641, + "tokens_seen": 2620915712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020788365095285858, + "loss": 1.3319, + "theoretical_loss": 3.3843757836605515, + "tokens_seen": 2620981248 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020786359077231694, + "loss": 1.2803, + "theoretical_loss": 3.3843691535326665, + "tokens_seen": 2621046784 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020784353059177533, + "loss": 1.2715, + "theoretical_loss": 3.384362523616974, + "tokens_seen": 2621112320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002078234704112337, + "loss": 1.383, + "theoretical_loss": 3.3843558939134626, + "tokens_seen": 2621177856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020780341023069208, + "loss": 1.1835, + "theoretical_loss": 3.384349264422119, + "tokens_seen": 2621243392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020778335005015045, + "loss": 1.2959, + "theoretical_loss": 3.3843426351429318, + "tokens_seen": 2621308928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020776328986960884, + "loss": 1.2128, + "theoretical_loss": 3.3843360060758885, + "tokens_seen": 2621374464 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002077432296890672, + "loss": 1.266, + "theoretical_loss": 3.3843293772209777, + "tokens_seen": 2621440000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020772316950852556, + "loss": 1.3024, + "theoretical_loss": 3.3843227485781866, + "tokens_seen": 2621505536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020770310932798395, + "loss": 1.2488, + "theoretical_loss": 3.384316120147503, + "tokens_seen": 2621571072 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002076830491474423, + "loss": 1.3177, + "theoretical_loss": 3.384309491928916, + "tokens_seen": 2621636608 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002076629889669007, + "loss": 1.3799, + "theoretical_loss": 3.3843028639224118, + "tokens_seen": 2621702144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020764292878635906, + "loss": 1.3233, + "theoretical_loss": 3.38429623612798, + "tokens_seen": 2621767680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020762286860581745, + "loss": 1.2874, + "theoretical_loss": 3.384289608545607, + "tokens_seen": 2621833216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020760280842527581, + "loss": 1.283, + "theoretical_loss": 3.384282981175282, + "tokens_seen": 2621898752 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002075827482447342, + "loss": 1.3398, + "theoretical_loss": 3.3842763540169924, + "tokens_seen": 2621964288 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002075626880641926, + "loss": 1.164, + "theoretical_loss": 3.3842697270707256, + "tokens_seen": 2622029824 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020754262788365098, + "loss": 1.3305, + "theoretical_loss": 3.3842631003364705, + "tokens_seen": 2622095360 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020752256770310934, + "loss": 1.3078, + "theoretical_loss": 3.3842564738142142, + "tokens_seen": 2622160896 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002075025075225677, + "loss": 1.3596, + "theoretical_loss": 3.384249847503945, + "tokens_seen": 2622226432 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002074824473420261, + "loss": 1.3373, + "theoretical_loss": 3.3842432214056504, + "tokens_seen": 2622291968 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020746238716148446, + "loss": 1.3418, + "theoretical_loss": 3.3842365955193188, + "tokens_seen": 2622357504 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020744232698094285, + "loss": 1.321, + "theoretical_loss": 3.3842299698449385, + "tokens_seen": 2622423040 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002074222668004012, + "loss": 1.2428, + "theoretical_loss": 3.3842233443824963, + "tokens_seen": 2622488576 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002074022066198596, + "loss": 1.2696, + "theoretical_loss": 3.384216719131981, + "tokens_seen": 2622554112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020738214643931796, + "loss": 1.268, + "theoretical_loss": 3.3842100940933806, + "tokens_seen": 2622619648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020736208625877635, + "loss": 1.24, + "theoretical_loss": 3.3842034692666823, + "tokens_seen": 2622685184 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002073420260782347, + "loss": 1.3166, + "theoretical_loss": 3.384196844651874, + "tokens_seen": 2622750720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020732196589769308, + "loss": 1.3988, + "theoretical_loss": 3.3841902202489447, + "tokens_seen": 2622816256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020730190571715146, + "loss": 1.2268, + "theoretical_loss": 3.3841835960578814, + "tokens_seen": 2622881792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020728184553660983, + "loss": 1.3014, + "theoretical_loss": 3.3841769720786727, + "tokens_seen": 2622947328 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020726178535606822, + "loss": 1.2907, + "theoretical_loss": 3.3841703483113057, + "tokens_seen": 2623012864 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020724172517552658, + "loss": 1.3183, + "theoretical_loss": 3.3841637247557688, + "tokens_seen": 2623078400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020722166499498497, + "loss": 1.1986, + "theoretical_loss": 3.38415710141205, + "tokens_seen": 2623143936 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 953489, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.287718653678894, + "objective/train/theoretical_loss": 3.3841504782801373, + "objective/train/tokens_used": 993668576, + "theoretical_loss": 3.3841504782801373, + "tokens_seen": 2623209472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020720160481444333, + "loss": 1.1881, + "theoretical_loss": 3.3841504782801373, + "tokens_seen": 2623209472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020718154463390172, + "loss": 1.3365, + "theoretical_loss": 3.3841438553600183, + "tokens_seen": 2623275008 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020716148445336008, + "loss": 1.258, + "theoretical_loss": 3.384137232651681, + "tokens_seen": 2623340544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020714142427281844, + "loss": 1.3091, + "theoretical_loss": 3.3841306101551134, + "tokens_seen": 2623406080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020712136409227683, + "loss": 1.3221, + "theoretical_loss": 3.3841239878703036, + "tokens_seen": 2623471616 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002071013039117352, + "loss": 1.1964, + "theoretical_loss": 3.3841173657972394, + "tokens_seen": 2623537152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020708124373119358, + "loss": 1.3042, + "theoretical_loss": 3.384110743935909, + "tokens_seen": 2623602688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020706118355065195, + "loss": 1.3658, + "theoretical_loss": 3.3841041222863, + "tokens_seen": 2623668224 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020704112337011034, + "loss": 1.3491, + "theoretical_loss": 3.3840975008484, + "tokens_seen": 2623733760 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002070210631895687, + "loss": 1.287, + "theoretical_loss": 3.3840908796221982, + "tokens_seen": 2623799296 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002070010030090271, + "loss": 1.4193, + "theoretical_loss": 3.384084258607681, + "tokens_seen": 2623864832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020698094282848545, + "loss": 1.3772, + "theoretical_loss": 3.384077637804838, + "tokens_seen": 2623930368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020696088264794384, + "loss": 1.4291, + "theoretical_loss": 3.384071017213655, + "tokens_seen": 2623995904 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002069408224674022, + "loss": 1.2768, + "theoretical_loss": 3.384064396834122, + "tokens_seen": 2624061440 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020692076228686056, + "loss": 1.3338, + "theoretical_loss": 3.384057776666226, + "tokens_seen": 2624126976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020690070210631895, + "loss": 1.2407, + "theoretical_loss": 3.3840511567099547, + "tokens_seen": 2624192512 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020688064192577732, + "loss": 1.3007, + "theoretical_loss": 3.384044536965297, + "tokens_seen": 2624258048 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002068605817452357, + "loss": 1.298, + "theoretical_loss": 3.3840379174322397, + "tokens_seen": 2624323584 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020684052156469407, + "loss": 1.2544, + "theoretical_loss": 3.384031298110772, + "tokens_seen": 2624389120 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020682046138415248, + "loss": 1.3135, + "theoretical_loss": 3.3840246790008806, + "tokens_seen": 2624454656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020680040120361085, + "loss": 1.2356, + "theoretical_loss": 3.3840180601025542, + "tokens_seen": 2624520192 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020678034102306924, + "loss": 1.2767, + "theoretical_loss": 3.3840114414157805, + "tokens_seen": 2624585728 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002067602808425276, + "loss": 1.2168, + "theoretical_loss": 3.3840048229405477, + "tokens_seen": 2624651264 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020674022066198596, + "loss": 1.3496, + "theoretical_loss": 3.3839982046768435, + "tokens_seen": 2624716800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020672016048144435, + "loss": 1.2074, + "theoretical_loss": 3.383991586624656, + "tokens_seen": 2624782336 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002067001003009027, + "loss": 1.4021, + "theoretical_loss": 3.383984968783973, + "tokens_seen": 2624847872 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002066800401203611, + "loss": 1.3027, + "theoretical_loss": 3.3839783511547825, + "tokens_seen": 2624913408 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020665997993981946, + "loss": 1.2421, + "theoretical_loss": 3.3839717337370727, + "tokens_seen": 2624978944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020663991975927785, + "loss": 1.3328, + "theoretical_loss": 3.3839651165308315, + "tokens_seen": 2625044480 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020661985957873621, + "loss": 1.3721, + "theoretical_loss": 3.383958499536047, + "tokens_seen": 2625110016 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002065997993981946, + "loss": 1.3472, + "theoretical_loss": 3.383951882752706, + "tokens_seen": 2625175552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020657973921765297, + "loss": 1.3602, + "theoretical_loss": 3.3839452661807985, + "tokens_seen": 2625241088 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020655967903711136, + "loss": 1.4061, + "theoretical_loss": 3.383938649820311, + "tokens_seen": 2625306624 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020653961885656972, + "loss": 1.2976, + "theoretical_loss": 3.3839320336712317, + "tokens_seen": 2625372160 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020651955867602808, + "loss": 1.3779, + "theoretical_loss": 3.3839254177335487, + "tokens_seen": 2625437696 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020649949849548647, + "loss": 1.417, + "theoretical_loss": 3.3839188020072495, + "tokens_seen": 2625503232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020647943831494483, + "loss": 1.2843, + "theoretical_loss": 3.3839121864923234, + "tokens_seen": 2625568768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020645937813440322, + "loss": 1.304, + "theoretical_loss": 3.383905571188757, + "tokens_seen": 2625634304 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020643931795386158, + "loss": 1.2843, + "theoretical_loss": 3.383898956096539, + "tokens_seen": 2625699840 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020641925777331997, + "loss": 1.237, + "theoretical_loss": 3.383892341215657, + "tokens_seen": 2625765376 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020639919759277833, + "loss": 1.3371, + "theoretical_loss": 3.383885726546099, + "tokens_seen": 2625830912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020637913741223672, + "loss": 1.3473, + "theoretical_loss": 3.383879112087853, + "tokens_seen": 2625896448 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020635907723169509, + "loss": 1.4216, + "theoretical_loss": 3.3838724978409074, + "tokens_seen": 2625961984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020633901705115345, + "loss": 1.4312, + "theoretical_loss": 3.38386588380525, + "tokens_seen": 2626027520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020631895687061184, + "loss": 1.2335, + "theoretical_loss": 3.383859269980868, + "tokens_seen": 2626093056 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002062988966900702, + "loss": 1.2849, + "theoretical_loss": 3.3838526563677505, + "tokens_seen": 2626158592 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002062788365095286, + "loss": 1.389, + "theoretical_loss": 3.383846042965885, + "tokens_seen": 2626224128 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020625877632898695, + "loss": 1.2948, + "theoretical_loss": 3.3838394297752594, + "tokens_seen": 2626289664 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020623871614844534, + "loss": 1.3282, + "theoretical_loss": 3.3838328167958616, + "tokens_seen": 2626355200 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002062186559679037, + "loss": 1.3534, + "theoretical_loss": 3.38382620402768, + "tokens_seen": 2626420736 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 954702, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0849006175994873, + "objective/train/theoretical_loss": 3.383819591470702, + "objective/train/tokens_used": 996945376, + "theoretical_loss": 3.383819591470702, + "tokens_seen": 2626486272 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002061985957873621, + "loss": 1.1477, + "theoretical_loss": 3.383819591470702, + "tokens_seen": 2626486272 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020617853560682046, + "loss": 1.2556, + "theoretical_loss": 3.383812979124916, + "tokens_seen": 2626551808 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020615847542627882, + "loss": 1.3105, + "theoretical_loss": 3.38380636699031, + "tokens_seen": 2626617344 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002061384152457372, + "loss": 1.2859, + "theoretical_loss": 3.383799755066872, + "tokens_seen": 2626682880 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020611835506519557, + "loss": 1.3241, + "theoretical_loss": 3.3837931433545894, + "tokens_seen": 2626748416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020609829488465396, + "loss": 1.3895, + "theoretical_loss": 3.383786531853451, + "tokens_seen": 2626813952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020607823470411235, + "loss": 1.2616, + "theoretical_loss": 3.3837799205634442, + "tokens_seen": 2626879488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020605817452357074, + "loss": 1.2748, + "theoretical_loss": 3.3837733094845572, + "tokens_seen": 2626945024 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002060381143430291, + "loss": 1.2992, + "theoretical_loss": 3.3837666986167783, + "tokens_seen": 2627010560 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002060180541624875, + "loss": 1.2374, + "theoretical_loss": 3.3837600879600953, + "tokens_seen": 2627076096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020599799398194585, + "loss": 1.353, + "theoretical_loss": 3.383753477514496, + "tokens_seen": 2627141632 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020597793380140424, + "loss": 1.3047, + "theoretical_loss": 3.383746867279968, + "tokens_seen": 2627207168 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002059578736208626, + "loss": 1.3592, + "theoretical_loss": 3.3837402572565005, + "tokens_seen": 2627272704 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020593781344032096, + "loss": 1.2548, + "theoretical_loss": 3.3837336474440805, + "tokens_seen": 2627338240 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020591775325977935, + "loss": 1.329, + "theoretical_loss": 3.383727037842696, + "tokens_seen": 2627403776 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020589769307923772, + "loss": 1.2599, + "theoretical_loss": 3.3837204284523357, + "tokens_seen": 2627469312 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002058776328986961, + "loss": 1.1678, + "theoretical_loss": 3.383713819272987, + "tokens_seen": 2627534848 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020585757271815447, + "loss": 1.3363, + "theoretical_loss": 3.3837072103046384, + "tokens_seen": 2627600384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020583751253761286, + "loss": 1.3702, + "theoretical_loss": 3.383700601547277, + "tokens_seen": 2627665920 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020581745235707122, + "loss": 1.2711, + "theoretical_loss": 3.383693993000892, + "tokens_seen": 2627731456 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002057973921765296, + "loss": 1.2179, + "theoretical_loss": 3.3836873846654703, + "tokens_seen": 2627796992 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020577733199598797, + "loss": 1.2412, + "theoretical_loss": 3.383680776541001, + "tokens_seen": 2627862528 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020575727181544633, + "loss": 1.2966, + "theoretical_loss": 3.383674168627471, + "tokens_seen": 2627928064 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020573721163490472, + "loss": 1.4539, + "theoretical_loss": 3.3836675609248688, + "tokens_seen": 2627993600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020571715145436308, + "loss": 1.3414, + "theoretical_loss": 3.3836609534331825, + "tokens_seen": 2628059136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020569709127382147, + "loss": 1.189, + "theoretical_loss": 3.3836543461524, + "tokens_seen": 2628124672 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020567703109327984, + "loss": 1.3724, + "theoretical_loss": 3.3836477390825097, + "tokens_seen": 2628190208 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020565697091273823, + "loss": 1.3481, + "theoretical_loss": 3.3836411322234987, + "tokens_seen": 2628255744 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002056369107321966, + "loss": 1.3553, + "theoretical_loss": 3.383634525575356, + "tokens_seen": 2628321280 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020561685055165498, + "loss": 1.292, + "theoretical_loss": 3.3836279191380694, + "tokens_seen": 2628386816 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020559679037111334, + "loss": 1.3067, + "theoretical_loss": 3.383621312911626, + "tokens_seen": 2628452352 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002055767301905717, + "loss": 1.2241, + "theoretical_loss": 3.3836147068960147, + "tokens_seen": 2628517888 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002055566700100301, + "loss": 1.2936, + "theoretical_loss": 3.3836081010912236, + "tokens_seen": 2628583424 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020553660982948845, + "loss": 1.2211, + "theoretical_loss": 3.38360149549724, + "tokens_seen": 2628648960 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020551654964894684, + "loss": 1.3436, + "theoretical_loss": 3.3835948901140527, + "tokens_seen": 2628714496 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002054964894684052, + "loss": 1.2761, + "theoretical_loss": 3.3835882849416494, + "tokens_seen": 2628780032 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002054764292878636, + "loss": 1.4263, + "theoretical_loss": 3.3835816799800176, + "tokens_seen": 2628845568 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020545636910732196, + "loss": 1.3702, + "theoretical_loss": 3.383575075229146, + "tokens_seen": 2628911104 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020543630892678035, + "loss": 1.3149, + "theoretical_loss": 3.383568470689023, + "tokens_seen": 2628976640 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002054162487462387, + "loss": 1.3477, + "theoretical_loss": 3.3835618663596354, + "tokens_seen": 2629042176 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053961885656971, + "loss": 1.2733, + "theoretical_loss": 3.383555262240972, + "tokens_seen": 2629107712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020537612838515546, + "loss": 1.3245, + "theoretical_loss": 3.3835486583330208, + "tokens_seen": 2629173248 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020535606820461382, + "loss": 1.2971, + "theoretical_loss": 3.38354205463577, + "tokens_seen": 2629238784 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053360080240722, + "loss": 1.39, + "theoretical_loss": 3.3835354511492066, + "tokens_seen": 2629304320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053159478435306, + "loss": 1.3068, + "theoretical_loss": 3.38352884787332, + "tokens_seen": 2629369856 + }, + { + "epoch": 0.59, + "learning_rate": 0.000205295887662989, + "loss": 1.3838, + "theoretical_loss": 3.3835222448080975, + "tokens_seen": 2629435392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020527582748244735, + "loss": 1.29, + "theoretical_loss": 3.383515641953527, + "tokens_seen": 2629500928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020525576730190574, + "loss": 1.2904, + "theoretical_loss": 3.3835090393095966, + "tokens_seen": 2629566464 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002052357071213641, + "loss": 1.3154, + "theoretical_loss": 3.383502436876295, + "tokens_seen": 2629632000 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002052156469408225, + "loss": 1.1901, + "theoretical_loss": 3.3834958346536093, + "tokens_seen": 2629697536 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 955785, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4545732736587524, + "objective/train/theoretical_loss": 3.3834892326415282, + "objective/train/tokens_used": 1000222176, + "theoretical_loss": 3.3834892326415282, + "tokens_seen": 2629763072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020519558676028086, + "loss": 1.3051, + "theoretical_loss": 3.3834892326415282, + "tokens_seen": 2629763072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020517552657973922, + "loss": 1.1722, + "theoretical_loss": 3.3834826308400396, + "tokens_seen": 2629828608 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002051554663991976, + "loss": 1.2645, + "theoretical_loss": 3.383476029249131, + "tokens_seen": 2629894144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020513540621865597, + "loss": 1.2748, + "theoretical_loss": 3.3834694278687913, + "tokens_seen": 2629959680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020511534603811436, + "loss": 1.3362, + "theoretical_loss": 3.3834628266990077, + "tokens_seen": 2630025216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020509528585757272, + "loss": 1.2211, + "theoretical_loss": 3.383456225739769, + "tokens_seen": 2630090752 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002050752256770311, + "loss": 1.3308, + "theoretical_loss": 3.3834496249910626, + "tokens_seen": 2630156288 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020505516549648947, + "loss": 1.2035, + "theoretical_loss": 3.383443024452877, + "tokens_seen": 2630221824 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020503510531594786, + "loss": 1.3142, + "theoretical_loss": 3.3834364241252, + "tokens_seen": 2630287360 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020501504513540622, + "loss": 1.1652, + "theoretical_loss": 3.3834298240080196, + "tokens_seen": 2630352896 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020499498495486459, + "loss": 1.2588, + "theoretical_loss": 3.383423224101324, + "tokens_seen": 2630418432 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020497492477432298, + "loss": 1.3933, + "theoretical_loss": 3.383416624405101, + "tokens_seen": 2630483968 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020495486459378134, + "loss": 1.3266, + "theoretical_loss": 3.383410024919339, + "tokens_seen": 2630549504 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020493480441323973, + "loss": 1.3421, + "theoretical_loss": 3.3834034256440257, + "tokens_seen": 2630615040 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002049147442326981, + "loss": 1.2816, + "theoretical_loss": 3.3833968265791494, + "tokens_seen": 2630680576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020489468405215648, + "loss": 1.3787, + "theoretical_loss": 3.383390227724698, + "tokens_seen": 2630746112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020487462387161484, + "loss": 1.2024, + "theoretical_loss": 3.38338362908066, + "tokens_seen": 2630811648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020485456369107323, + "loss": 1.359, + "theoretical_loss": 3.383377030647023, + "tokens_seen": 2630877184 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002048345035105316, + "loss": 1.2657, + "theoretical_loss": 3.3833704324237748, + "tokens_seen": 2630942720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020481444332998998, + "loss": 1.335, + "theoretical_loss": 3.3833638344109036, + "tokens_seen": 2631008256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020479438314944834, + "loss": 1.3214, + "theoretical_loss": 3.383357236608398, + "tokens_seen": 2631073792 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002047743229689067, + "loss": 1.3942, + "theoretical_loss": 3.3833506390162453, + "tokens_seen": 2631139328 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002047542627883651, + "loss": 1.2536, + "theoretical_loss": 3.3833440416344347, + "tokens_seen": 2631204864 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020473420260782346, + "loss": 1.2553, + "theoretical_loss": 3.3833374444629527, + "tokens_seen": 2631270400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020471414242728185, + "loss": 1.1939, + "theoretical_loss": 3.3833308475017883, + "tokens_seen": 2631335936 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002046940822467402, + "loss": 1.1219, + "theoretical_loss": 3.3833242507509294, + "tokens_seen": 2631401472 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002046740220661986, + "loss": 1.2789, + "theoretical_loss": 3.383317654210364, + "tokens_seen": 2631467008 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020465396188565696, + "loss": 1.259, + "theoretical_loss": 3.3833110578800807, + "tokens_seen": 2631532544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020463390170511535, + "loss": 1.2403, + "theoretical_loss": 3.3833044617600665, + "tokens_seen": 2631598080 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002046138415245737, + "loss": 1.3504, + "theoretical_loss": 3.3832978658503103, + "tokens_seen": 2631663616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020459378134403207, + "loss": 1.1628, + "theoretical_loss": 3.3832912701508, + "tokens_seen": 2631729152 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002045737211634905, + "loss": 1.2988, + "theoretical_loss": 3.383284674661523, + "tokens_seen": 2631794688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020455366098294885, + "loss": 1.2068, + "theoretical_loss": 3.3832780793824684, + "tokens_seen": 2631860224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020453360080240724, + "loss": 1.2834, + "theoretical_loss": 3.3832714843136236, + "tokens_seen": 2631925760 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002045135406218656, + "loss": 1.2207, + "theoretical_loss": 3.383264889454977, + "tokens_seen": 2631991296 + }, + { + "epoch": 0.6, + "learning_rate": 0.000204493480441324, + "loss": 1.1474, + "theoretical_loss": 3.383258294806516, + "tokens_seen": 2632056832 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020447342026078236, + "loss": 1.2967, + "theoretical_loss": 3.3832517003682296, + "tokens_seen": 2632122368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020445336008024075, + "loss": 1.2123, + "theoretical_loss": 3.3832451061401057, + "tokens_seen": 2632187904 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002044332998996991, + "loss": 1.3014, + "theoretical_loss": 3.383238512122132, + "tokens_seen": 2632253440 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002044132397191575, + "loss": 1.3797, + "theoretical_loss": 3.3832319183142965, + "tokens_seen": 2632318976 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020439317953861586, + "loss": 1.3559, + "theoretical_loss": 3.3832253247165873, + "tokens_seen": 2632384512 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020437311935807422, + "loss": 1.2383, + "theoretical_loss": 3.383218731328993, + "tokens_seen": 2632450048 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002043530591775326, + "loss": 1.316, + "theoretical_loss": 3.3832121381515012, + "tokens_seen": 2632515584 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020433299899699097, + "loss": 1.257, + "theoretical_loss": 3.3832055451841, + "tokens_seen": 2632581120 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020431293881644936, + "loss": 1.2435, + "theoretical_loss": 3.3831989524267776, + "tokens_seen": 2632646656 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020429287863590773, + "loss": 1.2417, + "theoretical_loss": 3.383192359879522, + "tokens_seen": 2632712192 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020427281845536611, + "loss": 1.1876, + "theoretical_loss": 3.3831857675423214, + "tokens_seen": 2632777728 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020425275827482448, + "loss": 1.342, + "theoretical_loss": 3.383179175415164, + "tokens_seen": 2632843264 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020423269809428287, + "loss": 1.1676, + "theoretical_loss": 3.383172583498037, + "tokens_seen": 2632908800 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020421263791374123, + "loss": 1.2602, + "theoretical_loss": 3.38316599179093, + "tokens_seen": 2632974336 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.7520842035644667, + "debugging/entropy-1-grams": 4.765424186883236, + "debugging/length": 508.6666666666667, + "debugging/num_segments": 6, + "debugging/score": 0.0044137430059683815, + "debugging/score_std": 0.0029234356048069474, + "epoch": 0.6, + "objective/train/docs_used": 956895, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2060573101043701, + "objective/train/theoretical_loss": 3.3831594002938297, + "objective/train/tokens_used": 1003498976, + "theoretical_loss": 3.3831594002938297, + "tokens_seen": 2633039872 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002041925777331996, + "loss": 1.281, + "theoretical_loss": 3.3831594002938297, + "tokens_seen": 2633039872 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020417251755265798, + "loss": 1.3121, + "theoretical_loss": 3.3831528090067247, + "tokens_seen": 2633105408 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020415245737211634, + "loss": 1.2675, + "theoretical_loss": 3.3831462179296032, + "tokens_seen": 2633170944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020413239719157473, + "loss": 1.2554, + "theoretical_loss": 3.3831396270624534, + "tokens_seen": 2633236480 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002041123370110331, + "loss": 1.2279, + "theoretical_loss": 3.3831330364052628, + "tokens_seen": 2633302016 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020409227683049148, + "loss": 1.2542, + "theoretical_loss": 3.38312644595802, + "tokens_seen": 2633367552 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020407221664994985, + "loss": 1.3532, + "theoretical_loss": 3.383119855720713, + "tokens_seen": 2633433088 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020405215646940823, + "loss": 1.3485, + "theoretical_loss": 3.38311326569333, + "tokens_seen": 2633498624 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002040320962888666, + "loss": 1.2305, + "theoretical_loss": 3.3831066758758586, + "tokens_seen": 2633564160 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020401203610832496, + "loss": 1.2319, + "theoretical_loss": 3.3831000862682874, + "tokens_seen": 2633629696 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020399197592778335, + "loss": 1.2706, + "theoretical_loss": 3.3830934968706043, + "tokens_seen": 2633695232 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002039719157472417, + "loss": 1.3699, + "theoretical_loss": 3.3830869076827974, + "tokens_seen": 2633760768 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002039518555667001, + "loss": 1.2627, + "theoretical_loss": 3.383080318704855, + "tokens_seen": 2633826304 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020393179538615846, + "loss": 1.237, + "theoretical_loss": 3.383073729936765, + "tokens_seen": 2633891840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020391173520561685, + "loss": 1.3676, + "theoretical_loss": 3.3830671413785147, + "tokens_seen": 2633957376 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020389167502507521, + "loss": 1.3265, + "theoretical_loss": 3.3830605530300932, + "tokens_seen": 2634022912 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002038716148445336, + "loss": 1.329, + "theoretical_loss": 3.383053964891489, + "tokens_seen": 2634088448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020385155466399197, + "loss": 1.2976, + "theoretical_loss": 3.383047376962689, + "tokens_seen": 2634153984 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020383149448345036, + "loss": 1.3263, + "theoretical_loss": 3.3830407892436822, + "tokens_seen": 2634219520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020381143430290874, + "loss": 1.3919, + "theoretical_loss": 3.3830342017344566, + "tokens_seen": 2634285056 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002037913741223671, + "loss": 1.2098, + "theoretical_loss": 3.3830276144349996, + "tokens_seen": 2634350592 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002037713139418255, + "loss": 1.2501, + "theoretical_loss": 3.3830210273452996, + "tokens_seen": 2634416128 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020375125376128386, + "loss": 1.3143, + "theoretical_loss": 3.3830144404653453, + "tokens_seen": 2634481664 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020373119358074225, + "loss": 1.2818, + "theoretical_loss": 3.3830078537951245, + "tokens_seen": 2634547200 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002037111334002006, + "loss": 1.3101, + "theoretical_loss": 3.383001267334625, + "tokens_seen": 2634612736 + }, + { + "epoch": 0.6, + "learning_rate": 0.000203691073219659, + "loss": 1.3533, + "theoretical_loss": 3.3829946810838347, + "tokens_seen": 2634678272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020367101303911736, + "loss": 1.2517, + "theoretical_loss": 3.3829880950427427, + "tokens_seen": 2634743808 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020365095285857575, + "loss": 1.2034, + "theoretical_loss": 3.3829815092113362, + "tokens_seen": 2634809344 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002036308926780341, + "loss": 1.3102, + "theoretical_loss": 3.382974923589604, + "tokens_seen": 2634874880 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020361083249749248, + "loss": 1.2457, + "theoretical_loss": 3.382968338177533, + "tokens_seen": 2634940416 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020359077231695086, + "loss": 1.3118, + "theoretical_loss": 3.382961752975113, + "tokens_seen": 2635005952 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020357071213640923, + "loss": 1.3523, + "theoretical_loss": 3.3829551679823306, + "tokens_seen": 2635071488 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020355065195586762, + "loss": 1.3211, + "theoretical_loss": 3.382948583199175, + "tokens_seen": 2635137024 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020353059177532598, + "loss": 1.4232, + "theoretical_loss": 3.3829419986256335, + "tokens_seen": 2635202560 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020351053159478437, + "loss": 1.3094, + "theoretical_loss": 3.382935414261695, + "tokens_seen": 2635268096 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020349047141424273, + "loss": 1.3112, + "theoretical_loss": 3.382928830107347, + "tokens_seen": 2635333632 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020347041123370112, + "loss": 1.2773, + "theoretical_loss": 3.3829222461625776, + "tokens_seen": 2635399168 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020345035105315948, + "loss": 1.2746, + "theoretical_loss": 3.3829156624273753, + "tokens_seen": 2635464704 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020343029087261784, + "loss": 1.243, + "theoretical_loss": 3.3829090789017284, + "tokens_seen": 2635530240 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020341023069207623, + "loss": 1.2504, + "theoretical_loss": 3.382902495585624, + "tokens_seen": 2635595776 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002033901705115346, + "loss": 1.2461, + "theoretical_loss": 3.3828959124790514, + "tokens_seen": 2635661312 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020337011033099298, + "loss": 1.2918, + "theoretical_loss": 3.382889329581998, + "tokens_seen": 2635726848 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020335005015045135, + "loss": 1.2903, + "theoretical_loss": 3.3828827468944525, + "tokens_seen": 2635792384 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020332998996990974, + "loss": 1.2851, + "theoretical_loss": 3.3828761644164023, + "tokens_seen": 2635857920 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002033099297893681, + "loss": 1.2285, + "theoretical_loss": 3.382869582147836, + "tokens_seen": 2635923456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002032898696088265, + "loss": 1.2742, + "theoretical_loss": 3.3828630000887414, + "tokens_seen": 2635988992 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020326980942828485, + "loss": 1.2928, + "theoretical_loss": 3.382856418239107, + "tokens_seen": 2636054528 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020324974924774324, + "loss": 1.101, + "theoretical_loss": 3.382849836598921, + "tokens_seen": 2636120064 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002032296890672016, + "loss": 1.2575, + "theoretical_loss": 3.382843255168171, + "tokens_seen": 2636185600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020320962888665996, + "loss": 1.1833, + "theoretical_loss": 3.382836673946845, + "tokens_seen": 2636251136 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 957995, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.201979160308838, + "objective/train/theoretical_loss": 3.3828300929349324, + "objective/train/tokens_used": 1006775776, + "theoretical_loss": 3.3828300929349324, + "tokens_seen": 2636316672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020318956870611835, + "loss": 1.264, + "theoretical_loss": 3.3828300929349324, + "tokens_seen": 2636316672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020316950852557672, + "loss": 1.2874, + "theoretical_loss": 3.38282351213242, + "tokens_seen": 2636382208 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002031494483450351, + "loss": 1.3076, + "theoretical_loss": 3.3828169315392964, + "tokens_seen": 2636447744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020312938816449347, + "loss": 1.3011, + "theoretical_loss": 3.38281035115555, + "tokens_seen": 2636513280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020310932798395186, + "loss": 1.2906, + "theoretical_loss": 3.3828037709811687, + "tokens_seen": 2636578816 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020308926780341022, + "loss": 1.3196, + "theoretical_loss": 3.3827971910161407, + "tokens_seen": 2636644352 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020306920762286864, + "loss": 1.2277, + "theoretical_loss": 3.3827906112604538, + "tokens_seen": 2636709888 + }, + { + "epoch": 0.6, + "learning_rate": 0.000203049147442327, + "loss": 1.3058, + "theoretical_loss": 3.3827840317140963, + "tokens_seen": 2636775424 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020302908726178536, + "loss": 1.2802, + "theoretical_loss": 3.3827774523770566, + "tokens_seen": 2636840960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020300902708124375, + "loss": 1.2996, + "theoretical_loss": 3.3827708732493225, + "tokens_seen": 2636906496 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002029889669007021, + "loss": 1.2672, + "theoretical_loss": 3.382764294330882, + "tokens_seen": 2636972032 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002029689067201605, + "loss": 1.2917, + "theoretical_loss": 3.3827577156217243, + "tokens_seen": 2637037568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020294884653961886, + "loss": 1.2406, + "theoretical_loss": 3.3827511371218364, + "tokens_seen": 2637103104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020292878635907725, + "loss": 1.2337, + "theoretical_loss": 3.382744558831207, + "tokens_seen": 2637168640 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020290872617853561, + "loss": 1.2899, + "theoretical_loss": 3.382737980749824, + "tokens_seen": 2637234176 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202888665997994, + "loss": 1.2923, + "theoretical_loss": 3.3827314028776754, + "tokens_seen": 2637299712 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020286860581745237, + "loss": 1.362, + "theoretical_loss": 3.38272482521475, + "tokens_seen": 2637365248 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020284854563691073, + "loss": 1.2281, + "theoretical_loss": 3.382718247761035, + "tokens_seen": 2637430784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020282848545636912, + "loss": 1.272, + "theoretical_loss": 3.382711670516519, + "tokens_seen": 2637496320 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020280842527582748, + "loss": 1.2332, + "theoretical_loss": 3.382705093481191, + "tokens_seen": 2637561856 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020278836509528587, + "loss": 1.221, + "theoretical_loss": 3.3826985166550374, + "tokens_seen": 2637627392 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020276830491474423, + "loss": 1.2161, + "theoretical_loss": 3.3826919400380477, + "tokens_seen": 2637692928 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020274824473420262, + "loss": 1.3571, + "theoretical_loss": 3.38268536363021, + "tokens_seen": 2637758464 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020272818455366098, + "loss": 1.2595, + "theoretical_loss": 3.382678787431512, + "tokens_seen": 2637824000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020270812437311937, + "loss": 1.3324, + "theoretical_loss": 3.3826722114419416, + "tokens_seen": 2637889536 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020268806419257773, + "loss": 1.2602, + "theoretical_loss": 3.3826656356614873, + "tokens_seen": 2637955072 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020266800401203612, + "loss": 1.2448, + "theoretical_loss": 3.3826590600901376, + "tokens_seen": 2638020608 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020264794383149449, + "loss": 1.2525, + "theoretical_loss": 3.3826524847278803, + "tokens_seen": 2638086144 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020262788365095285, + "loss": 1.1907, + "theoretical_loss": 3.3826459095747037, + "tokens_seen": 2638151680 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020260782347041124, + "loss": 1.2541, + "theoretical_loss": 3.3826393346305954, + "tokens_seen": 2638217216 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002025877632898696, + "loss": 1.3538, + "theoretical_loss": 3.3826327598955443, + "tokens_seen": 2638282752 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202567703109328, + "loss": 1.1886, + "theoretical_loss": 3.382626185369538, + "tokens_seen": 2638348288 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020254764292878635, + "loss": 1.312, + "theoretical_loss": 3.382619611052565, + "tokens_seen": 2638413824 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020252758274824474, + "loss": 1.2355, + "theoretical_loss": 3.3826130369446137, + "tokens_seen": 2638479360 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002025075225677031, + "loss": 1.3295, + "theoretical_loss": 3.382606463045672, + "tokens_seen": 2638544896 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002024874623871615, + "loss": 1.3521, + "theoretical_loss": 3.3825998893557276, + "tokens_seen": 2638610432 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020246740220661985, + "loss": 1.2493, + "theoretical_loss": 3.382593315874769, + "tokens_seen": 2638675968 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020244734202607822, + "loss": 1.3064, + "theoretical_loss": 3.382586742602785, + "tokens_seen": 2638741504 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002024272818455366, + "loss": 1.1987, + "theoretical_loss": 3.3825801695397626, + "tokens_seen": 2638807040 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020240722166499497, + "loss": 1.3061, + "theoretical_loss": 3.382573596685691, + "tokens_seen": 2638872576 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020238716148445336, + "loss": 1.2069, + "theoretical_loss": 3.382567024040558, + "tokens_seen": 2638938112 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020236710130391172, + "loss": 1.2174, + "theoretical_loss": 3.3825604516043515, + "tokens_seen": 2639003648 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002023470411233701, + "loss": 1.2628, + "theoretical_loss": 3.3825538793770598, + "tokens_seen": 2639069184 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002023269809428285, + "loss": 1.2979, + "theoretical_loss": 3.3825473073586716, + "tokens_seen": 2639134720 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002023069207622869, + "loss": 1.3032, + "theoretical_loss": 3.382540735549174, + "tokens_seen": 2639200256 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020228686058174525, + "loss": 1.1514, + "theoretical_loss": 3.382534163948556, + "tokens_seen": 2639265792 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020226680040120364, + "loss": 1.3321, + "theoretical_loss": 3.382527592556806, + "tokens_seen": 2639331328 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202246740220662, + "loss": 1.2436, + "theoretical_loss": 3.382521021373911, + "tokens_seen": 2639396864 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020222668004012036, + "loss": 1.2337, + "theoretical_loss": 3.382514450399861, + "tokens_seen": 2639462400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020220661985957875, + "loss": 1.2755, + "theoretical_loss": 3.3825078796346424, + "tokens_seen": 2639527936 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 959162, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3204405307769775, + "objective/train/theoretical_loss": 3.382501309078244, + "objective/train/tokens_used": 1010052576, + "theoretical_loss": 3.382501309078244, + "tokens_seen": 2639593472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020218655967903712, + "loss": 1.2452, + "theoretical_loss": 3.382501309078244, + "tokens_seen": 2639593472 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002021664994984955, + "loss": 1.2847, + "theoretical_loss": 3.382494738730654, + "tokens_seen": 2639659008 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020214643931795387, + "loss": 1.1747, + "theoretical_loss": 3.382488168591861, + "tokens_seen": 2639724544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020212637913741226, + "loss": 1.2614, + "theoretical_loss": 3.382481598661853, + "tokens_seen": 2639790080 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020210631895687062, + "loss": 1.2516, + "theoretical_loss": 3.3824750289406174, + "tokens_seen": 2639855616 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202086258776329, + "loss": 1.3555, + "theoretical_loss": 3.3824684594281433, + "tokens_seen": 2639921152 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020206619859578737, + "loss": 1.3245, + "theoretical_loss": 3.382461890124419, + "tokens_seen": 2639986688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020204613841524573, + "loss": 1.2297, + "theoretical_loss": 3.3824553210294317, + "tokens_seen": 2640052224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020202607823470412, + "loss": 1.2674, + "theoretical_loss": 3.38244875214317, + "tokens_seen": 2640117760 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020200601805416248, + "loss": 1.2367, + "theoretical_loss": 3.382442183465623, + "tokens_seen": 2640183296 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020198595787362087, + "loss": 1.1677, + "theoretical_loss": 3.382435614996777, + "tokens_seen": 2640248832 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020196589769307924, + "loss": 1.2467, + "theoretical_loss": 3.3824290467366223, + "tokens_seen": 2640314368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020194583751253763, + "loss": 1.2175, + "theoretical_loss": 3.382422478685146, + "tokens_seen": 2640379904 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201925777331996, + "loss": 1.3024, + "theoretical_loss": 3.3824159108423357, + "tokens_seen": 2640445440 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020190571715145438, + "loss": 1.2672, + "theoretical_loss": 3.382409343208181, + "tokens_seen": 2640510976 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020188565697091274, + "loss": 1.302, + "theoretical_loss": 3.382402775782669, + "tokens_seen": 2640576512 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002018655967903711, + "loss": 1.2346, + "theoretical_loss": 3.3823962085657886, + "tokens_seen": 2640642048 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002018455366098295, + "loss": 1.2787, + "theoretical_loss": 3.382389641557527, + "tokens_seen": 2640707584 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020182547642928785, + "loss": 1.13, + "theoretical_loss": 3.3823830747578736, + "tokens_seen": 2640773120 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020180541624874624, + "loss": 1.2116, + "theoretical_loss": 3.382376508166816, + "tokens_seen": 2640838656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002017853560682046, + "loss": 1.2709, + "theoretical_loss": 3.3823699417843427, + "tokens_seen": 2640904192 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201765295887663, + "loss": 1.415, + "theoretical_loss": 3.382363375610441, + "tokens_seen": 2640969728 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020174523570712136, + "loss": 1.297, + "theoretical_loss": 3.3823568096451004, + "tokens_seen": 2641035264 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020172517552657975, + "loss": 1.305, + "theoretical_loss": 3.3823502438883084, + "tokens_seen": 2641100800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002017051153460381, + "loss": 1.2318, + "theoretical_loss": 3.3823436783400527, + "tokens_seen": 2641166336 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002016850551654965, + "loss": 1.1838, + "theoretical_loss": 3.3823371130003226, + "tokens_seen": 2641231872 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020166499498495486, + "loss": 1.2678, + "theoretical_loss": 3.3823305478691053, + "tokens_seen": 2641297408 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020164493480441322, + "loss": 1.2903, + "theoretical_loss": 3.3823239829463896, + "tokens_seen": 2641362944 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002016248746238716, + "loss": 1.239, + "theoretical_loss": 3.382317418232164, + "tokens_seen": 2641428480 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020160481444332997, + "loss": 1.2758, + "theoretical_loss": 3.3823108537264157, + "tokens_seen": 2641494016 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020158475426278836, + "loss": 1.285, + "theoretical_loss": 3.382304289429134, + "tokens_seen": 2641559552 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020156469408224675, + "loss": 1.275, + "theoretical_loss": 3.3822977253403064, + "tokens_seen": 2641625088 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020154463390170514, + "loss": 1.3044, + "theoretical_loss": 3.382291161459921, + "tokens_seen": 2641690624 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002015245737211635, + "loss": 1.2368, + "theoretical_loss": 3.3822845977879665, + "tokens_seen": 2641756160 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002015045135406219, + "loss": 1.3448, + "theoretical_loss": 3.382278034324431, + "tokens_seen": 2641821696 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020148445336008026, + "loss": 1.3418, + "theoretical_loss": 3.3822714710693025, + "tokens_seen": 2641887232 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020146439317953862, + "loss": 1.292, + "theoretical_loss": 3.3822649080225693, + "tokens_seen": 2641952768 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201444332998997, + "loss": 1.2346, + "theoretical_loss": 3.38225834518422, + "tokens_seen": 2642018304 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020142427281845537, + "loss": 1.1476, + "theoretical_loss": 3.3822517825542424, + "tokens_seen": 2642083840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020140421263791376, + "loss": 1.1398, + "theoretical_loss": 3.3822452201326247, + "tokens_seen": 2642149376 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020138415245737212, + "loss": 1.2182, + "theoretical_loss": 3.382238657919355, + "tokens_seen": 2642214912 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002013640922768305, + "loss": 1.1945, + "theoretical_loss": 3.3822320959144223, + "tokens_seen": 2642280448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020134403209628887, + "loss": 1.2641, + "theoretical_loss": 3.3822255341178136, + "tokens_seen": 2642345984 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020132397191574726, + "loss": 1.3163, + "theoretical_loss": 3.382218972529518, + "tokens_seen": 2642411520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020130391173520562, + "loss": 1.2702, + "theoretical_loss": 3.3822124111495233, + "tokens_seen": 2642477056 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020128385155466399, + "loss": 1.2567, + "theoretical_loss": 3.3822058499778183, + "tokens_seen": 2642542592 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020126379137412238, + "loss": 1.2038, + "theoretical_loss": 3.3821992890143906, + "tokens_seen": 2642608128 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020124373119358074, + "loss": 1.276, + "theoretical_loss": 3.382192728259229, + "tokens_seen": 2642673664 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020122367101303913, + "loss": 1.2247, + "theoretical_loss": 3.3821861677123204, + "tokens_seen": 2642739200 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002012036108324975, + "loss": 1.2217, + "theoretical_loss": 3.382179607373655, + "tokens_seen": 2642804736 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 960167, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1384999752044678, + "objective/train/theoretical_loss": 3.38217304724322, + "objective/train/tokens_used": 1013329376, + "theoretical_loss": 3.38217304724322, + "tokens_seen": 2642870272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020118355065195588, + "loss": 1.2681, + "theoretical_loss": 3.38217304724322, + "tokens_seen": 2642870272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020116349047141424, + "loss": 1.3025, + "theoretical_loss": 3.382166487321003, + "tokens_seen": 2642935808 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020114343029087263, + "loss": 1.2767, + "theoretical_loss": 3.382159927606993, + "tokens_seen": 2643001344 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201123370110331, + "loss": 1.2844, + "theoretical_loss": 3.3821533681011786, + "tokens_seen": 2643066880 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020110330992978938, + "loss": 1.2512, + "theoretical_loss": 3.3821468088035473, + "tokens_seen": 2643132416 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020108324974924774, + "loss": 1.209, + "theoretical_loss": 3.3821402497140873, + "tokens_seen": 2643197952 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002010631895687061, + "loss": 1.2558, + "theoretical_loss": 3.3821336908327875, + "tokens_seen": 2643263488 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002010431293881645, + "loss": 1.323, + "theoretical_loss": 3.382127132159636, + "tokens_seen": 2643329024 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020102306920762286, + "loss": 1.265, + "theoretical_loss": 3.38212057369462, + "tokens_seen": 2643394560 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020100300902708125, + "loss": 1.2609, + "theoretical_loss": 3.382114015437729, + "tokens_seen": 2643460096 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002009829488465396, + "loss": 1.3538, + "theoretical_loss": 3.3821074573889502, + "tokens_seen": 2643525632 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200962888665998, + "loss": 1.2228, + "theoretical_loss": 3.382100899548273, + "tokens_seen": 2643591168 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020094282848545636, + "loss": 1.2541, + "theoretical_loss": 3.382094341915685, + "tokens_seen": 2643656704 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020092276830491475, + "loss": 1.3607, + "theoretical_loss": 3.382087784491174, + "tokens_seen": 2643722240 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002009027081243731, + "loss": 1.2506, + "theoretical_loss": 3.382081227274729, + "tokens_seen": 2643787776 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020088264794383147, + "loss": 1.3474, + "theoretical_loss": 3.382074670266338, + "tokens_seen": 2643853312 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020086258776328986, + "loss": 1.2217, + "theoretical_loss": 3.3820681134659893, + "tokens_seen": 2643918848 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020084252758274823, + "loss": 1.1686, + "theoretical_loss": 3.3820615568736705, + "tokens_seen": 2643984384 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020082246740220664, + "loss": 1.3207, + "theoretical_loss": 3.382055000489371, + "tokens_seen": 2644049920 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200802407221665, + "loss": 1.3747, + "theoretical_loss": 3.382048444313078, + "tokens_seen": 2644115456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002007823470411234, + "loss": 1.259, + "theoretical_loss": 3.38204188834478, + "tokens_seen": 2644180992 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020076228686058176, + "loss": 1.2697, + "theoretical_loss": 3.382035332584466, + "tokens_seen": 2644246528 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020074222668004015, + "loss": 1.2729, + "theoretical_loss": 3.382028777032123, + "tokens_seen": 2644312064 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002007221664994985, + "loss": 1.206, + "theoretical_loss": 3.3820222216877402, + "tokens_seen": 2644377600 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002007021063189569, + "loss": 1.3485, + "theoretical_loss": 3.3820156665513057, + "tokens_seen": 2644443136 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020068204613841526, + "loss": 1.3715, + "theoretical_loss": 3.3820091116228075, + "tokens_seen": 2644508672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020066198595787362, + "loss": 1.2732, + "theoretical_loss": 3.3820025569022336, + "tokens_seen": 2644574208 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200641925777332, + "loss": 1.248, + "theoretical_loss": 3.381996002389573, + "tokens_seen": 2644639744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020062186559679037, + "loss": 1.2418, + "theoretical_loss": 3.3819894480848136, + "tokens_seen": 2644705280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020060180541624876, + "loss": 1.3882, + "theoretical_loss": 3.381982893987943, + "tokens_seen": 2644770816 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020058174523570713, + "loss": 1.263, + "theoretical_loss": 3.3819763400989507, + "tokens_seen": 2644836352 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020056168505516551, + "loss": 1.2397, + "theoretical_loss": 3.381969786417824, + "tokens_seen": 2644901888 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020054162487462388, + "loss": 1.2616, + "theoretical_loss": 3.381963232944552, + "tokens_seen": 2644967424 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020052156469408227, + "loss": 1.2899, + "theoretical_loss": 3.3819566796791216, + "tokens_seen": 2645032960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020050150451354063, + "loss": 1.2283, + "theoretical_loss": 3.381950126621523, + "tokens_seen": 2645098496 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200481444332999, + "loss": 1.2003, + "theoretical_loss": 3.3819435737717423, + "tokens_seen": 2645164032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020046138415245738, + "loss": 1.4152, + "theoretical_loss": 3.3819370211297692, + "tokens_seen": 2645229568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020044132397191574, + "loss": 1.2271, + "theoretical_loss": 3.3819304686955913, + "tokens_seen": 2645295104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020042126379137413, + "loss": 1.3159, + "theoretical_loss": 3.381923916469198, + "tokens_seen": 2645360640 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002004012036108325, + "loss": 1.2204, + "theoretical_loss": 3.381917364450576, + "tokens_seen": 2645426176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020038114343029088, + "loss": 1.2656, + "theoretical_loss": 3.381910812639714, + "tokens_seen": 2645491712 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020036108324974925, + "loss": 1.1801, + "theoretical_loss": 3.381904261036601, + "tokens_seen": 2645557248 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020034102306920763, + "loss": 1.2248, + "theoretical_loss": 3.3818977096412244, + "tokens_seen": 2645622784 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200320962888666, + "loss": 1.2944, + "theoretical_loss": 3.3818911584535734, + "tokens_seen": 2645688320 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020030090270812436, + "loss": 1.207, + "theoretical_loss": 3.3818846074736357, + "tokens_seen": 2645753856 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020028084252758275, + "loss": 1.2354, + "theoretical_loss": 3.381878056701399, + "tokens_seen": 2645819392 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002002607823470411, + "loss": 1.2071, + "theoretical_loss": 3.3818715061368527, + "tokens_seen": 2645884928 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002002407221664995, + "loss": 1.2256, + "theoretical_loss": 3.3818649557799847, + "tokens_seen": 2645950464 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020022066198595786, + "loss": 1.1436, + "theoretical_loss": 3.3818584056307825, + "tokens_seen": 2646016000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020020060180541625, + "loss": 1.2193, + "theoretical_loss": 3.3818518556892356, + "tokens_seen": 2646081536 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 961281, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1973772048950195, + "objective/train/theoretical_loss": 3.3818453059553315, + "objective/train/tokens_used": 1016606176, + "theoretical_loss": 3.3818453059553315, + "tokens_seen": 2646147072 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020018054162487461, + "loss": 1.2111, + "theoretical_loss": 3.3818453059553315, + "tokens_seen": 2646147072 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200160481444333, + "loss": 1.1561, + "theoretical_loss": 3.381838756429058, + "tokens_seen": 2646212608 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020014042126379137, + "loss": 1.2872, + "theoretical_loss": 3.3818322071104046, + "tokens_seen": 2646278144 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020012036108324973, + "loss": 1.3475, + "theoretical_loss": 3.381825657999359, + "tokens_seen": 2646343680 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020010030090270812, + "loss": 1.2583, + "theoretical_loss": 3.3818191090959093, + "tokens_seen": 2646409216 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020008024072216648, + "loss": 1.2005, + "theoretical_loss": 3.381812560400044, + "tokens_seen": 2646474752 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002000601805416249, + "loss": 1.2885, + "theoretical_loss": 3.3818060119117517, + "tokens_seen": 2646540288 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020004012036108326, + "loss": 1.2989, + "theoretical_loss": 3.38179946363102, + "tokens_seen": 2646605824 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020002006018054165, + "loss": 1.2161, + "theoretical_loss": 3.3817929155578375, + "tokens_seen": 2646671360 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002, + "loss": 1.2512, + "theoretical_loss": 3.381786367692192, + "tokens_seen": 2646736896 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001999799398194584, + "loss": 1.2643, + "theoretical_loss": 3.381779820034073, + "tokens_seen": 2646802432 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019995987963891676, + "loss": 1.3153, + "theoretical_loss": 3.3817732725834677, + "tokens_seen": 2646867968 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019993981945837515, + "loss": 1.2022, + "theoretical_loss": 3.381766725340365, + "tokens_seen": 2646933504 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001999197592778335, + "loss": 1.3213, + "theoretical_loss": 3.3817601783047526, + "tokens_seen": 2646999040 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019989969909729187, + "loss": 1.1823, + "theoretical_loss": 3.3817536314766192, + "tokens_seen": 2647064576 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019987963891675026, + "loss": 1.2881, + "theoretical_loss": 3.3817470848559528, + "tokens_seen": 2647130112 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019985957873620863, + "loss": 1.2931, + "theoretical_loss": 3.381740538442742, + "tokens_seen": 2647195648 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019983951855566702, + "loss": 1.2177, + "theoretical_loss": 3.381733992236975, + "tokens_seen": 2647261184 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019981945837512538, + "loss": 1.265, + "theoretical_loss": 3.3817274462386404, + "tokens_seen": 2647326720 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019979939819458377, + "loss": 1.2663, + "theoretical_loss": 3.3817209004477258, + "tokens_seen": 2647392256 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019977933801404213, + "loss": 1.2891, + "theoretical_loss": 3.3817143548642195, + "tokens_seen": 2647457792 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019975927783350052, + "loss": 1.2522, + "theoretical_loss": 3.381707809488111, + "tokens_seen": 2647523328 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019973921765295888, + "loss": 1.2463, + "theoretical_loss": 3.381701264319387, + "tokens_seen": 2647588864 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019971915747241724, + "loss": 1.2642, + "theoretical_loss": 3.3816947193580367, + "tokens_seen": 2647654400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019969909729187563, + "loss": 1.272, + "theoretical_loss": 3.381688174604048, + "tokens_seen": 2647719936 + }, + { + "epoch": 0.6, + "learning_rate": 0.000199679037111334, + "loss": 1.2931, + "theoretical_loss": 3.38168163005741, + "tokens_seen": 2647785472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019965897693079238, + "loss": 1.1697, + "theoretical_loss": 3.3816750857181104, + "tokens_seen": 2647851008 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019963891675025075, + "loss": 1.3464, + "theoretical_loss": 3.381668541586137, + "tokens_seen": 2647916544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019961885656970914, + "loss": 1.1962, + "theoretical_loss": 3.381661997661479, + "tokens_seen": 2647982080 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001995987963891675, + "loss": 1.3695, + "theoretical_loss": 3.3816554539441244, + "tokens_seen": 2648047616 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001995787362086259, + "loss": 1.3091, + "theoretical_loss": 3.3816489104340612, + "tokens_seen": 2648113152 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019955867602808425, + "loss": 1.1998, + "theoretical_loss": 3.381642367131278, + "tokens_seen": 2648178688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019953861584754264, + "loss": 1.2733, + "theoretical_loss": 3.381635824035763, + "tokens_seen": 2648244224 + }, + { + "epoch": 0.61, + "learning_rate": 0.000199518555667001, + "loss": 1.2639, + "theoretical_loss": 3.3816292811475046, + "tokens_seen": 2648309760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019949849548645936, + "loss": 1.2089, + "theoretical_loss": 3.3816227384664908, + "tokens_seen": 2648375296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019947843530591775, + "loss": 1.1908, + "theoretical_loss": 3.3816161959927107, + "tokens_seen": 2648440832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019945837512537612, + "loss": 1.1739, + "theoretical_loss": 3.3816096537261515, + "tokens_seen": 2648506368 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001994383149448345, + "loss": 1.3263, + "theoretical_loss": 3.381603111666802, + "tokens_seen": 2648571904 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019941825476429287, + "loss": 1.2777, + "theoretical_loss": 3.381596569814651, + "tokens_seen": 2648637440 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019939819458375126, + "loss": 1.1741, + "theoretical_loss": 3.3815900281696867, + "tokens_seen": 2648702976 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019937813440320962, + "loss": 1.162, + "theoretical_loss": 3.3815834867318966, + "tokens_seen": 2648768512 + }, + { + "epoch": 0.61, + "learning_rate": 0.000199358074222668, + "loss": 1.2242, + "theoretical_loss": 3.3815769455012696, + "tokens_seen": 2648834048 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019933801404212637, + "loss": 1.2056, + "theoretical_loss": 3.381570404477794, + "tokens_seen": 2648899584 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019931795386158476, + "loss": 1.3062, + "theoretical_loss": 3.381563863661458, + "tokens_seen": 2648965120 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019929789368104315, + "loss": 1.2106, + "theoretical_loss": 3.3815573230522498, + "tokens_seen": 2649030656 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001992778335005015, + "loss": 1.1869, + "theoretical_loss": 3.3815507826501583, + "tokens_seen": 2649096192 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001992577733199599, + "loss": 1.3313, + "theoretical_loss": 3.381544242455171, + "tokens_seen": 2649161728 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019923771313941826, + "loss": 1.1158, + "theoretical_loss": 3.381537702467277, + "tokens_seen": 2649227264 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019921765295887665, + "loss": 1.2932, + "theoretical_loss": 3.3815311626864637, + "tokens_seen": 2649292800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019919759277833501, + "loss": 1.2912, + "theoretical_loss": 3.3815246231127203, + "tokens_seen": 2649358336 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 962524, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.213138461112976, + "objective/train/theoretical_loss": 3.381518083746035, + "objective/train/tokens_used": 1019882976, + "theoretical_loss": 3.381518083746035, + "tokens_seen": 2649423872 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001991775325977934, + "loss": 1.1698, + "theoretical_loss": 3.381518083746035, + "tokens_seen": 2649423872 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019915747241725177, + "loss": 1.3079, + "theoretical_loss": 3.3815115445863952, + "tokens_seen": 2649489408 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019913741223671013, + "loss": 1.2524, + "theoretical_loss": 3.3815050056337905, + "tokens_seen": 2649554944 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019911735205616852, + "loss": 1.2441, + "theoretical_loss": 3.3814984668882087, + "tokens_seen": 2649620480 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019909729187562688, + "loss": 1.138, + "theoretical_loss": 3.3814919283496376, + "tokens_seen": 2649686016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019907723169508527, + "loss": 1.3142, + "theoretical_loss": 3.3814853900180664, + "tokens_seen": 2649751552 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019905717151454363, + "loss": 1.2079, + "theoretical_loss": 3.3814788518934833, + "tokens_seen": 2649817088 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019903711133400202, + "loss": 1.3174, + "theoretical_loss": 3.3814723139758756, + "tokens_seen": 2649882624 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019901705115346038, + "loss": 1.2018, + "theoretical_loss": 3.381465776265233, + "tokens_seen": 2649948160 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019899699097291877, + "loss": 1.2357, + "theoretical_loss": 3.381459238761543, + "tokens_seen": 2650013696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019897693079237713, + "loss": 1.3107, + "theoretical_loss": 3.381452701464794, + "tokens_seen": 2650079232 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019895687061183552, + "loss": 1.2369, + "theoretical_loss": 3.3814461643749745, + "tokens_seen": 2650144768 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019893681043129389, + "loss": 1.3116, + "theoretical_loss": 3.381439627492073, + "tokens_seen": 2650210304 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019891675025075225, + "loss": 1.2234, + "theoretical_loss": 3.3814330908160777, + "tokens_seen": 2650275840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019889669007021064, + "loss": 1.2238, + "theoretical_loss": 3.3814265543469766, + "tokens_seen": 2650341376 + }, + { + "epoch": 0.61, + "learning_rate": 0.000198876629889669, + "loss": 1.3086, + "theoretical_loss": 3.381420018084759, + "tokens_seen": 2650406912 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001988565697091274, + "loss": 1.2966, + "theoretical_loss": 3.3814134820294117, + "tokens_seen": 2650472448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019883650952858575, + "loss": 1.187, + "theoretical_loss": 3.381406946180924, + "tokens_seen": 2650537984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019881644934804414, + "loss": 1.2762, + "theoretical_loss": 3.3814004105392845, + "tokens_seen": 2650603520 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001987963891675025, + "loss": 1.37, + "theoretical_loss": 3.3813938751044814, + "tokens_seen": 2650669056 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001987763289869609, + "loss": 1.2544, + "theoretical_loss": 3.381387339876502, + "tokens_seen": 2650734592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019875626880641925, + "loss": 1.3817, + "theoretical_loss": 3.381380804855336, + "tokens_seen": 2650800128 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019873620862587762, + "loss": 1.1681, + "theoretical_loss": 3.381374270040971, + "tokens_seen": 2650865664 + }, + { + "epoch": 0.61, + "learning_rate": 0.000198716148445336, + "loss": 1.2488, + "theoretical_loss": 3.381367735433396, + "tokens_seen": 2650931200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019869608826479437, + "loss": 1.2115, + "theoretical_loss": 3.381361201032598, + "tokens_seen": 2650996736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019867602808425276, + "loss": 1.1659, + "theoretical_loss": 3.381354666838567, + "tokens_seen": 2651062272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019865596790371112, + "loss": 1.4403, + "theoretical_loss": 3.3813481328512904, + "tokens_seen": 2651127808 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001986359077231695, + "loss": 1.2798, + "theoretical_loss": 3.3813415990707565, + "tokens_seen": 2651193344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019861584754262787, + "loss": 1.2298, + "theoretical_loss": 3.381335065496954, + "tokens_seen": 2651258880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019859578736208626, + "loss": 1.2182, + "theoretical_loss": 3.381328532129871, + "tokens_seen": 2651324416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019857572718154462, + "loss": 1.3436, + "theoretical_loss": 3.3813219989694963, + "tokens_seen": 2651389952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019855566700100304, + "loss": 1.1535, + "theoretical_loss": 3.381315466015818, + "tokens_seen": 2651455488 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001985356068204614, + "loss": 1.2648, + "theoretical_loss": 3.3813089332688238, + "tokens_seen": 2651521024 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001985356068204614, + "loss": 1.3039, + "theoretical_loss": 3.381302400728503, + "tokens_seen": 2651586560 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019851554663991976, + "loss": 1.2558, + "theoretical_loss": 3.3812958683948438, + "tokens_seen": 2651652096 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019849548645937815, + "loss": 1.1892, + "theoretical_loss": 3.381289336267834, + "tokens_seen": 2651717632 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019847542627883652, + "loss": 1.3813, + "theoretical_loss": 3.381282804347462, + "tokens_seen": 2651783168 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001984553660982949, + "loss": 1.275, + "theoretical_loss": 3.3812762726337167, + "tokens_seen": 2651848704 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019843530591775327, + "loss": 1.3222, + "theoretical_loss": 3.3812697411265864, + "tokens_seen": 2651914240 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019841524573721166, + "loss": 1.1559, + "theoretical_loss": 3.381263209826059, + "tokens_seen": 2651979776 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019839518555667002, + "loss": 1.3131, + "theoretical_loss": 3.3812566787321234, + "tokens_seen": 2652045312 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001983751253761284, + "loss": 1.3973, + "theoretical_loss": 3.381250147844767, + "tokens_seen": 2652110848 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019835506519558677, + "loss": 1.2412, + "theoretical_loss": 3.38124361716398, + "tokens_seen": 2652176384 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019833500501504513, + "loss": 1.2847, + "theoretical_loss": 3.3812370866897488, + "tokens_seen": 2652241920 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019831494483450352, + "loss": 1.1661, + "theoretical_loss": 3.3812305564220626, + "tokens_seen": 2652307456 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019829488465396188, + "loss": 1.3592, + "theoretical_loss": 3.38122402636091, + "tokens_seen": 2652372992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019827482447342027, + "loss": 1.2045, + "theoretical_loss": 3.3812174965062787, + "tokens_seen": 2652438528 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019825476429287864, + "loss": 1.2422, + "theoretical_loss": 3.381210966858158, + "tokens_seen": 2652504064 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019823470411233703, + "loss": 1.1748, + "theoretical_loss": 3.3812044374165353, + "tokens_seen": 2652569600 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001982146439317954, + "loss": 1.2377, + "theoretical_loss": 3.3811979081813996, + "tokens_seen": 2652635136 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 963694, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.239452600479126, + "objective/train/theoretical_loss": 3.381191379152739, + "objective/train/tokens_used": 1023159776, + "theoretical_loss": 3.381191379152739, + "tokens_seen": 2652700672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019819458375125378, + "loss": 1.3148, + "theoretical_loss": 3.381191379152739, + "tokens_seen": 2652700672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019817452357071214, + "loss": 1.2615, + "theoretical_loss": 3.381184850330542, + "tokens_seen": 2652766208 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001981544633901705, + "loss": 1.2956, + "theoretical_loss": 3.381178321714797, + "tokens_seen": 2652831744 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001981344032096289, + "loss": 1.2904, + "theoretical_loss": 3.3811717933054917, + "tokens_seen": 2652897280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019811434302908725, + "loss": 1.3746, + "theoretical_loss": 3.3811652651026156, + "tokens_seen": 2652962816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019809428284854564, + "loss": 1.1559, + "theoretical_loss": 3.3811587371061567, + "tokens_seen": 2653028352 + }, + { + "epoch": 0.61, + "learning_rate": 0.000198074222668004, + "loss": 1.4098, + "theoretical_loss": 3.381152209316103, + "tokens_seen": 2653093888 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001980541624874624, + "loss": 1.2169, + "theoretical_loss": 3.3811456817324426, + "tokens_seen": 2653159424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019803410230692076, + "loss": 1.2452, + "theoretical_loss": 3.3811391543551648, + "tokens_seen": 2653224960 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019801404212637915, + "loss": 1.1884, + "theoretical_loss": 3.3811326271842574, + "tokens_seen": 2653290496 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001979939819458375, + "loss": 1.2662, + "theoretical_loss": 3.381126100219709, + "tokens_seen": 2653356032 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001979739217652959, + "loss": 1.3056, + "theoretical_loss": 3.381119573461508, + "tokens_seen": 2653421568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019795386158475426, + "loss": 1.4675, + "theoretical_loss": 3.3811130469096424, + "tokens_seen": 2653487104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019793380140421262, + "loss": 1.2084, + "theoretical_loss": 3.381106520564101, + "tokens_seen": 2653552640 + }, + { + "epoch": 0.61, + "learning_rate": 0.000197913741223671, + "loss": 1.2547, + "theoretical_loss": 3.381099994424872, + "tokens_seen": 2653618176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019789368104312937, + "loss": 1.2575, + "theoretical_loss": 3.381093468491944, + "tokens_seen": 2653683712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019787362086258776, + "loss": 1.2532, + "theoretical_loss": 3.3810869427653047, + "tokens_seen": 2653749248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019785356068204612, + "loss": 1.3338, + "theoretical_loss": 3.3810804172449433, + "tokens_seen": 2653814784 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019783350050150451, + "loss": 1.2655, + "theoretical_loss": 3.381073891930848, + "tokens_seen": 2653880320 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001978134403209629, + "loss": 1.2915, + "theoretical_loss": 3.381067366823007, + "tokens_seen": 2653945856 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001977933801404213, + "loss": 1.2569, + "theoretical_loss": 3.3810608419214088, + "tokens_seen": 2654011392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019777331995987965, + "loss": 1.3045, + "theoretical_loss": 3.381054317226041, + "tokens_seen": 2654076928 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019775325977933802, + "loss": 1.2777, + "theoretical_loss": 3.3810477927368936, + "tokens_seen": 2654142464 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001977331995987964, + "loss": 1.3121, + "theoretical_loss": 3.381041268453954, + "tokens_seen": 2654208000 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019771313941825477, + "loss": 1.3335, + "theoretical_loss": 3.38103474437721, + "tokens_seen": 2654273536 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019769307923771316, + "loss": 1.263, + "theoretical_loss": 3.3810282205066517, + "tokens_seen": 2654339072 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019767301905717152, + "loss": 1.2393, + "theoretical_loss": 3.381021696842266, + "tokens_seen": 2654404608 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001976529588766299, + "loss": 1.3143, + "theoretical_loss": 3.3810151733840415, + "tokens_seen": 2654470144 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019763289869608827, + "loss": 1.2448, + "theoretical_loss": 3.3810086501319674, + "tokens_seen": 2654535680 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019761283851554666, + "loss": 1.3655, + "theoretical_loss": 3.381002127086031, + "tokens_seen": 2654601216 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019759277833500502, + "loss": 1.3713, + "theoretical_loss": 3.380995604246222, + "tokens_seen": 2654666752 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019757271815446339, + "loss": 1.345, + "theoretical_loss": 3.3809890816125274, + "tokens_seen": 2654732288 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019755265797392178, + "loss": 1.2556, + "theoretical_loss": 3.3809825591849365, + "tokens_seen": 2654797824 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019753259779338014, + "loss": 1.3493, + "theoretical_loss": 3.3809760369634376, + "tokens_seen": 2654863360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019751253761283853, + "loss": 1.3571, + "theoretical_loss": 3.380969514948019, + "tokens_seen": 2654928896 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001974924774322969, + "loss": 1.3099, + "theoretical_loss": 3.380962993138669, + "tokens_seen": 2654994432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019747241725175528, + "loss": 1.262, + "theoretical_loss": 3.380956471535376, + "tokens_seen": 2655059968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019745235707121364, + "loss": 1.2547, + "theoretical_loss": 3.3809499501381284, + "tokens_seen": 2655125504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019743229689067203, + "loss": 1.2093, + "theoretical_loss": 3.3809434289469147, + "tokens_seen": 2655191040 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001974122367101304, + "loss": 1.282, + "theoretical_loss": 3.3809369079617233, + "tokens_seen": 2655256576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019739217652958878, + "loss": 1.2582, + "theoretical_loss": 3.3809303871825427, + "tokens_seen": 2655322112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019737211634904714, + "loss": 1.263, + "theoretical_loss": 3.380923866609361, + "tokens_seen": 2655387648 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001973520561685055, + "loss": 1.2264, + "theoretical_loss": 3.3809173462421667, + "tokens_seen": 2655453184 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001973319959879639, + "loss": 1.3503, + "theoretical_loss": 3.3809108260809486, + "tokens_seen": 2655518720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019731193580742226, + "loss": 1.2345, + "theoretical_loss": 3.380904306125695, + "tokens_seen": 2655584256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019729187562688065, + "loss": 1.4298, + "theoretical_loss": 3.380897786376394, + "tokens_seen": 2655649792 + }, + { + "epoch": 0.61, + "learning_rate": 0.000197271815446339, + "loss": 1.3091, + "theoretical_loss": 3.380891266833034, + "tokens_seen": 2655715328 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001972517552657974, + "loss": 1.2336, + "theoretical_loss": 3.3808847474956036, + "tokens_seen": 2655780864 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019723169508525576, + "loss": 1.3105, + "theoretical_loss": 3.3808782283640912, + "tokens_seen": 2655846400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019721163490471415, + "loss": 1.3683, + "theoretical_loss": 3.3808717094384853, + "tokens_seen": 2655911936 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 964907, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.267056941986084, + "objective/train/theoretical_loss": 3.380865190718774, + "objective/train/tokens_used": 1026436576, + "theoretical_loss": 3.380865190718774, + "tokens_seen": 2655977472 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001971915747241725, + "loss": 1.2506, + "theoretical_loss": 3.380865190718774, + "tokens_seen": 2655977472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019717151454363087, + "loss": 1.3529, + "theoretical_loss": 3.380858672204946, + "tokens_seen": 2656043008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019715145436308926, + "loss": 1.2973, + "theoretical_loss": 3.38085215389699, + "tokens_seen": 2656108544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019713139418254763, + "loss": 1.3239, + "theoretical_loss": 3.3808456357948935, + "tokens_seen": 2656174080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019711133400200602, + "loss": 1.2802, + "theoretical_loss": 3.380839117898646, + "tokens_seen": 2656239616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019709127382146438, + "loss": 1.2926, + "theoretical_loss": 3.380832600208235, + "tokens_seen": 2656305152 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001970712136409228, + "loss": 1.297, + "theoretical_loss": 3.380826082723649, + "tokens_seen": 2656370688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019705115346038116, + "loss": 1.2773, + "theoretical_loss": 3.3808195654448774, + "tokens_seen": 2656436224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019703109327983955, + "loss": 1.2835, + "theoretical_loss": 3.3808130483719077, + "tokens_seen": 2656501760 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001970110330992979, + "loss": 1.2787, + "theoretical_loss": 3.380806531504729, + "tokens_seen": 2656567296 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001969909729187563, + "loss": 1.3826, + "theoretical_loss": 3.380800014843329, + "tokens_seen": 2656632832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019697091273821466, + "loss": 1.2839, + "theoretical_loss": 3.380793498387696, + "tokens_seen": 2656698368 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019695085255767302, + "loss": 1.3612, + "theoretical_loss": 3.3807869821378196, + "tokens_seen": 2656763904 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001969307923771314, + "loss": 1.263, + "theoretical_loss": 3.380780466093687, + "tokens_seen": 2656829440 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019691073219658977, + "loss": 1.3475, + "theoretical_loss": 3.3807739502552874, + "tokens_seen": 2656894976 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019689067201604816, + "loss": 1.3144, + "theoretical_loss": 3.380767434622609, + "tokens_seen": 2656960512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019687061183550652, + "loss": 1.1881, + "theoretical_loss": 3.38076091919564, + "tokens_seen": 2657026048 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019685055165496491, + "loss": 1.2939, + "theoretical_loss": 3.380754403974369, + "tokens_seen": 2657091584 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019683049147442328, + "loss": 1.34, + "theoretical_loss": 3.3807478889587848, + "tokens_seen": 2657157120 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019681043129388167, + "loss": 1.4229, + "theoretical_loss": 3.380741374148875, + "tokens_seen": 2657222656 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019679037111334003, + "loss": 1.2202, + "theoretical_loss": 3.3807348595446287, + "tokens_seen": 2657288192 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001967703109327984, + "loss": 1.2942, + "theoretical_loss": 3.3807283451460344, + "tokens_seen": 2657353728 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019675025075225678, + "loss": 1.2736, + "theoretical_loss": 3.38072183095308, + "tokens_seen": 2657419264 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019673019057171514, + "loss": 1.3408, + "theoretical_loss": 3.3807153169657544, + "tokens_seen": 2657484800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019671013039117353, + "loss": 1.3451, + "theoretical_loss": 3.3807088031840458, + "tokens_seen": 2657550336 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001966900702106319, + "loss": 1.2663, + "theoretical_loss": 3.380702289607943, + "tokens_seen": 2657615872 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019667001003009028, + "loss": 1.2517, + "theoretical_loss": 3.3806957762374337, + "tokens_seen": 2657681408 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019664994984954865, + "loss": 1.3547, + "theoretical_loss": 3.3806892630725067, + "tokens_seen": 2657746944 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019662988966900703, + "loss": 1.3876, + "theoretical_loss": 3.380682750113151, + "tokens_seen": 2657812480 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001966098294884654, + "loss": 1.2723, + "theoretical_loss": 3.380676237359354, + "tokens_seen": 2657878016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019658976930792376, + "loss": 1.1649, + "theoretical_loss": 3.3806697248111055, + "tokens_seen": 2657943552 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019656970912738215, + "loss": 1.243, + "theoretical_loss": 3.3806632124683924, + "tokens_seen": 2658009088 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001965496489468405, + "loss": 1.185, + "theoretical_loss": 3.3806567003312047, + "tokens_seen": 2658074624 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001965295887662989, + "loss": 1.1815, + "theoretical_loss": 3.3806501883995295, + "tokens_seen": 2658140160 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019650952858575726, + "loss": 1.321, + "theoretical_loss": 3.3806436766733556, + "tokens_seen": 2658205696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019648946840521565, + "loss": 1.3233, + "theoretical_loss": 3.380637165152672, + "tokens_seen": 2658271232 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019646940822467401, + "loss": 1.251, + "theoretical_loss": 3.380630653837467, + "tokens_seen": 2658336768 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001964493480441324, + "loss": 1.3714, + "theoretical_loss": 3.3806241427277284, + "tokens_seen": 2658402304 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019642928786359077, + "loss": 1.3176, + "theoretical_loss": 3.3806176318234455, + "tokens_seen": 2658467840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019640922768304913, + "loss": 1.2722, + "theoretical_loss": 3.3806111211246064, + "tokens_seen": 2658533376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019638916750250752, + "loss": 1.2552, + "theoretical_loss": 3.380604610631199, + "tokens_seen": 2658598912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019636910732196588, + "loss": 1.3284, + "theoretical_loss": 3.380598100343213, + "tokens_seen": 2658664448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019634904714142427, + "loss": 1.3202, + "theoretical_loss": 3.3805915902606354, + "tokens_seen": 2658729984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019632898696088263, + "loss": 1.2703, + "theoretical_loss": 3.3805850803834554, + "tokens_seen": 2658795520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019630892678034105, + "loss": 1.2509, + "theoretical_loss": 3.380578570711662, + "tokens_seen": 2658861056 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001962888665997994, + "loss": 1.3554, + "theoretical_loss": 3.3805720612452426, + "tokens_seen": 2658926592 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001962688064192578, + "loss": 1.3777, + "theoretical_loss": 3.380565551984186, + "tokens_seen": 2658992128 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019624874623871616, + "loss": 1.3427, + "theoretical_loss": 3.3805590429284815, + "tokens_seen": 2659057664 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019622868605817455, + "loss": 1.216, + "theoretical_loss": 3.3805525340781166, + "tokens_seen": 2659123200 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001962086258776329, + "loss": 1.2495, + "theoretical_loss": 3.38054602543308, + "tokens_seen": 2659188736 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 966064, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.339836597442627, + "objective/train/theoretical_loss": 3.38053951699336, + "objective/train/tokens_used": 1029713376, + "theoretical_loss": 3.38053951699336, + "tokens_seen": 2659254272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019618856569709127, + "loss": 1.3582, + "theoretical_loss": 3.38053951699336, + "tokens_seen": 2659254272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019616850551654966, + "loss": 1.1905, + "theoretical_loss": 3.3805330087589454, + "tokens_seen": 2659319808 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019614844533600803, + "loss": 1.1893, + "theoretical_loss": 3.3805265007298244, + "tokens_seen": 2659385344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019612838515546642, + "loss": 1.3688, + "theoretical_loss": 3.380519992905986, + "tokens_seen": 2659450880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019610832497492478, + "loss": 1.3907, + "theoretical_loss": 3.380513485287418, + "tokens_seen": 2659516416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019608826479438317, + "loss": 1.223, + "theoretical_loss": 3.380506977874109, + "tokens_seen": 2659581952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019606820461384153, + "loss": 1.3153, + "theoretical_loss": 3.3805004706660475, + "tokens_seen": 2659647488 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019604814443329992, + "loss": 1.156, + "theoretical_loss": 3.3804939636632225, + "tokens_seen": 2659713024 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019602808425275828, + "loss": 1.2414, + "theoretical_loss": 3.3804874568656214, + "tokens_seen": 2659778560 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019600802407221664, + "loss": 1.2743, + "theoretical_loss": 3.380480950273234, + "tokens_seen": 2659844096 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019598796389167503, + "loss": 1.2559, + "theoretical_loss": 3.3804744438860475, + "tokens_seen": 2659909632 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001959679037111334, + "loss": 1.3221, + "theoretical_loss": 3.380467937704051, + "tokens_seen": 2659975168 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019594784353059178, + "loss": 1.2654, + "theoretical_loss": 3.3804614317272335, + "tokens_seen": 2660040704 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019592778335005015, + "loss": 1.1443, + "theoretical_loss": 3.3804549259555823, + "tokens_seen": 2660106240 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019590772316950854, + "loss": 1.2079, + "theoretical_loss": 3.3804484203890866, + "tokens_seen": 2660171776 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001958876629889669, + "loss": 1.339, + "theoretical_loss": 3.3804419150277347, + "tokens_seen": 2660237312 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001958676028084253, + "loss": 1.3511, + "theoretical_loss": 3.3804354098715153, + "tokens_seen": 2660302848 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019584754262788365, + "loss": 1.2761, + "theoretical_loss": 3.3804289049204166, + "tokens_seen": 2660368384 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019582748244734204, + "loss": 1.2553, + "theoretical_loss": 3.380422400174427, + "tokens_seen": 2660433920 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001958074222668004, + "loss": 1.3845, + "theoretical_loss": 3.3804158956335355, + "tokens_seen": 2660499456 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019578736208625876, + "loss": 1.2584, + "theoretical_loss": 3.3804093912977304, + "tokens_seen": 2660564992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019576730190571715, + "loss": 1.148, + "theoretical_loss": 3.3804028871669995, + "tokens_seen": 2660630528 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019574724172517552, + "loss": 1.3555, + "theoretical_loss": 3.380396383241332, + "tokens_seen": 2660696064 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001957271815446339, + "loss": 1.1839, + "theoretical_loss": 3.3803898795207163, + "tokens_seen": 2660761600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019570712136409227, + "loss": 1.2782, + "theoretical_loss": 3.380383376005141, + "tokens_seen": 2660827136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019568706118355066, + "loss": 1.3598, + "theoretical_loss": 3.380376872694594, + "tokens_seen": 2660892672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019566700100300902, + "loss": 1.2497, + "theoretical_loss": 3.380370369589064, + "tokens_seen": 2660958208 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001956469408224674, + "loss": 1.2487, + "theoretical_loss": 3.3803638666885396, + "tokens_seen": 2661023744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019562688064192577, + "loss": 1.2954, + "theoretical_loss": 3.38035736399301, + "tokens_seen": 2661089280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019560682046138413, + "loss": 1.2797, + "theoretical_loss": 3.3803508615024627, + "tokens_seen": 2661154816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019558676028084252, + "loss": 1.2637, + "theoretical_loss": 3.380344359216886, + "tokens_seen": 2661220352 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001955667001003009, + "loss": 1.3957, + "theoretical_loss": 3.3803378571362694, + "tokens_seen": 2661285888 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001955466399197593, + "loss": 1.1916, + "theoretical_loss": 3.380331355260601, + "tokens_seen": 2661351424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019552657973921766, + "loss": 1.2735, + "theoretical_loss": 3.380324853589869, + "tokens_seen": 2661416960 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019550651955867605, + "loss": 1.2175, + "theoretical_loss": 3.380318352124062, + "tokens_seen": 2661482496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019548645937813441, + "loss": 1.2067, + "theoretical_loss": 3.3803118508631687, + "tokens_seen": 2661548032 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001954663991975928, + "loss": 1.2864, + "theoretical_loss": 3.3803053498071773, + "tokens_seen": 2661613568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019544633901705117, + "loss": 1.3328, + "theoretical_loss": 3.3802988489560764, + "tokens_seen": 2661679104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019542627883650953, + "loss": 1.1952, + "theoretical_loss": 3.3802923483098546, + "tokens_seen": 2661744640 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019540621865596792, + "loss": 1.3026, + "theoretical_loss": 3.3802858478685005, + "tokens_seen": 2661810176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019538615847542628, + "loss": 1.2715, + "theoretical_loss": 3.3802793476320026, + "tokens_seen": 2661875712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019536609829488467, + "loss": 1.2725, + "theoretical_loss": 3.380272847600349, + "tokens_seen": 2661941248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019534603811434303, + "loss": 1.2106, + "theoretical_loss": 3.380266347773529, + "tokens_seen": 2662006784 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019532597793380142, + "loss": 1.2455, + "theoretical_loss": 3.38025984815153, + "tokens_seen": 2662072320 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019530591775325978, + "loss": 1.1722, + "theoretical_loss": 3.3802533487343407, + "tokens_seen": 2662137856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019528585757271817, + "loss": 1.3761, + "theoretical_loss": 3.380246849521951, + "tokens_seen": 2662203392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019526579739217653, + "loss": 1.2397, + "theoretical_loss": 3.380240350514348, + "tokens_seen": 2662268928 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019524573721163492, + "loss": 1.2704, + "theoretical_loss": 3.38023385171152, + "tokens_seen": 2662334464 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019522567703109329, + "loss": 1.3713, + "theoretical_loss": 3.3802273531134563, + "tokens_seen": 2662400000 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019520561685055165, + "loss": 1.2335, + "theoretical_loss": 3.3802208547201458, + "tokens_seen": 2662465536 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 967352, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4056241512298584, + "objective/train/theoretical_loss": 3.3802143565315763, + "objective/train/tokens_used": 1032990176, + "theoretical_loss": 3.3802143565315763, + "tokens_seen": 2662531072 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019518555667001004, + "loss": 1.3436, + "theoretical_loss": 3.3802143565315763, + "tokens_seen": 2662531072 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001951654964894684, + "loss": 1.2872, + "theoretical_loss": 3.380207858547736, + "tokens_seen": 2662596608 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001951454363089268, + "loss": 1.1963, + "theoretical_loss": 3.380201360768614, + "tokens_seen": 2662662144 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019512537612838515, + "loss": 1.2783, + "theoretical_loss": 3.3801948631941987, + "tokens_seen": 2662727680 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019510531594784354, + "loss": 1.3089, + "theoretical_loss": 3.3801883658244782, + "tokens_seen": 2662793216 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001950852557673019, + "loss": 1.2035, + "theoretical_loss": 3.380181868659442, + "tokens_seen": 2662858752 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001950651955867603, + "loss": 1.2592, + "theoretical_loss": 3.3801753716990777, + "tokens_seen": 2662924288 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019504513540621865, + "loss": 1.2461, + "theoretical_loss": 3.380168874943374, + "tokens_seen": 2662989824 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019502507522567702, + "loss": 1.3318, + "theoretical_loss": 3.3801623783923196, + "tokens_seen": 2663055360 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001950050150451354, + "loss": 1.2828, + "theoretical_loss": 3.380155882045903, + "tokens_seen": 2663120896 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019498495486459377, + "loss": 1.3562, + "theoretical_loss": 3.3801493859041125, + "tokens_seen": 2663186432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019496489468405216, + "loss": 1.3365, + "theoretical_loss": 3.380142889966937, + "tokens_seen": 2663251968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019494483450351052, + "loss": 1.2243, + "theoretical_loss": 3.3801363942343645, + "tokens_seen": 2663317504 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001949247743229689, + "loss": 1.1459, + "theoretical_loss": 3.380129898706384, + "tokens_seen": 2663383040 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019490471414242727, + "loss": 1.2395, + "theoretical_loss": 3.3801234033829837, + "tokens_seen": 2663448576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019488465396188566, + "loss": 1.2678, + "theoretical_loss": 3.3801169082641525, + "tokens_seen": 2663514112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019486459378134402, + "loss": 1.2651, + "theoretical_loss": 3.3801104133498785, + "tokens_seen": 2663579648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019484453360080239, + "loss": 1.2437, + "theoretical_loss": 3.3801039186401507, + "tokens_seen": 2663645184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019482447342026077, + "loss": 1.1319, + "theoretical_loss": 3.3800974241349566, + "tokens_seen": 2663710720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019480441323971916, + "loss": 1.4213, + "theoretical_loss": 3.380090929834286, + "tokens_seen": 2663776256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019478435305917755, + "loss": 1.313, + "theoretical_loss": 3.380084435738127, + "tokens_seen": 2663841792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019476429287863592, + "loss": 1.2526, + "theoretical_loss": 3.380077941846468, + "tokens_seen": 2663907328 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001947442326980943, + "loss": 1.3835, + "theoretical_loss": 3.3800714481592973, + "tokens_seen": 2663972864 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019472417251755267, + "loss": 1.2958, + "theoretical_loss": 3.3800649546766035, + "tokens_seen": 2664038400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019470411233701106, + "loss": 1.2478, + "theoretical_loss": 3.380058461398376, + "tokens_seen": 2664103936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019468405215646942, + "loss": 1.2686, + "theoretical_loss": 3.380051968324602, + "tokens_seen": 2664169472 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001946639919759278, + "loss": 1.1908, + "theoretical_loss": 3.380045475455271, + "tokens_seen": 2664235008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019464393179538617, + "loss": 1.2109, + "theoretical_loss": 3.380038982790371, + "tokens_seen": 2664300544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019462387161484453, + "loss": 1.2909, + "theoretical_loss": 3.380032490329891, + "tokens_seen": 2664366080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019460381143430292, + "loss": 1.4067, + "theoretical_loss": 3.380025998073819, + "tokens_seen": 2664431616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019458375125376128, + "loss": 1.2901, + "theoretical_loss": 3.380019506022144, + "tokens_seen": 2664497152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019456369107321967, + "loss": 1.3075, + "theoretical_loss": 3.380013014174854, + "tokens_seen": 2664562688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019454363089267804, + "loss": 1.2623, + "theoretical_loss": 3.3800065225319385, + "tokens_seen": 2664628224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019452357071213642, + "loss": 1.2744, + "theoretical_loss": 3.380000031093385, + "tokens_seen": 2664693760 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001945035105315948, + "loss": 1.2315, + "theoretical_loss": 3.379993539859183, + "tokens_seen": 2664759296 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019448345035105318, + "loss": 1.3334, + "theoretical_loss": 3.37998704882932, + "tokens_seen": 2664824832 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019446339017051154, + "loss": 1.3622, + "theoretical_loss": 3.379980558003785, + "tokens_seen": 2664890368 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001944433299899699, + "loss": 1.3244, + "theoretical_loss": 3.379974067382567, + "tokens_seen": 2664955904 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001944232698094283, + "loss": 1.3377, + "theoretical_loss": 3.3799675769656536, + "tokens_seen": 2665021440 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019440320962888665, + "loss": 1.2742, + "theoretical_loss": 3.3799610867530343, + "tokens_seen": 2665086976 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019438314944834504, + "loss": 1.3016, + "theoretical_loss": 3.379954596744697, + "tokens_seen": 2665152512 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001943630892678034, + "loss": 1.2693, + "theoretical_loss": 3.3799481069406307, + "tokens_seen": 2665218048 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001943430290872618, + "loss": 1.2442, + "theoretical_loss": 3.3799416173408234, + "tokens_seen": 2665283584 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019432296890672016, + "loss": 1.3383, + "theoretical_loss": 3.3799351279452647, + "tokens_seen": 2665349120 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019430290872617855, + "loss": 1.3045, + "theoretical_loss": 3.379928638753942, + "tokens_seen": 2665414656 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001942828485456369, + "loss": 1.324, + "theoretical_loss": 3.379922149766844, + "tokens_seen": 2665480192 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001942627883650953, + "loss": 1.2815, + "theoretical_loss": 3.37991566098396, + "tokens_seen": 2665545728 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019424272818455366, + "loss": 1.2694, + "theoretical_loss": 3.3799091724052777, + "tokens_seen": 2665611264 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019422266800401202, + "loss": 1.3483, + "theoretical_loss": 3.379902684030786, + "tokens_seen": 2665676800 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001942026078234704, + "loss": 1.2199, + "theoretical_loss": 3.379896195860474, + "tokens_seen": 2665742336 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 968517, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1904771327972412, + "objective/train/theoretical_loss": 3.3798897078943293, + "objective/train/tokens_used": 1036266976, + "theoretical_loss": 3.3798897078943293, + "tokens_seen": 2665807872 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019418254764292877, + "loss": 1.1881, + "theoretical_loss": 3.3798897078943293, + "tokens_seen": 2665807872 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019416248746238716, + "loss": 1.3074, + "theoretical_loss": 3.379883220132341, + "tokens_seen": 2665873408 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019414242728184552, + "loss": 1.2525, + "theoretical_loss": 3.379876732574498, + "tokens_seen": 2665938944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019412236710130391, + "loss": 1.2101, + "theoretical_loss": 3.3798702452207876, + "tokens_seen": 2666004480 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019410230692076228, + "loss": 1.3973, + "theoretical_loss": 3.3798637580712, + "tokens_seen": 2666070016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019408224674022067, + "loss": 1.2646, + "theoretical_loss": 3.379857271125722, + "tokens_seen": 2666135552 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019406218655967905, + "loss": 1.2617, + "theoretical_loss": 3.379850784384344, + "tokens_seen": 2666201088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019404212637913742, + "loss": 1.3228, + "theoretical_loss": 3.379844297847053, + "tokens_seen": 2666266624 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001940220661985958, + "loss": 1.4132, + "theoretical_loss": 3.379837811513839, + "tokens_seen": 2666332160 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019400200601805417, + "loss": 1.3857, + "theoretical_loss": 3.3798313253846892, + "tokens_seen": 2666397696 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019398194583751256, + "loss": 1.2102, + "theoretical_loss": 3.3798248394595927, + "tokens_seen": 2666463232 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019396188565697092, + "loss": 1.3052, + "theoretical_loss": 3.3798183537385382, + "tokens_seen": 2666528768 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001939418254764293, + "loss": 1.3127, + "theoretical_loss": 3.3798118682215144, + "tokens_seen": 2666594304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019392176529588767, + "loss": 1.3574, + "theoretical_loss": 3.3798053829085095, + "tokens_seen": 2666659840 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019390170511534606, + "loss": 1.2186, + "theoretical_loss": 3.3797988977995126, + "tokens_seen": 2666725376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019388164493480442, + "loss": 1.2625, + "theoretical_loss": 3.3797924128945116, + "tokens_seen": 2666790912 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019386158475426279, + "loss": 1.1881, + "theoretical_loss": 3.379785928193495, + "tokens_seen": 2666856448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019384152457372117, + "loss": 1.294, + "theoretical_loss": 3.379779443696452, + "tokens_seen": 2666921984 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019382146439317954, + "loss": 1.2021, + "theoretical_loss": 3.379772959403371, + "tokens_seen": 2666987520 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019380140421263793, + "loss": 1.2596, + "theoretical_loss": 3.3797664753142405, + "tokens_seen": 2667053056 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001937813440320963, + "loss": 1.1942, + "theoretical_loss": 3.379759991429049, + "tokens_seen": 2667118592 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019376128385155468, + "loss": 1.4046, + "theoretical_loss": 3.379753507747785, + "tokens_seen": 2667184128 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019374122367101304, + "loss": 1.346, + "theoretical_loss": 3.3797470242704373, + "tokens_seen": 2667249664 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019372116349047143, + "loss": 1.1822, + "theoretical_loss": 3.3797405409969947, + "tokens_seen": 2667315200 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001937011033099298, + "loss": 1.2917, + "theoretical_loss": 3.379734057927445, + "tokens_seen": 2667380736 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019368104312938818, + "loss": 1.2507, + "theoretical_loss": 3.3797275750617772, + "tokens_seen": 2667446272 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019366098294884654, + "loss": 1.2957, + "theoretical_loss": 3.37972109239998, + "tokens_seen": 2667511808 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001936409227683049, + "loss": 1.3227, + "theoretical_loss": 3.379714609942042, + "tokens_seen": 2667577344 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001936208625877633, + "loss": 1.2974, + "theoretical_loss": 3.3797081276879517, + "tokens_seen": 2667642880 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019360080240722166, + "loss": 1.2986, + "theoretical_loss": 3.379701645637698, + "tokens_seen": 2667708416 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019358074222668005, + "loss": 1.3659, + "theoretical_loss": 3.3796951637912684, + "tokens_seen": 2667773952 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001935606820461384, + "loss": 1.3245, + "theoretical_loss": 3.3796886821486525, + "tokens_seen": 2667839488 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001935406218655968, + "loss": 1.3009, + "theoretical_loss": 3.379682200709839, + "tokens_seen": 2667905024 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019352056168505516, + "loss": 1.1695, + "theoretical_loss": 3.3796757194748155, + "tokens_seen": 2667970560 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019350050150451355, + "loss": 1.1743, + "theoretical_loss": 3.3796692384435714, + "tokens_seen": 2668036096 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001934804413239719, + "loss": 1.2831, + "theoretical_loss": 3.379662757616095, + "tokens_seen": 2668101632 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019346038114343027, + "loss": 1.328, + "theoretical_loss": 3.379656276992375, + "tokens_seen": 2668167168 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019344032096288866, + "loss": 1.2585, + "theoretical_loss": 3.3796497965724, + "tokens_seen": 2668232704 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019342026078234703, + "loss": 1.2463, + "theoretical_loss": 3.3796433163561588, + "tokens_seen": 2668298240 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019340020060180542, + "loss": 1.3478, + "theoretical_loss": 3.3796368363436393, + "tokens_seen": 2668363776 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019338014042126378, + "loss": 1.2647, + "theoretical_loss": 3.3796303565348307, + "tokens_seen": 2668429312 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019336008024072217, + "loss": 1.261, + "theoretical_loss": 3.3796238769297213, + "tokens_seen": 2668494848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019334002006018053, + "loss": 1.1797, + "theoretical_loss": 3.3796173975282997, + "tokens_seen": 2668560384 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019331995987963892, + "loss": 1.2988, + "theoretical_loss": 3.379610918330555, + "tokens_seen": 2668625920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001932998996990973, + "loss": 1.321, + "theoretical_loss": 3.3796044393364753, + "tokens_seen": 2668691456 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001932798395185557, + "loss": 1.3668, + "theoretical_loss": 3.3795979605460493, + "tokens_seen": 2668756992 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019325977933801406, + "loss": 1.1865, + "theoretical_loss": 3.3795914819592654, + "tokens_seen": 2668822528 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019323971915747242, + "loss": 1.151, + "theoretical_loss": 3.3795850035761124, + "tokens_seen": 2668888064 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001932196589769308, + "loss": 1.3627, + "theoretical_loss": 3.379578525396579, + "tokens_seen": 2668953600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019319959879638917, + "loss": 1.1747, + "theoretical_loss": 3.3795720474206536, + "tokens_seen": 2669019136 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 969669, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2841310501098633, + "objective/train/theoretical_loss": 3.379565569648325, + "objective/train/tokens_used": 1039543776, + "theoretical_loss": 3.379565569648325, + "tokens_seen": 2669084672 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019317953861584756, + "loss": 1.2036, + "theoretical_loss": 3.379565569648325, + "tokens_seen": 2669084672 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019315947843530592, + "loss": 1.1719, + "theoretical_loss": 3.3795590920795817, + "tokens_seen": 2669150208 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019313941825476431, + "loss": 1.3, + "theoretical_loss": 3.379552614714412, + "tokens_seen": 2669215744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019311935807422268, + "loss": 1.2812, + "theoretical_loss": 3.379546137552805, + "tokens_seen": 2669281280 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019309929789368107, + "loss": 1.2456, + "theoretical_loss": 3.3795396605947494, + "tokens_seen": 2669346816 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019307923771313943, + "loss": 1.2806, + "theoretical_loss": 3.379533183840233, + "tokens_seen": 2669412352 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001930591775325978, + "loss": 1.2262, + "theoretical_loss": 3.379526707289245, + "tokens_seen": 2669477888 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019303911735205618, + "loss": 1.3211, + "theoretical_loss": 3.3795202309417745, + "tokens_seen": 2669543424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019301905717151454, + "loss": 1.2184, + "theoretical_loss": 3.3795137547978094, + "tokens_seen": 2669608960 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019299899699097293, + "loss": 1.2998, + "theoretical_loss": 3.379507278857338, + "tokens_seen": 2669674496 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001929789368104313, + "loss": 1.3029, + "theoretical_loss": 3.3795008031203495, + "tokens_seen": 2669740032 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019295887662988968, + "loss": 1.155, + "theoretical_loss": 3.3794943275868325, + "tokens_seen": 2669805568 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019293881644934804, + "loss": 1.2162, + "theoretical_loss": 3.3794878522567755, + "tokens_seen": 2669871104 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019291875626880643, + "loss": 1.4354, + "theoretical_loss": 3.379481377130167, + "tokens_seen": 2669936640 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928986960882648, + "loss": 1.3583, + "theoretical_loss": 3.3794749022069954, + "tokens_seen": 2670002176 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019287863590772316, + "loss": 1.1993, + "theoretical_loss": 3.37946842748725, + "tokens_seen": 2670067712 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019285857572718155, + "loss": 1.3174, + "theoretical_loss": 3.379461952970919, + "tokens_seen": 2670133248 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928385155466399, + "loss": 1.3602, + "theoretical_loss": 3.3794554786579907, + "tokens_seen": 2670198784 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928184553660983, + "loss": 1.274, + "theoretical_loss": 3.3794490045484546, + "tokens_seen": 2670264320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019279839518555666, + "loss": 1.2479, + "theoretical_loss": 3.3794425306422986, + "tokens_seen": 2670329856 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019277833500501505, + "loss": 1.1793, + "theoretical_loss": 3.3794360569395114, + "tokens_seen": 2670395392 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001927582748244734, + "loss": 1.3215, + "theoretical_loss": 3.379429583440082, + "tokens_seen": 2670460928 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001927382146439318, + "loss": 1.2288, + "theoretical_loss": 3.3794231101439984, + "tokens_seen": 2670526464 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019271815446339017, + "loss": 1.2267, + "theoretical_loss": 3.37941663705125, + "tokens_seen": 2670592000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019269809428284853, + "loss": 1.2771, + "theoretical_loss": 3.3794101641618246, + "tokens_seen": 2670657536 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019267803410230692, + "loss": 1.2863, + "theoretical_loss": 3.3794036914757113, + "tokens_seen": 2670723072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019265797392176528, + "loss": 1.3475, + "theoretical_loss": 3.379397218992899, + "tokens_seen": 2670788608 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019263791374122367, + "loss": 1.2057, + "theoretical_loss": 3.3793907467133755, + "tokens_seen": 2670854144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019261785356068203, + "loss": 1.2344, + "theoretical_loss": 3.3793842746371303, + "tokens_seen": 2670919680 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019259779338014042, + "loss": 1.275, + "theoretical_loss": 3.3793778027641515, + "tokens_seen": 2670985216 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019257773319959878, + "loss": 1.1845, + "theoretical_loss": 3.3793713310944278, + "tokens_seen": 2671050752 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001925576730190572, + "loss": 1.3653, + "theoretical_loss": 3.3793648596279477, + "tokens_seen": 2671116288 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019253761283851556, + "loss": 1.2878, + "theoretical_loss": 3.3793583883647003, + "tokens_seen": 2671181824 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019251755265797395, + "loss": 1.3495, + "theoretical_loss": 3.3793519173046738, + "tokens_seen": 2671247360 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001924974924774323, + "loss": 1.2718, + "theoretical_loss": 3.379345446447857, + "tokens_seen": 2671312896 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019247743229689067, + "loss": 1.2726, + "theoretical_loss": 3.379338975794239, + "tokens_seen": 2671378432 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019245737211634906, + "loss": 1.3146, + "theoretical_loss": 3.3793325053438075, + "tokens_seen": 2671443968 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019243731193580743, + "loss": 1.3239, + "theoretical_loss": 3.3793260350965513, + "tokens_seen": 2671509504 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019241725175526582, + "loss": 1.3136, + "theoretical_loss": 3.3793195650524597, + "tokens_seen": 2671575040 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019239719157472418, + "loss": 1.2227, + "theoretical_loss": 3.379313095211521, + "tokens_seen": 2671640576 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019237713139418257, + "loss": 1.2616, + "theoretical_loss": 3.3793066255737236, + "tokens_seen": 2671706112 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019235707121364093, + "loss": 1.2453, + "theoretical_loss": 3.3793001561390565, + "tokens_seen": 2671771648 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019233701103309932, + "loss": 1.3529, + "theoretical_loss": 3.3792936869075083, + "tokens_seen": 2671837184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019231695085255768, + "loss": 1.2903, + "theoretical_loss": 3.3792872178790674, + "tokens_seen": 2671902720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019229689067201604, + "loss": 1.2907, + "theoretical_loss": 3.379280749053722, + "tokens_seen": 2671968256 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019227683049147443, + "loss": 1.2259, + "theoretical_loss": 3.379274280431462, + "tokens_seen": 2672033792 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001922567703109328, + "loss": 1.2363, + "theoretical_loss": 3.379267812012275, + "tokens_seen": 2672099328 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019223671013039118, + "loss": 1.3612, + "theoretical_loss": 3.3792613437961503, + "tokens_seen": 2672164864 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019221664994984955, + "loss": 1.2963, + "theoretical_loss": 3.3792548757830763, + "tokens_seen": 2672230400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019219658976930794, + "loss": 1.2402, + "theoretical_loss": 3.3792484079730416, + "tokens_seen": 2672295936 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 970931, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2702627182006836, + "objective/train/theoretical_loss": 3.3792419403660343, + "objective/train/tokens_used": 1042820576, + "theoretical_loss": 3.3792419403660343, + "tokens_seen": 2672361472 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001921765295887663, + "loss": 1.2611, + "theoretical_loss": 3.3792419403660343, + "tokens_seen": 2672361472 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001921564694082247, + "loss": 1.291, + "theoretical_loss": 3.379235472962044, + "tokens_seen": 2672427008 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019213640922768305, + "loss": 1.3085, + "theoretical_loss": 3.379229005761059, + "tokens_seen": 2672492544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019211634904714144, + "loss": 1.201, + "theoretical_loss": 3.3792225387630674, + "tokens_seen": 2672558080 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001920962888665998, + "loss": 1.3575, + "theoretical_loss": 3.3792160719680586, + "tokens_seen": 2672623616 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019207622868605816, + "loss": 1.3064, + "theoretical_loss": 3.379209605376021, + "tokens_seen": 2672689152 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019205616850551655, + "loss": 1.2878, + "theoretical_loss": 3.3792031389869432, + "tokens_seen": 2672754688 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019203610832497491, + "loss": 1.2692, + "theoretical_loss": 3.379196672800814, + "tokens_seen": 2672820224 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001920160481444333, + "loss": 1.2673, + "theoretical_loss": 3.379190206817622, + "tokens_seen": 2672885760 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019199598796389167, + "loss": 1.2377, + "theoretical_loss": 3.3791837410373553, + "tokens_seen": 2672951296 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019197592778335006, + "loss": 1.2548, + "theoretical_loss": 3.3791772754600036, + "tokens_seen": 2673016832 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019195586760280842, + "loss": 1.4395, + "theoretical_loss": 3.379170810085555, + "tokens_seen": 2673082368 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001919358074222668, + "loss": 1.2573, + "theoretical_loss": 3.379164344913998, + "tokens_seen": 2673147904 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019191574724172517, + "loss": 1.2524, + "theoretical_loss": 3.3791578799453212, + "tokens_seen": 2673213440 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019189568706118353, + "loss": 1.28, + "theoretical_loss": 3.3791514151795137, + "tokens_seen": 2673278976 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019187562688064192, + "loss": 1.3391, + "theoretical_loss": 3.3791449506165643, + "tokens_seen": 2673344512 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019185556670010028, + "loss": 1.3871, + "theoretical_loss": 3.3791384862564606, + "tokens_seen": 2673410048 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019183550651955867, + "loss": 1.3004, + "theoretical_loss": 3.3791320220991925, + "tokens_seen": 2673475584 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019181544633901706, + "loss": 1.3787, + "theoretical_loss": 3.379125558144748, + "tokens_seen": 2673541120 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019179538615847545, + "loss": 1.29, + "theoretical_loss": 3.3791190943931158, + "tokens_seen": 2673606656 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019177532597793381, + "loss": 1.3001, + "theoretical_loss": 3.3791126308442845, + "tokens_seen": 2673672192 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001917552657973922, + "loss": 1.2542, + "theoretical_loss": 3.3791061674982434, + "tokens_seen": 2673737728 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019173520561685057, + "loss": 1.322, + "theoretical_loss": 3.3790997043549806, + "tokens_seen": 2673803264 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019171514543630893, + "loss": 1.3317, + "theoretical_loss": 3.3790932414144845, + "tokens_seen": 2673868800 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019169508525576732, + "loss": 1.2408, + "theoretical_loss": 3.3790867786767445, + "tokens_seen": 2673934336 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019167502507522568, + "loss": 1.3049, + "theoretical_loss": 3.379080316141749, + "tokens_seen": 2673999872 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019165496489468407, + "loss": 1.2413, + "theoretical_loss": 3.379073853809486, + "tokens_seen": 2674065408 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019163490471414243, + "loss": 1.329, + "theoretical_loss": 3.3790673916799454, + "tokens_seen": 2674130944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019161484453360082, + "loss": 1.3022, + "theoretical_loss": 3.379060929753115, + "tokens_seen": 2674196480 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019159478435305918, + "loss": 1.3978, + "theoretical_loss": 3.3790544680289836, + "tokens_seen": 2674262016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019157472417251757, + "loss": 1.2625, + "theoretical_loss": 3.37904800650754, + "tokens_seen": 2674327552 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019155466399197593, + "loss": 1.3073, + "theoretical_loss": 3.3790415451887728, + "tokens_seen": 2674393088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019153460381143432, + "loss": 1.305, + "theoretical_loss": 3.3790350840726706, + "tokens_seen": 2674458624 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019151454363089269, + "loss": 1.241, + "theoretical_loss": 3.3790286231592224, + "tokens_seen": 2674524160 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019149448345035105, + "loss": 1.3244, + "theoretical_loss": 3.3790221624484165, + "tokens_seen": 2674589696 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019147442326980944, + "loss": 1.2436, + "theoretical_loss": 3.379015701940242, + "tokens_seen": 2674655232 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001914543630892678, + "loss": 1.3866, + "theoretical_loss": 3.3790092416346873, + "tokens_seen": 2674720768 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001914343029087262, + "loss": 1.1416, + "theoretical_loss": 3.379002781531741, + "tokens_seen": 2674786304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019141424272818455, + "loss": 1.2542, + "theoretical_loss": 3.378996321631392, + "tokens_seen": 2674851840 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019139418254764294, + "loss": 1.3604, + "theoretical_loss": 3.3789898619336287, + "tokens_seen": 2674917376 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001913741223671013, + "loss": 1.2589, + "theoretical_loss": 3.37898340243844, + "tokens_seen": 2674982912 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001913540621865597, + "loss": 1.1792, + "theoretical_loss": 3.378976943145815, + "tokens_seen": 2675048448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019133400200601805, + "loss": 1.1825, + "theoretical_loss": 3.3789704840557415, + "tokens_seen": 2675113984 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019131394182547642, + "loss": 1.1713, + "theoretical_loss": 3.3789640251682087, + "tokens_seen": 2675179520 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001912938816449348, + "loss": 1.3156, + "theoretical_loss": 3.378957566483205, + "tokens_seen": 2675245056 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019127382146439317, + "loss": 1.2106, + "theoretical_loss": 3.3789511080007197, + "tokens_seen": 2675310592 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019125376128385156, + "loss": 1.3434, + "theoretical_loss": 3.378944649720741, + "tokens_seen": 2675376128 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019123370110330992, + "loss": 1.2389, + "theoretical_loss": 3.3789381916432575, + "tokens_seen": 2675441664 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001912136409227683, + "loss": 1.2723, + "theoretical_loss": 3.3789317337682583, + "tokens_seen": 2675507200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019119358074222667, + "loss": 1.1906, + "theoretical_loss": 3.378925276095732, + "tokens_seen": 2675572736 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 972181, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.390811562538147, + "objective/train/theoretical_loss": 3.3789188186256665, + "objective/train/tokens_used": 1046097376, + "theoretical_loss": 3.3789188186256665, + "tokens_seen": 2675638272 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019117352056168506, + "loss": 1.3517, + "theoretical_loss": 3.3789188186256665, + "tokens_seen": 2675638272 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019115346038114342, + "loss": 1.2414, + "theoretical_loss": 3.3789123613580516, + "tokens_seen": 2675703808 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019113340020060178, + "loss": 1.3185, + "theoretical_loss": 3.3789059042928753, + "tokens_seen": 2675769344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019111334002006017, + "loss": 1.236, + "theoretical_loss": 3.378899447430127, + "tokens_seen": 2675834880 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019109327983951854, + "loss": 1.3724, + "theoretical_loss": 3.378892990769794, + "tokens_seen": 2675900416 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019107321965897693, + "loss": 1.3447, + "theoretical_loss": 3.378886534311867, + "tokens_seen": 2675965952 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019105315947843532, + "loss": 1.3479, + "theoretical_loss": 3.378880078056333, + "tokens_seen": 2676031488 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001910330992978937, + "loss": 1.2817, + "theoretical_loss": 3.378873622003182, + "tokens_seen": 2676097024 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019101303911735207, + "loss": 1.2947, + "theoretical_loss": 3.3788671661524012, + "tokens_seen": 2676162560 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019099297893681046, + "loss": 1.347, + "theoretical_loss": 3.3788607105039805, + "tokens_seen": 2676228096 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019097291875626882, + "loss": 1.3454, + "theoretical_loss": 3.378854255057908, + "tokens_seen": 2676293632 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001909528585757272, + "loss": 1.3315, + "theoretical_loss": 3.378847799814173, + "tokens_seen": 2676359168 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019093279839518557, + "loss": 1.1803, + "theoretical_loss": 3.3788413447727637, + "tokens_seen": 2676424704 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019091273821464393, + "loss": 1.2725, + "theoretical_loss": 3.378834889933669, + "tokens_seen": 2676490240 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019089267803410232, + "loss": 1.3039, + "theoretical_loss": 3.378828435296877, + "tokens_seen": 2676555776 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019087261785356068, + "loss": 1.2414, + "theoretical_loss": 3.378821980862378, + "tokens_seen": 2676621312 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019085255767301907, + "loss": 1.2447, + "theoretical_loss": 3.3788155266301585, + "tokens_seen": 2676686848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019083249749247744, + "loss": 1.3125, + "theoretical_loss": 3.378809072600209, + "tokens_seen": 2676752384 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019081243731193582, + "loss": 1.3048, + "theoretical_loss": 3.3788026187725175, + "tokens_seen": 2676817920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001907923771313942, + "loss": 1.2108, + "theoretical_loss": 3.378796165147073, + "tokens_seen": 2676883456 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019077231695085258, + "loss": 1.3193, + "theoretical_loss": 3.3787897117238637, + "tokens_seen": 2676948992 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019075225677031094, + "loss": 1.2521, + "theoretical_loss": 3.378783258502879, + "tokens_seen": 2677014528 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001907321965897693, + "loss": 1.3527, + "theoretical_loss": 3.3787768054841067, + "tokens_seen": 2677080064 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001907121364092277, + "loss": 1.2616, + "theoretical_loss": 3.3787703526675363, + "tokens_seen": 2677145600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019069207622868605, + "loss": 1.2198, + "theoretical_loss": 3.378763900053156, + "tokens_seen": 2677211136 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019067201604814444, + "loss": 1.1887, + "theoretical_loss": 3.3787574476409548, + "tokens_seen": 2677276672 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001906519558676028, + "loss": 1.2067, + "theoretical_loss": 3.378750995430922, + "tokens_seen": 2677342208 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001906318956870612, + "loss": 1.2238, + "theoretical_loss": 3.378744543423045, + "tokens_seen": 2677407744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019061183550651956, + "loss": 1.2413, + "theoretical_loss": 3.3787380916173135, + "tokens_seen": 2677473280 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019059177532597794, + "loss": 1.3954, + "theoretical_loss": 3.378731640013716, + "tokens_seen": 2677538816 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001905717151454363, + "loss": 1.2245, + "theoretical_loss": 3.3787251886122407, + "tokens_seen": 2677604352 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019055165496489467, + "loss": 1.1951, + "theoretical_loss": 3.3787187374128775, + "tokens_seen": 2677669888 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019053159478435306, + "loss": 1.2822, + "theoretical_loss": 3.378712286415614, + "tokens_seen": 2677735424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019051153460381142, + "loss": 1.149, + "theoretical_loss": 3.3787058356204396, + "tokens_seen": 2677800960 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001904914744232698, + "loss": 1.2555, + "theoretical_loss": 3.3786993850273426, + "tokens_seen": 2677866496 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019047141424272817, + "loss": 1.2333, + "theoretical_loss": 3.3786929346363115, + "tokens_seen": 2677932032 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019045135406218656, + "loss": 1.3094, + "theoretical_loss": 3.3786864844473357, + "tokens_seen": 2677997568 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019043129388164492, + "loss": 1.2878, + "theoretical_loss": 3.3786800344604035, + "tokens_seen": 2678063104 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001904112337011033, + "loss": 1.2926, + "theoretical_loss": 3.3786735846755036, + "tokens_seen": 2678128640 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019039117352056168, + "loss": 1.2735, + "theoretical_loss": 3.378667135092625, + "tokens_seen": 2678194176 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019037111334002007, + "loss": 1.4145, + "theoretical_loss": 3.3786606857117567, + "tokens_seen": 2678259712 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019035105315947843, + "loss": 1.3226, + "theoretical_loss": 3.3786542365328867, + "tokens_seen": 2678325248 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001903309929789368, + "loss": 1.3814, + "theoretical_loss": 3.3786477875560044, + "tokens_seen": 2678390784 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001903109327983952, + "loss": 1.2356, + "theoretical_loss": 3.378641338781098, + "tokens_seen": 2678456320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019029087261785357, + "loss": 1.3251, + "theoretical_loss": 3.378634890208156, + "tokens_seen": 2678521856 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019027081243731196, + "loss": 1.3197, + "theoretical_loss": 3.378628441837168, + "tokens_seen": 2678587392 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019025075225677032, + "loss": 1.2695, + "theoretical_loss": 3.3786219936681223, + "tokens_seen": 2678652928 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001902306920762287, + "loss": 1.2139, + "theoretical_loss": 3.3786155457010074, + "tokens_seen": 2678718464 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019021063189568707, + "loss": 1.4032, + "theoretical_loss": 3.3786090979358123, + "tokens_seen": 2678784000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019019057171514546, + "loss": 1.3137, + "theoretical_loss": 3.378602650372526, + "tokens_seen": 2678849536 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 973380, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1937341690063477, + "objective/train/theoretical_loss": 3.3785962030111367, + "objective/train/tokens_used": 1049374176, + "theoretical_loss": 3.3785962030111367, + "tokens_seen": 2678915072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019017051153460382, + "loss": 1.29, + "theoretical_loss": 3.3785962030111367, + "tokens_seen": 2678915072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019015045135406219, + "loss": 1.2726, + "theoretical_loss": 3.3785897558516336, + "tokens_seen": 2678980608 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019013039117352057, + "loss": 1.1968, + "theoretical_loss": 3.378583308894005, + "tokens_seen": 2679046144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019011033099297894, + "loss": 1.1832, + "theoretical_loss": 3.37857686213824, + "tokens_seen": 2679111680 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019009027081243733, + "loss": 1.2947, + "theoretical_loss": 3.3785704155843272, + "tokens_seen": 2679177216 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001900702106318957, + "loss": 1.3409, + "theoretical_loss": 3.378563969232255, + "tokens_seen": 2679242752 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019005015045135408, + "loss": 1.1919, + "theoretical_loss": 3.378557523082013, + "tokens_seen": 2679308288 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019003009027081244, + "loss": 1.2641, + "theoretical_loss": 3.378551077133589, + "tokens_seen": 2679373824 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019001003009027083, + "loss": 1.2909, + "theoretical_loss": 3.3785446313869727, + "tokens_seen": 2679439360 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001899899699097292, + "loss": 1.2462, + "theoretical_loss": 3.378538185842152, + "tokens_seen": 2679504896 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018996990972918758, + "loss": 1.2626, + "theoretical_loss": 3.378531740499116, + "tokens_seen": 2679570432 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018994984954864594, + "loss": 1.2488, + "theoretical_loss": 3.3785252953578535, + "tokens_seen": 2679635968 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001899297893681043, + "loss": 1.263, + "theoretical_loss": 3.378518850418353, + "tokens_seen": 2679701504 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001899097291875627, + "loss": 1.3377, + "theoretical_loss": 3.3785124056806035, + "tokens_seen": 2679767040 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018988966900702106, + "loss": 1.3389, + "theoretical_loss": 3.378505961144594, + "tokens_seen": 2679832576 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018986960882647945, + "loss": 1.2219, + "theoretical_loss": 3.3784995168103125, + "tokens_seen": 2679898112 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001898495486459378, + "loss": 1.1344, + "theoretical_loss": 3.378493072677748, + "tokens_seen": 2679963648 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001898294884653962, + "loss": 1.3718, + "theoretical_loss": 3.37848662874689, + "tokens_seen": 2680029184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018980942828485456, + "loss": 1.269, + "theoretical_loss": 3.378480185017726, + "tokens_seen": 2680094720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018978936810431295, + "loss": 1.3966, + "theoretical_loss": 3.378473741490246, + "tokens_seen": 2680160256 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001897693079237713, + "loss": 1.3913, + "theoretical_loss": 3.3784672981644386, + "tokens_seen": 2680225792 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018974924774322967, + "loss": 1.2809, + "theoretical_loss": 3.3784608550402915, + "tokens_seen": 2680291328 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018972918756268806, + "loss": 1.3273, + "theoretical_loss": 3.378454412117794, + "tokens_seen": 2680356864 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018970912738214643, + "loss": 1.3014, + "theoretical_loss": 3.378447969396935, + "tokens_seen": 2680422400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018968906720160481, + "loss": 1.3661, + "theoretical_loss": 3.3784415268777037, + "tokens_seen": 2680487936 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018966900702106318, + "loss": 1.3925, + "theoretical_loss": 3.3784350845600883, + "tokens_seen": 2680553472 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018964894684052157, + "loss": 1.1533, + "theoretical_loss": 3.378428642444077, + "tokens_seen": 2680619008 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018962888665997993, + "loss": 1.2372, + "theoretical_loss": 3.37842220052966, + "tokens_seen": 2680684544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018960882647943832, + "loss": 1.2516, + "theoretical_loss": 3.378415758816825, + "tokens_seen": 2680750080 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018958876629889668, + "loss": 1.2879, + "theoretical_loss": 3.378409317305561, + "tokens_seen": 2680815616 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018956870611835504, + "loss": 1.3018, + "theoretical_loss": 3.3784028759958566, + "tokens_seen": 2680881152 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018954864593781346, + "loss": 1.2701, + "theoretical_loss": 3.378396434887701, + "tokens_seen": 2680946688 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018952858575727182, + "loss": 1.2391, + "theoretical_loss": 3.3783899939810826, + "tokens_seen": 2681012224 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001895085255767302, + "loss": 1.3227, + "theoretical_loss": 3.3783835532759903, + "tokens_seen": 2681077760 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018948846539618857, + "loss": 1.1852, + "theoretical_loss": 3.378377112772413, + "tokens_seen": 2681143296 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018946840521564696, + "loss": 1.2738, + "theoretical_loss": 3.3783706724703393, + "tokens_seen": 2681208832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018944834503510532, + "loss": 1.2638, + "theoretical_loss": 3.3783642323697585, + "tokens_seen": 2681274368 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018942828485456371, + "loss": 1.2643, + "theoretical_loss": 3.3783577924706583, + "tokens_seen": 2681339904 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018940822467402208, + "loss": 1.2847, + "theoretical_loss": 3.378351352773028, + "tokens_seen": 2681405440 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018938816449348047, + "loss": 1.2619, + "theoretical_loss": 3.3783449132768566, + "tokens_seen": 2681470976 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018936810431293883, + "loss": 1.347, + "theoretical_loss": 3.378338473982133, + "tokens_seen": 2681536512 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001893480441323972, + "loss": 1.3082, + "theoretical_loss": 3.378332034888845, + "tokens_seen": 2681602048 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018932798395185558, + "loss": 1.2288, + "theoretical_loss": 3.3783255959969827, + "tokens_seen": 2681667584 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018930792377131394, + "loss": 1.3175, + "theoretical_loss": 3.378319157306534, + "tokens_seen": 2681733120 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018928786359077233, + "loss": 1.1632, + "theoretical_loss": 3.378312718817488, + "tokens_seen": 2681798656 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001892678034102307, + "loss": 1.378, + "theoretical_loss": 3.3783062805298334, + "tokens_seen": 2681864192 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018924774322968908, + "loss": 1.2017, + "theoretical_loss": 3.3782998424435586, + "tokens_seen": 2681929728 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018922768304914744, + "loss": 1.2376, + "theoretical_loss": 3.378293404558653, + "tokens_seen": 2681995264 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018920762286860583, + "loss": 1.2911, + "theoretical_loss": 3.3782869668751054, + "tokens_seen": 2682060800 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891875626880642, + "loss": 1.3274, + "theoretical_loss": 3.378280529392904, + "tokens_seen": 2682126336 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 974516, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2817437648773193, + "objective/train/theoretical_loss": 3.3782740921120378, + "objective/train/tokens_used": 1052650976, + "theoretical_loss": 3.3782740921120378, + "tokens_seen": 2682191872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018916750250752256, + "loss": 1.3338, + "theoretical_loss": 3.3782740921120378, + "tokens_seen": 2682191872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018914744232698095, + "loss": 1.2995, + "theoretical_loss": 3.378267655032496, + "tokens_seen": 2682257408 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891273821464393, + "loss": 1.3301, + "theoretical_loss": 3.378261218154267, + "tokens_seen": 2682322944 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891073219658977, + "loss": 1.2205, + "theoretical_loss": 3.3782547814773394, + "tokens_seen": 2682388480 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018908726178535606, + "loss": 1.201, + "theoretical_loss": 3.3782483450017025, + "tokens_seen": 2682454016 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018906720160481445, + "loss": 1.3596, + "theoretical_loss": 3.3782419087273445, + "tokens_seen": 2682519552 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001890471414242728, + "loss": 1.3146, + "theoretical_loss": 3.378235472654255, + "tokens_seen": 2682585088 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001890270812437312, + "loss": 1.1883, + "theoretical_loss": 3.3782290367824217, + "tokens_seen": 2682650624 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018900702106318956, + "loss": 1.2277, + "theoretical_loss": 3.3782226011118346, + "tokens_seen": 2682716160 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018898696088264793, + "loss": 1.1723, + "theoretical_loss": 3.3782161656424816, + "tokens_seen": 2682781696 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018896690070210632, + "loss": 1.2408, + "theoretical_loss": 3.3782097303743517, + "tokens_seen": 2682847232 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018894684052156468, + "loss": 1.3188, + "theoretical_loss": 3.3782032953074337, + "tokens_seen": 2682912768 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018892678034102307, + "loss": 1.3007, + "theoretical_loss": 3.378196860441717, + "tokens_seen": 2682978304 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018890672016048143, + "loss": 1.2857, + "theoretical_loss": 3.3781904257771895, + "tokens_seen": 2683043840 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018888665997993982, + "loss": 1.2931, + "theoretical_loss": 3.3781839913138403, + "tokens_seen": 2683109376 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018886659979939818, + "loss": 1.3403, + "theoretical_loss": 3.378177557051658, + "tokens_seen": 2683174912 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018884653961885657, + "loss": 1.3411, + "theoretical_loss": 3.378171122990632, + "tokens_seen": 2683240448 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018882647943831493, + "loss": 1.2592, + "theoretical_loss": 3.3781646891307506, + "tokens_seen": 2683305984 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018880641925777335, + "loss": 1.2874, + "theoretical_loss": 3.378158255472003, + "tokens_seen": 2683371520 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001887863590772317, + "loss": 1.2748, + "theoretical_loss": 3.378151822014378, + "tokens_seen": 2683437056 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018876629889669007, + "loss": 1.2951, + "theoretical_loss": 3.378145388757863, + "tokens_seen": 2683502592 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018874623871614846, + "loss": 1.3252, + "theoretical_loss": 3.378138955702449, + "tokens_seen": 2683568128 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018872617853560683, + "loss": 1.2277, + "theoretical_loss": 3.378132522848124, + "tokens_seen": 2683633664 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018870611835506522, + "loss": 1.2829, + "theoretical_loss": 3.3781260901948755, + "tokens_seen": 2683699200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018868605817452358, + "loss": 1.3172, + "theoretical_loss": 3.378119657742694, + "tokens_seen": 2683764736 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018866599799398197, + "loss": 1.2989, + "theoretical_loss": 3.3781132254915676, + "tokens_seen": 2683830272 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018864593781344033, + "loss": 1.3358, + "theoretical_loss": 3.378106793441485, + "tokens_seen": 2683895808 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018862587763289872, + "loss": 1.1881, + "theoretical_loss": 3.3781003615924354, + "tokens_seen": 2683961344 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018860581745235708, + "loss": 1.2759, + "theoretical_loss": 3.378093929944407, + "tokens_seen": 2684026880 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018858575727181544, + "loss": 1.3154, + "theoretical_loss": 3.3780874984973894, + "tokens_seen": 2684092416 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018856569709127383, + "loss": 1.3181, + "theoretical_loss": 3.378081067251371, + "tokens_seen": 2684157952 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001885456369107322, + "loss": 1.3171, + "theoretical_loss": 3.3780746362063403, + "tokens_seen": 2684223488 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018852557673019058, + "loss": 1.2124, + "theoretical_loss": 3.3780682053622866, + "tokens_seen": 2684289024 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018850551654964895, + "loss": 1.2622, + "theoretical_loss": 3.3780617747191988, + "tokens_seen": 2684354560 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018848545636910734, + "loss": 1.3523, + "theoretical_loss": 3.378055344277065, + "tokens_seen": 2684420096 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001884653961885657, + "loss": 1.333, + "theoretical_loss": 3.3780489140358747, + "tokens_seen": 2684485632 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001884453360080241, + "loss": 1.2463, + "theoretical_loss": 3.3780424839956167, + "tokens_seen": 2684551168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018842527582748245, + "loss": 1.4101, + "theoretical_loss": 3.3780360541562793, + "tokens_seen": 2684616704 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018840521564694084, + "loss": 1.2575, + "theoretical_loss": 3.3780296245178514, + "tokens_seen": 2684682240 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001883851554663992, + "loss": 1.3139, + "theoretical_loss": 3.3780231950803223, + "tokens_seen": 2684747776 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018836509528585756, + "loss": 1.2919, + "theoretical_loss": 3.3780167658436806, + "tokens_seen": 2684813312 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018834503510531595, + "loss": 1.296, + "theoretical_loss": 3.3780103368079155, + "tokens_seen": 2684878848 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018832497492477431, + "loss": 1.2919, + "theoretical_loss": 3.3780039079730146, + "tokens_seen": 2684944384 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001883049147442327, + "loss": 1.1995, + "theoretical_loss": 3.377997479338968, + "tokens_seen": 2685009920 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018828485456369107, + "loss": 1.2028, + "theoretical_loss": 3.377991050905764, + "tokens_seen": 2685075456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018826479438314946, + "loss": 1.2834, + "theoretical_loss": 3.3779846226733916, + "tokens_seen": 2685140992 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018824473420260782, + "loss": 1.2431, + "theoretical_loss": 3.3779781946418392, + "tokens_seen": 2685206528 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001882246740220662, + "loss": 1.3404, + "theoretical_loss": 3.377971766811096, + "tokens_seen": 2685272064 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018820461384152457, + "loss": 1.2511, + "theoretical_loss": 3.3779653391811504, + "tokens_seen": 2685337600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018818455366098293, + "loss": 1.1932, + "theoretical_loss": 3.377958911751992, + "tokens_seen": 2685403136 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 975809, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2196687459945679, + "objective/train/theoretical_loss": 3.3779524845236093, + "objective/train/tokens_used": 1055927776, + "theoretical_loss": 3.3779524845236093, + "tokens_seen": 2685468672 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018816449348044132, + "loss": 1.1863, + "theoretical_loss": 3.3779524845236093, + "tokens_seen": 2685468672 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018814443329989968, + "loss": 1.257, + "theoretical_loss": 3.3779460574959908, + "tokens_seen": 2685534208 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018812437311935807, + "loss": 1.2884, + "theoretical_loss": 3.3779396306691254, + "tokens_seen": 2685599744 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018810431293881643, + "loss": 1.3721, + "theoretical_loss": 3.3779332040430026, + "tokens_seen": 2685665280 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018808425275827482, + "loss": 1.3789, + "theoretical_loss": 3.3779267776176103, + "tokens_seen": 2685730816 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018806419257773319, + "loss": 1.2599, + "theoretical_loss": 3.377920351392938, + "tokens_seen": 2685796352 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001880441323971916, + "loss": 1.2731, + "theoretical_loss": 3.377913925368974, + "tokens_seen": 2685861888 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018802407221664997, + "loss": 1.3289, + "theoretical_loss": 3.3779074995457075, + "tokens_seen": 2685927424 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018800401203610833, + "loss": 1.2614, + "theoretical_loss": 3.377901073923127, + "tokens_seen": 2685992960 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018798395185556672, + "loss": 1.2449, + "theoretical_loss": 3.377894648501222, + "tokens_seen": 2686058496 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018796389167502508, + "loss": 1.2554, + "theoretical_loss": 3.3778882232799807, + "tokens_seen": 2686124032 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018794383149448347, + "loss": 1.2933, + "theoretical_loss": 3.3778817982593923, + "tokens_seen": 2686189568 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018792377131394183, + "loss": 1.2699, + "theoretical_loss": 3.3778753734394455, + "tokens_seen": 2686255104 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018790371113340022, + "loss": 1.3444, + "theoretical_loss": 3.3778689488201294, + "tokens_seen": 2686320640 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018788365095285858, + "loss": 1.3619, + "theoretical_loss": 3.377862524401432, + "tokens_seen": 2686386176 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018786359077231697, + "loss": 1.2479, + "theoretical_loss": 3.377856100183343, + "tokens_seen": 2686451712 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018784353059177533, + "loss": 1.253, + "theoretical_loss": 3.377849676165851, + "tokens_seen": 2686517248 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018782347041123372, + "loss": 1.1751, + "theoretical_loss": 3.377843252348945, + "tokens_seen": 2686582784 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018780341023069209, + "loss": 1.2837, + "theoretical_loss": 3.3778368287326135, + "tokens_seen": 2686648320 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018778335005015045, + "loss": 1.3053, + "theoretical_loss": 3.3778304053168453, + "tokens_seen": 2686713856 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018776328986960884, + "loss": 1.207, + "theoretical_loss": 3.3778239821016296, + "tokens_seen": 2686779392 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001877432296890672, + "loss": 1.3989, + "theoretical_loss": 3.377817559086955, + "tokens_seen": 2686844928 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001877231695085256, + "loss": 1.319, + "theoretical_loss": 3.3778111362728103, + "tokens_seen": 2686910464 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018770310932798395, + "loss": 1.2088, + "theoretical_loss": 3.377804713659185, + "tokens_seen": 2686976000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018768304914744234, + "loss": 1.2801, + "theoretical_loss": 3.377798291246067, + "tokens_seen": 2687041536 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001876629889669007, + "loss": 1.3427, + "theoretical_loss": 3.3777918690334454, + "tokens_seen": 2687107072 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001876429287863591, + "loss": 1.264, + "theoretical_loss": 3.37778544702131, + "tokens_seen": 2687172608 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018762286860581745, + "loss": 1.1464, + "theoretical_loss": 3.377779025209648, + "tokens_seen": 2687238144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018760280842527582, + "loss": 1.3858, + "theoretical_loss": 3.3777726035984497, + "tokens_seen": 2687303680 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001875827482447342, + "loss": 1.2449, + "theoretical_loss": 3.3777661821877034, + "tokens_seen": 2687369216 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018756268806419257, + "loss": 1.1721, + "theoretical_loss": 3.3777597609773977, + "tokens_seen": 2687434752 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018754262788365096, + "loss": 1.4086, + "theoretical_loss": 3.377753339967522, + "tokens_seen": 2687500288 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018752256770310932, + "loss": 1.2726, + "theoretical_loss": 3.3777469191580645, + "tokens_seen": 2687565824 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001875025075225677, + "loss": 1.3571, + "theoretical_loss": 3.3777404985490147, + "tokens_seen": 2687631360 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018748244734202607, + "loss": 1.3415, + "theoretical_loss": 3.377734078140361, + "tokens_seen": 2687696896 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018746238716148446, + "loss": 1.3169, + "theoretical_loss": 3.3777276579320925, + "tokens_seen": 2687762432 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018744232698094282, + "loss": 1.2626, + "theoretical_loss": 3.377721237924198, + "tokens_seen": 2687827968 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018742226680040118, + "loss": 1.1911, + "theoretical_loss": 3.3777148181166665, + "tokens_seen": 2687893504 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018740220661985957, + "loss": 1.1932, + "theoretical_loss": 3.3777083985094865, + "tokens_seen": 2687959040 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018738214643931794, + "loss": 1.2838, + "theoretical_loss": 3.377701979102647, + "tokens_seen": 2688024576 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018736208625877633, + "loss": 1.2482, + "theoretical_loss": 3.3776955598961376, + "tokens_seen": 2688090112 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001873420260782347, + "loss": 1.3211, + "theoretical_loss": 3.3776891408899457, + "tokens_seen": 2688155648 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018732196589769308, + "loss": 1.2152, + "theoretical_loss": 3.377682722084061, + "tokens_seen": 2688221184 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018730190571715147, + "loss": 1.2591, + "theoretical_loss": 3.3776763034784727, + "tokens_seen": 2688286720 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018728184553660986, + "loss": 1.3185, + "theoretical_loss": 3.3776698850731695, + "tokens_seen": 2688352256 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018726178535606822, + "loss": 1.355, + "theoretical_loss": 3.37766346686814, + "tokens_seen": 2688417792 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001872417251755266, + "loss": 1.3688, + "theoretical_loss": 3.377657048863373, + "tokens_seen": 2688483328 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018722166499498497, + "loss": 1.2499, + "theoretical_loss": 3.377650631058857, + "tokens_seen": 2688548864 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018720160481444333, + "loss": 1.2978, + "theoretical_loss": 3.377644213454582, + "tokens_seen": 2688614400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018718154463390172, + "loss": 1.3456, + "theoretical_loss": 3.377637796050536, + "tokens_seen": 2688679936 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 976981, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.30851411819458, + "objective/train/theoretical_loss": 3.3776313788467087, + "objective/train/tokens_used": 1059204576, + "theoretical_loss": 3.3776313788467087, + "tokens_seen": 2688745472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018716148445336008, + "loss": 1.3226, + "theoretical_loss": 3.3776313788467087, + "tokens_seen": 2688745472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018714142427281847, + "loss": 1.2651, + "theoretical_loss": 3.377624961843088, + "tokens_seen": 2688811008 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018712136409227684, + "loss": 1.1601, + "theoretical_loss": 3.377618545039663, + "tokens_seen": 2688876544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018710130391173522, + "loss": 1.3678, + "theoretical_loss": 3.377612128436423, + "tokens_seen": 2688942080 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001870812437311936, + "loss": 1.3709, + "theoretical_loss": 3.3776057120333567, + "tokens_seen": 2689007616 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018706118355065198, + "loss": 1.2364, + "theoretical_loss": 3.3775992958304526, + "tokens_seen": 2689073152 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018704112337011034, + "loss": 1.1755, + "theoretical_loss": 3.3775928798277, + "tokens_seen": 2689138688 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001870210631895687, + "loss": 1.395, + "theoretical_loss": 3.3775864640250877, + "tokens_seen": 2689204224 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001870010030090271, + "loss": 1.3095, + "theoretical_loss": 3.377580048422604, + "tokens_seen": 2689269760 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018698094282848545, + "loss": 1.2174, + "theoretical_loss": 3.3775736330202393, + "tokens_seen": 2689335296 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018696088264794384, + "loss": 1.1834, + "theoretical_loss": 3.377567217817981, + "tokens_seen": 2689400832 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001869408224674022, + "loss": 1.2467, + "theoretical_loss": 3.3775608028158186, + "tokens_seen": 2689466368 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001869207622868606, + "loss": 1.3043, + "theoretical_loss": 3.3775543880137406, + "tokens_seen": 2689531904 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018690070210631896, + "loss": 1.2881, + "theoretical_loss": 3.3775479734117364, + "tokens_seen": 2689597440 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018688064192577734, + "loss": 1.1605, + "theoretical_loss": 3.377541559009795, + "tokens_seen": 2689662976 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001868605817452357, + "loss": 1.1807, + "theoretical_loss": 3.3775351448079043, + "tokens_seen": 2689728512 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018684052156469407, + "loss": 1.2473, + "theoretical_loss": 3.3775287308060538, + "tokens_seen": 2689794048 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018682046138415246, + "loss": 1.2656, + "theoretical_loss": 3.3775223170042326, + "tokens_seen": 2689859584 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018680040120361082, + "loss": 1.3143, + "theoretical_loss": 3.3775159034024296, + "tokens_seen": 2689925120 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001867803410230692, + "loss": 1.3432, + "theoretical_loss": 3.3775094900006333, + "tokens_seen": 2689990656 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018676028084252757, + "loss": 1.2542, + "theoretical_loss": 3.377503076798833, + "tokens_seen": 2690056192 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018674022066198596, + "loss": 1.3057, + "theoretical_loss": 3.377496663797017, + "tokens_seen": 2690121728 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018672016048144432, + "loss": 1.3686, + "theoretical_loss": 3.3774902509951747, + "tokens_seen": 2690187264 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001867001003009027, + "loss": 1.3084, + "theoretical_loss": 3.377483838393295, + "tokens_seen": 2690252800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018668004012036108, + "loss": 1.2785, + "theoretical_loss": 3.3774774259913665, + "tokens_seen": 2690318336 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018665997993981946, + "loss": 1.3614, + "theoretical_loss": 3.377471013789378, + "tokens_seen": 2690383872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018663991975927783, + "loss": 1.385, + "theoretical_loss": 3.377464601787319, + "tokens_seen": 2690449408 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001866198595787362, + "loss": 1.3776, + "theoretical_loss": 3.3774581899851777, + "tokens_seen": 2690514944 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018659979939819458, + "loss": 1.1836, + "theoretical_loss": 3.3774517783829436, + "tokens_seen": 2690580480 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018657973921765294, + "loss": 1.3361, + "theoretical_loss": 3.3774453669806053, + "tokens_seen": 2690646016 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018655967903711136, + "loss": 1.2696, + "theoretical_loss": 3.3774389557781515, + "tokens_seen": 2690711552 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018653961885656972, + "loss": 1.3906, + "theoretical_loss": 3.3774325447755715, + "tokens_seen": 2690777088 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001865195586760281, + "loss": 1.2825, + "theoretical_loss": 3.377426133972854, + "tokens_seen": 2690842624 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018649949849548647, + "loss": 1.2613, + "theoretical_loss": 3.3774197233699876, + "tokens_seen": 2690908160 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018647943831494486, + "loss": 1.3839, + "theoretical_loss": 3.377413312966962, + "tokens_seen": 2690973696 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018645937813440322, + "loss": 1.3058, + "theoretical_loss": 3.3774069027637657, + "tokens_seen": 2691039232 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018643931795386159, + "loss": 1.2099, + "theoretical_loss": 3.377400492760387, + "tokens_seen": 2691104768 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018641925777331997, + "loss": 1.2881, + "theoretical_loss": 3.3773940829568154, + "tokens_seen": 2691170304 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018639919759277834, + "loss": 1.2965, + "theoretical_loss": 3.37738767335304, + "tokens_seen": 2691235840 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018637913741223673, + "loss": 1.2947, + "theoretical_loss": 3.3773812639490495, + "tokens_seen": 2691301376 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001863590772316951, + "loss": 1.3024, + "theoretical_loss": 3.377374854744833, + "tokens_seen": 2691366912 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018633901705115348, + "loss": 1.3444, + "theoretical_loss": 3.3773684457403785, + "tokens_seen": 2691432448 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018631895687061184, + "loss": 1.3721, + "theoretical_loss": 3.377362036935676, + "tokens_seen": 2691497984 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018629889669007023, + "loss": 1.3998, + "theoretical_loss": 3.3773556283307142, + "tokens_seen": 2691563520 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001862788365095286, + "loss": 1.2232, + "theoretical_loss": 3.377349219925481, + "tokens_seen": 2691629056 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018625877632898698, + "loss": 1.2628, + "theoretical_loss": 3.377342811719967, + "tokens_seen": 2691694592 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018623871614844534, + "loss": 1.3611, + "theoretical_loss": 3.3773364037141596, + "tokens_seen": 2691760128 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001862186559679037, + "loss": 1.3419, + "theoretical_loss": 3.377329995908049, + "tokens_seen": 2691825664 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001861985957873621, + "loss": 1.2183, + "theoretical_loss": 3.377323588301623, + "tokens_seen": 2691891200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018617853560682046, + "loss": 1.2865, + "theoretical_loss": 3.377317180894871, + "tokens_seen": 2691956736 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 978239, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.130199909210205, + "objective/train/theoretical_loss": 3.377310773687782, + "objective/train/tokens_used": 1062481376, + "theoretical_loss": 3.377310773687782, + "tokens_seen": 2692022272 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018615847542627885, + "loss": 1.2411, + "theoretical_loss": 3.377310773687782, + "tokens_seen": 2692022272 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001861384152457372, + "loss": 1.3594, + "theoretical_loss": 3.3773043666803444, + "tokens_seen": 2692087808 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001861183550651956, + "loss": 1.2251, + "theoretical_loss": 3.377297959872548, + "tokens_seen": 2692153344 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018609829488465396, + "loss": 1.2129, + "theoretical_loss": 3.377291553264381, + "tokens_seen": 2692218880 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018607823470411235, + "loss": 1.3608, + "theoretical_loss": 3.377285146855833, + "tokens_seen": 2692284416 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001860581745235707, + "loss": 1.2893, + "theoretical_loss": 3.377278740646892, + "tokens_seen": 2692349952 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018603811434302907, + "loss": 1.3382, + "theoretical_loss": 3.3772723346375475, + "tokens_seen": 2692415488 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018601805416248746, + "loss": 1.3454, + "theoretical_loss": 3.3772659288277884, + "tokens_seen": 2692481024 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018599799398194583, + "loss": 1.3267, + "theoretical_loss": 3.3772595232176035, + "tokens_seen": 2692546560 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018597793380140421, + "loss": 1.2875, + "theoretical_loss": 3.377253117806982, + "tokens_seen": 2692612096 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018595787362086258, + "loss": 1.3767, + "theoretical_loss": 3.3772467125959125, + "tokens_seen": 2692677632 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018593781344032097, + "loss": 1.2801, + "theoretical_loss": 3.3772403075843838, + "tokens_seen": 2692743168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018591775325977933, + "loss": 1.3334, + "theoretical_loss": 3.377233902772385, + "tokens_seen": 2692808704 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018589769307923772, + "loss": 1.438, + "theoretical_loss": 3.3772274981599057, + "tokens_seen": 2692874240 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018587763289869608, + "loss": 1.3733, + "theoretical_loss": 3.377221093746934, + "tokens_seen": 2692939776 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018585757271815444, + "loss": 1.2483, + "theoretical_loss": 3.3772146895334587, + "tokens_seen": 2693005312 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018583751253761283, + "loss": 1.3103, + "theoretical_loss": 3.377208285519469, + "tokens_seen": 2693070848 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001858174523570712, + "loss": 1.2367, + "theoretical_loss": 3.3772018817049543, + "tokens_seen": 2693136384 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001857973921765296, + "loss": 1.3222, + "theoretical_loss": 3.377195478089903, + "tokens_seen": 2693201920 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018577733199598797, + "loss": 1.3909, + "theoretical_loss": 3.3771890746743045, + "tokens_seen": 2693267456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018575727181544636, + "loss": 1.2548, + "theoretical_loss": 3.377182671458147, + "tokens_seen": 2693332992 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018573721163490472, + "loss": 1.3134, + "theoretical_loss": 3.3771762684414197, + "tokens_seen": 2693398528 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018571715145436311, + "loss": 1.3278, + "theoretical_loss": 3.3771698656241123, + "tokens_seen": 2693464064 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018569709127382148, + "loss": 1.1707, + "theoretical_loss": 3.377163463006213, + "tokens_seen": 2693529600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018567703109327987, + "loss": 1.3272, + "theoretical_loss": 3.3771570605877104, + "tokens_seen": 2693595136 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018565697091273823, + "loss": 1.3815, + "theoretical_loss": 3.377150658368594, + "tokens_seen": 2693660672 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001856369107321966, + "loss": 1.435, + "theoretical_loss": 3.3771442563488527, + "tokens_seen": 2693726208 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018561685055165498, + "loss": 1.2747, + "theoretical_loss": 3.377137854528476, + "tokens_seen": 2693791744 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018559679037111334, + "loss": 1.2391, + "theoretical_loss": 3.3771314529074514, + "tokens_seen": 2693857280 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018557673019057173, + "loss": 1.2485, + "theoretical_loss": 3.377125051485769, + "tokens_seen": 2693922816 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001855566700100301, + "loss": 1.3246, + "theoretical_loss": 3.377118650263417, + "tokens_seen": 2693988352 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018553660982948848, + "loss": 1.3096, + "theoretical_loss": 3.377112249240385, + "tokens_seen": 2694053888 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018551654964894684, + "loss": 1.2391, + "theoretical_loss": 3.377105848416662, + "tokens_seen": 2694119424 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018549648946840523, + "loss": 1.2733, + "theoretical_loss": 3.3770994477922365, + "tokens_seen": 2694184960 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001854764292878636, + "loss": 1.3157, + "theoretical_loss": 3.3770930473670977, + "tokens_seen": 2694250496 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018545636910732196, + "loss": 1.3173, + "theoretical_loss": 3.377086647141234, + "tokens_seen": 2694316032 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018543630892678035, + "loss": 1.3513, + "theoretical_loss": 3.3770802471146353, + "tokens_seen": 2694381568 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001854162487462387, + "loss": 1.3295, + "theoretical_loss": 3.37707384728729, + "tokens_seen": 2694447104 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001853961885656971, + "loss": 1.3679, + "theoretical_loss": 3.3770674476591864, + "tokens_seen": 2694512640 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018537612838515546, + "loss": 1.3452, + "theoretical_loss": 3.3770610482303147, + "tokens_seen": 2694578176 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018535606820461385, + "loss": 1.3549, + "theoretical_loss": 3.377054649000663, + "tokens_seen": 2694643712 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001853360080240722, + "loss": 1.2412, + "theoretical_loss": 3.3770482499702212, + "tokens_seen": 2694709248 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001853159478435306, + "loss": 1.2788, + "theoretical_loss": 3.3770418511389773, + "tokens_seen": 2694774784 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018529588766298896, + "loss": 1.2072, + "theoretical_loss": 3.37703545250692, + "tokens_seen": 2694840320 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018527582748244733, + "loss": 1.3047, + "theoretical_loss": 3.3770290540740393, + "tokens_seen": 2694905856 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018525576730190572, + "loss": 1.2673, + "theoretical_loss": 3.3770226558403236, + "tokens_seen": 2694971392 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018523570712136408, + "loss": 1.2546, + "theoretical_loss": 3.377016257805762, + "tokens_seen": 2695036928 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018521564694082247, + "loss": 1.3489, + "theoretical_loss": 3.377009859970343, + "tokens_seen": 2695102464 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018519558676028083, + "loss": 1.3243, + "theoretical_loss": 3.3770034623340566, + "tokens_seen": 2695168000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018517552657973922, + "loss": 1.2213, + "theoretical_loss": 3.3769970648968908, + "tokens_seen": 2695233536 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 979344, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.375655174255371, + "objective/train/theoretical_loss": 3.3769906676588346, + "objective/train/tokens_used": 1065758176, + "theoretical_loss": 3.3769906676588346, + "tokens_seen": 2695299072 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018515546639919758, + "loss": 1.3557, + "theoretical_loss": 3.3769906676588346, + "tokens_seen": 2695299072 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018513540621865597, + "loss": 1.2539, + "theoretical_loss": 3.3769842706198774, + "tokens_seen": 2695364608 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018511534603811433, + "loss": 1.2233, + "theoretical_loss": 3.376977873780008, + "tokens_seen": 2695430144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018509528585757272, + "loss": 1.293, + "theoretical_loss": 3.3769714771392154, + "tokens_seen": 2695495680 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018507522567703108, + "loss": 1.3252, + "theoretical_loss": 3.3769650806974885, + "tokens_seen": 2695561216 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018505516549648947, + "loss": 1.2073, + "theoretical_loss": 3.376958684454816, + "tokens_seen": 2695626752 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018503510531594786, + "loss": 1.2002, + "theoretical_loss": 3.3769522884111876, + "tokens_seen": 2695692288 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018501504513540623, + "loss": 1.3129, + "theoretical_loss": 3.3769458925665914, + "tokens_seen": 2695757824 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018499498495486462, + "loss": 1.2587, + "theoretical_loss": 3.3769394969210174, + "tokens_seen": 2695823360 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018497492477432298, + "loss": 1.2704, + "theoretical_loss": 3.376933101474453, + "tokens_seen": 2695888896 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018495486459378137, + "loss": 1.1931, + "theoretical_loss": 3.3769267062268886, + "tokens_seen": 2695954432 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018493480441323973, + "loss": 1.2588, + "theoretical_loss": 3.376920311178313, + "tokens_seen": 2696019968 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018491474423269812, + "loss": 1.1945, + "theoretical_loss": 3.3769139163287147, + "tokens_seen": 2696085504 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018489468405215648, + "loss": 1.2936, + "theoretical_loss": 3.3769075216780826, + "tokens_seen": 2696151040 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018487462387161484, + "loss": 1.2884, + "theoretical_loss": 3.376901127226406, + "tokens_seen": 2696216576 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018485456369107323, + "loss": 1.4073, + "theoretical_loss": 3.376894732973674, + "tokens_seen": 2696282112 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001848345035105316, + "loss": 1.3461, + "theoretical_loss": 3.376888338919875, + "tokens_seen": 2696347648 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018481444332998998, + "loss": 1.3021, + "theoretical_loss": 3.3768819450649987, + "tokens_seen": 2696413184 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018479438314944835, + "loss": 1.3153, + "theoretical_loss": 3.376875551409033, + "tokens_seen": 2696478720 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018477432296890674, + "loss": 1.3215, + "theoretical_loss": 3.3768691579519685, + "tokens_seen": 2696544256 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001847542627883651, + "loss": 1.257, + "theoretical_loss": 3.3768627646937928, + "tokens_seen": 2696609792 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001847342026078235, + "loss": 1.2458, + "theoretical_loss": 3.3768563716344957, + "tokens_seen": 2696675328 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018471414242728185, + "loss": 1.3325, + "theoretical_loss": 3.3768499787740653, + "tokens_seen": 2696740864 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018469408224674024, + "loss": 1.3164, + "theoretical_loss": 3.3768435861124915, + "tokens_seen": 2696806400 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001846740220661986, + "loss": 1.2158, + "theoretical_loss": 3.3768371936497625, + "tokens_seen": 2696871936 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018465396188565696, + "loss": 1.3166, + "theoretical_loss": 3.376830801385868, + "tokens_seen": 2696937472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018463390170511535, + "loss": 1.2954, + "theoretical_loss": 3.376824409320797, + "tokens_seen": 2697003008 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018461384152457371, + "loss": 1.2463, + "theoretical_loss": 3.3768180174545375, + "tokens_seen": 2697068544 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001845937813440321, + "loss": 1.2675, + "theoretical_loss": 3.376811625787079, + "tokens_seen": 2697134080 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018457372116349047, + "loss": 1.2966, + "theoretical_loss": 3.376805234318411, + "tokens_seen": 2697199616 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018455366098294886, + "loss": 1.2181, + "theoretical_loss": 3.3767988430485225, + "tokens_seen": 2697265152 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018453360080240722, + "loss": 1.2312, + "theoretical_loss": 3.3767924519774013, + "tokens_seen": 2697330688 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001845135406218656, + "loss": 1.2595, + "theoretical_loss": 3.3767860611050375, + "tokens_seen": 2697396224 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018449348044132397, + "loss": 1.2673, + "theoretical_loss": 3.3767796704314197, + "tokens_seen": 2697461760 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018447342026078233, + "loss": 1.2563, + "theoretical_loss": 3.376773279956537, + "tokens_seen": 2697527296 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018445336008024072, + "loss": 1.1277, + "theoretical_loss": 3.3767668896803786, + "tokens_seen": 2697592832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018443329989969908, + "loss": 1.3412, + "theoretical_loss": 3.3767604996029332, + "tokens_seen": 2697658368 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018441323971915747, + "loss": 1.3468, + "theoretical_loss": 3.3767541097241898, + "tokens_seen": 2697723904 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018439317953861583, + "loss": 1.2762, + "theoretical_loss": 3.376747720044137, + "tokens_seen": 2697789440 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018437311935807422, + "loss": 1.2268, + "theoretical_loss": 3.3767413305627647, + "tokens_seen": 2697854976 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018435305917753259, + "loss": 1.263, + "theoretical_loss": 3.3767349412800614, + "tokens_seen": 2697920512 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018433299899699098, + "loss": 1.2317, + "theoretical_loss": 3.376728552196016, + "tokens_seen": 2697986048 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018431293881644934, + "loss": 1.3815, + "theoretical_loss": 3.3767221633106175, + "tokens_seen": 2698051584 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018429287863590773, + "loss": 1.2771, + "theoretical_loss": 3.3767157746238547, + "tokens_seen": 2698117120 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018427281845536612, + "loss": 1.3258, + "theoretical_loss": 3.3767093861357176, + "tokens_seen": 2698182656 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018425275827482448, + "loss": 1.3029, + "theoretical_loss": 3.3767029978461944, + "tokens_seen": 2698248192 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018423269809428287, + "loss": 1.1993, + "theoretical_loss": 3.3766966097552737, + "tokens_seen": 2698313728 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018421263791374123, + "loss": 1.1792, + "theoretical_loss": 3.3766902218629458, + "tokens_seen": 2698379264 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018419257773319962, + "loss": 1.2025, + "theoretical_loss": 3.3766838341691985, + "tokens_seen": 2698444800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018417251755265798, + "loss": 1.2751, + "theoretical_loss": 3.3766774466740213, + "tokens_seen": 2698510336 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 980656, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4430756568908691, + "objective/train/theoretical_loss": 3.376671059377403, + "objective/train/tokens_used": 1069034976, + "theoretical_loss": 3.376671059377403, + "tokens_seen": 2698575872 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018415245737211637, + "loss": 1.4259, + "theoretical_loss": 3.376671059377403, + "tokens_seen": 2698575872 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018413239719157473, + "loss": 1.1579, + "theoretical_loss": 3.376664672279333, + "tokens_seen": 2698641408 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018411233701103312, + "loss": 1.3168, + "theoretical_loss": 3.3766582853797997, + "tokens_seen": 2698706944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018409227683049149, + "loss": 1.3789, + "theoretical_loss": 3.3766518986787926, + "tokens_seen": 2698772480 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018407221664994985, + "loss": 1.3199, + "theoretical_loss": 3.3766455121763004, + "tokens_seen": 2698838016 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018405215646940824, + "loss": 1.2997, + "theoretical_loss": 3.3766391258723125, + "tokens_seen": 2698903552 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001840320962888666, + "loss": 1.3263, + "theoretical_loss": 3.376632739766818, + "tokens_seen": 2698969088 + }, + { + "epoch": 0.64, + "learning_rate": 0.000184012036108325, + "loss": 1.4628, + "theoretical_loss": 3.376626353859805, + "tokens_seen": 2699034624 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018399197592778335, + "loss": 1.2597, + "theoretical_loss": 3.376619968151263, + "tokens_seen": 2699100160 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018397191574724174, + "loss": 1.2576, + "theoretical_loss": 3.376613582641182, + "tokens_seen": 2699165696 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001839518555667001, + "loss": 1.2593, + "theoretical_loss": 3.376607197329549, + "tokens_seen": 2699231232 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001839317953861585, + "loss": 1.3755, + "theoretical_loss": 3.376600812216355, + "tokens_seen": 2699296768 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018391173520561685, + "loss": 1.3459, + "theoretical_loss": 3.3765944273015878, + "tokens_seen": 2699362304 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018389167502507522, + "loss": 1.3657, + "theoretical_loss": 3.376588042585237, + "tokens_seen": 2699427840 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001838716148445336, + "loss": 1.2252, + "theoretical_loss": 3.376581658067291, + "tokens_seen": 2699493376 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018385155466399197, + "loss": 1.2095, + "theoretical_loss": 3.3765752737477395, + "tokens_seen": 2699558912 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018383149448345036, + "loss": 1.3188, + "theoretical_loss": 3.3765688896265713, + "tokens_seen": 2699624448 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018381143430290872, + "loss": 1.3971, + "theoretical_loss": 3.3765625057037756, + "tokens_seen": 2699689984 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001837913741223671, + "loss": 1.2705, + "theoretical_loss": 3.3765561219793403, + "tokens_seen": 2699755520 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018377131394182547, + "loss": 1.1753, + "theoretical_loss": 3.376549738453256, + "tokens_seen": 2699821056 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018375125376128386, + "loss": 1.3216, + "theoretical_loss": 3.376543355125511, + "tokens_seen": 2699886592 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018373119358074222, + "loss": 1.2014, + "theoretical_loss": 3.376536971996094, + "tokens_seen": 2699952128 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018371113340020058, + "loss": 1.3409, + "theoretical_loss": 3.376530589064995, + "tokens_seen": 2700017664 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018369107321965897, + "loss": 1.2144, + "theoretical_loss": 3.3765242063322014, + "tokens_seen": 2700083200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018367101303911734, + "loss": 1.2837, + "theoretical_loss": 3.376517823797704, + "tokens_seen": 2700148736 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018365095285857573, + "loss": 1.3162, + "theoretical_loss": 3.3765114414614907, + "tokens_seen": 2700214272 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001836308926780341, + "loss": 1.2485, + "theoretical_loss": 3.376505059323551, + "tokens_seen": 2700279808 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018361083249749248, + "loss": 1.2465, + "theoretical_loss": 3.3764986773838737, + "tokens_seen": 2700345344 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018359077231695084, + "loss": 1.3127, + "theoretical_loss": 3.3764922956424477, + "tokens_seen": 2700410880 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018357071213640923, + "loss": 1.2483, + "theoretical_loss": 3.3764859140992627, + "tokens_seen": 2700476416 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018355065195586762, + "loss": 1.3849, + "theoretical_loss": 3.376479532754307, + "tokens_seen": 2700541952 + }, + { + "epoch": 0.64, + "learning_rate": 0.000183530591775326, + "loss": 1.3264, + "theoretical_loss": 3.37647315160757, + "tokens_seen": 2700607488 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018351053159478437, + "loss": 1.32, + "theoretical_loss": 3.3764667706590408, + "tokens_seen": 2700673024 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018349047141424273, + "loss": 1.3413, + "theoretical_loss": 3.3764603899087082, + "tokens_seen": 2700738560 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018347041123370112, + "loss": 1.373, + "theoretical_loss": 3.376454009356561, + "tokens_seen": 2700804096 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018345035105315948, + "loss": 1.383, + "theoretical_loss": 3.3764476290025893, + "tokens_seen": 2700869632 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018343029087261787, + "loss": 1.2484, + "theoretical_loss": 3.3764412488467808, + "tokens_seen": 2700935168 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018341023069207623, + "loss": 1.3402, + "theoretical_loss": 3.3764348688891253, + "tokens_seen": 2701000704 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018339017051153462, + "loss": 1.3182, + "theoretical_loss": 3.3764284891296117, + "tokens_seen": 2701066240 + }, + { + "epoch": 0.64, + "learning_rate": 0.000183370110330993, + "loss": 1.1654, + "theoretical_loss": 3.376422109568229, + "tokens_seen": 2701131776 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018335005015045138, + "loss": 1.255, + "theoretical_loss": 3.376415730204966, + "tokens_seen": 2701197312 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018332998996990974, + "loss": 1.4261, + "theoretical_loss": 3.376409351039812, + "tokens_seen": 2701262848 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001833099297893681, + "loss": 1.3415, + "theoretical_loss": 3.3764029720727566, + "tokens_seen": 2701328384 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001832898696088265, + "loss": 1.2836, + "theoretical_loss": 3.376396593303788, + "tokens_seen": 2701393920 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018326980942828485, + "loss": 1.2847, + "theoretical_loss": 3.3763902147328952, + "tokens_seen": 2701459456 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018324974924774324, + "loss": 1.3633, + "theoretical_loss": 3.376383836360068, + "tokens_seen": 2701524992 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001832296890672016, + "loss": 1.2633, + "theoretical_loss": 3.3763774581852948, + "tokens_seen": 2701590528 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018320962888666, + "loss": 1.3437, + "theoretical_loss": 3.3763710802085645, + "tokens_seen": 2701656064 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018318956870611836, + "loss": 1.2914, + "theoretical_loss": 3.376364702429867, + "tokens_seen": 2701721600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018316950852557674, + "loss": 1.3869, + "theoretical_loss": 3.376358324849191, + "tokens_seen": 2701787136 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 981962, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.268550157546997, + "objective/train/theoretical_loss": 3.3763519474665245, + "objective/train/tokens_used": 1072311776, + "theoretical_loss": 3.3763519474665245, + "tokens_seen": 2701852672 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001831494483450351, + "loss": 1.2926, + "theoretical_loss": 3.3763519474665245, + "tokens_seen": 2701852672 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018312938816449347, + "loss": 1.2001, + "theoretical_loss": 3.3763455702818583, + "tokens_seen": 2701918208 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018310932798395186, + "loss": 1.3362, + "theoretical_loss": 3.37633919329518, + "tokens_seen": 2701983744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018308926780341022, + "loss": 1.3439, + "theoretical_loss": 3.37633281650648, + "tokens_seen": 2702049280 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001830692076228686, + "loss": 1.2448, + "theoretical_loss": 3.3763264399157458, + "tokens_seen": 2702114816 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018304914744232697, + "loss": 1.3015, + "theoretical_loss": 3.3763200635229675, + "tokens_seen": 2702180352 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018302908726178536, + "loss": 1.194, + "theoretical_loss": 3.376313687328134, + "tokens_seen": 2702245888 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018300902708124372, + "loss": 1.3356, + "theoretical_loss": 3.3763073113312343, + "tokens_seen": 2702311424 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001829889669007021, + "loss": 1.2151, + "theoretical_loss": 3.3763009355322575, + "tokens_seen": 2702376960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018296890672016048, + "loss": 1.2108, + "theoretical_loss": 3.3762945599311927, + "tokens_seen": 2702442496 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018294884653961886, + "loss": 1.3062, + "theoretical_loss": 3.376288184528028, + "tokens_seen": 2702508032 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018292878635907723, + "loss": 1.3143, + "theoretical_loss": 3.3762818093227542, + "tokens_seen": 2702573568 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001829087261785356, + "loss": 1.2904, + "theoretical_loss": 3.3762754343153594, + "tokens_seen": 2702639104 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018288866599799398, + "loss": 1.3086, + "theoretical_loss": 3.376269059505832, + "tokens_seen": 2702704640 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018286860581745234, + "loss": 1.2936, + "theoretical_loss": 3.3762626848941624, + "tokens_seen": 2702770176 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018284854563691073, + "loss": 1.2879, + "theoretical_loss": 3.376256310480339, + "tokens_seen": 2702835712 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001828284854563691, + "loss": 1.4082, + "theoretical_loss": 3.3762499362643505, + "tokens_seen": 2702901248 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018280842527582748, + "loss": 1.3968, + "theoretical_loss": 3.3762435622461866, + "tokens_seen": 2702966784 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018278836509528587, + "loss": 1.2412, + "theoretical_loss": 3.376237188425836, + "tokens_seen": 2703032320 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018276830491474426, + "loss": 1.2635, + "theoretical_loss": 3.376230814803288, + "tokens_seen": 2703097856 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018274824473420262, + "loss": 1.2619, + "theoretical_loss": 3.3762244413785316, + "tokens_seen": 2703163392 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018272818455366098, + "loss": 1.3502, + "theoretical_loss": 3.3762180681515557, + "tokens_seen": 2703228928 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018270812437311937, + "loss": 1.2567, + "theoretical_loss": 3.3762116951223495, + "tokens_seen": 2703294464 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018268806419257774, + "loss": 1.2372, + "theoretical_loss": 3.376205322290902, + "tokens_seen": 2703360000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018266800401203613, + "loss": 1.2614, + "theoretical_loss": 3.3761989496572022, + "tokens_seen": 2703425536 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001826479438314945, + "loss": 1.3517, + "theoretical_loss": 3.3761925772212398, + "tokens_seen": 2703491072 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018262788365095288, + "loss": 1.3059, + "theoretical_loss": 3.376186204983003, + "tokens_seen": 2703556608 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018260782347041124, + "loss": 1.3102, + "theoretical_loss": 3.3761798329424813, + "tokens_seen": 2703622144 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018258776328986963, + "loss": 1.1371, + "theoretical_loss": 3.376173461099664, + "tokens_seen": 2703687680 + }, + { + "epoch": 0.64, + "learning_rate": 0.000182567703109328, + "loss": 1.2977, + "theoretical_loss": 3.3761670894545395, + "tokens_seen": 2703753216 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018254764292878638, + "loss": 1.2429, + "theoretical_loss": 3.376160718007097, + "tokens_seen": 2703818752 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018252758274824474, + "loss": 1.3842, + "theoretical_loss": 3.3761543467573265, + "tokens_seen": 2703884288 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001825075225677031, + "loss": 1.3175, + "theoretical_loss": 3.3761479757052157, + "tokens_seen": 2703949824 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001824874623871615, + "loss": 1.3692, + "theoretical_loss": 3.376141604850755, + "tokens_seen": 2704015360 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018246740220661986, + "loss": 1.3141, + "theoretical_loss": 3.3761352341939324, + "tokens_seen": 2704080896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018244734202607825, + "loss": 1.3081, + "theoretical_loss": 3.3761288637347375, + "tokens_seen": 2704146432 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001824272818455366, + "loss": 1.3467, + "theoretical_loss": 3.3761224934731597, + "tokens_seen": 2704211968 + }, + { + "epoch": 0.64, + "learning_rate": 0.000182407221664995, + "loss": 1.3624, + "theoretical_loss": 3.3761161234091874, + "tokens_seen": 2704277504 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018238716148445336, + "loss": 1.337, + "theoretical_loss": 3.3761097535428104, + "tokens_seen": 2704343040 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018236710130391175, + "loss": 1.3665, + "theoretical_loss": 3.376103383874017, + "tokens_seen": 2704408576 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001823470411233701, + "loss": 1.3692, + "theoretical_loss": 3.376097014402797, + "tokens_seen": 2704474112 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018232698094282847, + "loss": 1.2274, + "theoretical_loss": 3.3760906451291386, + "tokens_seen": 2704539648 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018230692076228686, + "loss": 1.3324, + "theoretical_loss": 3.3760842760530316, + "tokens_seen": 2704605184 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018228686058174523, + "loss": 1.2264, + "theoretical_loss": 3.376077907174465, + "tokens_seen": 2704670720 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018226680040120361, + "loss": 1.46, + "theoretical_loss": 3.376071538493428, + "tokens_seen": 2704736256 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018224674022066198, + "loss": 1.3338, + "theoretical_loss": 3.376065170009909, + "tokens_seen": 2704801792 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018222668004012037, + "loss": 1.3225, + "theoretical_loss": 3.376058801723898, + "tokens_seen": 2704867328 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018220661985957873, + "loss": 1.2547, + "theoretical_loss": 3.376052433635383, + "tokens_seen": 2704932864 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018218655967903712, + "loss": 1.2694, + "theoretical_loss": 3.3760460657443545, + "tokens_seen": 2704998400 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018216649949849548, + "loss": 1.4207, + "theoretical_loss": 3.3760396980508007, + "tokens_seen": 2705063936 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 983087, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3899855613708496, + "objective/train/theoretical_loss": 3.3760333305547108, + "objective/train/tokens_used": 1075588576, + "theoretical_loss": 3.3760333305547108, + "tokens_seen": 2705129472 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018214643931795384, + "loss": 1.429, + "theoretical_loss": 3.3760333305547108, + "tokens_seen": 2705129472 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018212637913741223, + "loss": 1.2938, + "theoretical_loss": 3.376026963256074, + "tokens_seen": 2705195008 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001821063189568706, + "loss": 1.3034, + "theoretical_loss": 3.3760205961548793, + "tokens_seen": 2705260544 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018208625877632898, + "loss": 1.2755, + "theoretical_loss": 3.3760142292511155, + "tokens_seen": 2705326080 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018206619859578735, + "loss": 1.2975, + "theoretical_loss": 3.376007862544772, + "tokens_seen": 2705391616 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018204613841524576, + "loss": 1.25, + "theoretical_loss": 3.376001496035838, + "tokens_seen": 2705457152 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018202607823470412, + "loss": 1.3093, + "theoretical_loss": 3.375995129724303, + "tokens_seen": 2705522688 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001820060180541625, + "loss": 1.2637, + "theoretical_loss": 3.375988763610155, + "tokens_seen": 2705588224 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018198595787362088, + "loss": 1.2303, + "theoretical_loss": 3.375982397693384, + "tokens_seen": 2705653760 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018196589769307926, + "loss": 1.256, + "theoretical_loss": 3.3759760319739787, + "tokens_seen": 2705719296 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018194583751253763, + "loss": 1.2352, + "theoretical_loss": 3.3759696664519288, + "tokens_seen": 2705784832 + }, + { + "epoch": 0.64, + "learning_rate": 0.000181925777331996, + "loss": 1.3654, + "theoretical_loss": 3.3759633011272223, + "tokens_seen": 2705850368 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018190571715145438, + "loss": 1.2762, + "theoretical_loss": 3.3759569359998487, + "tokens_seen": 2705915904 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018188565697091274, + "loss": 1.3371, + "theoretical_loss": 3.3759505710697977, + "tokens_seen": 2705981440 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018186559679037113, + "loss": 1.3019, + "theoretical_loss": 3.375944206337058, + "tokens_seen": 2706046976 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001818455366098295, + "loss": 1.2163, + "theoretical_loss": 3.3759378418016186, + "tokens_seen": 2706112512 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018182547642928788, + "loss": 1.2684, + "theoretical_loss": 3.375931477463469, + "tokens_seen": 2706178048 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018180541624874624, + "loss": 1.2597, + "theoretical_loss": 3.375925113322598, + "tokens_seen": 2706243584 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018178535606820463, + "loss": 1.2853, + "theoretical_loss": 3.3759187493789944, + "tokens_seen": 2706309120 + }, + { + "epoch": 0.64, + "learning_rate": 0.000181765295887663, + "loss": 1.2427, + "theoretical_loss": 3.375912385632648, + "tokens_seen": 2706374656 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018174523570712136, + "loss": 1.2836, + "theoretical_loss": 3.375906022083547, + "tokens_seen": 2706440192 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018172517552657975, + "loss": 1.3165, + "theoretical_loss": 3.3758996587316816, + "tokens_seen": 2706505728 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001817051153460381, + "loss": 1.3351, + "theoretical_loss": 3.37589329557704, + "tokens_seen": 2706571264 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001816850551654965, + "loss": 1.288, + "theoretical_loss": 3.375886932619612, + "tokens_seen": 2706636800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018166499498495486, + "loss": 1.4257, + "theoretical_loss": 3.375880569859387, + "tokens_seen": 2706702336 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018164493480441325, + "loss": 1.3389, + "theoretical_loss": 3.3758742072963526, + "tokens_seen": 2706767872 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001816248746238716, + "loss": 1.3796, + "theoretical_loss": 3.3758678449304993, + "tokens_seen": 2706833408 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018160481444333, + "loss": 1.3624, + "theoretical_loss": 3.375861482761815, + "tokens_seen": 2706898944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018158475426278836, + "loss": 1.3978, + "theoretical_loss": 3.3758551207902903, + "tokens_seen": 2706964480 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018156469408224673, + "loss": 1.3236, + "theoretical_loss": 3.3758487590159136, + "tokens_seen": 2707030016 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018154463390170512, + "loss": 1.3101, + "theoretical_loss": 3.3758423974386735, + "tokens_seen": 2707095552 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018152457372116348, + "loss": 1.4664, + "theoretical_loss": 3.37583603605856, + "tokens_seen": 2707161088 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018150451354062187, + "loss": 1.3504, + "theoretical_loss": 3.3758296748755616, + "tokens_seen": 2707226624 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018148445336008023, + "loss": 1.2378, + "theoretical_loss": 3.375823313889668, + "tokens_seen": 2707292160 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018146439317953862, + "loss": 1.2975, + "theoretical_loss": 3.375816953100868, + "tokens_seen": 2707357696 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018144433299899698, + "loss": 1.3198, + "theoretical_loss": 3.37581059250915, + "tokens_seen": 2707423232 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018142427281845537, + "loss": 1.4959, + "theoretical_loss": 3.3758042321145045, + "tokens_seen": 2707488768 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018140421263791373, + "loss": 1.3353, + "theoretical_loss": 3.3757978719169195, + "tokens_seen": 2707554304 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018138415245737212, + "loss": 1.3857, + "theoretical_loss": 3.3757915119163853, + "tokens_seen": 2707619840 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018136409227683048, + "loss": 1.2183, + "theoretical_loss": 3.37578515211289, + "tokens_seen": 2707685376 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018134403209628885, + "loss": 1.3534, + "theoretical_loss": 3.3757787925064227, + "tokens_seen": 2707750912 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018132397191574724, + "loss": 1.2593, + "theoretical_loss": 3.375772433096973, + "tokens_seen": 2707816448 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018130391173520563, + "loss": 1.3121, + "theoretical_loss": 3.3757660738845296, + "tokens_seen": 2707881984 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018128385155466401, + "loss": 1.1856, + "theoretical_loss": 3.375759714869082, + "tokens_seen": 2707947520 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018126379137412238, + "loss": 1.3425, + "theoretical_loss": 3.3757533560506197, + "tokens_seen": 2708013056 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018124373119358077, + "loss": 1.2115, + "theoretical_loss": 3.375746997429131, + "tokens_seen": 2708078592 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018122367101303913, + "loss": 1.3329, + "theoretical_loss": 3.375740639004606, + "tokens_seen": 2708144128 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018120361083249752, + "loss": 1.3617, + "theoretical_loss": 3.3757342807770323, + "tokens_seen": 2708209664 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018118355065195588, + "loss": 1.4144, + "theoretical_loss": 3.3757279227464005, + "tokens_seen": 2708275200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018116349047141424, + "loss": 1.3063, + "theoretical_loss": 3.3757215649126993, + "tokens_seen": 2708340736 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 984234, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.212406039237976, + "objective/train/theoretical_loss": 3.375715207275918, + "objective/train/tokens_used": 1078865376, + "theoretical_loss": 3.375715207275918, + "tokens_seen": 2708406272 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018114343029087263, + "loss": 1.3512, + "theoretical_loss": 3.375715207275918, + "tokens_seen": 2708406272 + }, + { + "epoch": 0.64, + "learning_rate": 0.000181123370110331, + "loss": 1.3705, + "theoretical_loss": 3.3757088498360446, + "tokens_seen": 2708471808 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018110330992978938, + "loss": 1.355, + "theoretical_loss": 3.37570249259307, + "tokens_seen": 2708537344 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018108324974924775, + "loss": 1.2609, + "theoretical_loss": 3.3756961355469817, + "tokens_seen": 2708602880 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018106318956870613, + "loss": 1.3343, + "theoretical_loss": 3.3756897786977698, + "tokens_seen": 2708668416 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001810431293881645, + "loss": 1.2499, + "theoretical_loss": 3.3756834220454235, + "tokens_seen": 2708733952 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001810230692076229, + "loss": 1.1874, + "theoretical_loss": 3.3756770655899313, + "tokens_seen": 2708799488 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018100300902708125, + "loss": 1.4343, + "theoretical_loss": 3.375670709331283, + "tokens_seen": 2708865024 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001809829488465396, + "loss": 1.3782, + "theoretical_loss": 3.3756643532694675, + "tokens_seen": 2708930560 + }, + { + "epoch": 0.64, + "learning_rate": 0.000180962888665998, + "loss": 1.4183, + "theoretical_loss": 3.375657997404474, + "tokens_seen": 2708996096 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018094282848545636, + "loss": 1.3337, + "theoretical_loss": 3.375651641736291, + "tokens_seen": 2709061632 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018092276830491475, + "loss": 1.4367, + "theoretical_loss": 3.3756452862649087, + "tokens_seen": 2709127168 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018090270812437311, + "loss": 1.3099, + "theoretical_loss": 3.3756389309903154, + "tokens_seen": 2709192704 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001808826479438315, + "loss": 1.3732, + "theoretical_loss": 3.375632575912501, + "tokens_seen": 2709258240 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018086258776328987, + "loss": 1.3269, + "theoretical_loss": 3.375626221031454, + "tokens_seen": 2709323776 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018084252758274826, + "loss": 1.2156, + "theoretical_loss": 3.3756198663471637, + "tokens_seen": 2709389312 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018082246740220662, + "loss": 1.4591, + "theoretical_loss": 3.375613511859619, + "tokens_seen": 2709454848 + }, + { + "epoch": 0.64, + "learning_rate": 0.000180802407221665, + "loss": 1.4467, + "theoretical_loss": 3.37560715756881, + "tokens_seen": 2709520384 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018078234704112337, + "loss": 1.2421, + "theoretical_loss": 3.375600803474725, + "tokens_seen": 2709585920 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018076228686058173, + "loss": 1.38, + "theoretical_loss": 3.3755944495773536, + "tokens_seen": 2709651456 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018074222668004012, + "loss": 1.3948, + "theoretical_loss": 3.3755880958766844, + "tokens_seen": 2709716992 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018072216649949848, + "loss": 1.3127, + "theoretical_loss": 3.375581742372707, + "tokens_seen": 2709782528 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018070210631895687, + "loss": 1.3331, + "theoretical_loss": 3.37557538906541, + "tokens_seen": 2709848064 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018068204613841523, + "loss": 1.207, + "theoretical_loss": 3.3755690359547836, + "tokens_seen": 2709913600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018066198595787362, + "loss": 1.3121, + "theoretical_loss": 3.375562683040816, + "tokens_seen": 2709979136 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018064192577733199, + "loss": 1.2909, + "theoretical_loss": 3.375556330323497, + "tokens_seen": 2710044672 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018062186559679038, + "loss": 1.3125, + "theoretical_loss": 3.375549977802815, + "tokens_seen": 2710110208 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018060180541624874, + "loss": 1.34, + "theoretical_loss": 3.37554362547876, + "tokens_seen": 2710175744 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001805817452357071, + "loss": 1.358, + "theoretical_loss": 3.3755372733513207, + "tokens_seen": 2710241280 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001805616850551655, + "loss": 1.2636, + "theoretical_loss": 3.3755309214204865, + "tokens_seen": 2710306816 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018054162487462388, + "loss": 1.3878, + "theoretical_loss": 3.375524569686246, + "tokens_seen": 2710372352 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018052156469408227, + "loss": 1.3536, + "theoretical_loss": 3.375518218148589, + "tokens_seen": 2710437888 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018050150451354063, + "loss": 1.266, + "theoretical_loss": 3.3755118668075044, + "tokens_seen": 2710503424 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018048144433299902, + "loss": 1.2186, + "theoretical_loss": 3.3755055156629816, + "tokens_seen": 2710568960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018046138415245738, + "loss": 1.3233, + "theoretical_loss": 3.375499164715009, + "tokens_seen": 2710634496 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018044132397191577, + "loss": 1.3231, + "theoretical_loss": 3.375492813963577, + "tokens_seen": 2710700032 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018042126379137413, + "loss": 1.472, + "theoretical_loss": 3.3754864634086736, + "tokens_seen": 2710765568 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018040120361083252, + "loss": 1.3516, + "theoretical_loss": 3.3754801130502887, + "tokens_seen": 2710831104 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018038114343029088, + "loss": 1.3306, + "theoretical_loss": 3.3754737628884115, + "tokens_seen": 2710896640 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018036108324974925, + "loss": 1.4419, + "theoretical_loss": 3.3754674129230304, + "tokens_seen": 2710962176 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018034102306920764, + "loss": 1.2292, + "theoretical_loss": 3.375461063154135, + "tokens_seen": 2711027712 + }, + { + "epoch": 0.64, + "learning_rate": 0.000180320962888666, + "loss": 1.2546, + "theoretical_loss": 3.375454713581715, + "tokens_seen": 2711093248 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001803009027081244, + "loss": 1.3587, + "theoretical_loss": 3.3754483642057584, + "tokens_seen": 2711158784 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018028084252758275, + "loss": 1.3103, + "theoretical_loss": 3.375442015026256, + "tokens_seen": 2711224320 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018026078234704114, + "loss": 1.1019, + "theoretical_loss": 3.3754356660431952, + "tokens_seen": 2711289856 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001802407221664995, + "loss": 1.3324, + "theoretical_loss": 3.3754293172565664, + "tokens_seen": 2711355392 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001802206619859579, + "loss": 1.3489, + "theoretical_loss": 3.375422968666358, + "tokens_seen": 2711420928 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018020060180541625, + "loss": 1.4152, + "theoretical_loss": 3.3754166202725604, + "tokens_seen": 2711486464 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018018054162487462, + "loss": 1.4148, + "theoretical_loss": 3.3754102720751615, + "tokens_seen": 2711552000 + }, + { + "epoch": 0.64, + "learning_rate": 0.000180160481444333, + "loss": 1.3362, + "theoretical_loss": 3.375403924074151, + "tokens_seen": 2711617536 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 985372, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3839335441589355, + "objective/train/theoretical_loss": 3.375397576269518, + "objective/train/tokens_used": 1082142176, + "theoretical_loss": 3.375397576269518, + "tokens_seen": 2711683072 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018014042126379137, + "loss": 1.298, + "theoretical_loss": 3.375397576269518, + "tokens_seen": 2711683072 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018012036108324976, + "loss": 1.2714, + "theoretical_loss": 3.3753912286612513, + "tokens_seen": 2711748608 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018010030090270812, + "loss": 1.2916, + "theoretical_loss": 3.3753848812493406, + "tokens_seen": 2711814144 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001800802407221665, + "loss": 1.2888, + "theoretical_loss": 3.3753785340337754, + "tokens_seen": 2711879680 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018006018054162487, + "loss": 1.2693, + "theoretical_loss": 3.375372187014544, + "tokens_seen": 2711945216 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018004012036108326, + "loss": 1.2787, + "theoretical_loss": 3.375365840191636, + "tokens_seen": 2712010752 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018002006018054162, + "loss": 1.2967, + "theoretical_loss": 3.375359493565041, + "tokens_seen": 2712076288 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017999999999999998, + "loss": 1.1786, + "theoretical_loss": 3.3753531471347475, + "tokens_seen": 2712141824 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017997993981945837, + "loss": 1.4275, + "theoretical_loss": 3.375346800900745, + "tokens_seen": 2712207360 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017995987963891674, + "loss": 1.2934, + "theoretical_loss": 3.3753404548630224, + "tokens_seen": 2712272896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017993981945837513, + "loss": 1.3358, + "theoretical_loss": 3.375334109021569, + "tokens_seen": 2712338432 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001799197592778335, + "loss": 1.3402, + "theoretical_loss": 3.375327763376375, + "tokens_seen": 2712403968 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017989969909729188, + "loss": 1.3088, + "theoretical_loss": 3.3753214179274282, + "tokens_seen": 2712469504 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017987963891675024, + "loss": 1.3327, + "theoretical_loss": 3.3753150726747183, + "tokens_seen": 2712535040 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017985957873620863, + "loss": 1.344, + "theoretical_loss": 3.3753087276182345, + "tokens_seen": 2712600576 + }, + { + "epoch": 0.64, + "learning_rate": 0.000179839518555667, + "loss": 1.5, + "theoretical_loss": 3.375302382757966, + "tokens_seen": 2712666112 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017981945837512538, + "loss": 1.4291, + "theoretical_loss": 3.375296038093902, + "tokens_seen": 2712731648 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017979939819458377, + "loss": 1.2942, + "theoretical_loss": 3.3752896936260313, + "tokens_seen": 2712797184 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017977933801404213, + "loss": 1.3567, + "theoretical_loss": 3.3752833493543437, + "tokens_seen": 2712862720 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017975927783350052, + "loss": 1.44, + "theoretical_loss": 3.3752770052788286, + "tokens_seen": 2712928256 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017973921765295888, + "loss": 1.2012, + "theoretical_loss": 3.3752706613994743, + "tokens_seen": 2712993792 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017971915747241727, + "loss": 1.2104, + "theoretical_loss": 3.3752643177162702, + "tokens_seen": 2713059328 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017969909729187563, + "loss": 1.324, + "theoretical_loss": 3.375257974229206, + "tokens_seen": 2713124864 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017967903711133402, + "loss": 1.1641, + "theoretical_loss": 3.375251630938271, + "tokens_seen": 2713190400 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017965897693079239, + "loss": 1.2993, + "theoretical_loss": 3.3752452878434536, + "tokens_seen": 2713255936 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017963891675025078, + "loss": 1.4211, + "theoretical_loss": 3.3752389449447433, + "tokens_seen": 2713321472 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017961885656970914, + "loss": 1.3962, + "theoretical_loss": 3.37523260224213, + "tokens_seen": 2713387008 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001795987963891675, + "loss": 1.3953, + "theoretical_loss": 3.375226259735602, + "tokens_seen": 2713452544 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001795787362086259, + "loss": 1.299, + "theoretical_loss": 3.3752199174251487, + "tokens_seen": 2713518080 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017955867602808425, + "loss": 1.3269, + "theoretical_loss": 3.37521357531076, + "tokens_seen": 2713583616 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017953861584754264, + "loss": 1.184, + "theoretical_loss": 3.375207233392424, + "tokens_seen": 2713649152 + }, + { + "epoch": 0.64, + "learning_rate": 0.000179518555667001, + "loss": 1.2643, + "theoretical_loss": 3.3752008916701306, + "tokens_seen": 2713714688 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001794984954864594, + "loss": 1.5399, + "theoretical_loss": 3.375194550143869, + "tokens_seen": 2713780224 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017947843530591775, + "loss": 1.3362, + "theoretical_loss": 3.3751882088136282, + "tokens_seen": 2713845760 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017945837512537614, + "loss": 1.2977, + "theoretical_loss": 3.375181867679397, + "tokens_seen": 2713911296 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001794383149448345, + "loss": 1.2893, + "theoretical_loss": 3.375175526741166, + "tokens_seen": 2713976832 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017941825476429287, + "loss": 1.3033, + "theoretical_loss": 3.375169185998923, + "tokens_seen": 2714042368 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017939819458375126, + "loss": 1.3452, + "theoretical_loss": 3.3751628454526577, + "tokens_seen": 2714107904 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017937813440320962, + "loss": 1.2127, + "theoretical_loss": 3.3751565051023595, + "tokens_seen": 2714173440 + }, + { + "epoch": 0.64, + "learning_rate": 0.000179358074222668, + "loss": 1.3915, + "theoretical_loss": 3.3751501649480176, + "tokens_seen": 2714238976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017933801404212637, + "loss": 1.1754, + "theoretical_loss": 3.3751438249896206, + "tokens_seen": 2714304512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017931795386158476, + "loss": 1.2748, + "theoretical_loss": 3.375137485227158, + "tokens_seen": 2714370048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017929789368104312, + "loss": 1.3254, + "theoretical_loss": 3.3751311456606197, + "tokens_seen": 2714435584 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001792778335005015, + "loss": 1.3941, + "theoretical_loss": 3.375124806289994, + "tokens_seen": 2714501120 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017925777331995988, + "loss": 1.3558, + "theoretical_loss": 3.3751184671152705, + "tokens_seen": 2714566656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017923771313941826, + "loss": 1.3026, + "theoretical_loss": 3.375112128136439, + "tokens_seen": 2714632192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017921765295887663, + "loss": 1.359, + "theoretical_loss": 3.3751057893534875, + "tokens_seen": 2714697728 + }, + { + "epoch": 0.65, + "learning_rate": 0.000179197592778335, + "loss": 1.3657, + "theoretical_loss": 3.375099450766406, + "tokens_seen": 2714763264 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017917753259779338, + "loss": 1.351, + "theoretical_loss": 3.3750931123751835, + "tokens_seen": 2714828800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017915747241725174, + "loss": 1.3502, + "theoretical_loss": 3.375086774179809, + "tokens_seen": 2714894336 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 986641, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2018595933914185, + "objective/train/theoretical_loss": 3.375080436180273, + "objective/train/tokens_used": 1085418976, + "theoretical_loss": 3.375080436180273, + "tokens_seen": 2714959872 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017913741223671013, + "loss": 1.2518, + "theoretical_loss": 3.375080436180273, + "tokens_seen": 2714959872 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001791173520561685, + "loss": 1.2654, + "theoretical_loss": 3.3750740983765626, + "tokens_seen": 2715025408 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017909729187562688, + "loss": 1.3174, + "theoretical_loss": 3.375067760768669, + "tokens_seen": 2715090944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017907723169508524, + "loss": 1.3371, + "theoretical_loss": 3.37506142335658, + "tokens_seen": 2715156480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017905717151454363, + "loss": 1.3157, + "theoretical_loss": 3.3750550861402857, + "tokens_seen": 2715222016 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017903711133400202, + "loss": 1.14, + "theoretical_loss": 3.375048749119775, + "tokens_seen": 2715287552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017901705115346038, + "loss": 1.1996, + "theoretical_loss": 3.3750424122950373, + "tokens_seen": 2715353088 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017899699097291877, + "loss": 1.2348, + "theoretical_loss": 3.3750360756660616, + "tokens_seen": 2715418624 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017897693079237714, + "loss": 1.3652, + "theoretical_loss": 3.3750297392328372, + "tokens_seen": 2715484160 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017895687061183553, + "loss": 1.357, + "theoretical_loss": 3.375023402995353, + "tokens_seen": 2715549696 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001789368104312939, + "loss": 1.2461, + "theoretical_loss": 3.3750170669535993, + "tokens_seen": 2715615232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017891675025075228, + "loss": 1.1629, + "theoretical_loss": 3.375010731107564, + "tokens_seen": 2715680768 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017889669007021064, + "loss": 1.1867, + "theoretical_loss": 3.375004395457237, + "tokens_seen": 2715746304 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017887662988966903, + "loss": 1.2205, + "theoretical_loss": 3.3749980600026075, + "tokens_seen": 2715811840 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001788565697091274, + "loss": 1.3899, + "theoretical_loss": 3.3749917247436647, + "tokens_seen": 2715877376 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017883650952858578, + "loss": 1.3079, + "theoretical_loss": 3.374985389680398, + "tokens_seen": 2715942912 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017881644934804414, + "loss": 1.2199, + "theoretical_loss": 3.3749790548127963, + "tokens_seen": 2716008448 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001787963891675025, + "loss": 1.2889, + "theoretical_loss": 3.3749727201408493, + "tokens_seen": 2716073984 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001787763289869609, + "loss": 1.2586, + "theoretical_loss": 3.3749663856645453, + "tokens_seen": 2716139520 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017875626880641926, + "loss": 1.3, + "theoretical_loss": 3.3749600513838747, + "tokens_seen": 2716205056 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017873620862587765, + "loss": 1.2731, + "theoretical_loss": 3.374953717298826, + "tokens_seen": 2716270592 + }, + { + "epoch": 0.65, + "learning_rate": 0.000178716148445336, + "loss": 1.2522, + "theoretical_loss": 3.374947383409389, + "tokens_seen": 2716336128 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001786960882647944, + "loss": 1.1864, + "theoretical_loss": 3.3749410497155523, + "tokens_seen": 2716401664 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017867602808425276, + "loss": 1.3336, + "theoretical_loss": 3.374934716217305, + "tokens_seen": 2716467200 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017865596790371115, + "loss": 1.2186, + "theoretical_loss": 3.3749283829146375, + "tokens_seen": 2716532736 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001786359077231695, + "loss": 1.2264, + "theoretical_loss": 3.374922049807538, + "tokens_seen": 2716598272 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017861584754262787, + "loss": 1.3095, + "theoretical_loss": 3.374915716895996, + "tokens_seen": 2716663808 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017859578736208626, + "loss": 1.2308, + "theoretical_loss": 3.374909384180001, + "tokens_seen": 2716729344 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017857572718154462, + "loss": 1.2784, + "theoretical_loss": 3.374903051659542, + "tokens_seen": 2716794880 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017855566700100301, + "loss": 1.2729, + "theoretical_loss": 3.374896719334608, + "tokens_seen": 2716860416 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017853560682046138, + "loss": 1.3034, + "theoretical_loss": 3.374890387205189, + "tokens_seen": 2716925952 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017851554663991977, + "loss": 1.2636, + "theoretical_loss": 3.3748840552712736, + "tokens_seen": 2716991488 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017849548645937813, + "loss": 1.3777, + "theoretical_loss": 3.374877723532851, + "tokens_seen": 2717057024 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017847542627883652, + "loss": 1.3283, + "theoretical_loss": 3.374871391989911, + "tokens_seen": 2717122560 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017845536609829488, + "loss": 1.2638, + "theoretical_loss": 3.3748650606424424, + "tokens_seen": 2717188096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017843530591775324, + "loss": 1.3265, + "theoretical_loss": 3.3748587294904344, + "tokens_seen": 2717253632 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017841524573721163, + "loss": 1.2737, + "theoretical_loss": 3.3748523985338768, + "tokens_seen": 2717319168 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017839518555667, + "loss": 1.2463, + "theoretical_loss": 3.3748460677727583, + "tokens_seen": 2717384704 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017837512537612838, + "loss": 1.3187, + "theoretical_loss": 3.3748397372070684, + "tokens_seen": 2717450240 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017835506519558675, + "loss": 1.2911, + "theoretical_loss": 3.374833406836796, + "tokens_seen": 2717515776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017833500501504513, + "loss": 1.339, + "theoretical_loss": 3.3748270766619313, + "tokens_seen": 2717581312 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001783149448345035, + "loss": 1.2923, + "theoretical_loss": 3.3748207466824622, + "tokens_seen": 2717646848 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001782948846539619, + "loss": 1.279, + "theoretical_loss": 3.374814416898379, + "tokens_seen": 2717712384 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017827482447342028, + "loss": 1.1514, + "theoretical_loss": 3.374808087309671, + "tokens_seen": 2717777920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017825476429287866, + "loss": 1.2629, + "theoretical_loss": 3.3748017579163267, + "tokens_seen": 2717843456 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017823470411233703, + "loss": 1.3609, + "theoretical_loss": 3.3747954287183357, + "tokens_seen": 2717908992 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001782146439317954, + "loss": 1.3436, + "theoretical_loss": 3.374789099715687, + "tokens_seen": 2717974528 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017819458375125378, + "loss": 1.2496, + "theoretical_loss": 3.3747827709083706, + "tokens_seen": 2718040064 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017817452357071214, + "loss": 1.2843, + "theoretical_loss": 3.3747764422963753, + "tokens_seen": 2718105600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017815446339017053, + "loss": 1.2061, + "theoretical_loss": 3.3747701138796904, + "tokens_seen": 2718171136 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 987833, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4549930095672607, + "objective/train/theoretical_loss": 3.3747637856583053, + "objective/train/tokens_used": 1088695776, + "theoretical_loss": 3.3747637856583053, + "tokens_seen": 2718236672 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001781344032096289, + "loss": 1.3678, + "theoretical_loss": 3.3747637856583053, + "tokens_seen": 2718236672 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017811434302908728, + "loss": 1.2925, + "theoretical_loss": 3.374757457632209, + "tokens_seen": 2718302208 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017809428284854564, + "loss": 1.3231, + "theoretical_loss": 3.3747511298013904, + "tokens_seen": 2718367744 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017807422266800403, + "loss": 1.3365, + "theoretical_loss": 3.3747448021658397, + "tokens_seen": 2718433280 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001780541624874624, + "loss": 1.2816, + "theoretical_loss": 3.374738474725546, + "tokens_seen": 2718498816 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017803410230692076, + "loss": 1.2356, + "theoretical_loss": 3.3747321474804974, + "tokens_seen": 2718564352 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017801404212637915, + "loss": 1.1705, + "theoretical_loss": 3.374725820430685, + "tokens_seen": 2718629888 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001779939819458375, + "loss": 1.288, + "theoretical_loss": 3.3747194935760962, + "tokens_seen": 2718695424 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001779739217652959, + "loss": 1.2488, + "theoretical_loss": 3.374713166916722, + "tokens_seen": 2718760960 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017795386158475426, + "loss": 1.2265, + "theoretical_loss": 3.3747068404525504, + "tokens_seen": 2718826496 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017793380140421265, + "loss": 1.2195, + "theoretical_loss": 3.3747005141835715, + "tokens_seen": 2718892032 + }, + { + "epoch": 0.65, + "learning_rate": 0.000177913741223671, + "loss": 1.3235, + "theoretical_loss": 3.374694188109774, + "tokens_seen": 2718957568 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001778936810431294, + "loss": 1.3751, + "theoretical_loss": 3.3746878622311476, + "tokens_seen": 2719023104 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017787362086258776, + "loss": 1.3078, + "theoretical_loss": 3.3746815365476808, + "tokens_seen": 2719088640 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017785356068204613, + "loss": 1.2639, + "theoretical_loss": 3.3746752110593636, + "tokens_seen": 2719154176 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017783350050150452, + "loss": 1.2618, + "theoretical_loss": 3.3746688857661855, + "tokens_seen": 2719219712 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017781344032096288, + "loss": 1.258, + "theoretical_loss": 3.3746625606681353, + "tokens_seen": 2719285248 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017779338014042127, + "loss": 1.3238, + "theoretical_loss": 3.374656235765202, + "tokens_seen": 2719350784 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017777331995987963, + "loss": 1.2617, + "theoretical_loss": 3.3746499110573756, + "tokens_seen": 2719416320 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017775325977933802, + "loss": 1.2599, + "theoretical_loss": 3.3746435865446447, + "tokens_seen": 2719481856 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017773319959879638, + "loss": 1.2598, + "theoretical_loss": 3.374637262226999, + "tokens_seen": 2719547392 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017771313941825477, + "loss": 1.3305, + "theoretical_loss": 3.374630938104428, + "tokens_seen": 2719612928 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017769307923771313, + "loss": 1.3416, + "theoretical_loss": 3.3746246141769207, + "tokens_seen": 2719678464 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017767301905717152, + "loss": 1.329, + "theoretical_loss": 3.3746182904444657, + "tokens_seen": 2719744000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017765295887662988, + "loss": 1.3004, + "theoretical_loss": 3.374611966907054, + "tokens_seen": 2719809536 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017763289869608825, + "loss": 1.2819, + "theoretical_loss": 3.3746056435646725, + "tokens_seen": 2719875072 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017761283851554664, + "loss": 1.192, + "theoretical_loss": 3.3745993204173126, + "tokens_seen": 2719940608 + }, + { + "epoch": 0.65, + "learning_rate": 0.000177592778335005, + "loss": 1.2836, + "theoretical_loss": 3.374592997464963, + "tokens_seen": 2720006144 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001775727181544634, + "loss": 1.3179, + "theoretical_loss": 3.3745866747076123, + "tokens_seen": 2720071680 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017755265797392175, + "loss": 1.3395, + "theoretical_loss": 3.37458035214525, + "tokens_seen": 2720137216 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017753259779338017, + "loss": 1.3639, + "theoretical_loss": 3.3745740297778664, + "tokens_seen": 2720202752 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017751253761283853, + "loss": 1.3331, + "theoretical_loss": 3.3745677076054497, + "tokens_seen": 2720268288 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017749247743229692, + "loss": 1.3166, + "theoretical_loss": 3.3745613856279895, + "tokens_seen": 2720333824 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017747241725175528, + "loss": 1.3033, + "theoretical_loss": 3.374555063845475, + "tokens_seen": 2720399360 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017745235707121364, + "loss": 1.2054, + "theoretical_loss": 3.374548742257896, + "tokens_seen": 2720464896 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017743229689067203, + "loss": 1.3122, + "theoretical_loss": 3.3745424208652413, + "tokens_seen": 2720530432 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001774122367101304, + "loss": 1.2983, + "theoretical_loss": 3.3745360996675, + "tokens_seen": 2720595968 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017739217652958878, + "loss": 1.344, + "theoretical_loss": 3.374529778664662, + "tokens_seen": 2720661504 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017737211634904715, + "loss": 1.1931, + "theoretical_loss": 3.374523457856716, + "tokens_seen": 2720727040 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017735205616850553, + "loss": 1.308, + "theoretical_loss": 3.374517137243652, + "tokens_seen": 2720792576 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001773319959879639, + "loss": 1.2847, + "theoretical_loss": 3.3745108168254587, + "tokens_seen": 2720858112 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017731193580742229, + "loss": 1.4261, + "theoretical_loss": 3.3745044966021256, + "tokens_seen": 2720923648 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017729187562688065, + "loss": 1.266, + "theoretical_loss": 3.374498176573642, + "tokens_seen": 2720989184 + }, + { + "epoch": 0.65, + "learning_rate": 0.000177271815446339, + "loss": 1.4059, + "theoretical_loss": 3.374491856739997, + "tokens_seen": 2721054720 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001772517552657974, + "loss": 1.1772, + "theoretical_loss": 3.3744855371011804, + "tokens_seen": 2721120256 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017723169508525576, + "loss": 1.2322, + "theoretical_loss": 3.3744792176571807, + "tokens_seen": 2721185792 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017721163490471415, + "loss": 1.2207, + "theoretical_loss": 3.3744728984079884, + "tokens_seen": 2721251328 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017719157472417251, + "loss": 1.3456, + "theoretical_loss": 3.3744665793535917, + "tokens_seen": 2721316864 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001771715145436309, + "loss": 1.368, + "theoretical_loss": 3.37446026049398, + "tokens_seen": 2721382400 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017715145436308927, + "loss": 1.283, + "theoretical_loss": 3.3744539418291435, + "tokens_seen": 2721447936 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 988960, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1977958679199219, + "objective/train/theoretical_loss": 3.3744476233590706, + "objective/train/tokens_used": 1091972576, + "theoretical_loss": 3.3744476233590706, + "tokens_seen": 2721513472 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017713139418254765, + "loss": 1.1915, + "theoretical_loss": 3.3744476233590706, + "tokens_seen": 2721513472 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017711133400200602, + "loss": 1.3639, + "theoretical_loss": 3.374441305083751, + "tokens_seen": 2721579008 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001770912738214644, + "loss": 1.2568, + "theoretical_loss": 3.3744349870031742, + "tokens_seen": 2721644544 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017707121364092277, + "loss": 1.3649, + "theoretical_loss": 3.3744286691173286, + "tokens_seen": 2721710080 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017705115346038113, + "loss": 1.321, + "theoretical_loss": 3.3744223514262046, + "tokens_seen": 2721775616 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017703109327983952, + "loss": 1.3016, + "theoretical_loss": 3.374416033929791, + "tokens_seen": 2721841152 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017701103309929788, + "loss": 1.2809, + "theoretical_loss": 3.3744097166280773, + "tokens_seen": 2721906688 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017699097291875627, + "loss": 1.2408, + "theoretical_loss": 3.3744033995210523, + "tokens_seen": 2721972224 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017697091273821463, + "loss": 1.285, + "theoretical_loss": 3.374397082608706, + "tokens_seen": 2722037760 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017695085255767302, + "loss": 1.3707, + "theoretical_loss": 3.3743907658910275, + "tokens_seen": 2722103296 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017693079237713139, + "loss": 1.3192, + "theoretical_loss": 3.374384449368006, + "tokens_seen": 2722168832 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017691073219658978, + "loss": 1.3878, + "theoretical_loss": 3.3743781330396305, + "tokens_seen": 2722234368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017689067201604814, + "loss": 1.301, + "theoretical_loss": 3.374371816905891, + "tokens_seen": 2722299904 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001768706118355065, + "loss": 1.2924, + "theoretical_loss": 3.374365500966776, + "tokens_seen": 2722365440 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001768505516549649, + "loss": 1.1991, + "theoretical_loss": 3.374359185222276, + "tokens_seen": 2722430976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017683049147442325, + "loss": 1.3, + "theoretical_loss": 3.374352869672379, + "tokens_seen": 2722496512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017681043129388164, + "loss": 1.2906, + "theoretical_loss": 3.374346554317075, + "tokens_seen": 2722562048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017679037111334003, + "loss": 1.2052, + "theoretical_loss": 3.3743402391563535, + "tokens_seen": 2722627584 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017677031093279842, + "loss": 1.2511, + "theoretical_loss": 3.3743339241902035, + "tokens_seen": 2722693120 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017675025075225678, + "loss": 1.4108, + "theoretical_loss": 3.3743276094186143, + "tokens_seen": 2722758656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017673019057171517, + "loss": 1.4201, + "theoretical_loss": 3.3743212948415753, + "tokens_seen": 2722824192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017671013039117353, + "loss": 1.2491, + "theoretical_loss": 3.3743149804590757, + "tokens_seen": 2722889728 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017669007021063192, + "loss": 1.2665, + "theoretical_loss": 3.374308666271105, + "tokens_seen": 2722955264 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017667001003009028, + "loss": 1.2304, + "theoretical_loss": 3.374302352277653, + "tokens_seen": 2723020800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017664994984954865, + "loss": 1.1812, + "theoretical_loss": 3.374296038478708, + "tokens_seen": 2723086336 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017662988966900704, + "loss": 1.3866, + "theoretical_loss": 3.37428972487426, + "tokens_seen": 2723151872 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001766098294884654, + "loss": 1.2546, + "theoretical_loss": 3.3742834114642983, + "tokens_seen": 2723217408 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001765897693079238, + "loss": 1.2385, + "theoretical_loss": 3.374277098248812, + "tokens_seen": 2723282944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017656970912738215, + "loss": 1.283, + "theoretical_loss": 3.3742707852277904, + "tokens_seen": 2723348480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017654964894684054, + "loss": 1.2118, + "theoretical_loss": 3.3742644724012227, + "tokens_seen": 2723414016 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001765295887662989, + "loss": 1.1496, + "theoretical_loss": 3.3742581597690986, + "tokens_seen": 2723479552 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001765095285857573, + "loss": 1.1907, + "theoretical_loss": 3.374251847331408, + "tokens_seen": 2723545088 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017648946840521565, + "loss": 1.3318, + "theoretical_loss": 3.374245535088139, + "tokens_seen": 2723610624 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017646940822467402, + "loss": 1.3166, + "theoretical_loss": 3.3742392230392815, + "tokens_seen": 2723676160 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001764493480441324, + "loss": 1.3096, + "theoretical_loss": 3.3742329111848246, + "tokens_seen": 2723741696 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017642928786359077, + "loss": 1.3086, + "theoretical_loss": 3.3742265995247585, + "tokens_seen": 2723807232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017640922768304916, + "loss": 1.2076, + "theoretical_loss": 3.3742202880590715, + "tokens_seen": 2723872768 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017638916750250752, + "loss": 1.3116, + "theoretical_loss": 3.3742139767877535, + "tokens_seen": 2723938304 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001763691073219659, + "loss": 1.2382, + "theoretical_loss": 3.3742076657107933, + "tokens_seen": 2724003840 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017634904714142427, + "loss": 1.3277, + "theoretical_loss": 3.374201354828181, + "tokens_seen": 2724069376 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017632898696088266, + "loss": 1.3191, + "theoretical_loss": 3.3741950441399053, + "tokens_seen": 2724134912 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017630892678034102, + "loss": 1.348, + "theoretical_loss": 3.3741887336459557, + "tokens_seen": 2724200448 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017628886659979938, + "loss": 1.3347, + "theoretical_loss": 3.374182423346322, + "tokens_seen": 2724265984 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017626880641925777, + "loss": 1.3784, + "theoretical_loss": 3.374176113240993, + "tokens_seen": 2724331520 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017624874623871614, + "loss": 1.3103, + "theoretical_loss": 3.3741698033299583, + "tokens_seen": 2724397056 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017622868605817452, + "loss": 1.3243, + "theoretical_loss": 3.374163493613207, + "tokens_seen": 2724462592 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001762086258776329, + "loss": 1.226, + "theoretical_loss": 3.3741571840907287, + "tokens_seen": 2724528128 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017618856569709128, + "loss": 1.2321, + "theoretical_loss": 3.3741508747625124, + "tokens_seen": 2724593664 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017616850551654964, + "loss": 1.288, + "theoretical_loss": 3.374144565628548, + "tokens_seen": 2724659200 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017614844533600803, + "loss": 1.3012, + "theoretical_loss": 3.3741382566888243, + "tokens_seen": 2724724736 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 990115, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3105227947235107, + "objective/train/theoretical_loss": 3.374131947943331, + "objective/train/tokens_used": 1095249376, + "theoretical_loss": 3.374131947943331, + "tokens_seen": 2724790272 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001761283851554664, + "loss": 1.33, + "theoretical_loss": 3.374131947943331, + "tokens_seen": 2724790272 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017610832497492478, + "loss": 1.2385, + "theoretical_loss": 3.3741256393920573, + "tokens_seen": 2724855808 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017608826479438314, + "loss": 1.2984, + "theoretical_loss": 3.3741193310349926, + "tokens_seen": 2724921344 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001760682046138415, + "loss": 1.1929, + "theoretical_loss": 3.3741130228721263, + "tokens_seen": 2724986880 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017604814443329992, + "loss": 1.2643, + "theoretical_loss": 3.374106714903448, + "tokens_seen": 2725052416 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017602808425275828, + "loss": 1.2704, + "theoretical_loss": 3.3741004071289464, + "tokens_seen": 2725117952 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017600802407221667, + "loss": 1.3131, + "theoretical_loss": 3.374094099548611, + "tokens_seen": 2725183488 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017598796389167503, + "loss": 1.2973, + "theoretical_loss": 3.374087792162432, + "tokens_seen": 2725249024 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017596790371113342, + "loss": 1.2333, + "theoretical_loss": 3.3740814849703975, + "tokens_seen": 2725314560 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017594784353059179, + "loss": 1.2635, + "theoretical_loss": 3.3740751779724976, + "tokens_seen": 2725380096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017592778335005018, + "loss": 1.316, + "theoretical_loss": 3.3740688711687215, + "tokens_seen": 2725445632 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017590772316950854, + "loss": 1.3542, + "theoretical_loss": 3.3740625645590585, + "tokens_seen": 2725511168 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001758876629889669, + "loss": 1.3193, + "theoretical_loss": 3.374056258143498, + "tokens_seen": 2725576704 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001758676028084253, + "loss": 1.2329, + "theoretical_loss": 3.37404995192203, + "tokens_seen": 2725642240 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017584754262788365, + "loss": 1.2795, + "theoretical_loss": 3.3740436458946426, + "tokens_seen": 2725707776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017582748244734204, + "loss": 1.3081, + "theoretical_loss": 3.374037340061326, + "tokens_seen": 2725773312 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001758074222668004, + "loss": 1.333, + "theoretical_loss": 3.3740310344220696, + "tokens_seen": 2725838848 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001757873620862588, + "loss": 1.1846, + "theoretical_loss": 3.3740247289768623, + "tokens_seen": 2725904384 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017576730190571715, + "loss": 1.224, + "theoretical_loss": 3.3740184237256936, + "tokens_seen": 2725969920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017574724172517554, + "loss": 1.3963, + "theoretical_loss": 3.374012118668553, + "tokens_seen": 2726035456 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001757271815446339, + "loss": 1.29, + "theoretical_loss": 3.37400581380543, + "tokens_seen": 2726100992 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017570712136409227, + "loss": 1.2437, + "theoretical_loss": 3.373999509136314, + "tokens_seen": 2726166528 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017568706118355066, + "loss": 1.3018, + "theoretical_loss": 3.373993204661194, + "tokens_seen": 2726232064 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017566700100300902, + "loss": 1.2156, + "theoretical_loss": 3.373986900380059, + "tokens_seen": 2726297600 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001756469408224674, + "loss": 1.177, + "theoretical_loss": 3.3739805962928995, + "tokens_seen": 2726363136 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017562688064192577, + "loss": 1.3073, + "theoretical_loss": 3.373974292399704, + "tokens_seen": 2726428672 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017560682046138416, + "loss": 1.3015, + "theoretical_loss": 3.3739679887004623, + "tokens_seen": 2726494208 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017558676028084252, + "loss": 1.2945, + "theoretical_loss": 3.3739616851951633, + "tokens_seen": 2726559744 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001755667001003009, + "loss": 1.2696, + "theoretical_loss": 3.373955381883797, + "tokens_seen": 2726625280 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017554663991975927, + "loss": 1.321, + "theoretical_loss": 3.373949078766352, + "tokens_seen": 2726690816 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017552657973921766, + "loss": 1.3923, + "theoretical_loss": 3.373942775842819, + "tokens_seen": 2726756352 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017550651955867603, + "loss": 1.2874, + "theoretical_loss": 3.3739364731131856, + "tokens_seen": 2726821888 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001754864593781344, + "loss": 1.2264, + "theoretical_loss": 3.3739301705774425, + "tokens_seen": 2726887424 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017546639919759278, + "loss": 1.2742, + "theoretical_loss": 3.373923868235579, + "tokens_seen": 2726952960 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017544633901705114, + "loss": 1.2955, + "theoretical_loss": 3.3739175660875835, + "tokens_seen": 2727018496 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017542627883650953, + "loss": 1.2603, + "theoretical_loss": 3.3739112641334463, + "tokens_seen": 2727084032 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001754062186559679, + "loss": 1.2998, + "theoretical_loss": 3.3739049623731563, + "tokens_seen": 2727149568 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017538615847542628, + "loss": 1.1594, + "theoretical_loss": 3.373898660806703, + "tokens_seen": 2727215104 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017536609829488464, + "loss": 1.236, + "theoretical_loss": 3.373892359434076, + "tokens_seen": 2727280640 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017534603811434303, + "loss": 1.3297, + "theoretical_loss": 3.3738860582552648, + "tokens_seen": 2727346176 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001753259779338014, + "loss": 1.238, + "theoretical_loss": 3.373879757270258, + "tokens_seen": 2727411712 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017530591775325976, + "loss": 1.3286, + "theoretical_loss": 3.373873456479046, + "tokens_seen": 2727477248 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017528585757271817, + "loss": 1.2257, + "theoretical_loss": 3.373867155881617, + "tokens_seen": 2727542784 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017526579739217654, + "loss": 1.3354, + "theoretical_loss": 3.3738608554779614, + "tokens_seen": 2727608320 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017524573721163493, + "loss": 1.2808, + "theoretical_loss": 3.3738545552680685, + "tokens_seen": 2727673856 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001752256770310933, + "loss": 1.1986, + "theoretical_loss": 3.373848255251927, + "tokens_seen": 2727739392 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017520561685055168, + "loss": 1.31, + "theoretical_loss": 3.373841955429527, + "tokens_seen": 2727804928 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017518555667001004, + "loss": 1.2716, + "theoretical_loss": 3.3738356558008578, + "tokens_seen": 2727870464 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017516549648946843, + "loss": 1.2646, + "theoretical_loss": 3.373829356365908, + "tokens_seen": 2727936000 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001751454363089268, + "loss": 1.2781, + "theoretical_loss": 3.373823057124668, + "tokens_seen": 2728001536 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 991320, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.324868083000183, + "objective/train/theoretical_loss": 3.3738167580771266, + "objective/train/tokens_used": 1098526176, + "theoretical_loss": 3.3738167580771266, + "tokens_seen": 2728067072 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017512537612838518, + "loss": 1.3126, + "theoretical_loss": 3.3738167580771266, + "tokens_seen": 2728067072 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017510531594784354, + "loss": 1.289, + "theoretical_loss": 3.3738104592232734, + "tokens_seen": 2728132608 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001750852557673019, + "loss": 1.2124, + "theoretical_loss": 3.3738041605630977, + "tokens_seen": 2728198144 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001750651955867603, + "loss": 1.2613, + "theoretical_loss": 3.373797862096589, + "tokens_seen": 2728263680 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017504513540621866, + "loss": 1.272, + "theoretical_loss": 3.3737915638237364, + "tokens_seen": 2728329216 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017502507522567705, + "loss": 1.2761, + "theoretical_loss": 3.3737852657445297, + "tokens_seen": 2728394752 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001750050150451354, + "loss": 1.4161, + "theoretical_loss": 3.373778967858958, + "tokens_seen": 2728460288 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001749849548645938, + "loss": 1.2137, + "theoretical_loss": 3.3737726701670114, + "tokens_seen": 2728525824 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017496489468405216, + "loss": 1.3148, + "theoretical_loss": 3.373766372668678, + "tokens_seen": 2728591360 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017494483450351055, + "loss": 1.2861, + "theoretical_loss": 3.3737600753639483, + "tokens_seen": 2728656896 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001749247743229689, + "loss": 1.2961, + "theoretical_loss": 3.3737537782528113, + "tokens_seen": 2728722432 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017490471414242727, + "loss": 1.4394, + "theoretical_loss": 3.3737474813352564, + "tokens_seen": 2728787968 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017488465396188566, + "loss": 1.2621, + "theoretical_loss": 3.373741184611273, + "tokens_seen": 2728853504 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017486459378134402, + "loss": 1.3395, + "theoretical_loss": 3.3737348880808504, + "tokens_seen": 2728919040 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017484453360080241, + "loss": 1.2651, + "theoretical_loss": 3.373728591743978, + "tokens_seen": 2728984576 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017482447342026078, + "loss": 1.3592, + "theoretical_loss": 3.3737222956006456, + "tokens_seen": 2729050112 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017480441323971917, + "loss": 1.3493, + "theoretical_loss": 3.373715999650842, + "tokens_seen": 2729115648 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017478435305917753, + "loss": 1.3399, + "theoretical_loss": 3.3737097038945576, + "tokens_seen": 2729181184 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017476429287863592, + "loss": 1.1831, + "theoretical_loss": 3.3737034083317803, + "tokens_seen": 2729246720 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017474423269809428, + "loss": 1.2721, + "theoretical_loss": 3.3736971129625006, + "tokens_seen": 2729312256 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017472417251755264, + "loss": 1.3091, + "theoretical_loss": 3.373690817786708, + "tokens_seen": 2729377792 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017470411233701103, + "loss": 1.3027, + "theoretical_loss": 3.3736845228043912, + "tokens_seen": 2729443328 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001746840521564694, + "loss": 1.2578, + "theoretical_loss": 3.37367822801554, + "tokens_seen": 2729508864 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017466399197592778, + "loss": 1.2958, + "theoretical_loss": 3.3736719334201437, + "tokens_seen": 2729574400 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017464393179538614, + "loss": 1.2738, + "theoretical_loss": 3.373665639018192, + "tokens_seen": 2729639936 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017462387161484453, + "loss": 1.2351, + "theoretical_loss": 3.373659344809674, + "tokens_seen": 2729705472 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001746038114343029, + "loss": 1.2604, + "theoretical_loss": 3.3736530507945792, + "tokens_seen": 2729771008 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017458375125376129, + "loss": 1.3026, + "theoretical_loss": 3.373646756972897, + "tokens_seen": 2729836544 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017456369107321965, + "loss": 1.2746, + "theoretical_loss": 3.373640463344617, + "tokens_seen": 2729902080 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017454363089267806, + "loss": 1.2176, + "theoretical_loss": 3.373634169909728, + "tokens_seen": 2729967616 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017452357071213643, + "loss": 1.3601, + "theoretical_loss": 3.37362787666822, + "tokens_seen": 2730033152 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001745035105315948, + "loss": 1.2489, + "theoretical_loss": 3.3736215836200825, + "tokens_seen": 2730098688 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017448345035105318, + "loss": 1.3858, + "theoretical_loss": 3.3736152907653043, + "tokens_seen": 2730164224 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017446339017051154, + "loss": 1.3124, + "theoretical_loss": 3.373608998103876, + "tokens_seen": 2730229760 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017444332998996993, + "loss": 1.2893, + "theoretical_loss": 3.3736027056357853, + "tokens_seen": 2730295296 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001744232698094283, + "loss": 1.2237, + "theoretical_loss": 3.3735964133610232, + "tokens_seen": 2730360832 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017440320962888668, + "loss": 1.3181, + "theoretical_loss": 3.373590121279578, + "tokens_seen": 2730426368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017438314944834504, + "loss": 1.3088, + "theoretical_loss": 3.37358382939144, + "tokens_seen": 2730491904 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017436308926780343, + "loss": 1.3373, + "theoretical_loss": 3.373577537696598, + "tokens_seen": 2730557440 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001743430290872618, + "loss": 1.2452, + "theoretical_loss": 3.3735712461950413, + "tokens_seen": 2730622976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017432296890672016, + "loss": 1.2236, + "theoretical_loss": 3.37356495488676, + "tokens_seen": 2730688512 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017430290872617855, + "loss": 1.1967, + "theoretical_loss": 3.3735586637717434, + "tokens_seen": 2730754048 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001742828485456369, + "loss": 1.352, + "theoretical_loss": 3.3735523728499803, + "tokens_seen": 2730819584 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001742627883650953, + "loss": 1.2489, + "theoretical_loss": 3.373546082121461, + "tokens_seen": 2730885120 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017424272818455366, + "loss": 1.2995, + "theoretical_loss": 3.373539791586174, + "tokens_seen": 2730950656 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017422266800401205, + "loss": 1.1804, + "theoretical_loss": 3.373533501244109, + "tokens_seen": 2731016192 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001742026078234704, + "loss": 1.3571, + "theoretical_loss": 3.373527211095256, + "tokens_seen": 2731081728 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001741825476429288, + "loss": 1.283, + "theoretical_loss": 3.373520921139604, + "tokens_seen": 2731147264 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017416248746238716, + "loss": 1.2738, + "theoretical_loss": 3.3735146313771427, + "tokens_seen": 2731212800 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017414242728184553, + "loss": 1.2861, + "theoretical_loss": 3.373508341807861, + "tokens_seen": 2731278336 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 992345, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4288108348846436, + "objective/train/theoretical_loss": 3.3735020524317485, + "objective/train/tokens_used": 1101802976, + "theoretical_loss": 3.3735020524317485, + "tokens_seen": 2731343872 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017412236710130392, + "loss": 1.3499, + "theoretical_loss": 3.3735020524317485, + "tokens_seen": 2731343872 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017410230692076228, + "loss": 1.2654, + "theoretical_loss": 3.373495763248795, + "tokens_seen": 2731409408 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017408224674022067, + "loss": 1.3258, + "theoretical_loss": 3.3734894742589896, + "tokens_seen": 2731474944 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017406218655967903, + "loss": 1.28, + "theoretical_loss": 3.3734831854623217, + "tokens_seen": 2731540480 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017404212637913742, + "loss": 1.26, + "theoretical_loss": 3.373476896858781, + "tokens_seen": 2731606016 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017402206619859578, + "loss": 1.3133, + "theoretical_loss": 3.3734706084483568, + "tokens_seen": 2731671552 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017400200601805417, + "loss": 1.3426, + "theoretical_loss": 3.3734643202310384, + "tokens_seen": 2731737088 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017398194583751253, + "loss": 1.1334, + "theoretical_loss": 3.3734580322068157, + "tokens_seen": 2731802624 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017396188565697092, + "loss": 1.3165, + "theoretical_loss": 3.3734517443756777, + "tokens_seen": 2731868160 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017394182547642928, + "loss": 1.247, + "theoretical_loss": 3.3734454567376138, + "tokens_seen": 2731933696 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017392176529588765, + "loss": 1.2913, + "theoretical_loss": 3.373439169292614, + "tokens_seen": 2731999232 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017390170511534604, + "loss": 1.3227, + "theoretical_loss": 3.373432882040667, + "tokens_seen": 2732064768 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001738816449348044, + "loss": 1.3313, + "theoretical_loss": 3.3734265949817623, + "tokens_seen": 2732130304 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001738615847542628, + "loss": 1.3121, + "theoretical_loss": 3.37342030811589, + "tokens_seen": 2732195840 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017384152457372115, + "loss": 1.27, + "theoretical_loss": 3.3734140214430393, + "tokens_seen": 2732261376 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017382146439317954, + "loss": 1.3096, + "theoretical_loss": 3.373407734963199, + "tokens_seen": 2732326912 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001738014042126379, + "loss": 1.2898, + "theoretical_loss": 3.3734014486763595, + "tokens_seen": 2732392448 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017378134403209632, + "loss": 1.2323, + "theoretical_loss": 3.3733951625825096, + "tokens_seen": 2732457984 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017376128385155468, + "loss": 1.3225, + "theoretical_loss": 3.373388876681639, + "tokens_seen": 2732523520 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017374122367101304, + "loss": 1.3805, + "theoretical_loss": 3.373382590973737, + "tokens_seen": 2732589056 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017372116349047143, + "loss": 1.2703, + "theoretical_loss": 3.3733763054587933, + "tokens_seen": 2732654592 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001737011033099298, + "loss": 1.3053, + "theoretical_loss": 3.373370020136797, + "tokens_seen": 2732720128 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017368104312938818, + "loss": 1.3391, + "theoretical_loss": 3.373363735007738, + "tokens_seen": 2732785664 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017366098294884655, + "loss": 1.2892, + "theoretical_loss": 3.3733574500716053, + "tokens_seen": 2732851200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017364092276830493, + "loss": 1.2695, + "theoretical_loss": 3.3733511653283887, + "tokens_seen": 2732916736 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001736208625877633, + "loss": 1.2421, + "theoretical_loss": 3.3733448807780775, + "tokens_seen": 2732982272 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017360080240722169, + "loss": 1.4435, + "theoretical_loss": 3.373338596420661, + "tokens_seen": 2733047808 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017358074222668005, + "loss": 1.2732, + "theoretical_loss": 3.373332312256129, + "tokens_seen": 2733113344 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001735606820461384, + "loss": 1.2349, + "theoretical_loss": 3.3733260282844704, + "tokens_seen": 2733178880 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001735406218655968, + "loss": 1.3886, + "theoretical_loss": 3.3733197445056753, + "tokens_seen": 2733244416 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017352056168505516, + "loss": 1.2251, + "theoretical_loss": 3.3733134609197326, + "tokens_seen": 2733309952 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017350050150451355, + "loss": 1.2146, + "theoretical_loss": 3.373307177526632, + "tokens_seen": 2733375488 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017348044132397191, + "loss": 1.1172, + "theoretical_loss": 3.3733008943263636, + "tokens_seen": 2733441024 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001734603811434303, + "loss": 1.3583, + "theoretical_loss": 3.3732946113189155, + "tokens_seen": 2733506560 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017344032096288867, + "loss": 1.3257, + "theoretical_loss": 3.3732883285042785, + "tokens_seen": 2733572096 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017342026078234705, + "loss": 1.3769, + "theoretical_loss": 3.373282045882441, + "tokens_seen": 2733637632 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017340020060180542, + "loss": 1.2443, + "theoretical_loss": 3.3732757634533934, + "tokens_seen": 2733703168 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001733801404212638, + "loss": 1.2918, + "theoretical_loss": 3.373269481217124, + "tokens_seen": 2733768704 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017336008024072217, + "loss": 1.2832, + "theoretical_loss": 3.3732631991736235, + "tokens_seen": 2733834240 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017334002006018053, + "loss": 1.3632, + "theoretical_loss": 3.373256917322881, + "tokens_seen": 2733899776 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017331995987963892, + "loss": 1.2299, + "theoretical_loss": 3.3732506356648853, + "tokens_seen": 2733965312 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017329989969909728, + "loss": 1.3592, + "theoretical_loss": 3.3732443541996266, + "tokens_seen": 2734030848 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017327983951855567, + "loss": 1.3288, + "theoretical_loss": 3.373238072927094, + "tokens_seen": 2734096384 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017325977933801403, + "loss": 1.2295, + "theoretical_loss": 3.3732317918472767, + "tokens_seen": 2734161920 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017323971915747242, + "loss": 1.2117, + "theoretical_loss": 3.3732255109601654, + "tokens_seen": 2734227456 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017321965897693079, + "loss": 1.2221, + "theoretical_loss": 3.373219230265748, + "tokens_seen": 2734292992 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017319959879638917, + "loss": 1.3648, + "theoretical_loss": 3.373212949764015, + "tokens_seen": 2734358528 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017317953861584754, + "loss": 1.3264, + "theoretical_loss": 3.3732066694549556, + "tokens_seen": 2734424064 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001731594784353059, + "loss": 1.3121, + "theoretical_loss": 3.373200389338559, + "tokens_seen": 2734489600 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001731394182547643, + "loss": 1.372, + "theoretical_loss": 3.3731941094148152, + "tokens_seen": 2734555136 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 993449, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1959069967269897, + "objective/train/theoretical_loss": 3.373187829683713, + "objective/train/tokens_used": 1105079776, + "theoretical_loss": 3.373187829683713, + "tokens_seen": 2734620672 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017311935807422265, + "loss": 1.2498, + "theoretical_loss": 3.373187829683713, + "tokens_seen": 2734620672 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017309929789368104, + "loss": 1.2493, + "theoretical_loss": 3.373181550145243, + "tokens_seen": 2734686208 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001730792377131394, + "loss": 1.2668, + "theoretical_loss": 3.373175270799393, + "tokens_seen": 2734751744 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001730591775325978, + "loss": 1.1627, + "theoretical_loss": 3.373168991646154, + "tokens_seen": 2734817280 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017303911735205618, + "loss": 1.2443, + "theoretical_loss": 3.3731627126855144, + "tokens_seen": 2734882816 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017301905717151457, + "loss": 1.3004, + "theoretical_loss": 3.3731564339174644, + "tokens_seen": 2734948352 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017299899699097293, + "loss": 1.2801, + "theoretical_loss": 3.3731501553419934, + "tokens_seen": 2735013888 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017297893681043132, + "loss": 1.2171, + "theoretical_loss": 3.3731438769590905, + "tokens_seen": 2735079424 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017295887662988968, + "loss": 1.1735, + "theoretical_loss": 3.3731375987687455, + "tokens_seen": 2735144960 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017293881644934805, + "loss": 1.3391, + "theoretical_loss": 3.373131320770948, + "tokens_seen": 2735210496 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017291875626880644, + "loss": 1.1408, + "theoretical_loss": 3.373125042965687, + "tokens_seen": 2735276032 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001728986960882648, + "loss": 1.3259, + "theoretical_loss": 3.373118765352952, + "tokens_seen": 2735341568 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001728786359077232, + "loss": 1.2743, + "theoretical_loss": 3.373112487932733, + "tokens_seen": 2735407104 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017285857572718155, + "loss": 1.2686, + "theoretical_loss": 3.373106210705019, + "tokens_seen": 2735472640 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017283851554663994, + "loss": 1.2688, + "theoretical_loss": 3.3730999336698, + "tokens_seen": 2735538176 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001728184553660983, + "loss": 1.2706, + "theoretical_loss": 3.373093656827065, + "tokens_seen": 2735603712 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001727983951855567, + "loss": 1.2556, + "theoretical_loss": 3.3730873801768038, + "tokens_seen": 2735669248 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017277833500501505, + "loss": 1.2062, + "theoretical_loss": 3.3730811037190054, + "tokens_seen": 2735734784 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017275827482447342, + "loss": 1.2474, + "theoretical_loss": 3.37307482745366, + "tokens_seen": 2735800320 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001727382146439318, + "loss": 1.4083, + "theoretical_loss": 3.3730685513807566, + "tokens_seen": 2735865856 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017271815446339017, + "loss": 1.2722, + "theoretical_loss": 3.373062275500285, + "tokens_seen": 2735931392 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017269809428284856, + "loss": 1.1815, + "theoretical_loss": 3.3730559998122343, + "tokens_seen": 2735996928 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017267803410230692, + "loss": 1.2276, + "theoretical_loss": 3.3730497243165942, + "tokens_seen": 2736062464 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001726579739217653, + "loss": 1.2762, + "theoretical_loss": 3.373043449013354, + "tokens_seen": 2736128000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017263791374122367, + "loss": 1.3627, + "theoretical_loss": 3.3730371739025036, + "tokens_seen": 2736193536 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017261785356068206, + "loss": 1.2273, + "theoretical_loss": 3.373030898984032, + "tokens_seen": 2736259072 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017259779338014042, + "loss": 1.2698, + "theoretical_loss": 3.3730246242579294, + "tokens_seen": 2736324608 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017257773319959878, + "loss": 1.3509, + "theoretical_loss": 3.373018349724185, + "tokens_seen": 2736390144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017255767301905717, + "loss": 1.1038, + "theoretical_loss": 3.3730120753827877, + "tokens_seen": 2736455680 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017253761283851554, + "loss": 1.2128, + "theoretical_loss": 3.3730058012337274, + "tokens_seen": 2736521216 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017251755265797392, + "loss": 1.3022, + "theoretical_loss": 3.3729995272769937, + "tokens_seen": 2736586752 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001724974924774323, + "loss": 1.2019, + "theoretical_loss": 3.3729932535125764, + "tokens_seen": 2736652288 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017247743229689068, + "loss": 1.2342, + "theoretical_loss": 3.3729869799404644, + "tokens_seen": 2736717824 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017245737211634904, + "loss": 1.2242, + "theoretical_loss": 3.372980706560648, + "tokens_seen": 2736783360 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017243731193580743, + "loss": 1.3011, + "theoretical_loss": 3.3729744333731153, + "tokens_seen": 2736848896 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001724172517552658, + "loss": 1.3484, + "theoretical_loss": 3.372968160377857, + "tokens_seen": 2736914432 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017239719157472418, + "loss": 1.3555, + "theoretical_loss": 3.3729618875748626, + "tokens_seen": 2736979968 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017237713139418254, + "loss": 1.2507, + "theoretical_loss": 3.3729556149641207, + "tokens_seen": 2737045504 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001723570712136409, + "loss": 1.3588, + "theoretical_loss": 3.372949342545622, + "tokens_seen": 2737111040 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001723370110330993, + "loss": 1.2777, + "theoretical_loss": 3.372943070319355, + "tokens_seen": 2737176576 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017231695085255766, + "loss": 1.3673, + "theoretical_loss": 3.3729367982853096, + "tokens_seen": 2737242112 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017229689067201607, + "loss": 1.2261, + "theoretical_loss": 3.3729305264434752, + "tokens_seen": 2737307648 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017227683049147443, + "loss": 1.3322, + "theoretical_loss": 3.3729242547938414, + "tokens_seen": 2737373184 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017225677031093282, + "loss": 1.4461, + "theoretical_loss": 3.372917983336398, + "tokens_seen": 2737438720 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017223671013039119, + "loss": 1.388, + "theoretical_loss": 3.372911712071134, + "tokens_seen": 2737504256 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017221664994984958, + "loss": 1.2688, + "theoretical_loss": 3.372905440998039, + "tokens_seen": 2737569792 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017219658976930794, + "loss": 1.2697, + "theoretical_loss": 3.3728991701171025, + "tokens_seen": 2737635328 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001721765295887663, + "loss": 1.2351, + "theoretical_loss": 3.3728928994283147, + "tokens_seen": 2737700864 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001721564694082247, + "loss": 1.2055, + "theoretical_loss": 3.3728866289316644, + "tokens_seen": 2737766400 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017213640922768305, + "loss": 1.2692, + "theoretical_loss": 3.3728803586271408, + "tokens_seen": 2737831936 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 994721, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2009656429290771, + "objective/train/theoretical_loss": 3.3728740885147346, + "objective/train/tokens_used": 1108356576, + "theoretical_loss": 3.3728740885147346, + "tokens_seen": 2737897472 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017211634904714144, + "loss": 1.1759, + "theoretical_loss": 3.3728740885147346, + "tokens_seen": 2737897472 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001720962888665998, + "loss": 1.2683, + "theoretical_loss": 3.3728678185944343, + "tokens_seen": 2737963008 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001720762286860582, + "loss": 1.311, + "theoretical_loss": 3.3728615488662292, + "tokens_seen": 2738028544 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017205616850551655, + "loss": 1.2569, + "theoretical_loss": 3.37285527933011, + "tokens_seen": 2738094080 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017203610832497494, + "loss": 1.2878, + "theoretical_loss": 3.3728490099860653, + "tokens_seen": 2738159616 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001720160481444333, + "loss": 1.2336, + "theoretical_loss": 3.372842740834085, + "tokens_seen": 2738225152 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017199598796389167, + "loss": 1.3294, + "theoretical_loss": 3.3728364718741584, + "tokens_seen": 2738290688 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017197592778335006, + "loss": 1.1971, + "theoretical_loss": 3.3728302031062753, + "tokens_seen": 2738356224 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017195586760280842, + "loss": 1.2158, + "theoretical_loss": 3.372823934530425, + "tokens_seen": 2738421760 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001719358074222668, + "loss": 1.2721, + "theoretical_loss": 3.372817666146597, + "tokens_seen": 2738487296 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017191574724172517, + "loss": 1.2244, + "theoretical_loss": 3.3728113979547807, + "tokens_seen": 2738552832 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017189568706118356, + "loss": 1.24, + "theoretical_loss": 3.3728051299549655, + "tokens_seen": 2738618368 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017187562688064192, + "loss": 1.1594, + "theoretical_loss": 3.3727988621471416, + "tokens_seen": 2738683904 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001718555667001003, + "loss": 1.3453, + "theoretical_loss": 3.3727925945312984, + "tokens_seen": 2738749440 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017183550651955867, + "loss": 1.3149, + "theoretical_loss": 3.372786327107425, + "tokens_seen": 2738814976 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017181544633901706, + "loss": 1.2239, + "theoretical_loss": 3.3727800598755113, + "tokens_seen": 2738880512 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017179538615847543, + "loss": 1.3059, + "theoretical_loss": 3.372773792835546, + "tokens_seen": 2738946048 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001717753259779338, + "loss": 1.2291, + "theoretical_loss": 3.37276752598752, + "tokens_seen": 2739011584 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017175526579739218, + "loss": 1.1973, + "theoretical_loss": 3.372761259331422, + "tokens_seen": 2739077120 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017173520561685054, + "loss": 1.3296, + "theoretical_loss": 3.3727549928672413, + "tokens_seen": 2739142656 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017171514543630893, + "loss": 1.3692, + "theoretical_loss": 3.372748726594968, + "tokens_seen": 2739208192 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001716950852557673, + "loss": 1.2796, + "theoretical_loss": 3.372742460514591, + "tokens_seen": 2739273728 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017167502507522568, + "loss": 1.3916, + "theoretical_loss": 3.3727361946261003, + "tokens_seen": 2739339264 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017165496489468404, + "loss": 1.3163, + "theoretical_loss": 3.372729928929486, + "tokens_seen": 2739404800 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017163490471414243, + "loss": 1.2901, + "theoretical_loss": 3.372723663424736, + "tokens_seen": 2739470336 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001716148445336008, + "loss": 1.2249, + "theoretical_loss": 3.3727173981118415, + "tokens_seen": 2739535872 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017159478435305916, + "loss": 1.2136, + "theoretical_loss": 3.372711132990791, + "tokens_seen": 2739601408 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017157472417251755, + "loss": 1.3136, + "theoretical_loss": 3.3727048680615743, + "tokens_seen": 2739666944 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001715546639919759, + "loss": 1.2873, + "theoretical_loss": 3.3726986033241815, + "tokens_seen": 2739732480 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017153460381143433, + "loss": 1.2386, + "theoretical_loss": 3.3726923387786014, + "tokens_seen": 2739798016 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001715145436308927, + "loss": 1.2803, + "theoretical_loss": 3.372686074424824, + "tokens_seen": 2739863552 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017149448345035108, + "loss": 1.272, + "theoretical_loss": 3.372679810262839, + "tokens_seen": 2739929088 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017147442326980944, + "loss": 1.3538, + "theoretical_loss": 3.3726735462926345, + "tokens_seen": 2739994624 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017145436308926783, + "loss": 1.2966, + "theoretical_loss": 3.3726672825142017, + "tokens_seen": 2740060160 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001714343029087262, + "loss": 1.3439, + "theoretical_loss": 3.3726610189275297, + "tokens_seen": 2740125696 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017141424272818455, + "loss": 1.3288, + "theoretical_loss": 3.372654755532608, + "tokens_seen": 2740191232 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017139418254764294, + "loss": 1.4029, + "theoretical_loss": 3.3726484923294255, + "tokens_seen": 2740256768 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001713741223671013, + "loss": 1.2156, + "theoretical_loss": 3.3726422293179725, + "tokens_seen": 2740322304 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001713540621865597, + "loss": 1.2599, + "theoretical_loss": 3.3726359664982386, + "tokens_seen": 2740387840 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017133400200601806, + "loss": 1.237, + "theoretical_loss": 3.3726297038702135, + "tokens_seen": 2740453376 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017131394182547645, + "loss": 1.2351, + "theoretical_loss": 3.372623441433886, + "tokens_seen": 2740518912 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001712938816449348, + "loss": 1.2672, + "theoretical_loss": 3.3726171791892456, + "tokens_seen": 2740584448 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001712738214643932, + "loss": 1.2791, + "theoretical_loss": 3.3726109171362824, + "tokens_seen": 2740649984 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017125376128385156, + "loss": 1.3374, + "theoretical_loss": 3.3726046552749858, + "tokens_seen": 2740715520 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017123370110330995, + "loss": 1.2464, + "theoretical_loss": 3.3725983936053456, + "tokens_seen": 2740781056 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001712136409227683, + "loss": 1.2517, + "theoretical_loss": 3.372592132127351, + "tokens_seen": 2740846592 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017119358074222667, + "loss": 1.1275, + "theoretical_loss": 3.3725858708409913, + "tokens_seen": 2740912128 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017117352056168506, + "loss": 1.2055, + "theoretical_loss": 3.3725796097462566, + "tokens_seen": 2740977664 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017115346038114342, + "loss": 1.1924, + "theoretical_loss": 3.3725733488431366, + "tokens_seen": 2741043200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017113340020060181, + "loss": 1.3295, + "theoretical_loss": 3.37256708813162, + "tokens_seen": 2741108736 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 996102, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3912932872772217, + "objective/train/theoretical_loss": 3.372560827611697, + "objective/train/tokens_used": 1111633376, + "theoretical_loss": 3.372560827611697, + "tokens_seen": 2741174272 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017111334002006018, + "loss": 1.3562, + "theoretical_loss": 3.372560827611697, + "tokens_seen": 2741174272 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017109327983951857, + "loss": 1.2341, + "theoretical_loss": 3.3725545672833572, + "tokens_seen": 2741239808 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017107321965897693, + "loss": 1.3187, + "theoretical_loss": 3.37254830714659, + "tokens_seen": 2741305344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017105315947843532, + "loss": 1.3165, + "theoretical_loss": 3.3725420472013843, + "tokens_seen": 2741370880 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017103309929789368, + "loss": 1.2562, + "theoretical_loss": 3.372535787447731, + "tokens_seen": 2741436416 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017101303911735204, + "loss": 1.3679, + "theoretical_loss": 3.372529527885619, + "tokens_seen": 2741501952 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017099297893681043, + "loss": 1.1858, + "theoretical_loss": 3.372523268515037, + "tokens_seen": 2741567488 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001709729187562688, + "loss": 1.3546, + "theoretical_loss": 3.372517009335976, + "tokens_seen": 2741633024 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017095285857572718, + "loss": 1.2919, + "theoretical_loss": 3.3725107503484253, + "tokens_seen": 2741698560 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017093279839518554, + "loss": 1.3078, + "theoretical_loss": 3.372504491552373, + "tokens_seen": 2741764096 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017091273821464393, + "loss": 1.1745, + "theoretical_loss": 3.372498232947811, + "tokens_seen": 2741829632 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001708926780341023, + "loss": 1.3325, + "theoretical_loss": 3.372491974534727, + "tokens_seen": 2741895168 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017087261785356069, + "loss": 1.2115, + "theoretical_loss": 3.3724857163131112, + "tokens_seen": 2741960704 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017085255767301905, + "loss": 1.2321, + "theoretical_loss": 3.372479458282953, + "tokens_seen": 2742026240 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001708324974924774, + "loss": 1.1983, + "theoretical_loss": 3.3724732004442424, + "tokens_seen": 2742091776 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001708124373119358, + "loss": 1.198, + "theoretical_loss": 3.3724669427969687, + "tokens_seen": 2742157312 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001707923771313942, + "loss": 1.3119, + "theoretical_loss": 3.3724606853411214, + "tokens_seen": 2742222848 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017077231695085258, + "loss": 1.2454, + "theoretical_loss": 3.37245442807669, + "tokens_seen": 2742288384 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017075225677031094, + "loss": 1.2738, + "theoretical_loss": 3.3724481710036645, + "tokens_seen": 2742353920 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017073219658976933, + "loss": 1.2454, + "theoretical_loss": 3.372441914122034, + "tokens_seen": 2742419456 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001707121364092277, + "loss": 1.264, + "theoretical_loss": 3.372435657431788, + "tokens_seen": 2742484992 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017069207622868608, + "loss": 1.2531, + "theoretical_loss": 3.3724294009329165, + "tokens_seen": 2742550528 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017067201604814444, + "loss": 1.2194, + "theoretical_loss": 3.372423144625409, + "tokens_seen": 2742616064 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017065195586760283, + "loss": 1.2529, + "theoretical_loss": 3.372416888509255, + "tokens_seen": 2742681600 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001706318956870612, + "loss": 1.258, + "theoretical_loss": 3.372410632584444, + "tokens_seen": 2742747136 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017061183550651956, + "loss": 1.258, + "theoretical_loss": 3.3724043768509655, + "tokens_seen": 2742812672 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017059177532597795, + "loss": 1.2722, + "theoretical_loss": 3.3723981213088092, + "tokens_seen": 2742878208 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001705717151454363, + "loss": 1.3189, + "theoretical_loss": 3.372391865957965, + "tokens_seen": 2742943744 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001705516549648947, + "loss": 1.2494, + "theoretical_loss": 3.3723856107984216, + "tokens_seen": 2743009280 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017053159478435306, + "loss": 1.195, + "theoretical_loss": 3.3723793558301693, + "tokens_seen": 2743074816 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017051153460381145, + "loss": 1.1838, + "theoretical_loss": 3.3723731010531974, + "tokens_seen": 2743140352 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001704914744232698, + "loss": 1.329, + "theoretical_loss": 3.372366846467496, + "tokens_seen": 2743205888 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001704714142427282, + "loss": 1.2309, + "theoretical_loss": 3.372360592073054, + "tokens_seen": 2743271424 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017045135406218656, + "loss": 1.2451, + "theoretical_loss": 3.3723543378698615, + "tokens_seen": 2743336960 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017043129388164493, + "loss": 1.3045, + "theoretical_loss": 3.3723480838579074, + "tokens_seen": 2743402496 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017041123370110332, + "loss": 1.2715, + "theoretical_loss": 3.372341830037182, + "tokens_seen": 2743468032 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017039117352056168, + "loss": 1.2904, + "theoretical_loss": 3.3723355764076746, + "tokens_seen": 2743533568 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017037111334002007, + "loss": 1.1532, + "theoretical_loss": 3.372329322969375, + "tokens_seen": 2743599104 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017035105315947843, + "loss": 1.1719, + "theoretical_loss": 3.372323069722272, + "tokens_seen": 2743664640 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017033099297893682, + "loss": 1.2843, + "theoretical_loss": 3.3723168166663564, + "tokens_seen": 2743730176 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017031093279839518, + "loss": 1.2821, + "theoretical_loss": 3.3723105638016166, + "tokens_seen": 2743795712 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017029087261785357, + "loss": 1.2686, + "theoretical_loss": 3.372304311128043, + "tokens_seen": 2743861248 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017027081243731193, + "loss": 1.226, + "theoretical_loss": 3.372298058645625, + "tokens_seen": 2743926784 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017025075225677032, + "loss": 1.2063, + "theoretical_loss": 3.372291806354352, + "tokens_seen": 2743992320 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017023069207622868, + "loss": 1.177, + "theoretical_loss": 3.372285554254214, + "tokens_seen": 2744057856 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017021063189568705, + "loss": 1.2426, + "theoretical_loss": 3.3722793023452, + "tokens_seen": 2744123392 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017019057171514544, + "loss": 1.278, + "theoretical_loss": 3.3722730506273, + "tokens_seen": 2744188928 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001701705115346038, + "loss": 1.1527, + "theoretical_loss": 3.3722667991005033, + "tokens_seen": 2744254464 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001701504513540622, + "loss": 1.2925, + "theoretical_loss": 3.3722605477648, + "tokens_seen": 2744320000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017013039117352055, + "loss": 1.3233, + "theoretical_loss": 3.372254296620179, + "tokens_seen": 2744385536 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 997363, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2676897048950195, + "objective/train/theoretical_loss": 3.372248045666631, + "objective/train/tokens_used": 1114910176, + "theoretical_loss": 3.372248045666631, + "tokens_seen": 2744451072 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017011033099297894, + "loss": 1.3492, + "theoretical_loss": 3.372248045666631, + "tokens_seen": 2744451072 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001700902708124373, + "loss": 1.1548, + "theoretical_loss": 3.372241794904144, + "tokens_seen": 2744516608 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001700702106318957, + "loss": 1.3303, + "theoretical_loss": 3.372235544332709, + "tokens_seen": 2744582144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017005015045135405, + "loss": 1.2691, + "theoretical_loss": 3.372229293952315, + "tokens_seen": 2744647680 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017003009027081244, + "loss": 1.2684, + "theoretical_loss": 3.3722230437629515, + "tokens_seen": 2744713216 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017001003009027083, + "loss": 1.2141, + "theoretical_loss": 3.3722167937646086, + "tokens_seen": 2744778752 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001699899699097292, + "loss": 1.2941, + "theoretical_loss": 3.3722105439572756, + "tokens_seen": 2744844288 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016996990972918758, + "loss": 1.1786, + "theoretical_loss": 3.372204294340942, + "tokens_seen": 2744909824 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016994984954864594, + "loss": 1.2765, + "theoretical_loss": 3.3721980449155966, + "tokens_seen": 2744975360 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016992978936810433, + "loss": 1.3603, + "theoretical_loss": 3.372191795681231, + "tokens_seen": 2745040896 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001699097291875627, + "loss": 1.2026, + "theoretical_loss": 3.372185546637833, + "tokens_seen": 2745106432 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016988966900702109, + "loss": 1.3169, + "theoretical_loss": 3.3721792977853933, + "tokens_seen": 2745171968 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016986960882647945, + "loss": 1.3251, + "theoretical_loss": 3.372173049123901, + "tokens_seen": 2745237504 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001698495486459378, + "loss": 1.3857, + "theoretical_loss": 3.372166800653346, + "tokens_seen": 2745303040 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001698294884653962, + "loss": 1.2291, + "theoretical_loss": 3.3721605523737175, + "tokens_seen": 2745368576 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016980942828485456, + "loss": 1.343, + "theoretical_loss": 3.3721543042850053, + "tokens_seen": 2745434112 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016978936810431295, + "loss": 1.2625, + "theoretical_loss": 3.372148056387199, + "tokens_seen": 2745499648 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001697693079237713, + "loss": 1.3129, + "theoretical_loss": 3.3721418086802886, + "tokens_seen": 2745565184 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001697492477432297, + "loss": 1.2101, + "theoretical_loss": 3.3721355611642627, + "tokens_seen": 2745630720 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016972918756268807, + "loss": 1.3214, + "theoretical_loss": 3.372129313839112, + "tokens_seen": 2745696256 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016970912738214645, + "loss": 1.3275, + "theoretical_loss": 3.372123066704826, + "tokens_seen": 2745761792 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016968906720160482, + "loss": 1.1856, + "theoretical_loss": 3.3721168197613935, + "tokens_seen": 2745827328 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001696690070210632, + "loss": 1.2177, + "theoretical_loss": 3.372110573008805, + "tokens_seen": 2745892864 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016964894684052157, + "loss": 1.2518, + "theoretical_loss": 3.372104326447049, + "tokens_seen": 2745958400 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016962888665997993, + "loss": 1.3004, + "theoretical_loss": 3.3720980800761167, + "tokens_seen": 2746023936 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016960882647943832, + "loss": 1.234, + "theoretical_loss": 3.372091833895996, + "tokens_seen": 2746089472 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016958876629889668, + "loss": 1.1374, + "theoretical_loss": 3.3720855879066782, + "tokens_seen": 2746155008 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016956870611835507, + "loss": 1.2979, + "theoretical_loss": 3.372079342108152, + "tokens_seen": 2746220544 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016954864593781343, + "loss": 1.2945, + "theoretical_loss": 3.372073096500406, + "tokens_seen": 2746286080 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016952858575727182, + "loss": 1.2146, + "theoretical_loss": 3.3720668510834324, + "tokens_seen": 2746351616 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016950852557673019, + "loss": 1.3769, + "theoretical_loss": 3.3720606058572185, + "tokens_seen": 2746417152 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016948846539618857, + "loss": 1.2489, + "theoretical_loss": 3.372054360821755, + "tokens_seen": 2746482688 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016946840521564694, + "loss": 1.2534, + "theoretical_loss": 3.372048115977031, + "tokens_seen": 2746548224 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001694483450351053, + "loss": 1.1976, + "theoretical_loss": 3.372041871323037, + "tokens_seen": 2746613760 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001694282848545637, + "loss": 1.3825, + "theoretical_loss": 3.3720356268597618, + "tokens_seen": 2746679296 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016940822467402205, + "loss": 1.2015, + "theoretical_loss": 3.372029382587195, + "tokens_seen": 2746744832 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016938816449348044, + "loss": 1.2684, + "theoretical_loss": 3.3720231385053268, + "tokens_seen": 2746810368 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001693681043129388, + "loss": 1.3052, + "theoretical_loss": 3.3720168946141467, + "tokens_seen": 2746875904 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001693480441323972, + "loss": 1.2937, + "theoretical_loss": 3.3720106509136434, + "tokens_seen": 2746941440 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016932798395185555, + "loss": 1.2156, + "theoretical_loss": 3.372004407403808, + "tokens_seen": 2747006976 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016930792377131394, + "loss": 1.2643, + "theoretical_loss": 3.371998164084629, + "tokens_seen": 2747072512 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016928786359077233, + "loss": 1.3483, + "theoretical_loss": 3.3719919209560967, + "tokens_seen": 2747138048 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016926780341023072, + "loss": 1.2874, + "theoretical_loss": 3.3719856780182003, + "tokens_seen": 2747203584 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016924774322968908, + "loss": 1.2107, + "theoretical_loss": 3.3719794352709296, + "tokens_seen": 2747269120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016922768304914745, + "loss": 1.2744, + "theoretical_loss": 3.3719731927142744, + "tokens_seen": 2747334656 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016920762286860584, + "loss": 1.1012, + "theoretical_loss": 3.3719669503482237, + "tokens_seen": 2747400192 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001691875626880642, + "loss": 1.2836, + "theoretical_loss": 3.371960708172768, + "tokens_seen": 2747465728 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001691675025075226, + "loss": 1.3203, + "theoretical_loss": 3.3719544661878964, + "tokens_seen": 2747531264 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016914744232698095, + "loss": 1.185, + "theoretical_loss": 3.3719482243935985, + "tokens_seen": 2747596800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016912738214643934, + "loss": 1.3491, + "theoretical_loss": 3.3719419827898647, + "tokens_seen": 2747662336 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 998534, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0819231271743774, + "objective/train/theoretical_loss": 3.3719357413766833, + "objective/train/tokens_used": 1118186976, + "theoretical_loss": 3.3719357413766833, + "tokens_seen": 2747727872 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001691073219658977, + "loss": 1.1673, + "theoretical_loss": 3.3719357413766833, + "tokens_seen": 2747727872 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001690872617853561, + "loss": 1.2326, + "theoretical_loss": 3.371929500154045, + "tokens_seen": 2747793408 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016906720160481445, + "loss": 1.1831, + "theoretical_loss": 3.3719232591219392, + "tokens_seen": 2747858944 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016904714142427281, + "loss": 1.3285, + "theoretical_loss": 3.3719170182803557, + "tokens_seen": 2747924480 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001690270812437312, + "loss": 1.2737, + "theoretical_loss": 3.3719107776292834, + "tokens_seen": 2747990016 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016900702106318957, + "loss": 1.375, + "theoretical_loss": 3.371904537168713, + "tokens_seen": 2748055552 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016898696088264796, + "loss": 1.2779, + "theoretical_loss": 3.3718982968986326, + "tokens_seen": 2748121088 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016896690070210632, + "loss": 1.2167, + "theoretical_loss": 3.371892056819034, + "tokens_seen": 2748186624 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001689468405215647, + "loss": 1.3189, + "theoretical_loss": 3.371885816929905, + "tokens_seen": 2748252160 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016892678034102307, + "loss": 1.2824, + "theoretical_loss": 3.3718795772312355, + "tokens_seen": 2748317696 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016890672016048146, + "loss": 1.2821, + "theoretical_loss": 3.371873337723016, + "tokens_seen": 2748383232 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016888665997993982, + "loss": 1.2036, + "theoretical_loss": 3.371867098405236, + "tokens_seen": 2748448768 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016886659979939818, + "loss": 1.2633, + "theoretical_loss": 3.3718608592778847, + "tokens_seen": 2748514304 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016884653961885657, + "loss": 1.3038, + "theoretical_loss": 3.371854620340952, + "tokens_seen": 2748579840 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016882647943831494, + "loss": 1.225, + "theoretical_loss": 3.3718483815944267, + "tokens_seen": 2748645376 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016880641925777332, + "loss": 1.2285, + "theoretical_loss": 3.3718421430382994, + "tokens_seen": 2748710912 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001687863590772317, + "loss": 1.2754, + "theoretical_loss": 3.3718359046725603, + "tokens_seen": 2748776448 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016876629889669008, + "loss": 1.3156, + "theoretical_loss": 3.371829666497198, + "tokens_seen": 2748841984 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016874623871614844, + "loss": 1.1659, + "theoretical_loss": 3.3718234285122017, + "tokens_seen": 2748907520 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016872617853560683, + "loss": 1.2346, + "theoretical_loss": 3.3718171907175627, + "tokens_seen": 2748973056 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001687061183550652, + "loss": 1.3037, + "theoretical_loss": 3.371810953113269, + "tokens_seen": 2749038592 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016868605817452355, + "loss": 1.2666, + "theoretical_loss": 3.371804715699312, + "tokens_seen": 2749104128 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016866599799398194, + "loss": 1.2406, + "theoretical_loss": 3.3717984784756796, + "tokens_seen": 2749169664 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001686459378134403, + "loss": 1.3559, + "theoretical_loss": 3.3717922414423622, + "tokens_seen": 2749235200 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001686258776328987, + "loss": 1.2628, + "theoretical_loss": 3.3717860045993495, + "tokens_seen": 2749300736 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016860581745235706, + "loss": 1.2768, + "theoretical_loss": 3.3717797679466317, + "tokens_seen": 2749366272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016858575727181544, + "loss": 1.215, + "theoretical_loss": 3.3717735314841972, + "tokens_seen": 2749431808 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001685656970912738, + "loss": 1.2588, + "theoretical_loss": 3.3717672952120368, + "tokens_seen": 2749497344 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001685456369107322, + "loss": 1.3582, + "theoretical_loss": 3.3717610591301392, + "tokens_seen": 2749562880 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016852557673019059, + "loss": 1.2727, + "theoretical_loss": 3.371754823238495, + "tokens_seen": 2749628416 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016850551654964897, + "loss": 1.2612, + "theoretical_loss": 3.3717485875370934, + "tokens_seen": 2749693952 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016848545636910734, + "loss": 1.3237, + "theoretical_loss": 3.3717423520259238, + "tokens_seen": 2749759488 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001684653961885657, + "loss": 1.263, + "theoretical_loss": 3.371736116704976, + "tokens_seen": 2749825024 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001684453360080241, + "loss": 1.2925, + "theoretical_loss": 3.3717298815742405, + "tokens_seen": 2749890560 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016842527582748245, + "loss": 1.3057, + "theoretical_loss": 3.371723646633706, + "tokens_seen": 2749956096 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016840521564694084, + "loss": 1.2431, + "theoretical_loss": 3.3717174118833624, + "tokens_seen": 2750021632 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001683851554663992, + "loss": 1.3273, + "theoretical_loss": 3.371711177323199, + "tokens_seen": 2750087168 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001683650952858576, + "loss": 1.3489, + "theoretical_loss": 3.3717049429532064, + "tokens_seen": 2750152704 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016834503510531595, + "loss": 1.3423, + "theoretical_loss": 3.3716987087733736, + "tokens_seen": 2750218240 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016832497492477434, + "loss": 1.2318, + "theoretical_loss": 3.3716924747836905, + "tokens_seen": 2750283776 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001683049147442327, + "loss": 1.2561, + "theoretical_loss": 3.371686240984147, + "tokens_seen": 2750349312 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016828485456369107, + "loss": 1.3194, + "theoretical_loss": 3.3716800073747315, + "tokens_seen": 2750414848 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016826479438314946, + "loss": 1.3696, + "theoretical_loss": 3.371673773955435, + "tokens_seen": 2750480384 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016824473420260782, + "loss": 1.348, + "theoretical_loss": 3.3716675407262473, + "tokens_seen": 2750545920 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001682246740220662, + "loss": 1.3902, + "theoretical_loss": 3.371661307687157, + "tokens_seen": 2750611456 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016820461384152457, + "loss": 1.4415, + "theoretical_loss": 3.371655074838155, + "tokens_seen": 2750676992 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016818455366098296, + "loss": 1.3532, + "theoretical_loss": 3.3716488421792294, + "tokens_seen": 2750742528 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016816449348044132, + "loss": 1.1759, + "theoretical_loss": 3.3716426097103716, + "tokens_seen": 2750808064 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001681444332998997, + "loss": 1.217, + "theoretical_loss": 3.37163637743157, + "tokens_seen": 2750873600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016812437311935807, + "loss": 1.3303, + "theoretical_loss": 3.371630145342815, + "tokens_seen": 2750939136 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 999260, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1894795894622803, + "objective/train/theoretical_loss": 3.3716239134440955, + "objective/train/tokens_used": 1121463776, + "theoretical_loss": 3.3716239134440955, + "tokens_seen": 2751004672 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016810431293881646, + "loss": 1.2375, + "theoretical_loss": 3.3716239134440955, + "tokens_seen": 2751004672 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016808425275827483, + "loss": 1.2298, + "theoretical_loss": 3.3716176817354024, + "tokens_seen": 2751070208 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001680641925777332, + "loss": 1.2025, + "theoretical_loss": 3.3716114502167245, + "tokens_seen": 2751135744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016804413239719158, + "loss": 1.1846, + "theoretical_loss": 3.3716052188880514, + "tokens_seen": 2751201280 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016802407221664994, + "loss": 1.256, + "theoretical_loss": 3.371598987749373, + "tokens_seen": 2751266816 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016800401203610833, + "loss": 1.2829, + "theoretical_loss": 3.371592756800679, + "tokens_seen": 2751332352 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001679839518555667, + "loss": 1.2404, + "theoretical_loss": 3.3715865260419595, + "tokens_seen": 2751397888 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016796389167502508, + "loss": 1.3218, + "theoretical_loss": 3.3715802954732035, + "tokens_seen": 2751463424 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016794383149448344, + "loss": 1.1767, + "theoretical_loss": 3.3715740650944013, + "tokens_seen": 2751528960 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016792377131394183, + "loss": 1.2308, + "theoretical_loss": 3.371567834905542, + "tokens_seen": 2751594496 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001679037111334002, + "loss": 1.2704, + "theoretical_loss": 3.3715616049066153, + "tokens_seen": 2751660032 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016788365095285856, + "loss": 1.1846, + "theoretical_loss": 3.3715553750976115, + "tokens_seen": 2751725568 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016786359077231695, + "loss": 1.2159, + "theoretical_loss": 3.37154914547852, + "tokens_seen": 2751791104 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001678435305917753, + "loss": 1.1793, + "theoretical_loss": 3.37154291604933, + "tokens_seen": 2751856640 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001678234704112337, + "loss": 1.3151, + "theoretical_loss": 3.371536686810032, + "tokens_seen": 2751922176 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016780341023069206, + "loss": 1.3483, + "theoretical_loss": 3.3715304577606147, + "tokens_seen": 2751987712 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016778335005015048, + "loss": 1.2376, + "theoretical_loss": 3.3715242289010687, + "tokens_seen": 2752053248 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016776328986960884, + "loss": 1.343, + "theoretical_loss": 3.3715180002313834, + "tokens_seen": 2752118784 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016774322968906723, + "loss": 1.3001, + "theoretical_loss": 3.3715117717515484, + "tokens_seen": 2752184320 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001677231695085256, + "loss": 1.2814, + "theoretical_loss": 3.3715055434615535, + "tokens_seen": 2752249856 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016770310932798395, + "loss": 1.2549, + "theoretical_loss": 3.371499315361388, + "tokens_seen": 2752315392 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016768304914744234, + "loss": 1.2115, + "theoretical_loss": 3.3714930874510425, + "tokens_seen": 2752380928 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001676629889669007, + "loss": 1.2776, + "theoretical_loss": 3.371486859730506, + "tokens_seen": 2752446464 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001676429287863591, + "loss": 1.2282, + "theoretical_loss": 3.371480632199768, + "tokens_seen": 2752512000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016762286860581746, + "loss": 1.2643, + "theoretical_loss": 3.3714744048588186, + "tokens_seen": 2752577536 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016760280842527584, + "loss": 1.4276, + "theoretical_loss": 3.371468177707648, + "tokens_seen": 2752643072 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001675827482447342, + "loss": 1.3795, + "theoretical_loss": 3.3714619507462444, + "tokens_seen": 2752708608 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001675626880641926, + "loss": 1.4465, + "theoretical_loss": 3.371455723974599, + "tokens_seen": 2752774144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016754262788365096, + "loss": 1.2546, + "theoretical_loss": 3.371449497392701, + "tokens_seen": 2752839680 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016752256770310935, + "loss": 1.2639, + "theoretical_loss": 3.3714432710005395, + "tokens_seen": 2752905216 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001675025075225677, + "loss": 1.1964, + "theoretical_loss": 3.371437044798105, + "tokens_seen": 2752970752 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016748244734202607, + "loss": 1.2831, + "theoretical_loss": 3.371430818785387, + "tokens_seen": 2753036288 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016746238716148446, + "loss": 1.3247, + "theoretical_loss": 3.371424592962375, + "tokens_seen": 2753101824 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016744232698094282, + "loss": 1.2814, + "theoretical_loss": 3.371418367329059, + "tokens_seen": 2753167360 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001674222668004012, + "loss": 1.2264, + "theoretical_loss": 3.371412141885428, + "tokens_seen": 2753232896 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016740220661985958, + "loss": 1.3073, + "theoretical_loss": 3.371405916631473, + "tokens_seen": 2753298432 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016738214643931797, + "loss": 1.2968, + "theoretical_loss": 3.3713996915671824, + "tokens_seen": 2753363968 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016736208625877633, + "loss": 1.1687, + "theoretical_loss": 3.371393466692547, + "tokens_seen": 2753429504 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016734202607823472, + "loss": 1.2647, + "theoretical_loss": 3.3713872420075557, + "tokens_seen": 2753495040 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016732196589769308, + "loss": 1.337, + "theoretical_loss": 3.3713810175121983, + "tokens_seen": 2753560576 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016730190571715144, + "loss": 1.4151, + "theoretical_loss": 3.371374793206465, + "tokens_seen": 2753626112 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016728184553660983, + "loss": 1.3095, + "theoretical_loss": 3.3713685690903445, + "tokens_seen": 2753691648 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001672617853560682, + "loss": 1.3294, + "theoretical_loss": 3.3713623451638277, + "tokens_seen": 2753757184 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016724172517552658, + "loss": 1.3321, + "theoretical_loss": 3.3713561214269037, + "tokens_seen": 2753822720 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016722166499498494, + "loss": 1.2653, + "theoretical_loss": 3.3713498978795626, + "tokens_seen": 2753888256 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016720160481444333, + "loss": 1.2522, + "theoretical_loss": 3.3713436745217935, + "tokens_seen": 2753953792 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001671815446339017, + "loss": 1.214, + "theoretical_loss": 3.371337451353587, + "tokens_seen": 2754019328 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016716148445336009, + "loss": 1.26, + "theoretical_loss": 3.3713312283749315, + "tokens_seen": 2754084864 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016714142427281845, + "loss": 1.2422, + "theoretical_loss": 3.3713250055858177, + "tokens_seen": 2754150400 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001671213640922768, + "loss": 1.3388, + "theoretical_loss": 3.3713187829862354, + "tokens_seen": 2754215936 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1000409, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.376333475112915, + "objective/train/theoretical_loss": 3.371312560576174, + "objective/train/tokens_used": 1124740576, + "theoretical_loss": 3.371312560576174, + "tokens_seen": 2754281472 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001671013039117352, + "loss": 1.2714, + "theoretical_loss": 3.371312560576174, + "tokens_seen": 2754281472 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016708124373119356, + "loss": 1.3123, + "theoretical_loss": 3.371306338355623, + "tokens_seen": 2754347008 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016706118355065195, + "loss": 1.1889, + "theoretical_loss": 3.3713001163245724, + "tokens_seen": 2754412544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016704112337011034, + "loss": 1.2765, + "theoretical_loss": 3.3712938944830118, + "tokens_seen": 2754478080 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016702106318956873, + "loss": 1.2297, + "theoretical_loss": 3.3712876728309316, + "tokens_seen": 2754543616 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001670010030090271, + "loss": 1.1897, + "theoretical_loss": 3.3712814513683202, + "tokens_seen": 2754609152 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016698094282848548, + "loss": 1.2181, + "theoretical_loss": 3.3712752300951685, + "tokens_seen": 2754674688 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016696088264794384, + "loss": 1.2743, + "theoretical_loss": 3.371269009011465, + "tokens_seen": 2754740224 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016694082246740223, + "loss": 1.3309, + "theoretical_loss": 3.371262788117201, + "tokens_seen": 2754805760 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001669207622868606, + "loss": 1.3581, + "theoretical_loss": 3.3712565674123653, + "tokens_seen": 2754871296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016690070210631896, + "loss": 1.2551, + "theoretical_loss": 3.3712503468969475, + "tokens_seen": 2754936832 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016688064192577735, + "loss": 1.2573, + "theoretical_loss": 3.3712441265709376, + "tokens_seen": 2755002368 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001668605817452357, + "loss": 1.2758, + "theoretical_loss": 3.3712379064343256, + "tokens_seen": 2755067904 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001668405215646941, + "loss": 1.22, + "theoretical_loss": 3.371231686487101, + "tokens_seen": 2755133440 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016682046138415246, + "loss": 1.3182, + "theoretical_loss": 3.371225466729253, + "tokens_seen": 2755198976 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016680040120361085, + "loss": 1.355, + "theoretical_loss": 3.371219247160772, + "tokens_seen": 2755264512 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001667803410230692, + "loss": 1.3263, + "theoretical_loss": 3.371213027781647, + "tokens_seen": 2755330048 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001667602808425276, + "loss": 1.3499, + "theoretical_loss": 3.371206808591869, + "tokens_seen": 2755395584 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016674022066198596, + "loss": 1.3696, + "theoretical_loss": 3.3712005895914263, + "tokens_seen": 2755461120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016672016048144433, + "loss": 1.3301, + "theoretical_loss": 3.3711943707803096, + "tokens_seen": 2755526656 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016670010030090271, + "loss": 1.3457, + "theoretical_loss": 3.371188152158509, + "tokens_seen": 2755592192 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016668004012036108, + "loss": 1.2773, + "theoretical_loss": 3.371181933726013, + "tokens_seen": 2755657728 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016665997993981947, + "loss": 1.3321, + "theoretical_loss": 3.3711757154828117, + "tokens_seen": 2755723264 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016663991975927783, + "loss": 1.2237, + "theoretical_loss": 3.371169497428895, + "tokens_seen": 2755788800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016661985957873622, + "loss": 1.2962, + "theoretical_loss": 3.371163279564253, + "tokens_seen": 2755854336 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016659979939819458, + "loss": 1.2718, + "theoretical_loss": 3.371157061888875, + "tokens_seen": 2755919872 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016657973921765297, + "loss": 1.2753, + "theoretical_loss": 3.371150844402751, + "tokens_seen": 2755985408 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016655967903711133, + "loss": 1.2329, + "theoretical_loss": 3.3711446271058705, + "tokens_seen": 2756050944 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016653961885656972, + "loss": 1.2437, + "theoretical_loss": 3.3711384099982236, + "tokens_seen": 2756116480 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016651955867602808, + "loss": 1.3679, + "theoretical_loss": 3.3711321930797995, + "tokens_seen": 2756182016 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016649949849548645, + "loss": 1.2775, + "theoretical_loss": 3.371125976350588, + "tokens_seen": 2756247552 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016647943831494484, + "loss": 1.2927, + "theoretical_loss": 3.371119759810579, + "tokens_seen": 2756313088 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001664593781344032, + "loss": 1.1863, + "theoretical_loss": 3.3711135434597628, + "tokens_seen": 2756378624 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001664393179538616, + "loss": 1.3011, + "theoretical_loss": 3.371107327298129, + "tokens_seen": 2756444160 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016641925777331995, + "loss": 1.2203, + "theoretical_loss": 3.3711011113256664, + "tokens_seen": 2756509696 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016639919759277834, + "loss": 1.2104, + "theoretical_loss": 3.371094895542365, + "tokens_seen": 2756575232 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001663791374122367, + "loss": 1.3844, + "theoretical_loss": 3.3710886799482154, + "tokens_seen": 2756640768 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001663590772316951, + "loss": 1.3288, + "theoretical_loss": 3.371082464543207, + "tokens_seen": 2756706304 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016633901705115345, + "loss": 1.2656, + "theoretical_loss": 3.371076249327329, + "tokens_seen": 2756771840 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016631895687061181, + "loss": 1.2298, + "theoretical_loss": 3.3710700343005717, + "tokens_seen": 2756837376 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001662988966900702, + "loss": 1.3095, + "theoretical_loss": 3.3710638194629245, + "tokens_seen": 2756902912 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001662788365095286, + "loss": 1.2716, + "theoretical_loss": 3.3710576048143777, + "tokens_seen": 2756968448 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016625877632898698, + "loss": 1.2727, + "theoretical_loss": 3.3710513903549204, + "tokens_seen": 2757033984 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016623871614844534, + "loss": 1.1839, + "theoretical_loss": 3.371045176084543, + "tokens_seen": 2757099520 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016621865596790373, + "loss": 1.3455, + "theoretical_loss": 3.3710389620032344, + "tokens_seen": 2757165056 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001661985957873621, + "loss": 1.2874, + "theoretical_loss": 3.371032748110985, + "tokens_seen": 2757230592 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016617853560682049, + "loss": 1.3689, + "theoretical_loss": 3.3710265344077843, + "tokens_seen": 2757296128 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016615847542627885, + "loss": 1.376, + "theoretical_loss": 3.3710203208936225, + "tokens_seen": 2757361664 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001661384152457372, + "loss": 1.331, + "theoretical_loss": 3.371014107568489, + "tokens_seen": 2757427200 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001661183550651956, + "loss": 1.2802, + "theoretical_loss": 3.371007894432373, + "tokens_seen": 2757492736 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1001599, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3787121772766113, + "objective/train/theoretical_loss": 3.3710016814852652, + "objective/train/tokens_used": 1128017376, + "theoretical_loss": 3.3710016814852652, + "tokens_seen": 2757558272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016609829488465396, + "loss": 1.4272, + "theoretical_loss": 3.3710016814852652, + "tokens_seen": 2757558272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016607823470411235, + "loss": 1.2472, + "theoretical_loss": 3.3709954687271555, + "tokens_seen": 2757623808 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001660581745235707, + "loss": 1.3416, + "theoretical_loss": 3.3709892561580324, + "tokens_seen": 2757689344 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001660381143430291, + "loss": 1.3559, + "theoretical_loss": 3.3709830437778865, + "tokens_seen": 2757754880 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016601805416248746, + "loss": 1.213, + "theoretical_loss": 3.3709768315867077, + "tokens_seen": 2757820416 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016599799398194585, + "loss": 1.3054, + "theoretical_loss": 3.3709706195844857, + "tokens_seen": 2757885952 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016597793380140422, + "loss": 1.2594, + "theoretical_loss": 3.3709644077712095, + "tokens_seen": 2757951488 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001659578736208626, + "loss": 1.1767, + "theoretical_loss": 3.3709581961468698, + "tokens_seen": 2758017024 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016593781344032097, + "loss": 1.2584, + "theoretical_loss": 3.370951984711456, + "tokens_seen": 2758082560 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016591775325977933, + "loss": 1.2527, + "theoretical_loss": 3.3709457734649577, + "tokens_seen": 2758148096 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016589769307923772, + "loss": 1.2631, + "theoretical_loss": 3.370939562407365, + "tokens_seen": 2758213632 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016587763289869608, + "loss": 1.2045, + "theoretical_loss": 3.370933351538668, + "tokens_seen": 2758279168 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016585757271815447, + "loss": 1.2822, + "theoretical_loss": 3.3709271408588553, + "tokens_seen": 2758344704 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016583751253761283, + "loss": 1.2555, + "theoretical_loss": 3.3709209303679177, + "tokens_seen": 2758410240 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016581745235707122, + "loss": 1.271, + "theoretical_loss": 3.3709147200658443, + "tokens_seen": 2758475776 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016579739217652959, + "loss": 1.2972, + "theoretical_loss": 3.3709085099526255, + "tokens_seen": 2758541312 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016577733199598797, + "loss": 1.2692, + "theoretical_loss": 3.3709023000282503, + "tokens_seen": 2758606848 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016575727181544634, + "loss": 1.2596, + "theoretical_loss": 3.3708960902927094, + "tokens_seen": 2758672384 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001657372116349047, + "loss": 1.4131, + "theoretical_loss": 3.370889880745992, + "tokens_seen": 2758737920 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001657171514543631, + "loss": 1.3679, + "theoretical_loss": 3.370883671388088, + "tokens_seen": 2758803456 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016569709127382145, + "loss": 1.2896, + "theoretical_loss": 3.3708774622189868, + "tokens_seen": 2758868992 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016567703109327984, + "loss": 1.3551, + "theoretical_loss": 3.370871253238679, + "tokens_seen": 2758934528 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001656569709127382, + "loss": 1.3649, + "theoretical_loss": 3.3708650444471537, + "tokens_seen": 2759000064 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001656369107321966, + "loss": 1.2963, + "theoretical_loss": 3.370858835844401, + "tokens_seen": 2759065600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016561685055165495, + "loss": 1.3216, + "theoretical_loss": 3.37085262743041, + "tokens_seen": 2759131136 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016559679037111334, + "loss": 1.2045, + "theoretical_loss": 3.3708464192051717, + "tokens_seen": 2759196672 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001655767301905717, + "loss": 1.2393, + "theoretical_loss": 3.370840211168675, + "tokens_seen": 2759262208 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016555667001003007, + "loss": 1.2352, + "theoretical_loss": 3.3708340033209097, + "tokens_seen": 2759327744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016553660982948848, + "loss": 1.2635, + "theoretical_loss": 3.370827795661866, + "tokens_seen": 2759393280 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016551654964894685, + "loss": 1.2759, + "theoretical_loss": 3.370821588191533, + "tokens_seen": 2759458816 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016549648946840524, + "loss": 1.324, + "theoretical_loss": 3.3708153809099013, + "tokens_seen": 2759524352 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001654764292878636, + "loss": 1.1488, + "theoretical_loss": 3.3708091738169603, + "tokens_seen": 2759589888 + }, + { + "epoch": 0.67, + "learning_rate": 0.000165456369107322, + "loss": 1.2354, + "theoretical_loss": 3.3708029669127, + "tokens_seen": 2759655424 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016543630892678035, + "loss": 1.2702, + "theoretical_loss": 3.37079676019711, + "tokens_seen": 2759720960 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016541624874623874, + "loss": 1.272, + "theoretical_loss": 3.3707905536701794, + "tokens_seen": 2759786496 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001653961885656971, + "loss": 1.2732, + "theoretical_loss": 3.3707843473318992, + "tokens_seen": 2759852032 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001653761283851555, + "loss": 1.3023, + "theoretical_loss": 3.3707781411822584, + "tokens_seen": 2759917568 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016535606820461385, + "loss": 1.2426, + "theoretical_loss": 3.3707719352212475, + "tokens_seen": 2759983104 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016533600802407221, + "loss": 1.2797, + "theoretical_loss": 3.370765729448855, + "tokens_seen": 2760048640 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001653159478435306, + "loss": 1.2381, + "theoretical_loss": 3.3707595238650723, + "tokens_seen": 2760114176 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016529588766298897, + "loss": 1.2533, + "theoretical_loss": 3.370753318469888, + "tokens_seen": 2760179712 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016527582748244736, + "loss": 1.3088, + "theoretical_loss": 3.3707471132632922, + "tokens_seen": 2760245248 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016525576730190572, + "loss": 1.1988, + "theoretical_loss": 3.3707409082452746, + "tokens_seen": 2760310784 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001652357071213641, + "loss": 1.3281, + "theoretical_loss": 3.3707347034158257, + "tokens_seen": 2760376320 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016521564694082247, + "loss": 1.251, + "theoretical_loss": 3.3707284987749344, + "tokens_seen": 2760441856 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016519558676028086, + "loss": 1.3619, + "theoretical_loss": 3.370722294322591, + "tokens_seen": 2760507392 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016517552657973922, + "loss": 1.1338, + "theoretical_loss": 3.370716090058785, + "tokens_seen": 2760572928 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016515546639919758, + "loss": 1.2411, + "theoretical_loss": 3.3707098859835063, + "tokens_seen": 2760638464 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016513540621865597, + "loss": 1.2467, + "theoretical_loss": 3.370703682096745, + "tokens_seen": 2760704000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016511534603811433, + "loss": 1.3318, + "theoretical_loss": 3.3706974783984904, + "tokens_seen": 2760769536 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1002769, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3618190288543701, + "objective/train/theoretical_loss": 3.3706912748887325, + "objective/train/tokens_used": 1131294176, + "theoretical_loss": 3.3706912748887325, + "tokens_seen": 2760835072 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016509528585757272, + "loss": 1.3901, + "theoretical_loss": 3.3706912748887325, + "tokens_seen": 2760835072 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001650752256770311, + "loss": 1.3447, + "theoretical_loss": 3.370685071567461, + "tokens_seen": 2760900608 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016505516549648948, + "loss": 1.2878, + "theoretical_loss": 3.3706788684346662, + "tokens_seen": 2760966144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016503510531594784, + "loss": 1.2157, + "theoretical_loss": 3.3706726654903374, + "tokens_seen": 2761031680 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016501504513540623, + "loss": 1.2263, + "theoretical_loss": 3.3706664627344645, + "tokens_seen": 2761097216 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001649949849548646, + "loss": 1.2767, + "theoretical_loss": 3.3706602601670372, + "tokens_seen": 2761162752 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016497492477432295, + "loss": 1.2408, + "theoretical_loss": 3.3706540577880455, + "tokens_seen": 2761228288 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016495486459378134, + "loss": 1.2725, + "theoretical_loss": 3.3706478555974795, + "tokens_seen": 2761293824 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001649348044132397, + "loss": 1.2328, + "theoretical_loss": 3.370641653595328, + "tokens_seen": 2761359360 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001649147442326981, + "loss": 1.342, + "theoretical_loss": 3.370635451781582, + "tokens_seen": 2761424896 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016489468405215646, + "loss": 1.3471, + "theoretical_loss": 3.3706292501562305, + "tokens_seen": 2761490432 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016487462387161484, + "loss": 1.2588, + "theoretical_loss": 3.3706230487192634, + "tokens_seen": 2761555968 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001648545636910732, + "loss": 1.405, + "theoretical_loss": 3.3706168474706706, + "tokens_seen": 2761621504 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001648345035105316, + "loss": 1.2964, + "theoretical_loss": 3.3706106464104426, + "tokens_seen": 2761687040 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016481444332998996, + "loss": 1.3046, + "theoretical_loss": 3.370604445538568, + "tokens_seen": 2761752576 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016479438314944835, + "loss": 1.2115, + "theoretical_loss": 3.3705982448550373, + "tokens_seen": 2761818112 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016477432296890674, + "loss": 1.3285, + "theoretical_loss": 3.3705920443598405, + "tokens_seen": 2761883648 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001647542627883651, + "loss": 1.2677, + "theoretical_loss": 3.3705858440529664, + "tokens_seen": 2761949184 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001647342026078235, + "loss": 1.3254, + "theoretical_loss": 3.370579643934406, + "tokens_seen": 2762014720 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016471414242728185, + "loss": 1.2949, + "theoretical_loss": 3.370573444004149, + "tokens_seen": 2762080256 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016469408224674024, + "loss": 1.2499, + "theoretical_loss": 3.370567244262184, + "tokens_seen": 2762145792 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001646740220661986, + "loss": 1.2353, + "theoretical_loss": 3.370561044708502, + "tokens_seen": 2762211328 + }, + { + "epoch": 0.67, + "learning_rate": 0.000164653961885657, + "loss": 1.2908, + "theoretical_loss": 3.3705548453430927, + "tokens_seen": 2762276864 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016463390170511535, + "loss": 1.3118, + "theoretical_loss": 3.370548646165946, + "tokens_seen": 2762342400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016461384152457374, + "loss": 1.2551, + "theoretical_loss": 3.3705424471770504, + "tokens_seen": 2762407936 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001645937813440321, + "loss": 1.2791, + "theoretical_loss": 3.3705362483763976, + "tokens_seen": 2762473472 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016457372116349047, + "loss": 1.295, + "theoretical_loss": 3.370530049763976, + "tokens_seen": 2762539008 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016455366098294886, + "loss": 1.2676, + "theoretical_loss": 3.3705238513397764, + "tokens_seen": 2762604544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016453360080240722, + "loss": 1.2614, + "theoretical_loss": 3.370517653103788, + "tokens_seen": 2762670080 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001645135406218656, + "loss": 1.2387, + "theoretical_loss": 3.3705114550560005, + "tokens_seen": 2762735616 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016449348044132397, + "loss": 1.2367, + "theoretical_loss": 3.3705052571964043, + "tokens_seen": 2762801152 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016447342026078236, + "loss": 1.3554, + "theoretical_loss": 3.370499059524989, + "tokens_seen": 2762866688 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016445336008024072, + "loss": 1.3788, + "theoretical_loss": 3.3704928620417443, + "tokens_seen": 2762932224 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001644332998996991, + "loss": 1.2971, + "theoretical_loss": 3.37048666474666, + "tokens_seen": 2762997760 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016441323971915747, + "loss": 1.398, + "theoretical_loss": 3.370480467639726, + "tokens_seen": 2763063296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016439317953861586, + "loss": 1.2954, + "theoretical_loss": 3.3704742707209325, + "tokens_seen": 2763128832 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016437311935807423, + "loss": 1.3137, + "theoretical_loss": 3.370468073990269, + "tokens_seen": 2763194368 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001643530591775326, + "loss": 1.1932, + "theoretical_loss": 3.370461877447725, + "tokens_seen": 2763259904 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016433299899699098, + "loss": 1.2378, + "theoretical_loss": 3.3704556810932904, + "tokens_seen": 2763325440 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016431293881644934, + "loss": 1.2293, + "theoretical_loss": 3.3704494849269553, + "tokens_seen": 2763390976 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016429287863590773, + "loss": 1.1782, + "theoretical_loss": 3.3704432889487097, + "tokens_seen": 2763456512 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001642728184553661, + "loss": 1.2989, + "theoretical_loss": 3.3704370931585435, + "tokens_seen": 2763522048 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016425275827482448, + "loss": 1.1369, + "theoretical_loss": 3.370430897556446, + "tokens_seen": 2763587584 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016423269809428284, + "loss": 1.1962, + "theoretical_loss": 3.370424702142407, + "tokens_seen": 2763653120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016421263791374123, + "loss": 1.2644, + "theoretical_loss": 3.3704185069164168, + "tokens_seen": 2763718656 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001641925777331996, + "loss": 1.3315, + "theoretical_loss": 3.370412311878465, + "tokens_seen": 2763784192 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016417251755265796, + "loss": 1.1844, + "theoretical_loss": 3.3704061170285415, + "tokens_seen": 2763849728 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016415245737211635, + "loss": 1.2556, + "theoretical_loss": 3.370399922366636, + "tokens_seen": 2763915264 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001641323971915747, + "loss": 1.2724, + "theoretical_loss": 3.3703937278927385, + "tokens_seen": 2763980800 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001641123370110331, + "loss": 1.3688, + "theoretical_loss": 3.3703875336068387, + "tokens_seen": 2764046336 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1003856, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1871845722198486, + "objective/train/theoretical_loss": 3.370381339508927, + "objective/train/tokens_used": 1134570976, + "theoretical_loss": 3.370381339508927, + "tokens_seen": 2764111872 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016409227683049146, + "loss": 1.2468, + "theoretical_loss": 3.370381339508927, + "tokens_seen": 2764111872 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016407221664994985, + "loss": 1.3024, + "theoretical_loss": 3.370375145598992, + "tokens_seen": 2764177408 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001640521564694082, + "loss": 1.2677, + "theoretical_loss": 3.370368951877025, + "tokens_seen": 2764242944 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016403209628886663, + "loss": 1.2023, + "theoretical_loss": 3.3703627583430147, + "tokens_seen": 2764308480 + }, + { + "epoch": 0.68, + "learning_rate": 0.000164012036108325, + "loss": 1.2856, + "theoretical_loss": 3.3703565649969516, + "tokens_seen": 2764374016 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016399197592778335, + "loss": 1.3119, + "theoretical_loss": 3.370350371838825, + "tokens_seen": 2764439552 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016397191574724174, + "loss": 1.3218, + "theoretical_loss": 3.370344178868625, + "tokens_seen": 2764505088 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001639518555667001, + "loss": 1.293, + "theoretical_loss": 3.370337986086342, + "tokens_seen": 2764570624 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001639317953861585, + "loss": 1.4112, + "theoretical_loss": 3.3703317934919648, + "tokens_seen": 2764636160 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016391173520561686, + "loss": 1.3238, + "theoretical_loss": 3.3703256010854843, + "tokens_seen": 2764701696 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016389167502507524, + "loss": 1.2879, + "theoretical_loss": 3.3703194088668895, + "tokens_seen": 2764767232 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001638716148445336, + "loss": 1.2413, + "theoretical_loss": 3.3703132168361707, + "tokens_seen": 2764832768 + }, + { + "epoch": 0.68, + "learning_rate": 0.000163851554663992, + "loss": 1.2491, + "theoretical_loss": 3.3703070249933176, + "tokens_seen": 2764898304 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016383149448345036, + "loss": 1.2682, + "theoretical_loss": 3.3703008333383204, + "tokens_seen": 2764963840 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016381143430290875, + "loss": 1.3225, + "theoretical_loss": 3.370294641871168, + "tokens_seen": 2765029376 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001637913741223671, + "loss": 1.3117, + "theoretical_loss": 3.3702884505918513, + "tokens_seen": 2765094912 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016377131394182547, + "loss": 1.2282, + "theoretical_loss": 3.3702822595003594, + "tokens_seen": 2765160448 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016375125376128386, + "loss": 1.3778, + "theoretical_loss": 3.3702760685966826, + "tokens_seen": 2765225984 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016373119358074222, + "loss": 1.327, + "theoretical_loss": 3.3702698778808107, + "tokens_seen": 2765291520 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001637111334002006, + "loss": 1.3243, + "theoretical_loss": 3.3702636873527334, + "tokens_seen": 2765357056 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016369107321965898, + "loss": 1.3506, + "theoretical_loss": 3.3702574970124406, + "tokens_seen": 2765422592 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016367101303911736, + "loss": 1.3616, + "theoretical_loss": 3.370251306859922, + "tokens_seen": 2765488128 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016365095285857573, + "loss": 1.2166, + "theoretical_loss": 3.370245116895168, + "tokens_seen": 2765553664 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016363089267803412, + "loss": 1.427, + "theoretical_loss": 3.370238927118168, + "tokens_seen": 2765619200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016361083249749248, + "loss": 1.2037, + "theoretical_loss": 3.370232737528912, + "tokens_seen": 2765684736 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016359077231695084, + "loss": 1.2782, + "theoretical_loss": 3.3702265481273894, + "tokens_seen": 2765750272 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016357071213640923, + "loss": 1.2883, + "theoretical_loss": 3.370220358913591, + "tokens_seen": 2765815808 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001635506519558676, + "loss": 1.2342, + "theoretical_loss": 3.3702141698875057, + "tokens_seen": 2765881344 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016353059177532598, + "loss": 1.3131, + "theoretical_loss": 3.3702079810491243, + "tokens_seen": 2765946880 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016351053159478434, + "loss": 1.2925, + "theoretical_loss": 3.3702017923984355, + "tokens_seen": 2766012416 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016349047141424273, + "loss": 1.394, + "theoretical_loss": 3.3701956039354304, + "tokens_seen": 2766077952 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001634704112337011, + "loss": 1.2878, + "theoretical_loss": 3.3701894156600978, + "tokens_seen": 2766143488 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016345035105315949, + "loss": 1.4733, + "theoretical_loss": 3.370183227572428, + "tokens_seen": 2766209024 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016343029087261785, + "loss": 1.1931, + "theoretical_loss": 3.370177039672411, + "tokens_seen": 2766274560 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001634102306920762, + "loss": 1.23, + "theoretical_loss": 3.3701708519600366, + "tokens_seen": 2766340096 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001633901705115346, + "loss": 1.2989, + "theoretical_loss": 3.3701646644352943, + "tokens_seen": 2766405632 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016337011033099296, + "loss": 1.3169, + "theoretical_loss": 3.3701584770981747, + "tokens_seen": 2766471168 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016335005015045135, + "loss": 1.115, + "theoretical_loss": 3.370152289948667, + "tokens_seen": 2766536704 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001633299899699097, + "loss": 1.3471, + "theoretical_loss": 3.3701461029867614, + "tokens_seen": 2766602240 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001633099297893681, + "loss": 1.3133, + "theoretical_loss": 3.370139916212448, + "tokens_seen": 2766667776 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016328986960882646, + "loss": 1.2611, + "theoretical_loss": 3.370133729625716, + "tokens_seen": 2766733312 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016326980942828488, + "loss": 1.2984, + "theoretical_loss": 3.3701275432265554, + "tokens_seen": 2766798848 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016324974924774324, + "loss": 1.2212, + "theoretical_loss": 3.370121357014957, + "tokens_seen": 2766864384 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016322968906720163, + "loss": 1.2986, + "theoretical_loss": 3.370115170990909, + "tokens_seen": 2766929920 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016320962888666, + "loss": 1.2524, + "theoretical_loss": 3.370108985154403, + "tokens_seen": 2766995456 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016318956870611836, + "loss": 1.3288, + "theoretical_loss": 3.370102799505428, + "tokens_seen": 2767060992 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016316950852557675, + "loss": 1.2687, + "theoretical_loss": 3.3700966140439736, + "tokens_seen": 2767126528 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001631494483450351, + "loss": 1.1719, + "theoretical_loss": 3.3700904287700304, + "tokens_seen": 2767192064 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001631293881644935, + "loss": 1.2824, + "theoretical_loss": 3.370084243683588, + "tokens_seen": 2767257600 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016310932798395186, + "loss": 1.3743, + "theoretical_loss": 3.3700780587846357, + "tokens_seen": 2767323136 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1005007, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3620854616165161, + "objective/train/theoretical_loss": 3.3700718740731643, + "objective/train/tokens_used": 1137847776, + "theoretical_loss": 3.3700718740731643, + "tokens_seen": 2767388672 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016308926780341025, + "loss": 1.217, + "theoretical_loss": 3.3700718740731643, + "tokens_seen": 2767388672 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001630692076228686, + "loss": 1.1842, + "theoretical_loss": 3.370065689549163, + "tokens_seen": 2767454208 + }, + { + "epoch": 0.68, + "learning_rate": 0.000163049147442327, + "loss": 1.3029, + "theoretical_loss": 3.3700595052126223, + "tokens_seen": 2767519744 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016302908726178536, + "loss": 1.266, + "theoretical_loss": 3.3700533210635313, + "tokens_seen": 2767585280 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016300902708124373, + "loss": 1.2676, + "theoretical_loss": 3.3700471371018805, + "tokens_seen": 2767650816 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016298896690070211, + "loss": 1.3573, + "theoretical_loss": 3.3700409533276594, + "tokens_seen": 2767716352 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016296890672016048, + "loss": 1.252, + "theoretical_loss": 3.370034769740858, + "tokens_seen": 2767781888 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016294884653961887, + "loss": 1.4028, + "theoretical_loss": 3.3700285863414665, + "tokens_seen": 2767847424 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016292878635907723, + "loss": 1.1998, + "theoretical_loss": 3.3700224031294748, + "tokens_seen": 2767912960 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016290872617853562, + "loss": 1.3296, + "theoretical_loss": 3.370016220104872, + "tokens_seen": 2767978496 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016288866599799398, + "loss": 1.2872, + "theoretical_loss": 3.3700100372676487, + "tokens_seen": 2768044032 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016286860581745237, + "loss": 1.2329, + "theoretical_loss": 3.3700038546177944, + "tokens_seen": 2768109568 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016284854563691073, + "loss": 1.3292, + "theoretical_loss": 3.369997672155299, + "tokens_seen": 2768175104 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016282848545636912, + "loss": 1.1957, + "theoretical_loss": 3.369991489880153, + "tokens_seen": 2768240640 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016280842527582748, + "loss": 1.34, + "theoretical_loss": 3.369985307792346, + "tokens_seen": 2768306176 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016278836509528585, + "loss": 1.2732, + "theoretical_loss": 3.3699791258918674, + "tokens_seen": 2768371712 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016276830491474423, + "loss": 1.2435, + "theoretical_loss": 3.3699729441787074, + "tokens_seen": 2768437248 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001627482447342026, + "loss": 1.3087, + "theoretical_loss": 3.3699667626528558, + "tokens_seen": 2768502784 + }, + { + "epoch": 0.68, + "learning_rate": 0.000162728184553661, + "loss": 1.3298, + "theoretical_loss": 3.3699605813143023, + "tokens_seen": 2768568320 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016270812437311935, + "loss": 1.3429, + "theoretical_loss": 3.3699544001630377, + "tokens_seen": 2768633856 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016268806419257774, + "loss": 1.3856, + "theoretical_loss": 3.369948219199051, + "tokens_seen": 2768699392 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001626680040120361, + "loss": 1.146, + "theoretical_loss": 3.3699420384223324, + "tokens_seen": 2768764928 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001626479438314945, + "loss": 1.1669, + "theoretical_loss": 3.3699358578328718, + "tokens_seen": 2768830464 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016262788365095285, + "loss": 1.2574, + "theoretical_loss": 3.369929677430659, + "tokens_seen": 2768896000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016260782347041121, + "loss": 1.2857, + "theoretical_loss": 3.369923497215684, + "tokens_seen": 2768961536 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001625877632898696, + "loss": 1.3022, + "theoretical_loss": 3.3699173171879364, + "tokens_seen": 2769027072 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016256770310932797, + "loss": 1.3353, + "theoretical_loss": 3.369911137347407, + "tokens_seen": 2769092608 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016254764292878636, + "loss": 1.2525, + "theoretical_loss": 3.3699049576940845, + "tokens_seen": 2769158144 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016252758274824474, + "loss": 1.299, + "theoretical_loss": 3.3698987782279595, + "tokens_seen": 2769223680 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016250752256770313, + "loss": 1.2903, + "theoretical_loss": 3.369892598949021, + "tokens_seen": 2769289216 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001624874623871615, + "loss": 1.2865, + "theoretical_loss": 3.3698864198572607, + "tokens_seen": 2769354752 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016246740220661989, + "loss": 1.2789, + "theoretical_loss": 3.369880240952667, + "tokens_seen": 2769420288 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016244734202607825, + "loss": 1.2236, + "theoretical_loss": 3.3698740622352306, + "tokens_seen": 2769485824 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001624272818455366, + "loss": 1.315, + "theoretical_loss": 3.3698678837049405, + "tokens_seen": 2769551360 + }, + { + "epoch": 0.68, + "learning_rate": 0.000162407221664995, + "loss": 1.3916, + "theoretical_loss": 3.3698617053617874, + "tokens_seen": 2769616896 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016238716148445336, + "loss": 1.4239, + "theoretical_loss": 3.369855527205761, + "tokens_seen": 2769682432 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016236710130391175, + "loss": 1.2843, + "theoretical_loss": 3.3698493492368513, + "tokens_seen": 2769747968 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001623470411233701, + "loss": 1.2763, + "theoretical_loss": 3.369843171455048, + "tokens_seen": 2769813504 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001623269809428285, + "loss": 1.3035, + "theoretical_loss": 3.369836993860341, + "tokens_seen": 2769879040 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016230692076228686, + "loss": 1.2769, + "theoretical_loss": 3.36983081645272, + "tokens_seen": 2769944576 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016228686058174525, + "loss": 1.2634, + "theoretical_loss": 3.369824639232175, + "tokens_seen": 2770010112 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016226680040120362, + "loss": 1.2869, + "theoretical_loss": 3.3698184621986966, + "tokens_seen": 2770075648 + }, + { + "epoch": 0.68, + "learning_rate": 0.000162246740220662, + "loss": 1.2722, + "theoretical_loss": 3.3698122853522743, + "tokens_seen": 2770141184 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016222668004012037, + "loss": 1.4375, + "theoretical_loss": 3.369806108692898, + "tokens_seen": 2770206720 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016220661985957873, + "loss": 1.2202, + "theoretical_loss": 3.369799932220557, + "tokens_seen": 2770272256 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016218655967903712, + "loss": 1.2148, + "theoretical_loss": 3.369793755935242, + "tokens_seen": 2770337792 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016216649949849548, + "loss": 1.2942, + "theoretical_loss": 3.3697875798369425, + "tokens_seen": 2770403328 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016214643931795387, + "loss": 1.3072, + "theoretical_loss": 3.3697814039256486, + "tokens_seen": 2770468864 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016212637913741223, + "loss": 1.2976, + "theoretical_loss": 3.3697752282013504, + "tokens_seen": 2770534400 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016210631895687062, + "loss": 1.3175, + "theoretical_loss": 3.3697690526640374, + "tokens_seen": 2770599936 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1006223, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.198311448097229, + "objective/train/theoretical_loss": 3.3697628773136996, + "objective/train/tokens_used": 1141124576, + "theoretical_loss": 3.3697628773136996, + "tokens_seen": 2770665472 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016208625877632898, + "loss": 1.1907, + "theoretical_loss": 3.3697628773136996, + "tokens_seen": 2770665472 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016206619859578737, + "loss": 1.23, + "theoretical_loss": 3.3697567021503274, + "tokens_seen": 2770731008 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016204613841524574, + "loss": 1.3371, + "theoretical_loss": 3.36975052717391, + "tokens_seen": 2770796544 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001620260782347041, + "loss": 1.3134, + "theoretical_loss": 3.369744352384438, + "tokens_seen": 2770862080 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001620060180541625, + "loss": 1.4237, + "theoretical_loss": 3.3697381777819007, + "tokens_seen": 2770927616 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016198595787362085, + "loss": 1.3607, + "theoretical_loss": 3.3697320033662885, + "tokens_seen": 2770993152 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016196589769307924, + "loss": 1.2428, + "theoretical_loss": 3.369725829137591, + "tokens_seen": 2771058688 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001619458375125376, + "loss": 1.25, + "theoretical_loss": 3.369719655095798, + "tokens_seen": 2771124224 + }, + { + "epoch": 0.68, + "learning_rate": 0.000161925777331996, + "loss": 1.2553, + "theoretical_loss": 3.3697134812409, + "tokens_seen": 2771189760 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016190571715145435, + "loss": 1.3469, + "theoretical_loss": 3.3697073075728863, + "tokens_seen": 2771255296 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016188565697091274, + "loss": 1.199, + "theoretical_loss": 3.3697011340917475, + "tokens_seen": 2771320832 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001618655967903711, + "loss": 1.1764, + "theoretical_loss": 3.3696949607974727, + "tokens_seen": 2771386368 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016184553660982947, + "loss": 1.3439, + "theoretical_loss": 3.3696887876900528, + "tokens_seen": 2771451904 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016182547642928786, + "loss": 1.3476, + "theoretical_loss": 3.3696826147694767, + "tokens_seen": 2771517440 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016180541624874622, + "loss": 1.308, + "theoretical_loss": 3.369676442035735, + "tokens_seen": 2771582976 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016178535606820464, + "loss": 1.3253, + "theoretical_loss": 3.369670269488817, + "tokens_seen": 2771648512 + }, + { + "epoch": 0.68, + "learning_rate": 0.000161765295887663, + "loss": 1.2715, + "theoretical_loss": 3.3696640971287137, + "tokens_seen": 2771714048 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001617452357071214, + "loss": 1.2819, + "theoretical_loss": 3.369657924955414, + "tokens_seen": 2771779584 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016172517552657975, + "loss": 1.2083, + "theoretical_loss": 3.3696517529689087, + "tokens_seen": 2771845120 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016170511534603814, + "loss": 1.2164, + "theoretical_loss": 3.3696455811691868, + "tokens_seen": 2771910656 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001616850551654965, + "loss": 1.2479, + "theoretical_loss": 3.369639409556239, + "tokens_seen": 2771976192 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001616649949849549, + "loss": 1.2754, + "theoretical_loss": 3.3696332381300547, + "tokens_seen": 2772041728 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016164493480441325, + "loss": 1.284, + "theoretical_loss": 3.369627066890624, + "tokens_seen": 2772107264 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016162487462387161, + "loss": 1.3108, + "theoretical_loss": 3.369620895837937, + "tokens_seen": 2772172800 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016160481444333, + "loss": 1.2698, + "theoretical_loss": 3.3696147249719837, + "tokens_seen": 2772238336 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016158475426278837, + "loss": 1.2472, + "theoretical_loss": 3.3696085542927534, + "tokens_seen": 2772303872 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016156469408224676, + "loss": 1.2051, + "theoretical_loss": 3.369602383800237, + "tokens_seen": 2772369408 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016154463390170512, + "loss": 1.1932, + "theoretical_loss": 3.3695962134944235, + "tokens_seen": 2772434944 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001615245737211635, + "loss": 1.2558, + "theoretical_loss": 3.3695900433753034, + "tokens_seen": 2772500480 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016150451354062187, + "loss": 1.2355, + "theoretical_loss": 3.369583873442867, + "tokens_seen": 2772566016 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016148445336008026, + "loss": 1.2555, + "theoretical_loss": 3.369577703697103, + "tokens_seen": 2772631552 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016146439317953862, + "loss": 1.2524, + "theoretical_loss": 3.3695715341380024, + "tokens_seen": 2772697088 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016144433299899698, + "loss": 1.1769, + "theoretical_loss": 3.369565364765555, + "tokens_seen": 2772762624 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016142427281845537, + "loss": 1.1677, + "theoretical_loss": 3.3695591955797504, + "tokens_seen": 2772828160 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016140421263791373, + "loss": 1.2685, + "theoretical_loss": 3.3695530265805784, + "tokens_seen": 2772893696 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016138415245737212, + "loss": 1.2464, + "theoretical_loss": 3.3695468577680296, + "tokens_seen": 2772959232 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016136409227683049, + "loss": 1.3843, + "theoretical_loss": 3.3695406891420934, + "tokens_seen": 2773024768 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016134403209628888, + "loss": 1.1621, + "theoretical_loss": 3.3695345207027603, + "tokens_seen": 2773090304 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016132397191574724, + "loss": 1.3214, + "theoretical_loss": 3.3695283524500197, + "tokens_seen": 2773155840 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016130391173520563, + "loss": 1.2589, + "theoretical_loss": 3.369522184383862, + "tokens_seen": 2773221376 + }, + { + "epoch": 0.68, + "learning_rate": 0.000161283851554664, + "loss": 1.2999, + "theoretical_loss": 3.369516016504276, + "tokens_seen": 2773286912 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016126379137412235, + "loss": 1.226, + "theoretical_loss": 3.369509848811253, + "tokens_seen": 2773352448 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016124373119358074, + "loss": 1.3334, + "theoretical_loss": 3.3695036813047827, + "tokens_seen": 2773417984 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001612236710130391, + "loss": 1.2409, + "theoretical_loss": 3.3694975139848546, + "tokens_seen": 2773483520 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001612036108324975, + "loss": 1.302, + "theoretical_loss": 3.3694913468514596, + "tokens_seen": 2773549056 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016118355065195585, + "loss": 1.3112, + "theoretical_loss": 3.369485179904586, + "tokens_seen": 2773614592 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016116349047141424, + "loss": 1.2147, + "theoretical_loss": 3.369479013144225, + "tokens_seen": 2773680128 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001611434302908726, + "loss": 1.2544, + "theoretical_loss": 3.3694728465703663, + "tokens_seen": 2773745664 + }, + { + "epoch": 0.68, + "learning_rate": 0.000161123370110331, + "loss": 1.3686, + "theoretical_loss": 3.3694666801829998, + "tokens_seen": 2773811200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016110330992978936, + "loss": 1.3114, + "theoretical_loss": 3.369460513982115, + "tokens_seen": 2773876736 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1007399, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2168593406677246, + "objective/train/theoretical_loss": 3.369454347967703, + "objective/train/tokens_used": 1144401376, + "theoretical_loss": 3.369454347967703, + "tokens_seen": 2773942272 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016108324974924775, + "loss": 1.1862, + "theoretical_loss": 3.369454347967703, + "tokens_seen": 2773942272 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001610631895687061, + "loss": 1.2611, + "theoretical_loss": 3.3694481821397524, + "tokens_seen": 2774007808 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016104312938816447, + "loss": 1.3657, + "theoretical_loss": 3.369442016498254, + "tokens_seen": 2774073344 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001610230692076229, + "loss": 1.2533, + "theoretical_loss": 3.369435851043198, + "tokens_seen": 2774138880 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016100300902708125, + "loss": 1.3528, + "theoretical_loss": 3.3694296857745734, + "tokens_seen": 2774204416 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016098294884653964, + "loss": 1.3286, + "theoretical_loss": 3.369423520692371, + "tokens_seen": 2774269952 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160962888665998, + "loss": 1.2963, + "theoretical_loss": 3.36941735579658, + "tokens_seen": 2774335488 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001609428284854564, + "loss": 1.2819, + "theoretical_loss": 3.369411191087191, + "tokens_seen": 2774401024 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016092276830491475, + "loss": 1.306, + "theoretical_loss": 3.369405026564194, + "tokens_seen": 2774466560 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016090270812437314, + "loss": 1.2781, + "theoretical_loss": 3.3693988622275786, + "tokens_seen": 2774532096 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001608826479438315, + "loss": 1.2475, + "theoretical_loss": 3.369392698077335, + "tokens_seen": 2774597632 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016086258776328987, + "loss": 1.2191, + "theoretical_loss": 3.3693865341134526, + "tokens_seen": 2774663168 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016084252758274826, + "loss": 1.1975, + "theoretical_loss": 3.3693803703359224, + "tokens_seen": 2774728704 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016082246740220662, + "loss": 1.3225, + "theoretical_loss": 3.369374206744733, + "tokens_seen": 2774794240 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160802407221665, + "loss": 1.2348, + "theoretical_loss": 3.369368043339876, + "tokens_seen": 2774859776 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016078234704112337, + "loss": 1.2765, + "theoretical_loss": 3.36936188012134, + "tokens_seen": 2774925312 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016076228686058176, + "loss": 1.288, + "theoretical_loss": 3.3693557170891157, + "tokens_seen": 2774990848 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016074222668004012, + "loss": 1.3271, + "theoretical_loss": 3.3693495542431924, + "tokens_seen": 2775056384 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001607221664994985, + "loss": 1.2797, + "theoretical_loss": 3.369343391583561, + "tokens_seen": 2775121920 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016070210631895687, + "loss": 1.242, + "theoretical_loss": 3.369337229110211, + "tokens_seen": 2775187456 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016068204613841526, + "loss": 1.1628, + "theoretical_loss": 3.369331066823132, + "tokens_seen": 2775252992 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016066198595787363, + "loss": 1.2762, + "theoretical_loss": 3.3693249047223146, + "tokens_seen": 2775318528 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160641925777332, + "loss": 1.1142, + "theoretical_loss": 3.3693187428077485, + "tokens_seen": 2775384064 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016062186559679038, + "loss": 1.2228, + "theoretical_loss": 3.3693125810794236, + "tokens_seen": 2775449600 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016060180541624874, + "loss": 1.2553, + "theoretical_loss": 3.3693064195373297, + "tokens_seen": 2775515136 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016058174523570713, + "loss": 1.2869, + "theoretical_loss": 3.3693002581814575, + "tokens_seen": 2775580672 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001605616850551655, + "loss": 1.24, + "theoretical_loss": 3.369294097011796, + "tokens_seen": 2775646208 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016054162487462388, + "loss": 1.2091, + "theoretical_loss": 3.3692879360283357, + "tokens_seen": 2775711744 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016052156469408224, + "loss": 1.178, + "theoretical_loss": 3.369281775231067, + "tokens_seen": 2775777280 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016050150451354063, + "loss": 1.2619, + "theoretical_loss": 3.369275614619979, + "tokens_seen": 2775842816 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160481444332999, + "loss": 1.2809, + "theoretical_loss": 3.3692694541950625, + "tokens_seen": 2775908352 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016046138415245736, + "loss": 1.203, + "theoretical_loss": 3.3692632939563065, + "tokens_seen": 2775973888 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016044132397191575, + "loss": 1.2157, + "theoretical_loss": 3.369257133903702, + "tokens_seen": 2776039424 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001604212637913741, + "loss": 1.1821, + "theoretical_loss": 3.3692509740372385, + "tokens_seen": 2776104960 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001604012036108325, + "loss": 1.3248, + "theoretical_loss": 3.369244814356906, + "tokens_seen": 2776170496 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016038114343029086, + "loss": 1.2996, + "theoretical_loss": 3.3692386548626945, + "tokens_seen": 2776236032 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016036108324974925, + "loss": 1.2649, + "theoretical_loss": 3.369232495554594, + "tokens_seen": 2776301568 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001603410230692076, + "loss": 1.2395, + "theoretical_loss": 3.369226336432594, + "tokens_seen": 2776367104 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160320962888666, + "loss": 1.3442, + "theoretical_loss": 3.3692201774966857, + "tokens_seen": 2776432640 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016030090270812436, + "loss": 1.3217, + "theoretical_loss": 3.3692140187468578, + "tokens_seen": 2776498176 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016028084252758275, + "loss": 1.2607, + "theoretical_loss": 3.369207860183101, + "tokens_seen": 2776563712 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016026078234704114, + "loss": 1.3561, + "theoretical_loss": 3.3692017018054052, + "tokens_seen": 2776629248 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001602407221664995, + "loss": 1.2089, + "theoretical_loss": 3.36919554361376, + "tokens_seen": 2776694784 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001602206619859579, + "loss": 1.2782, + "theoretical_loss": 3.369189385608156, + "tokens_seen": 2776760320 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016020060180541626, + "loss": 1.1468, + "theoretical_loss": 3.369183227788583, + "tokens_seen": 2776825856 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016018054162487464, + "loss": 1.2281, + "theoretical_loss": 3.3691770701550303, + "tokens_seen": 2776891392 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160160481444333, + "loss": 1.2822, + "theoretical_loss": 3.369170912707489, + "tokens_seen": 2776956928 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001601404212637914, + "loss": 1.2566, + "theoretical_loss": 3.369164755445948, + "tokens_seen": 2777022464 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016012036108324976, + "loss": 1.3422, + "theoretical_loss": 3.369158598370398, + "tokens_seen": 2777088000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016010030090270815, + "loss": 1.2698, + "theoretical_loss": 3.3691524414808294, + "tokens_seen": 2777153536 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1008708, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2562955617904663, + "objective/train/theoretical_loss": 3.3691462847772313, + "objective/train/tokens_used": 1147678176, + "theoretical_loss": 3.3691462847772313, + "tokens_seen": 2777219072 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001600802407221665, + "loss": 1.2311, + "theoretical_loss": 3.3691462847772313, + "tokens_seen": 2777219072 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016006018054162487, + "loss": 1.2917, + "theoretical_loss": 3.369140128259594, + "tokens_seen": 2777284608 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016004012036108326, + "loss": 1.1815, + "theoretical_loss": 3.3691339719279076, + "tokens_seen": 2777350144 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016002006018054162, + "loss": 1.2521, + "theoretical_loss": 3.3691278157821616, + "tokens_seen": 2777415680 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016, + "loss": 1.242, + "theoretical_loss": 3.369121659822347, + "tokens_seen": 2777481216 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015997993981945838, + "loss": 1.3357, + "theoretical_loss": 3.3691155040484526, + "tokens_seen": 2777546752 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015995987963891676, + "loss": 1.2895, + "theoretical_loss": 3.3691093484604693, + "tokens_seen": 2777612288 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015993981945837513, + "loss": 1.3231, + "theoretical_loss": 3.3691031930583866, + "tokens_seen": 2777677824 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015991975927783352, + "loss": 1.3878, + "theoretical_loss": 3.369097037842195, + "tokens_seen": 2777743360 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015989969909729188, + "loss": 1.2995, + "theoretical_loss": 3.369090882811884, + "tokens_seen": 2777808896 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015987963891675024, + "loss": 1.3225, + "theoretical_loss": 3.369084727967444, + "tokens_seen": 2777874432 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015985957873620863, + "loss": 1.182, + "theoretical_loss": 3.3690785733088644, + "tokens_seen": 2777939968 + }, + { + "epoch": 0.68, + "learning_rate": 0.000159839518555667, + "loss": 1.2515, + "theoretical_loss": 3.369072418836136, + "tokens_seen": 2778005504 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015981945837512538, + "loss": 1.3079, + "theoretical_loss": 3.3690662645492484, + "tokens_seen": 2778071040 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015979939819458374, + "loss": 1.1674, + "theoretical_loss": 3.3690601104481916, + "tokens_seen": 2778136576 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015977933801404213, + "loss": 1.2726, + "theoretical_loss": 3.3690539565329556, + "tokens_seen": 2778202112 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001597592778335005, + "loss": 1.1346, + "theoretical_loss": 3.3690478028035304, + "tokens_seen": 2778267648 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015973921765295888, + "loss": 1.3612, + "theoretical_loss": 3.3690416492599056, + "tokens_seen": 2778333184 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015971915747241725, + "loss": 1.2974, + "theoretical_loss": 3.369035495902072, + "tokens_seen": 2778398720 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001596990972918756, + "loss": 1.2414, + "theoretical_loss": 3.3690293427300197, + "tokens_seen": 2778464256 + }, + { + "epoch": 0.68, + "learning_rate": 0.000159679037111334, + "loss": 1.1859, + "theoretical_loss": 3.3690231897437375, + "tokens_seen": 2778529792 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015965897693079236, + "loss": 1.2945, + "theoretical_loss": 3.369017036943217, + "tokens_seen": 2778595328 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015963891675025075, + "loss": 1.338, + "theoretical_loss": 3.3690108843284468, + "tokens_seen": 2778660864 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001596188565697091, + "loss": 1.3367, + "theoretical_loss": 3.3690047318994174, + "tokens_seen": 2778726400 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001595987963891675, + "loss": 1.2002, + "theoretical_loss": 3.368998579656119, + "tokens_seen": 2778791936 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015957873620862586, + "loss": 1.2901, + "theoretical_loss": 3.368992427598542, + "tokens_seen": 2778857472 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015955867602808425, + "loss": 1.3218, + "theoretical_loss": 3.3689862757266753, + "tokens_seen": 2778923008 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015953861584754262, + "loss": 1.2427, + "theoretical_loss": 3.36898012404051, + "tokens_seen": 2778988544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015951855566700103, + "loss": 1.3137, + "theoretical_loss": 3.368973972540035, + "tokens_seen": 2779054080 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001594984954864594, + "loss": 1.1387, + "theoretical_loss": 3.3689678212252416, + "tokens_seen": 2779119616 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015947843530591776, + "loss": 1.3117, + "theoretical_loss": 3.368961670096119, + "tokens_seen": 2779185152 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015945837512537615, + "loss": 1.2547, + "theoretical_loss": 3.3689555191526575, + "tokens_seen": 2779250688 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001594383149448345, + "loss": 1.3175, + "theoretical_loss": 3.368949368394847, + "tokens_seen": 2779316224 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001594182547642929, + "loss": 1.2757, + "theoretical_loss": 3.3689432178226775, + "tokens_seen": 2779381760 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015939819458375126, + "loss": 1.2866, + "theoretical_loss": 3.368937067436139, + "tokens_seen": 2779447296 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015937813440320965, + "loss": 1.2128, + "theoretical_loss": 3.3689309172352218, + "tokens_seen": 2779512832 + }, + { + "epoch": 0.68, + "learning_rate": 0.000159358074222668, + "loss": 1.413, + "theoretical_loss": 3.3689247672199154, + "tokens_seen": 2779578368 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001593380140421264, + "loss": 1.2921, + "theoretical_loss": 3.3689186173902104, + "tokens_seen": 2779643904 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015931795386158476, + "loss": 1.2924, + "theoretical_loss": 3.3689124677460964, + "tokens_seen": 2779709440 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015929789368104313, + "loss": 1.2705, + "theoretical_loss": 3.3689063182875634, + "tokens_seen": 2779774976 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015927783350050151, + "loss": 1.1901, + "theoretical_loss": 3.368900169014602, + "tokens_seen": 2779840512 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015925777331995988, + "loss": 1.2478, + "theoretical_loss": 3.3688940199272017, + "tokens_seen": 2779906048 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015923771313941827, + "loss": 1.2087, + "theoretical_loss": 3.3688878710253523, + "tokens_seen": 2779971584 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015921765295887663, + "loss": 1.2486, + "theoretical_loss": 3.3688817223090446, + "tokens_seen": 2780037120 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015919759277833502, + "loss": 1.3353, + "theoretical_loss": 3.3688755737782685, + "tokens_seen": 2780102656 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015917753259779338, + "loss": 1.2924, + "theoretical_loss": 3.368869425433013, + "tokens_seen": 2780168192 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015915747241725177, + "loss": 1.0903, + "theoretical_loss": 3.3688632772732694, + "tokens_seen": 2780233728 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015913741223671013, + "loss": 1.3159, + "theoretical_loss": 3.368857129299027, + "tokens_seen": 2780299264 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001591173520561685, + "loss": 1.3081, + "theoretical_loss": 3.368850981510276, + "tokens_seen": 2780364800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015909729187562688, + "loss": 1.2354, + "theoretical_loss": 3.3688448339070063, + "tokens_seen": 2780430336 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1010001, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4214930534362793, + "objective/train/theoretical_loss": 3.3688386864892084, + "objective/train/tokens_used": 1150954976, + "theoretical_loss": 3.3688386864892084, + "tokens_seen": 2780495872 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015907723169508525, + "loss": 1.3234, + "theoretical_loss": 3.3688386864892084, + "tokens_seen": 2780495872 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015905717151454363, + "loss": 1.3184, + "theoretical_loss": 3.368832539256872, + "tokens_seen": 2780561408 + }, + { + "epoch": 0.69, + "learning_rate": 0.000159037111334002, + "loss": 1.3075, + "theoretical_loss": 3.368826392209987, + "tokens_seen": 2780626944 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015901705115346039, + "loss": 1.1758, + "theoretical_loss": 3.3688202453485436, + "tokens_seen": 2780692480 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015899699097291875, + "loss": 1.2902, + "theoretical_loss": 3.368814098672532, + "tokens_seen": 2780758016 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015897693079237714, + "loss": 1.2318, + "theoretical_loss": 3.3688079521819416, + "tokens_seen": 2780823552 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001589568706118355, + "loss": 1.2731, + "theoretical_loss": 3.3688018058767635, + "tokens_seen": 2780889088 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001589368104312939, + "loss": 1.2808, + "theoretical_loss": 3.368795659756987, + "tokens_seen": 2780954624 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015891675025075225, + "loss": 1.229, + "theoretical_loss": 3.368789513822602, + "tokens_seen": 2781020160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015889669007021061, + "loss": 1.2981, + "theoretical_loss": 3.368783368073599, + "tokens_seen": 2781085696 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158876629889669, + "loss": 1.3202, + "theoretical_loss": 3.368777222509968, + "tokens_seen": 2781151232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015885656970912737, + "loss": 1.2645, + "theoretical_loss": 3.3687710771316985, + "tokens_seen": 2781216768 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015883650952858575, + "loss": 1.2931, + "theoretical_loss": 3.3687649319387813, + "tokens_seen": 2781282304 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015881644934804412, + "loss": 1.1619, + "theoretical_loss": 3.368758786931206, + "tokens_seen": 2781347840 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001587963891675025, + "loss": 1.3567, + "theoretical_loss": 3.3687526421089626, + "tokens_seen": 2781413376 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001587763289869609, + "loss": 1.2666, + "theoretical_loss": 3.3687464974720416, + "tokens_seen": 2781478912 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015875626880641929, + "loss": 1.3107, + "theoretical_loss": 3.3687403530204323, + "tokens_seen": 2781544448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015873620862587765, + "loss": 1.2805, + "theoretical_loss": 3.3687342087541254, + "tokens_seen": 2781609984 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158716148445336, + "loss": 1.2343, + "theoretical_loss": 3.3687280646731104, + "tokens_seen": 2781675520 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001586960882647944, + "loss": 1.323, + "theoretical_loss": 3.3687219207773778, + "tokens_seen": 2781741056 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015867602808425276, + "loss": 1.2224, + "theoretical_loss": 3.368715777066918, + "tokens_seen": 2781806592 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015865596790371115, + "loss": 1.2962, + "theoretical_loss": 3.36870963354172, + "tokens_seen": 2781872128 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001586359077231695, + "loss": 1.326, + "theoretical_loss": 3.3687034902017743, + "tokens_seen": 2781937664 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001586158475426279, + "loss": 1.3369, + "theoretical_loss": 3.368697347047071, + "tokens_seen": 2782003200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015859578736208626, + "loss": 1.2518, + "theoretical_loss": 3.3686912040776007, + "tokens_seen": 2782068736 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015857572718154465, + "loss": 1.3001, + "theoretical_loss": 3.3686850612933528, + "tokens_seen": 2782134272 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015855566700100302, + "loss": 1.3071, + "theoretical_loss": 3.368678918694317, + "tokens_seen": 2782199808 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001585356068204614, + "loss": 1.2761, + "theoretical_loss": 3.3686727762804844, + "tokens_seen": 2782265344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015851554663991977, + "loss": 1.2678, + "theoretical_loss": 3.3686666340518445, + "tokens_seen": 2782330880 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015849548645937813, + "loss": 1.328, + "theoretical_loss": 3.368660492008387, + "tokens_seen": 2782396416 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015847542627883652, + "loss": 1.2283, + "theoretical_loss": 3.368654350150103, + "tokens_seen": 2782461952 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015845536609829488, + "loss": 1.2909, + "theoretical_loss": 3.3686482084769813, + "tokens_seen": 2782527488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015843530591775327, + "loss": 1.3173, + "theoretical_loss": 3.3686420669890125, + "tokens_seen": 2782593024 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015841524573721163, + "loss": 1.2632, + "theoretical_loss": 3.3686359256861866, + "tokens_seen": 2782658560 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015839518555667002, + "loss": 1.2622, + "theoretical_loss": 3.368629784568494, + "tokens_seen": 2782724096 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015837512537612838, + "loss": 1.3249, + "theoretical_loss": 3.3686236436359245, + "tokens_seen": 2782789632 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015835506519558677, + "loss": 1.2992, + "theoretical_loss": 3.368617502888468, + "tokens_seen": 2782855168 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015833500501504514, + "loss": 1.3158, + "theoretical_loss": 3.3686113623261154, + "tokens_seen": 2782920704 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001583149448345035, + "loss": 1.2475, + "theoretical_loss": 3.3686052219488554, + "tokens_seen": 2782986240 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001582948846539619, + "loss": 1.2853, + "theoretical_loss": 3.368599081756679, + "tokens_seen": 2783051776 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015827482447342025, + "loss": 1.315, + "theoretical_loss": 3.3685929417495757, + "tokens_seen": 2783117312 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015825476429287864, + "loss": 1.1589, + "theoretical_loss": 3.3685868019275365, + "tokens_seen": 2783182848 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158234704112337, + "loss": 1.3116, + "theoretical_loss": 3.36858066229055, + "tokens_seen": 2783248384 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001582146439317954, + "loss": 1.2277, + "theoretical_loss": 3.3685745228386077, + "tokens_seen": 2783313920 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015819458375125375, + "loss": 1.2877, + "theoretical_loss": 3.3685683835716986, + "tokens_seen": 2783379456 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015817452357071214, + "loss": 1.2382, + "theoretical_loss": 3.368562244489814, + "tokens_seen": 2783444992 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001581544633901705, + "loss": 1.3183, + "theoretical_loss": 3.3685561055929427, + "tokens_seen": 2783510528 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015813440320962887, + "loss": 1.3484, + "theoretical_loss": 3.3685499668810754, + "tokens_seen": 2783576064 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015811434302908726, + "loss": 1.4689, + "theoretical_loss": 3.368543828354202, + "tokens_seen": 2783641600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015809428284854562, + "loss": 1.256, + "theoretical_loss": 3.3685376900123125, + "tokens_seen": 2783707136 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1011173, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2660366296768188, + "objective/train/theoretical_loss": 3.3685315518553973, + "objective/train/tokens_used": 1154231776, + "theoretical_loss": 3.3685315518553973, + "tokens_seen": 2783772672 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158074222668004, + "loss": 1.2205, + "theoretical_loss": 3.3685315518553973, + "tokens_seen": 2783772672 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015805416248746237, + "loss": 1.2328, + "theoretical_loss": 3.3685254138834457, + "tokens_seen": 2783838208 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015803410230692076, + "loss": 1.3744, + "theoretical_loss": 3.368519276096449, + "tokens_seen": 2783903744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015801404212637915, + "loss": 1.263, + "theoretical_loss": 3.3685131384943965, + "tokens_seen": 2783969280 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015799398194583754, + "loss": 1.2344, + "theoretical_loss": 3.368507001077278, + "tokens_seen": 2784034816 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001579739217652959, + "loss": 1.2693, + "theoretical_loss": 3.3685008638450844, + "tokens_seen": 2784100352 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001579538615847543, + "loss": 1.35, + "theoretical_loss": 3.368494726797805, + "tokens_seen": 2784165888 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015793380140421265, + "loss": 1.1348, + "theoretical_loss": 3.3684885899354304, + "tokens_seen": 2784231424 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015791374122367101, + "loss": 1.301, + "theoretical_loss": 3.3684824532579505, + "tokens_seen": 2784296960 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001578936810431294, + "loss": 1.3771, + "theoretical_loss": 3.368476316765355, + "tokens_seen": 2784362496 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015787362086258777, + "loss": 1.1526, + "theoretical_loss": 3.368470180457635, + "tokens_seen": 2784428032 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015785356068204616, + "loss": 1.3135, + "theoretical_loss": 3.3684640443347793, + "tokens_seen": 2784493568 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015783350050150452, + "loss": 1.2889, + "theoretical_loss": 3.3684579083967785, + "tokens_seen": 2784559104 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001578134403209629, + "loss": 1.2195, + "theoretical_loss": 3.368451772643623, + "tokens_seen": 2784624640 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015779338014042127, + "loss": 1.2382, + "theoretical_loss": 3.368445637075303, + "tokens_seen": 2784690176 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015777331995987966, + "loss": 1.2356, + "theoretical_loss": 3.368439501691808, + "tokens_seen": 2784755712 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015775325977933802, + "loss": 1.2887, + "theoretical_loss": 3.3684333664931283, + "tokens_seen": 2784821248 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015773319959879638, + "loss": 1.2124, + "theoretical_loss": 3.368427231479254, + "tokens_seen": 2784886784 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015771313941825477, + "loss": 1.2323, + "theoretical_loss": 3.368421096650175, + "tokens_seen": 2784952320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015769307923771313, + "loss": 1.2965, + "theoretical_loss": 3.3684149620058816, + "tokens_seen": 2785017856 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015767301905717152, + "loss": 1.3576, + "theoretical_loss": 3.3684088275463644, + "tokens_seen": 2785083392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015765295887662989, + "loss": 1.273, + "theoretical_loss": 3.3684026932716122, + "tokens_seen": 2785148928 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015763289869608828, + "loss": 1.3343, + "theoretical_loss": 3.3683965591816163, + "tokens_seen": 2785214464 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015761283851554664, + "loss": 1.2379, + "theoretical_loss": 3.3683904252763663, + "tokens_seen": 2785280000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015759277833500503, + "loss": 1.1643, + "theoretical_loss": 3.368384291555852, + "tokens_seen": 2785345536 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001575727181544634, + "loss": 1.3233, + "theoretical_loss": 3.3683781580200645, + "tokens_seen": 2785411072 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015755265797392175, + "loss": 1.2772, + "theoretical_loss": 3.3683720246689925, + "tokens_seen": 2785476608 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015753259779338014, + "loss": 1.2327, + "theoretical_loss": 3.368365891502627, + "tokens_seen": 2785542144 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001575125376128385, + "loss": 1.3708, + "theoretical_loss": 3.368359758520958, + "tokens_seen": 2785607680 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001574924774322969, + "loss": 1.3292, + "theoretical_loss": 3.3683536257239752, + "tokens_seen": 2785673216 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015747241725175525, + "loss": 1.2999, + "theoretical_loss": 3.368347493111669, + "tokens_seen": 2785738752 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015745235707121364, + "loss": 1.2749, + "theoretical_loss": 3.36834136068403, + "tokens_seen": 2785804288 + }, + { + "epoch": 0.69, + "learning_rate": 0.000157432296890672, + "loss": 1.2429, + "theoretical_loss": 3.368335228441047, + "tokens_seen": 2785869824 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001574122367101304, + "loss": 1.2681, + "theoretical_loss": 3.3683290963827113, + "tokens_seen": 2785935360 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015739217652958876, + "loss": 1.3929, + "theoretical_loss": 3.3683229645090123, + "tokens_seen": 2786000896 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015737211634904715, + "loss": 1.3789, + "theoretical_loss": 3.36831683281994, + "tokens_seen": 2786066432 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001573520561685055, + "loss": 1.2159, + "theoretical_loss": 3.368310701315486, + "tokens_seen": 2786131968 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015733199598796387, + "loss": 1.3449, + "theoretical_loss": 3.368304569995638, + "tokens_seen": 2786197504 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015731193580742226, + "loss": 1.3518, + "theoretical_loss": 3.3682984388603883, + "tokens_seen": 2786263040 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015729187562688062, + "loss": 1.2081, + "theoretical_loss": 3.3682923079097256, + "tokens_seen": 2786328576 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015727181544633904, + "loss": 1.2804, + "theoretical_loss": 3.36828617714364, + "tokens_seen": 2786394112 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001572517552657974, + "loss": 1.2806, + "theoretical_loss": 3.3682800465621225, + "tokens_seen": 2786459648 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001572316950852558, + "loss": 1.2719, + "theoretical_loss": 3.3682739161651627, + "tokens_seen": 2786525184 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015721163490471415, + "loss": 1.2245, + "theoretical_loss": 3.3682677859527512, + "tokens_seen": 2786590720 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015719157472417254, + "loss": 1.3396, + "theoretical_loss": 3.368261655924877, + "tokens_seen": 2786656256 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001571715145436309, + "loss": 1.2403, + "theoretical_loss": 3.3682555260815312, + "tokens_seen": 2786721792 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015715145436308927, + "loss": 1.2992, + "theoretical_loss": 3.368249396422703, + "tokens_seen": 2786787328 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015713139418254766, + "loss": 1.3695, + "theoretical_loss": 3.368243266948384, + "tokens_seen": 2786852864 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015711133400200602, + "loss": 1.3909, + "theoretical_loss": 3.368237137658563, + "tokens_seen": 2786918400 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001570912738214644, + "loss": 1.4028, + "theoretical_loss": 3.3682310085532303, + "tokens_seen": 2786983936 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1012303, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2136070728302002, + "objective/train/theoretical_loss": 3.3682248796323764, + "objective/train/tokens_used": 1157508576, + "theoretical_loss": 3.3682248796323764, + "tokens_seen": 2787049472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015707121364092277, + "loss": 1.2652, + "theoretical_loss": 3.3682248796323764, + "tokens_seen": 2787049472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015705115346038116, + "loss": 1.298, + "theoretical_loss": 3.368218750895991, + "tokens_seen": 2787115008 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015703109327983952, + "loss": 1.363, + "theoretical_loss": 3.3682126223440645, + "tokens_seen": 2787180544 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001570110330992979, + "loss": 1.188, + "theoretical_loss": 3.368206493976587, + "tokens_seen": 2787246080 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015699097291875627, + "loss": 1.3281, + "theoretical_loss": 3.368200365793548, + "tokens_seen": 2787311616 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015697091273821466, + "loss": 1.2751, + "theoretical_loss": 3.368194237794939, + "tokens_seen": 2787377152 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015695085255767303, + "loss": 1.3443, + "theoretical_loss": 3.368188109980749, + "tokens_seen": 2787442688 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001569307923771314, + "loss": 1.2638, + "theoretical_loss": 3.368181982350968, + "tokens_seen": 2787508224 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015691073219658978, + "loss": 1.2809, + "theoretical_loss": 3.368175854905587, + "tokens_seen": 2787573760 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015689067201604814, + "loss": 1.3232, + "theoretical_loss": 3.368169727644595, + "tokens_seen": 2787639296 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015687061183550653, + "loss": 1.2255, + "theoretical_loss": 3.3681636005679834, + "tokens_seen": 2787704832 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001568505516549649, + "loss": 1.2868, + "theoretical_loss": 3.3681574736757414, + "tokens_seen": 2787770368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015683049147442328, + "loss": 1.2943, + "theoretical_loss": 3.368151346967859, + "tokens_seen": 2787835904 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015681043129388164, + "loss": 1.2692, + "theoretical_loss": 3.368145220444327, + "tokens_seen": 2787901440 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015679037111334003, + "loss": 1.1873, + "theoretical_loss": 3.3681390941051355, + "tokens_seen": 2787966976 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001567703109327984, + "loss": 1.3214, + "theoretical_loss": 3.368132967950274, + "tokens_seen": 2788032512 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015675025075225676, + "loss": 1.2472, + "theoretical_loss": 3.3681268419797328, + "tokens_seen": 2788098048 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015673019057171515, + "loss": 1.3471, + "theoretical_loss": 3.3681207161935025, + "tokens_seen": 2788163584 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001567101303911735, + "loss": 1.179, + "theoretical_loss": 3.368114590591573, + "tokens_seen": 2788229120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001566900702106319, + "loss": 1.2836, + "theoretical_loss": 3.3681084651739335, + "tokens_seen": 2788294656 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015667001003009026, + "loss": 1.2969, + "theoretical_loss": 3.3681023399405756, + "tokens_seen": 2788360192 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015664994984954865, + "loss": 1.3795, + "theoretical_loss": 3.3680962148914886, + "tokens_seen": 2788425728 + }, + { + "epoch": 0.69, + "learning_rate": 0.000156629889669007, + "loss": 1.2702, + "theoretical_loss": 3.368090090026663, + "tokens_seen": 2788491264 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001566098294884654, + "loss": 1.4187, + "theoretical_loss": 3.3680839653460883, + "tokens_seen": 2788556800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015658976930792376, + "loss": 1.2504, + "theoretical_loss": 3.368077840849755, + "tokens_seen": 2788622336 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015656970912738212, + "loss": 1.2987, + "theoretical_loss": 3.3680717165376537, + "tokens_seen": 2788687872 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015654964894684051, + "loss": 1.2605, + "theoretical_loss": 3.368065592409774, + "tokens_seen": 2788753408 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001565295887662989, + "loss": 1.2618, + "theoretical_loss": 3.368059468466106, + "tokens_seen": 2788818944 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001565095285857573, + "loss": 1.3242, + "theoretical_loss": 3.36805334470664, + "tokens_seen": 2788884480 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015648946840521565, + "loss": 1.3215, + "theoretical_loss": 3.3680472211313663, + "tokens_seen": 2788950016 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015646940822467404, + "loss": 1.2712, + "theoretical_loss": 3.3680410977402744, + "tokens_seen": 2789015552 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001564493480441324, + "loss": 1.2236, + "theoretical_loss": 3.368034974533355, + "tokens_seen": 2789081088 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001564292878635908, + "loss": 1.3979, + "theoretical_loss": 3.3680288515105983, + "tokens_seen": 2789146624 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015640922768304916, + "loss": 1.2403, + "theoretical_loss": 3.368022728671994, + "tokens_seen": 2789212160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015638916750250755, + "loss": 1.2464, + "theoretical_loss": 3.3680166060175325, + "tokens_seen": 2789277696 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001563691073219659, + "loss": 1.2973, + "theoretical_loss": 3.368010483547204, + "tokens_seen": 2789343232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015634904714142427, + "loss": 1.2604, + "theoretical_loss": 3.368004361260998, + "tokens_seen": 2789408768 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015632898696088266, + "loss": 1.262, + "theoretical_loss": 3.367998239158906, + "tokens_seen": 2789474304 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015630892678034102, + "loss": 1.2161, + "theoretical_loss": 3.3679921172409166, + "tokens_seen": 2789539840 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001562888665997994, + "loss": 1.3997, + "theoretical_loss": 3.367985995507021, + "tokens_seen": 2789605376 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015626880641925778, + "loss": 1.3152, + "theoretical_loss": 3.367979873957209, + "tokens_seen": 2789670912 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015624874623871616, + "loss": 1.2825, + "theoretical_loss": 3.3679737525914706, + "tokens_seen": 2789736448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015622868605817453, + "loss": 1.309, + "theoretical_loss": 3.3679676314097957, + "tokens_seen": 2789801984 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015620862587763292, + "loss": 1.2371, + "theoretical_loss": 3.3679615104121754, + "tokens_seen": 2789867520 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015618856569709128, + "loss": 1.2543, + "theoretical_loss": 3.367955389598599, + "tokens_seen": 2789933056 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015616850551654964, + "loss": 1.2357, + "theoretical_loss": 3.367949268969057, + "tokens_seen": 2789998592 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015614844533600803, + "loss": 1.2691, + "theoretical_loss": 3.3679431485235396, + "tokens_seen": 2790064128 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001561283851554664, + "loss": 1.2776, + "theoretical_loss": 3.3679370282620362, + "tokens_seen": 2790129664 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015610832497492478, + "loss": 1.3179, + "theoretical_loss": 3.3679309081845377, + "tokens_seen": 2790195200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015608826479438314, + "loss": 1.3128, + "theoretical_loss": 3.3679247882910346, + "tokens_seen": 2790260736 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1013591, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.435854434967041, + "objective/train/theoretical_loss": 3.367918668581516, + "objective/train/tokens_used": 1160785376, + "theoretical_loss": 3.367918668581516, + "tokens_seen": 2790326272 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015606820461384153, + "loss": 1.4045, + "theoretical_loss": 3.367918668581516, + "tokens_seen": 2790326272 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001560481444332999, + "loss": 1.3343, + "theoretical_loss": 3.3679125490559727, + "tokens_seen": 2790391808 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015602808425275828, + "loss": 1.2761, + "theoretical_loss": 3.3679064297143944, + "tokens_seen": 2790457344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015600802407221665, + "loss": 1.3228, + "theoretical_loss": 3.367900310556772, + "tokens_seen": 2790522880 + }, + { + "epoch": 0.69, + "learning_rate": 0.000155987963891675, + "loss": 1.2184, + "theoretical_loss": 3.367894191583095, + "tokens_seen": 2790588416 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001559679037111334, + "loss": 1.2595, + "theoretical_loss": 3.367888072793354, + "tokens_seen": 2790653952 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015594784353059176, + "loss": 1.239, + "theoretical_loss": 3.3678819541875384, + "tokens_seen": 2790719488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015592778335005015, + "loss": 1.3567, + "theoretical_loss": 3.3678758357656395, + "tokens_seen": 2790785024 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001559077231695085, + "loss": 1.3545, + "theoretical_loss": 3.3678697175276464, + "tokens_seen": 2790850560 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001558876629889669, + "loss": 1.3513, + "theoretical_loss": 3.3678635994735497, + "tokens_seen": 2790916096 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015586760280842526, + "loss": 1.2292, + "theoretical_loss": 3.3678574816033393, + "tokens_seen": 2790981632 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015584754262788365, + "loss": 1.2174, + "theoretical_loss": 3.3678513639170062, + "tokens_seen": 2791047168 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015582748244734202, + "loss": 1.395, + "theoretical_loss": 3.3678452464145394, + "tokens_seen": 2791112704 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001558074222668004, + "loss": 1.3019, + "theoretical_loss": 3.36783912909593, + "tokens_seen": 2791178240 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015578736208625877, + "loss": 1.3322, + "theoretical_loss": 3.3678330119611672, + "tokens_seen": 2791243776 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015576730190571716, + "loss": 1.2482, + "theoretical_loss": 3.3678268950102423, + "tokens_seen": 2791309312 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015574724172517555, + "loss": 1.2258, + "theoretical_loss": 3.3678207782431446, + "tokens_seen": 2791374848 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001557271815446339, + "loss": 1.2809, + "theoretical_loss": 3.3678146616598648, + "tokens_seen": 2791440384 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001557071213640923, + "loss": 1.1845, + "theoretical_loss": 3.3678085452603925, + "tokens_seen": 2791505920 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015568706118355066, + "loss": 1.2645, + "theoretical_loss": 3.367802429044718, + "tokens_seen": 2791571456 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015566700100300905, + "loss": 1.3305, + "theoretical_loss": 3.367796313012832, + "tokens_seen": 2791636992 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001556469408224674, + "loss": 1.3637, + "theoretical_loss": 3.367790197164724, + "tokens_seen": 2791702528 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001556268806419258, + "loss": 1.2409, + "theoretical_loss": 3.3677840815003846, + "tokens_seen": 2791768064 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015560682046138416, + "loss": 1.3536, + "theoretical_loss": 3.3677779660198035, + "tokens_seen": 2791833600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015558676028084252, + "loss": 1.3528, + "theoretical_loss": 3.3677718507229715, + "tokens_seen": 2791899136 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015556670010030091, + "loss": 1.2712, + "theoretical_loss": 3.3677657356098782, + "tokens_seen": 2791964672 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015554663991975928, + "loss": 1.2894, + "theoretical_loss": 3.3677596206805145, + "tokens_seen": 2792030208 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015552657973921767, + "loss": 1.2682, + "theoretical_loss": 3.3677535059348696, + "tokens_seen": 2792095744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015550651955867603, + "loss": 1.3866, + "theoretical_loss": 3.367747391372934, + "tokens_seen": 2792161280 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015548645937813442, + "loss": 1.3225, + "theoretical_loss": 3.3677412769946984, + "tokens_seen": 2792226816 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015546639919759278, + "loss": 1.3357, + "theoretical_loss": 3.3677351628001526, + "tokens_seen": 2792292352 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015544633901705117, + "loss": 1.2411, + "theoretical_loss": 3.367729048789286, + "tokens_seen": 2792357888 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015542627883650953, + "loss": 1.2915, + "theoretical_loss": 3.3677229349620905, + "tokens_seen": 2792423424 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001554062186559679, + "loss": 1.2711, + "theoretical_loss": 3.367716821318555, + "tokens_seen": 2792488960 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015538615847542628, + "loss": 1.2748, + "theoretical_loss": 3.3677107078586697, + "tokens_seen": 2792554496 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015536609829488465, + "loss": 1.3145, + "theoretical_loss": 3.367704594582425, + "tokens_seen": 2792620032 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015534603811434303, + "loss": 1.2352, + "theoretical_loss": 3.3676984814898114, + "tokens_seen": 2792685568 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001553259779338014, + "loss": 1.273, + "theoretical_loss": 3.3676923685808187, + "tokens_seen": 2792751104 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015530591775325979, + "loss": 1.2772, + "theoretical_loss": 3.3676862558554372, + "tokens_seen": 2792816640 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015528585757271815, + "loss": 1.2598, + "theoretical_loss": 3.367680143313657, + "tokens_seen": 2792882176 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015526579739217654, + "loss": 1.3922, + "theoretical_loss": 3.367674030955468, + "tokens_seen": 2792947712 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001552457372116349, + "loss": 1.3467, + "theoretical_loss": 3.367667918780861, + "tokens_seen": 2793013248 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001552256770310933, + "loss": 1.1551, + "theoretical_loss": 3.367661806789826, + "tokens_seen": 2793078784 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015520561685055165, + "loss": 1.2571, + "theoretical_loss": 3.3676556949823526, + "tokens_seen": 2793144320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015518555667001001, + "loss": 1.2757, + "theoretical_loss": 3.367649583358432, + "tokens_seen": 2793209856 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001551654964894684, + "loss": 1.2331, + "theoretical_loss": 3.3676434719180532, + "tokens_seen": 2793275392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015514543630892677, + "loss": 1.2455, + "theoretical_loss": 3.367637360661208, + "tokens_seen": 2793340928 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015512537612838515, + "loss": 1.2727, + "theoretical_loss": 3.367631249587885, + "tokens_seen": 2793406464 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015510531594784352, + "loss": 1.2624, + "theoretical_loss": 3.3676251386980747, + "tokens_seen": 2793472000 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001550852557673019, + "loss": 1.3668, + "theoretical_loss": 3.3676190279917675, + "tokens_seen": 2793537536 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1014727, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2322803735733032, + "objective/train/theoretical_loss": 3.3676129174689535, + "objective/train/tokens_used": 1164062176, + "theoretical_loss": 3.3676129174689535, + "tokens_seen": 2793603072 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015506519558676027, + "loss": 1.2469, + "theoretical_loss": 3.3676129174689535, + "tokens_seen": 2793603072 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015504513540621866, + "loss": 1.2831, + "theoretical_loss": 3.3676068071296235, + "tokens_seen": 2793668608 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015502507522567705, + "loss": 1.1917, + "theoretical_loss": 3.367600696973767, + "tokens_seen": 2793734144 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001550050150451354, + "loss": 1.1831, + "theoretical_loss": 3.3675945870013746, + "tokens_seen": 2793799680 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001549849548645938, + "loss": 1.3836, + "theoretical_loss": 3.3675884772124363, + "tokens_seen": 2793865216 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015496489468405216, + "loss": 1.1708, + "theoretical_loss": 3.367582367606942, + "tokens_seen": 2793930752 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015494483450351055, + "loss": 1.3289, + "theoretical_loss": 3.367576258184882, + "tokens_seen": 2793996288 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001549247743229689, + "loss": 1.279, + "theoretical_loss": 3.367570148946247, + "tokens_seen": 2794061824 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001549047141424273, + "loss": 1.2905, + "theoretical_loss": 3.3675640398910267, + "tokens_seen": 2794127360 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015488465396188566, + "loss": 1.2941, + "theoretical_loss": 3.3675579310192116, + "tokens_seen": 2794192896 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015486459378134405, + "loss": 1.2935, + "theoretical_loss": 3.3675518223307916, + "tokens_seen": 2794258432 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015484453360080242, + "loss": 1.1678, + "theoretical_loss": 3.367545713825757, + "tokens_seen": 2794323968 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001548244734202608, + "loss": 1.259, + "theoretical_loss": 3.367539605504098, + "tokens_seen": 2794389504 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015480441323971917, + "loss": 1.3381, + "theoretical_loss": 3.3675334973658044, + "tokens_seen": 2794455040 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015478435305917753, + "loss": 1.2109, + "theoretical_loss": 3.3675273894108675, + "tokens_seen": 2794520576 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015476429287863592, + "loss": 1.2585, + "theoretical_loss": 3.3675212816392763, + "tokens_seen": 2794586112 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015474423269809428, + "loss": 1.2944, + "theoretical_loss": 3.3675151740510216, + "tokens_seen": 2794651648 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015472417251755267, + "loss": 1.2783, + "theoretical_loss": 3.3675090666460936, + "tokens_seen": 2794717184 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015470411233701103, + "loss": 1.3082, + "theoretical_loss": 3.3675029594244825, + "tokens_seen": 2794782720 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015468405215646942, + "loss": 1.3727, + "theoretical_loss": 3.367496852386178, + "tokens_seen": 2794848256 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015466399197592778, + "loss": 1.2887, + "theoretical_loss": 3.367490745531171, + "tokens_seen": 2794913792 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015464393179538617, + "loss": 1.3228, + "theoretical_loss": 3.3674846388594513, + "tokens_seen": 2794979328 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015462387161484454, + "loss": 1.3273, + "theoretical_loss": 3.367478532371009, + "tokens_seen": 2795044864 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001546038114343029, + "loss": 1.2028, + "theoretical_loss": 3.367472426065835, + "tokens_seen": 2795110400 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001545837512537613, + "loss": 1.3059, + "theoretical_loss": 3.3674663199439183, + "tokens_seen": 2795175936 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015456369107321965, + "loss": 1.2816, + "theoretical_loss": 3.3674602140052503, + "tokens_seen": 2795241472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015454363089267804, + "loss": 1.3267, + "theoretical_loss": 3.3674541082498206, + "tokens_seen": 2795307008 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001545235707121364, + "loss": 1.3369, + "theoretical_loss": 3.3674480026776195, + "tokens_seen": 2795372544 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001545035105315948, + "loss": 1.2822, + "theoretical_loss": 3.367441897288637, + "tokens_seen": 2795438080 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015448345035105315, + "loss": 1.3787, + "theoretical_loss": 3.3674357920828637, + "tokens_seen": 2795503616 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015446339017051154, + "loss": 1.3471, + "theoretical_loss": 3.3674296870602896, + "tokens_seen": 2795569152 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001544433299899699, + "loss": 1.3749, + "theoretical_loss": 3.367423582220905, + "tokens_seen": 2795634688 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015442326980942827, + "loss": 1.2505, + "theoretical_loss": 3.3674174775647, + "tokens_seen": 2795700224 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015440320962888666, + "loss": 1.232, + "theoretical_loss": 3.367411373091665, + "tokens_seen": 2795765760 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015438314944834502, + "loss": 1.3499, + "theoretical_loss": 3.3674052688017895, + "tokens_seen": 2795831296 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001543630892678034, + "loss": 1.2694, + "theoretical_loss": 3.367399164695065, + "tokens_seen": 2795896832 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015434302908726177, + "loss": 1.342, + "theoretical_loss": 3.36739306077148, + "tokens_seen": 2795962368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015432296890672016, + "loss": 1.3576, + "theoretical_loss": 3.3673869570310266, + "tokens_seen": 2796027904 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015430290872617852, + "loss": 1.2752, + "theoretical_loss": 3.367380853473694, + "tokens_seen": 2796093440 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001542828485456369, + "loss": 1.3021, + "theoretical_loss": 3.367374750099472, + "tokens_seen": 2796158976 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001542627883650953, + "loss": 1.2407, + "theoretical_loss": 3.367368646908352, + "tokens_seen": 2796224512 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001542427281845537, + "loss": 1.2935, + "theoretical_loss": 3.3673625439003234, + "tokens_seen": 2796290048 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015422266800401205, + "loss": 1.3069, + "theoretical_loss": 3.3673564410753767, + "tokens_seen": 2796355584 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015420260782347041, + "loss": 1.3316, + "theoretical_loss": 3.3673503384335017, + "tokens_seen": 2796421120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001541825476429288, + "loss": 1.3127, + "theoretical_loss": 3.367344235974689, + "tokens_seen": 2796486656 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015416248746238717, + "loss": 1.335, + "theoretical_loss": 3.3673381336989285, + "tokens_seen": 2796552192 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015414242728184555, + "loss": 1.2738, + "theoretical_loss": 3.3673320316062108, + "tokens_seen": 2796617728 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015412236710130392, + "loss": 1.3322, + "theoretical_loss": 3.367325929696526, + "tokens_seen": 2796683264 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001541023069207623, + "loss": 1.2551, + "theoretical_loss": 3.3673198279698644, + "tokens_seen": 2796748800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015408224674022067, + "loss": 1.3306, + "theoretical_loss": 3.367313726426216, + "tokens_seen": 2796814336 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1015946, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2699360847473145, + "objective/train/theoretical_loss": 3.367307625065571, + "objective/train/tokens_used": 1167338976, + "theoretical_loss": 3.367307625065571, + "tokens_seen": 2796879872 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015406218655967906, + "loss": 1.2302, + "theoretical_loss": 3.367307625065571, + "tokens_seen": 2796879872 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015404212637913742, + "loss": 1.199, + "theoretical_loss": 3.36730152388792, + "tokens_seen": 2796945408 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015402206619859578, + "loss": 1.3447, + "theoretical_loss": 3.367295422893253, + "tokens_seen": 2797010944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015400200601805417, + "loss": 1.3696, + "theoretical_loss": 3.36728932208156, + "tokens_seen": 2797076480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015398194583751253, + "loss": 1.3815, + "theoretical_loss": 3.3672832214528317, + "tokens_seen": 2797142016 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015396188565697092, + "loss": 1.2788, + "theoretical_loss": 3.3672771210070573, + "tokens_seen": 2797207552 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015394182547642929, + "loss": 1.2762, + "theoretical_loss": 3.3672710207442287, + "tokens_seen": 2797273088 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015392176529588768, + "loss": 1.3541, + "theoretical_loss": 3.3672649206643346, + "tokens_seen": 2797338624 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015390170511534604, + "loss": 1.2917, + "theoretical_loss": 3.367258820767366, + "tokens_seen": 2797404160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015388164493480443, + "loss": 1.4599, + "theoretical_loss": 3.367252721053313, + "tokens_seen": 2797469696 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001538615847542628, + "loss": 1.2909, + "theoretical_loss": 3.367246621522166, + "tokens_seen": 2797535232 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015384152457372115, + "loss": 1.305, + "theoretical_loss": 3.367240522173915, + "tokens_seen": 2797600768 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015382146439317954, + "loss": 1.3089, + "theoretical_loss": 3.3672344230085502, + "tokens_seen": 2797666304 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001538014042126379, + "loss": 1.3421, + "theoretical_loss": 3.367228324026062, + "tokens_seen": 2797731840 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001537813440320963, + "loss": 1.2542, + "theoretical_loss": 3.36722222522644, + "tokens_seen": 2797797376 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015376128385155465, + "loss": 1.2337, + "theoretical_loss": 3.3672161266096756, + "tokens_seen": 2797862912 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015374122367101304, + "loss": 1.2396, + "theoretical_loss": 3.3672100281757578, + "tokens_seen": 2797928448 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001537211634904714, + "loss": 1.3445, + "theoretical_loss": 3.367203929924678, + "tokens_seen": 2797993984 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001537011033099298, + "loss": 1.2104, + "theoretical_loss": 3.3671978318564255, + "tokens_seen": 2798059520 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015368104312938816, + "loss": 1.2544, + "theoretical_loss": 3.367191733970991, + "tokens_seen": 2798125056 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015366098294884655, + "loss": 1.2602, + "theoretical_loss": 3.3671856362683648, + "tokens_seen": 2798190592 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001536409227683049, + "loss": 1.3109, + "theoretical_loss": 3.3671795387485366, + "tokens_seen": 2798256128 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015362086258776327, + "loss": 1.2553, + "theoretical_loss": 3.3671734414114978, + "tokens_seen": 2798321664 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015360080240722166, + "loss": 1.2937, + "theoretical_loss": 3.367167344257237, + "tokens_seen": 2798387200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015358074222668002, + "loss": 1.2817, + "theoretical_loss": 3.3671612472857455, + "tokens_seen": 2798452736 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001535606820461384, + "loss": 1.3663, + "theoretical_loss": 3.3671551504970134, + "tokens_seen": 2798518272 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015354062186559677, + "loss": 1.3485, + "theoretical_loss": 3.367149053891031, + "tokens_seen": 2798583808 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001535205616850552, + "loss": 1.2589, + "theoretical_loss": 3.3671429574677885, + "tokens_seen": 2798649344 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015350050150451355, + "loss": 1.2109, + "theoretical_loss": 3.367136861227276, + "tokens_seen": 2798714880 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015348044132397194, + "loss": 1.3117, + "theoretical_loss": 3.3671307651694837, + "tokens_seen": 2798780416 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001534603811434303, + "loss": 1.3473, + "theoretical_loss": 3.3671246692944017, + "tokens_seen": 2798845952 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015344032096288867, + "loss": 1.3155, + "theoretical_loss": 3.367118573602021, + "tokens_seen": 2798911488 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015342026078234706, + "loss": 1.2024, + "theoretical_loss": 3.367112478092331, + "tokens_seen": 2798977024 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015340020060180542, + "loss": 1.3188, + "theoretical_loss": 3.3671063827653227, + "tokens_seen": 2799042560 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001533801404212638, + "loss": 1.2507, + "theoretical_loss": 3.3671002876209855, + "tokens_seen": 2799108096 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015336008024072217, + "loss": 1.2107, + "theoretical_loss": 3.3670941926593105, + "tokens_seen": 2799173632 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015334002006018056, + "loss": 1.2544, + "theoretical_loss": 3.3670880978802873, + "tokens_seen": 2799239168 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015331995987963892, + "loss": 1.319, + "theoretical_loss": 3.3670820032839064, + "tokens_seen": 2799304704 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001532998996990973, + "loss": 1.2526, + "theoretical_loss": 3.367075908870158, + "tokens_seen": 2799370240 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015327983951855567, + "loss": 1.258, + "theoretical_loss": 3.3670698146390325, + "tokens_seen": 2799435776 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015325977933801406, + "loss": 1.3511, + "theoretical_loss": 3.36706372059052, + "tokens_seen": 2799501312 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015323971915747243, + "loss": 1.2584, + "theoretical_loss": 3.367057626724611, + "tokens_seen": 2799566848 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001532196589769308, + "loss": 1.2545, + "theoretical_loss": 3.367051533041295, + "tokens_seen": 2799632384 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015319959879638918, + "loss": 1.276, + "theoretical_loss": 3.367045439540563, + "tokens_seen": 2799697920 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015317953861584754, + "loss": 1.2717, + "theoretical_loss": 3.3670393462224055, + "tokens_seen": 2799763456 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015315947843530593, + "loss": 1.2771, + "theoretical_loss": 3.367033253086812, + "tokens_seen": 2799828992 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001531394182547643, + "loss": 1.2478, + "theoretical_loss": 3.367027160133773, + "tokens_seen": 2799894528 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015311935807422268, + "loss": 1.216, + "theoretical_loss": 3.367021067363279, + "tokens_seen": 2799960064 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015309929789368104, + "loss": 1.1972, + "theoretical_loss": 3.36701497477532, + "tokens_seen": 2800025600 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015307923771313943, + "loss": 1.279, + "theoretical_loss": 3.3670088823698867, + "tokens_seen": 2800091136 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1017141, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3148716688156128, + "objective/train/theoretical_loss": 3.3670027901469686, + "objective/train/tokens_used": 1170615776, + "theoretical_loss": 3.3670027901469686, + "tokens_seen": 2800156672 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001530591775325978, + "loss": 1.2839, + "theoretical_loss": 3.3670027901469686, + "tokens_seen": 2800156672 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015303911735205616, + "loss": 1.3135, + "theoretical_loss": 3.3669966981065564, + "tokens_seen": 2800222208 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015301905717151455, + "loss": 1.204, + "theoretical_loss": 3.3669906062486405, + "tokens_seen": 2800287744 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001529989969909729, + "loss": 1.4353, + "theoretical_loss": 3.366984514573211, + "tokens_seen": 2800353280 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001529789368104313, + "loss": 1.1759, + "theoretical_loss": 3.3669784230802584, + "tokens_seen": 2800418816 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015295887662988966, + "loss": 1.2574, + "theoretical_loss": 3.366972331769772, + "tokens_seen": 2800484352 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015293881644934805, + "loss": 1.2388, + "theoretical_loss": 3.3669662406417435, + "tokens_seen": 2800549888 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001529187562688064, + "loss": 1.266, + "theoretical_loss": 3.3669601496961623, + "tokens_seen": 2800615424 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001528986960882648, + "loss": 1.2993, + "theoretical_loss": 3.3669540589330187, + "tokens_seen": 2800680960 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015287863590772316, + "loss": 1.2379, + "theoretical_loss": 3.366947968352303, + "tokens_seen": 2800746496 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015285857572718152, + "loss": 1.2036, + "theoretical_loss": 3.3669418779540057, + "tokens_seen": 2800812032 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015283851554663991, + "loss": 1.1584, + "theoretical_loss": 3.3669357877381167, + "tokens_seen": 2800877568 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015281845536609828, + "loss": 1.294, + "theoretical_loss": 3.366929697704627, + "tokens_seen": 2800943104 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015279839518555667, + "loss": 1.2787, + "theoretical_loss": 3.3669236078535256, + "tokens_seen": 2801008640 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015277833500501503, + "loss": 1.3286, + "theoretical_loss": 3.3669175181848043, + "tokens_seen": 2801074176 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015275827482447344, + "loss": 1.3035, + "theoretical_loss": 3.3669114286984523, + "tokens_seen": 2801139712 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001527382146439318, + "loss": 1.2963, + "theoretical_loss": 3.36690533939446, + "tokens_seen": 2801205248 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001527181544633902, + "loss": 1.2752, + "theoretical_loss": 3.3668992502728177, + "tokens_seen": 2801270784 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015269809428284856, + "loss": 1.2226, + "theoretical_loss": 3.3668931613335165, + "tokens_seen": 2801336320 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015267803410230695, + "loss": 1.3268, + "theoretical_loss": 3.3668870725765454, + "tokens_seen": 2801401856 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001526579739217653, + "loss": 1.1744, + "theoretical_loss": 3.3668809840018956, + "tokens_seen": 2801467392 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015263791374122367, + "loss": 1.2586, + "theoretical_loss": 3.366874895609557, + "tokens_seen": 2801532928 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015261785356068206, + "loss": 1.3905, + "theoretical_loss": 3.36686880739952, + "tokens_seen": 2801598464 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015259779338014042, + "loss": 1.2298, + "theoretical_loss": 3.3668627193717744, + "tokens_seen": 2801664000 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001525777331995988, + "loss": 1.2986, + "theoretical_loss": 3.366856631526311, + "tokens_seen": 2801729536 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015255767301905717, + "loss": 1.2971, + "theoretical_loss": 3.36685054386312, + "tokens_seen": 2801795072 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015253761283851556, + "loss": 1.3904, + "theoretical_loss": 3.366844456382192, + "tokens_seen": 2801860608 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015251755265797393, + "loss": 1.2592, + "theoretical_loss": 3.3668383690835166, + "tokens_seen": 2801926144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015249749247743232, + "loss": 1.2643, + "theoretical_loss": 3.3668322819670844, + "tokens_seen": 2801991680 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015247743229689068, + "loss": 1.2977, + "theoretical_loss": 3.3668261950328855, + "tokens_seen": 2802057216 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015245737211634904, + "loss": 1.2586, + "theoretical_loss": 3.3668201082809106, + "tokens_seen": 2802122752 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015243731193580743, + "loss": 1.3086, + "theoretical_loss": 3.36681402171115, + "tokens_seen": 2802188288 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001524172517552658, + "loss": 1.2223, + "theoretical_loss": 3.3668079353235933, + "tokens_seen": 2802253824 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015239719157472418, + "loss": 1.2393, + "theoretical_loss": 3.3668018491182314, + "tokens_seen": 2802319360 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015237713139418254, + "loss": 1.289, + "theoretical_loss": 3.3667957630950545, + "tokens_seen": 2802384896 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015235707121364093, + "loss": 1.35, + "theoretical_loss": 3.3667896772540526, + "tokens_seen": 2802450432 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001523370110330993, + "loss": 1.3308, + "theoretical_loss": 3.3667835915952162, + "tokens_seen": 2802515968 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015231695085255768, + "loss": 1.2928, + "theoretical_loss": 3.3667775061185354, + "tokens_seen": 2802581504 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015229689067201605, + "loss": 1.3302, + "theoretical_loss": 3.366771420824001, + "tokens_seen": 2802647040 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001522768304914744, + "loss": 1.2223, + "theoretical_loss": 3.3667653357116025, + "tokens_seen": 2802712576 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001522567703109328, + "loss": 1.3194, + "theoretical_loss": 3.366759250781331, + "tokens_seen": 2802778112 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015223671013039116, + "loss": 1.3334, + "theoretical_loss": 3.3667531660331766, + "tokens_seen": 2802843648 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015221664994984955, + "loss": 1.3514, + "theoretical_loss": 3.3667470814671288, + "tokens_seen": 2802909184 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001521965897693079, + "loss": 1.2199, + "theoretical_loss": 3.3667409970831788, + "tokens_seen": 2802974720 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001521765295887663, + "loss": 1.2347, + "theoretical_loss": 3.3667349128813164, + "tokens_seen": 2803040256 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015215646940822466, + "loss": 1.3234, + "theoretical_loss": 3.3667288288615325, + "tokens_seen": 2803105792 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015213640922768305, + "loss": 1.2668, + "theoretical_loss": 3.3667227450238166, + "tokens_seen": 2803171328 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015211634904714142, + "loss": 1.2806, + "theoretical_loss": 3.3667166613681596, + "tokens_seen": 2803236864 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001520962888665998, + "loss": 1.2325, + "theoretical_loss": 3.3667105778945516, + "tokens_seen": 2803302400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015207622868605817, + "loss": 1.3488, + "theoretical_loss": 3.366704494602983, + "tokens_seen": 2803367936 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1018418, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.270719289779663, + "objective/train/theoretical_loss": 3.3666984114934437, + "objective/train/tokens_used": 1173892576, + "theoretical_loss": 3.3666984114934437, + "tokens_seen": 2803433472 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015205616850551653, + "loss": 1.2654, + "theoretical_loss": 3.3666984114934437, + "tokens_seen": 2803433472 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015203610832497492, + "loss": 1.1946, + "theoretical_loss": 3.3666923285659243, + "tokens_seen": 2803499008 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001520160481444333, + "loss": 1.2658, + "theoretical_loss": 3.366686245820415, + "tokens_seen": 2803564544 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001519959879638917, + "loss": 1.2379, + "theoretical_loss": 3.3666801632569063, + "tokens_seen": 2803630080 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015197592778335006, + "loss": 1.1718, + "theoretical_loss": 3.3666740808753883, + "tokens_seen": 2803695616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015195586760280845, + "loss": 1.2403, + "theoretical_loss": 3.3666679986758514, + "tokens_seen": 2803761152 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001519358074222668, + "loss": 1.3886, + "theoretical_loss": 3.366661916658286, + "tokens_seen": 2803826688 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001519157472417252, + "loss": 1.3762, + "theoretical_loss": 3.366655834822682, + "tokens_seen": 2803892224 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015189568706118356, + "loss": 1.3187, + "theoretical_loss": 3.36664975316903, + "tokens_seen": 2803957760 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015187562688064192, + "loss": 1.2707, + "theoretical_loss": 3.3666436716973203, + "tokens_seen": 2804023296 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015185556670010031, + "loss": 1.3147, + "theoretical_loss": 3.366637590407543, + "tokens_seen": 2804088832 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015183550651955868, + "loss": 1.3437, + "theoretical_loss": 3.366631509299689, + "tokens_seen": 2804154368 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015181544633901707, + "loss": 1.3232, + "theoretical_loss": 3.366625428373748, + "tokens_seen": 2804219904 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015179538615847543, + "loss": 1.3253, + "theoretical_loss": 3.366619347629711, + "tokens_seen": 2804285440 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015177532597793382, + "loss": 1.2332, + "theoretical_loss": 3.366613267067567, + "tokens_seen": 2804350976 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015175526579739218, + "loss": 1.2617, + "theoretical_loss": 3.3666071866873075, + "tokens_seen": 2804416512 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015173520561685057, + "loss": 1.2785, + "theoretical_loss": 3.3666011064889223, + "tokens_seen": 2804482048 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015171514543630893, + "loss": 1.3013, + "theoretical_loss": 3.366595026472402, + "tokens_seen": 2804547584 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001516950852557673, + "loss": 1.2512, + "theoretical_loss": 3.3665889466377363, + "tokens_seen": 2804613120 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015167502507522568, + "loss": 1.3265, + "theoretical_loss": 3.3665828669849165, + "tokens_seen": 2804678656 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015165496489468404, + "loss": 1.1703, + "theoretical_loss": 3.366576787513932, + "tokens_seen": 2804744192 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015163490471414243, + "loss": 1.2439, + "theoretical_loss": 3.366570708224774, + "tokens_seen": 2804809728 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001516148445336008, + "loss": 1.2812, + "theoretical_loss": 3.366564629117432, + "tokens_seen": 2804875264 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015159478435305919, + "loss": 1.2602, + "theoretical_loss": 3.366558550191896, + "tokens_seen": 2804940800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015157472417251755, + "loss": 1.307, + "theoretical_loss": 3.3665524714481574, + "tokens_seen": 2805006336 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015155466399197594, + "loss": 1.1957, + "theoretical_loss": 3.3665463928862063, + "tokens_seen": 2805071872 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001515346038114343, + "loss": 1.3938, + "theoretical_loss": 3.3665403145060324, + "tokens_seen": 2805137408 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001515145436308927, + "loss": 1.2709, + "theoretical_loss": 3.3665342363076265, + "tokens_seen": 2805202944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015149448345035105, + "loss": 1.331, + "theoretical_loss": 3.366528158290979, + "tokens_seen": 2805268480 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001514744232698094, + "loss": 1.302, + "theoretical_loss": 3.36652208045608, + "tokens_seen": 2805334016 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001514543630892678, + "loss": 1.3629, + "theoretical_loss": 3.3665160028029195, + "tokens_seen": 2805399552 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015143430290872617, + "loss": 1.2956, + "theoretical_loss": 3.3665099253314885, + "tokens_seen": 2805465088 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015141424272818455, + "loss": 1.2546, + "theoretical_loss": 3.3665038480417766, + "tokens_seen": 2805530624 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015139418254764292, + "loss": 1.3184, + "theoretical_loss": 3.3664977709337744, + "tokens_seen": 2805596160 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001513741223671013, + "loss": 1.2493, + "theoretical_loss": 3.3664916940074727, + "tokens_seen": 2805661696 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015135406218655967, + "loss": 1.3128, + "theoretical_loss": 3.3664856172628617, + "tokens_seen": 2805727232 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015133400200601806, + "loss": 1.1913, + "theoretical_loss": 3.366479540699931, + "tokens_seen": 2805792768 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015131394182547642, + "loss": 1.3337, + "theoretical_loss": 3.3664734643186716, + "tokens_seen": 2805858304 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015129388164493478, + "loss": 1.259, + "theoretical_loss": 3.3664673881190734, + "tokens_seen": 2805923840 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001512738214643932, + "loss": 1.2508, + "theoretical_loss": 3.366461312101127, + "tokens_seen": 2805989376 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015125376128385156, + "loss": 1.3314, + "theoretical_loss": 3.3664552362648226, + "tokens_seen": 2806054912 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015123370110330995, + "loss": 1.3179, + "theoretical_loss": 3.3664491606101508, + "tokens_seen": 2806120448 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001512136409227683, + "loss": 1.3026, + "theoretical_loss": 3.366443085137101, + "tokens_seen": 2806185984 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001511935807422267, + "loss": 1.3589, + "theoretical_loss": 3.3664370098456655, + "tokens_seen": 2806251520 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015117352056168506, + "loss": 1.2968, + "theoretical_loss": 3.3664309347358325, + "tokens_seen": 2806317056 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015115346038114345, + "loss": 1.2514, + "theoretical_loss": 3.3664248598075934, + "tokens_seen": 2806382592 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015113340020060182, + "loss": 1.2259, + "theoretical_loss": 3.3664187850609384, + "tokens_seen": 2806448128 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001511133400200602, + "loss": 1.2635, + "theoretical_loss": 3.366412710495858, + "tokens_seen": 2806513664 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015109327983951857, + "loss": 1.2431, + "theoretical_loss": 3.366406636112342, + "tokens_seen": 2806579200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015107321965897693, + "loss": 1.2988, + "theoretical_loss": 3.366400561910381, + "tokens_seen": 2806644736 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1019695, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.300621747970581, + "objective/train/theoretical_loss": 3.366394487889966, + "objective/train/tokens_used": 1177169376, + "theoretical_loss": 3.366394487889966, + "tokens_seen": 2806710272 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015105315947843532, + "loss": 1.2623, + "theoretical_loss": 3.366394487889966, + "tokens_seen": 2806710272 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015103309929789368, + "loss": 1.2336, + "theoretical_loss": 3.366388414051086, + "tokens_seen": 2806775808 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015101303911735207, + "loss": 1.2963, + "theoretical_loss": 3.366382340393732, + "tokens_seen": 2806841344 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015099297893681043, + "loss": 1.2037, + "theoretical_loss": 3.3663762669178947, + "tokens_seen": 2806906880 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015097291875626882, + "loss": 1.2767, + "theoretical_loss": 3.366370193623564, + "tokens_seen": 2806972416 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015095285857572718, + "loss": 1.1683, + "theoretical_loss": 3.3663641205107306, + "tokens_seen": 2807037952 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015093279839518557, + "loss": 1.329, + "theoretical_loss": 3.3663580475793844, + "tokens_seen": 2807103488 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015091273821464394, + "loss": 1.211, + "theoretical_loss": 3.366351974829516, + "tokens_seen": 2807169024 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001508926780341023, + "loss": 1.3223, + "theoretical_loss": 3.3663459022611155, + "tokens_seen": 2807234560 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001508726178535607, + "loss": 1.256, + "theoretical_loss": 3.366339829874174, + "tokens_seen": 2807300096 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015085255767301905, + "loss": 1.2948, + "theoretical_loss": 3.3663337576686807, + "tokens_seen": 2807365632 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015083249749247744, + "loss": 1.371, + "theoretical_loss": 3.3663276856446265, + "tokens_seen": 2807431168 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001508124373119358, + "loss": 1.2891, + "theoretical_loss": 3.3663216138020022, + "tokens_seen": 2807496704 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001507923771313942, + "loss": 1.3914, + "theoretical_loss": 3.3663155421407973, + "tokens_seen": 2807562240 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015077231695085255, + "loss": 1.3284, + "theoretical_loss": 3.3663094706610024, + "tokens_seen": 2807627776 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015075225677031094, + "loss": 1.2942, + "theoretical_loss": 3.366303399362608, + "tokens_seen": 2807693312 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001507321965897693, + "loss": 1.3384, + "theoretical_loss": 3.366297328245605, + "tokens_seen": 2807758848 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015071213640922767, + "loss": 1.24, + "theoretical_loss": 3.3662912573099826, + "tokens_seen": 2807824384 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015069207622868606, + "loss": 1.2714, + "theoretical_loss": 3.366285186555732, + "tokens_seen": 2807889920 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015067201604814442, + "loss": 1.2375, + "theoretical_loss": 3.3662791159828434, + "tokens_seen": 2807955456 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001506519558676028, + "loss": 1.2936, + "theoretical_loss": 3.366273045591307, + "tokens_seen": 2808020992 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015063189568706117, + "loss": 1.2367, + "theoretical_loss": 3.3662669753811127, + "tokens_seen": 2808086528 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015061183550651956, + "loss": 1.2781, + "theoretical_loss": 3.3662609053522514, + "tokens_seen": 2808152064 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015059177532597792, + "loss": 1.3548, + "theoretical_loss": 3.366254835504714, + "tokens_seen": 2808217600 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001505717151454363, + "loss": 1.238, + "theoretical_loss": 3.3662487658384896, + "tokens_seen": 2808283136 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015055165496489467, + "loss": 1.2755, + "theoretical_loss": 3.366242696353569, + "tokens_seen": 2808348672 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015053159478435304, + "loss": 1.2329, + "theoretical_loss": 3.3662366270499433, + "tokens_seen": 2808414208 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015051153460381145, + "loss": 1.2944, + "theoretical_loss": 3.366230557927602, + "tokens_seen": 2808479744 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015049147442326981, + "loss": 1.3628, + "theoretical_loss": 3.366224488986536, + "tokens_seen": 2808545280 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001504714142427282, + "loss": 1.334, + "theoretical_loss": 3.366218420226735, + "tokens_seen": 2808610816 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015045135406218657, + "loss": 1.3177, + "theoretical_loss": 3.3662123516481905, + "tokens_seen": 2808676352 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015043129388164495, + "loss": 1.3237, + "theoretical_loss": 3.366206283250891, + "tokens_seen": 2808741888 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015041123370110332, + "loss": 1.2799, + "theoretical_loss": 3.366200215034829, + "tokens_seen": 2808807424 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001503911735205617, + "loss": 1.291, + "theoretical_loss": 3.3661941469999936, + "tokens_seen": 2808872960 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015037111334002007, + "loss": 1.3049, + "theoretical_loss": 3.366188079146375, + "tokens_seen": 2808938496 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015035105315947846, + "loss": 1.2636, + "theoretical_loss": 3.3661820114739642, + "tokens_seen": 2809004032 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015033099297893682, + "loss": 1.2467, + "theoretical_loss": 3.3661759439827508, + "tokens_seen": 2809069568 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015031093279839518, + "loss": 1.2171, + "theoretical_loss": 3.366169876672726, + "tokens_seen": 2809135104 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015029087261785357, + "loss": 1.1793, + "theoretical_loss": 3.36616380954388, + "tokens_seen": 2809200640 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015027081243731193, + "loss": 1.1923, + "theoretical_loss": 3.366157742596203, + "tokens_seen": 2809266176 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015025075225677032, + "loss": 1.3145, + "theoretical_loss": 3.3661516758296854, + "tokens_seen": 2809331712 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015023069207622869, + "loss": 1.3295, + "theoretical_loss": 3.3661456092443176, + "tokens_seen": 2809397248 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015021063189568707, + "loss": 1.3785, + "theoretical_loss": 3.3661395428400898, + "tokens_seen": 2809462784 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015019057171514544, + "loss": 1.2697, + "theoretical_loss": 3.366133476616992, + "tokens_seen": 2809528320 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015017051153460383, + "loss": 1.3382, + "theoretical_loss": 3.3661274105750154, + "tokens_seen": 2809593856 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001501504513540622, + "loss": 1.2608, + "theoretical_loss": 3.3661213447141503, + "tokens_seen": 2809659392 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015013039117352055, + "loss": 1.326, + "theoretical_loss": 3.3661152790343865, + "tokens_seen": 2809724928 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015011033099297894, + "loss": 1.1812, + "theoretical_loss": 3.3661092135357142, + "tokens_seen": 2809790464 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001500902708124373, + "loss": 1.3251, + "theoretical_loss": 3.3661031482181247, + "tokens_seen": 2809856000 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001500702106318957, + "loss": 1.3088, + "theoretical_loss": 3.3660970830816077, + "tokens_seen": 2809921536 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1020904, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3573384284973145, + "objective/train/theoretical_loss": 3.366091018126154, + "objective/train/tokens_used": 1180446176, + "theoretical_loss": 3.366091018126154, + "tokens_seen": 2809987072 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015005015045135405, + "loss": 1.3532, + "theoretical_loss": 3.366091018126154, + "tokens_seen": 2809987072 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015003009027081244, + "loss": 1.2438, + "theoretical_loss": 3.366084953351753, + "tokens_seen": 2810052608 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001500100300902708, + "loss": 1.3242, + "theoretical_loss": 3.3660788887583966, + "tokens_seen": 2810118144 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001499899699097292, + "loss": 1.2805, + "theoretical_loss": 3.366072824346074, + "tokens_seen": 2810183680 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014996990972918756, + "loss": 1.3058, + "theoretical_loss": 3.3660667601147756, + "tokens_seen": 2810249216 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014994984954864595, + "loss": 1.2142, + "theoretical_loss": 3.3660606960644928, + "tokens_seen": 2810314752 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001499297893681043, + "loss": 1.2758, + "theoretical_loss": 3.366054632195215, + "tokens_seen": 2810380288 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014990972918756267, + "loss": 1.2353, + "theoretical_loss": 3.3660485685069323, + "tokens_seen": 2810445824 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014988966900702106, + "loss": 1.2517, + "theoretical_loss": 3.3660425049996365, + "tokens_seen": 2810511360 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014986960882647942, + "loss": 1.3745, + "theoretical_loss": 3.366036441673317, + "tokens_seen": 2810576896 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001498495486459378, + "loss": 1.2953, + "theoretical_loss": 3.3660303785279635, + "tokens_seen": 2810642432 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014982948846539617, + "loss": 1.2827, + "theoretical_loss": 3.3660243155635676, + "tokens_seen": 2810707968 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014980942828485456, + "loss": 1.3026, + "theoretical_loss": 3.3660182527801195, + "tokens_seen": 2810773504 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014978936810431293, + "loss": 1.2254, + "theoretical_loss": 3.366012190177609, + "tokens_seen": 2810839040 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014976930792377134, + "loss": 1.3262, + "theoretical_loss": 3.3660061277560276, + "tokens_seen": 2810904576 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001497492477432297, + "loss": 1.3479, + "theoretical_loss": 3.366000065515364, + "tokens_seen": 2810970112 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014972918756268807, + "loss": 1.207, + "theoretical_loss": 3.3659940034556097, + "tokens_seen": 2811035648 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014970912738214646, + "loss": 1.3488, + "theoretical_loss": 3.365987941576755, + "tokens_seen": 2811101184 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014968906720160482, + "loss": 1.3395, + "theoretical_loss": 3.36598187987879, + "tokens_seen": 2811166720 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001496690070210632, + "loss": 1.3602, + "theoretical_loss": 3.3659758183617057, + "tokens_seen": 2811232256 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014964894684052157, + "loss": 1.364, + "theoretical_loss": 3.3659697570254914, + "tokens_seen": 2811297792 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014962888665997996, + "loss": 1.3224, + "theoretical_loss": 3.365963695870138, + "tokens_seen": 2811363328 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014960882647943832, + "loss": 1.242, + "theoretical_loss": 3.365957634895637, + "tokens_seen": 2811428864 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001495887662988967, + "loss": 1.4712, + "theoretical_loss": 3.365951574101977, + "tokens_seen": 2811494400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014956870611835507, + "loss": 1.3002, + "theoretical_loss": 3.365945513489149, + "tokens_seen": 2811559936 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014954864593781344, + "loss": 1.3035, + "theoretical_loss": 3.365939453057144, + "tokens_seen": 2811625472 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014952858575727182, + "loss": 1.2132, + "theoretical_loss": 3.365933392805952, + "tokens_seen": 2811691008 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001495085255767302, + "loss": 1.283, + "theoretical_loss": 3.3659273327355628, + "tokens_seen": 2811756544 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014948846539618858, + "loss": 1.1784, + "theoretical_loss": 3.3659212728459678, + "tokens_seen": 2811822080 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014946840521564694, + "loss": 1.2441, + "theoretical_loss": 3.365915213137157, + "tokens_seen": 2811887616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014944834503510533, + "loss": 1.3213, + "theoretical_loss": 3.3659091536091204, + "tokens_seen": 2811953152 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001494282848545637, + "loss": 1.3185, + "theoretical_loss": 3.365903094261849, + "tokens_seen": 2812018688 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014940822467402208, + "loss": 1.2277, + "theoretical_loss": 3.365897035095333, + "tokens_seen": 2812084224 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014938816449348044, + "loss": 1.3234, + "theoretical_loss": 3.3658909761095623, + "tokens_seen": 2812149760 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014936810431293883, + "loss": 1.1999, + "theoretical_loss": 3.3658849173045278, + "tokens_seen": 2812215296 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001493480441323972, + "loss": 1.3286, + "theoretical_loss": 3.3658788586802197, + "tokens_seen": 2812280832 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014932798395185556, + "loss": 1.2499, + "theoretical_loss": 3.3658728002366285, + "tokens_seen": 2812346368 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014930792377131394, + "loss": 1.3928, + "theoretical_loss": 3.365866741973745, + "tokens_seen": 2812411904 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001492878635907723, + "loss": 1.3609, + "theoretical_loss": 3.365860683891559, + "tokens_seen": 2812477440 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001492678034102307, + "loss": 1.2247, + "theoretical_loss": 3.3658546259900612, + "tokens_seen": 2812542976 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014924774322968906, + "loss": 1.2921, + "theoretical_loss": 3.3658485682692416, + "tokens_seen": 2812608512 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014922768304914745, + "loss": 1.3862, + "theoretical_loss": 3.365842510729091, + "tokens_seen": 2812674048 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001492076228686058, + "loss": 1.3808, + "theoretical_loss": 3.3658364533696, + "tokens_seen": 2812739584 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001491875626880642, + "loss": 1.3087, + "theoretical_loss": 3.3658303961907583, + "tokens_seen": 2812805120 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014916750250752256, + "loss": 1.3023, + "theoretical_loss": 3.3658243391925566, + "tokens_seen": 2812870656 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014914744232698092, + "loss": 1.2785, + "theoretical_loss": 3.3658182823749856, + "tokens_seen": 2812936192 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001491273821464393, + "loss": 1.3601, + "theoretical_loss": 3.365812225738036, + "tokens_seen": 2813001728 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014910732196589768, + "loss": 1.2484, + "theoretical_loss": 3.365806169281697, + "tokens_seen": 2813067264 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014908726178535607, + "loss": 1.2908, + "theoretical_loss": 3.3658001130059603, + "tokens_seen": 2813132800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014906720160481443, + "loss": 1.3212, + "theoretical_loss": 3.3657940569108153, + "tokens_seen": 2813198336 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1022064, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3037389516830444, + "objective/train/theoretical_loss": 3.365788000996253, + "objective/train/tokens_used": 1183722976, + "theoretical_loss": 3.365788000996253, + "tokens_seen": 2813263872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014904714142427282, + "loss": 1.2881, + "theoretical_loss": 3.365788000996253, + "tokens_seen": 2813263872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014902708124373118, + "loss": 1.247, + "theoretical_loss": 3.3657819452622637, + "tokens_seen": 2813329408 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001490070210631896, + "loss": 1.3213, + "theoretical_loss": 3.3657758897088375, + "tokens_seen": 2813394944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014898696088264796, + "loss": 1.3377, + "theoretical_loss": 3.3657698343359654, + "tokens_seen": 2813460480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014896690070210635, + "loss": 1.347, + "theoretical_loss": 3.3657637791436374, + "tokens_seen": 2813526016 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001489468405215647, + "loss": 1.2703, + "theoretical_loss": 3.3657577241318437, + "tokens_seen": 2813591552 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014892678034102307, + "loss": 1.2967, + "theoretical_loss": 3.3657516693005753, + "tokens_seen": 2813657088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014890672016048146, + "loss": 1.3306, + "theoretical_loss": 3.365745614649822, + "tokens_seen": 2813722624 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014888665997993982, + "loss": 1.1877, + "theoretical_loss": 3.365739560179575, + "tokens_seen": 2813788160 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001488665997993982, + "loss": 1.2836, + "theoretical_loss": 3.3657335058898243, + "tokens_seen": 2813853696 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014884653961885657, + "loss": 1.2961, + "theoretical_loss": 3.36572745178056, + "tokens_seen": 2813919232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014882647943831496, + "loss": 1.1976, + "theoretical_loss": 3.365721397851773, + "tokens_seen": 2813984768 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014880641925777333, + "loss": 1.2404, + "theoretical_loss": 3.365715344103453, + "tokens_seen": 2814050304 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014878635907723172, + "loss": 1.351, + "theoretical_loss": 3.3657092905355914, + "tokens_seen": 2814115840 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014876629889669008, + "loss": 1.2728, + "theoretical_loss": 3.365703237148178, + "tokens_seen": 2814181376 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014874623871614844, + "loss": 1.3906, + "theoretical_loss": 3.365697183941203, + "tokens_seen": 2814246912 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014872617853560683, + "loss": 1.2368, + "theoretical_loss": 3.3656911309146578, + "tokens_seen": 2814312448 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001487061183550652, + "loss": 1.2756, + "theoretical_loss": 3.3656850780685317, + "tokens_seen": 2814377984 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014868605817452358, + "loss": 1.2397, + "theoretical_loss": 3.365679025402816, + "tokens_seen": 2814443520 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014866599799398194, + "loss": 1.1949, + "theoretical_loss": 3.3656729729175003, + "tokens_seen": 2814509056 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014864593781344033, + "loss": 1.2854, + "theoretical_loss": 3.3656669206125756, + "tokens_seen": 2814574592 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001486258776328987, + "loss": 1.2674, + "theoretical_loss": 3.3656608684880323, + "tokens_seen": 2814640128 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014860581745235708, + "loss": 1.3069, + "theoretical_loss": 3.3656548165438607, + "tokens_seen": 2814705664 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014858575727181545, + "loss": 1.3918, + "theoretical_loss": 3.365648764780051, + "tokens_seen": 2814771200 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001485656970912738, + "loss": 1.2786, + "theoretical_loss": 3.3656427131965945, + "tokens_seen": 2814836736 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001485456369107322, + "loss": 1.3064, + "theoretical_loss": 3.3656366617934803, + "tokens_seen": 2814902272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014852557673019056, + "loss": 1.2267, + "theoretical_loss": 3.3656306105706997, + "tokens_seen": 2814967808 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014850551654964895, + "loss": 1.2111, + "theoretical_loss": 3.365624559528243, + "tokens_seen": 2815033344 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001484854563691073, + "loss": 1.2221, + "theoretical_loss": 3.3656185086661004, + "tokens_seen": 2815098880 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001484653961885657, + "loss": 1.3426, + "theoretical_loss": 3.3656124579842626, + "tokens_seen": 2815164416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014844533600802406, + "loss": 1.2264, + "theoretical_loss": 3.36560640748272, + "tokens_seen": 2815229952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014842527582748245, + "loss": 1.2896, + "theoretical_loss": 3.3656003571614628, + "tokens_seen": 2815295488 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014840521564694081, + "loss": 1.3565, + "theoretical_loss": 3.3655943070204817, + "tokens_seen": 2815361024 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001483851554663992, + "loss": 1.4383, + "theoretical_loss": 3.365588257059767, + "tokens_seen": 2815426560 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014836509528585757, + "loss": 1.2021, + "theoretical_loss": 3.365582207279309, + "tokens_seen": 2815492096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014834503510531593, + "loss": 1.4061, + "theoretical_loss": 3.3655761576790986, + "tokens_seen": 2815557632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014832497492477432, + "loss": 1.3142, + "theoretical_loss": 3.365570108259126, + "tokens_seen": 2815623168 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014830491474423268, + "loss": 1.2672, + "theoretical_loss": 3.365564059019381, + "tokens_seen": 2815688704 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014828485456369107, + "loss": 1.2992, + "theoretical_loss": 3.365558009959855, + "tokens_seen": 2815754240 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014826479438314946, + "loss": 1.4026, + "theoretical_loss": 3.365551961080538, + "tokens_seen": 2815819776 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014824473420260785, + "loss": 1.3509, + "theoretical_loss": 3.36554591238142, + "tokens_seen": 2815885312 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001482246740220662, + "loss": 1.2679, + "theoretical_loss": 3.3655398638624927, + "tokens_seen": 2815950848 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001482046138415246, + "loss": 1.3021, + "theoretical_loss": 3.3655338155237455, + "tokens_seen": 2816016384 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014818455366098296, + "loss": 1.2219, + "theoretical_loss": 3.3655277673651685, + "tokens_seen": 2816081920 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014816449348044132, + "loss": 1.1718, + "theoretical_loss": 3.3655217193867535, + "tokens_seen": 2816147456 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014814443329989971, + "loss": 1.3033, + "theoretical_loss": 3.3655156715884895, + "tokens_seen": 2816212992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014812437311935808, + "loss": 1.2286, + "theoretical_loss": 3.365509623970368, + "tokens_seen": 2816278528 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014810431293881647, + "loss": 1.2077, + "theoretical_loss": 3.365503576532379, + "tokens_seen": 2816344064 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014808425275827483, + "loss": 1.2522, + "theoretical_loss": 3.365497529274513, + "tokens_seen": 2816409600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014806419257773322, + "loss": 1.4091, + "theoretical_loss": 3.3654914821967603, + "tokens_seen": 2816475136 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1023297, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.332075834274292, + "objective/train/theoretical_loss": 3.3654854352991115, + "objective/train/tokens_used": 1186999776, + "theoretical_loss": 3.3654854352991115, + "tokens_seen": 2816540672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014804413239719158, + "loss": 1.3627, + "theoretical_loss": 3.3654854352991115, + "tokens_seen": 2816540672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014802407221664997, + "loss": 1.3526, + "theoretical_loss": 3.3654793885815573, + "tokens_seen": 2816606208 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014800401203610833, + "loss": 1.3273, + "theoretical_loss": 3.3654733420440874, + "tokens_seen": 2816671744 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001479839518555667, + "loss": 1.2575, + "theoretical_loss": 3.365467295686693, + "tokens_seen": 2816737280 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014796389167502508, + "loss": 1.2802, + "theoretical_loss": 3.365461249509364, + "tokens_seen": 2816802816 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014794383149448344, + "loss": 1.2967, + "theoretical_loss": 3.3654552035120915, + "tokens_seen": 2816868352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014792377131394183, + "loss": 1.3133, + "theoretical_loss": 3.3654491576948655, + "tokens_seen": 2816933888 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001479037111334002, + "loss": 1.3794, + "theoretical_loss": 3.3654431120576764, + "tokens_seen": 2816999424 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014788365095285859, + "loss": 1.2313, + "theoretical_loss": 3.365437066600515, + "tokens_seen": 2817064960 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014786359077231695, + "loss": 1.3032, + "theoretical_loss": 3.365431021323371, + "tokens_seen": 2817130496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014784353059177534, + "loss": 1.1861, + "theoretical_loss": 3.365424976226236, + "tokens_seen": 2817196032 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001478234704112337, + "loss": 1.3505, + "theoretical_loss": 3.365418931309099, + "tokens_seen": 2817261568 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001478034102306921, + "loss": 1.2523, + "theoretical_loss": 3.365412886571952, + "tokens_seen": 2817327104 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014778335005015045, + "loss": 1.21, + "theoretical_loss": 3.3654068420147842, + "tokens_seen": 2817392640 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001477632898696088, + "loss": 1.3126, + "theoretical_loss": 3.3654007976375873, + "tokens_seen": 2817458176 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001477432296890672, + "loss": 1.3921, + "theoretical_loss": 3.3653947534403503, + "tokens_seen": 2817523712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014772316950852556, + "loss": 1.3034, + "theoretical_loss": 3.3653887094230646, + "tokens_seen": 2817589248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014770310932798395, + "loss": 1.26, + "theoretical_loss": 3.3653826655857206, + "tokens_seen": 2817654784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014768304914744232, + "loss": 1.2252, + "theoretical_loss": 3.3653766219283083, + "tokens_seen": 2817720320 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001476629889669007, + "loss": 1.2532, + "theoretical_loss": 3.3653705784508188, + "tokens_seen": 2817785856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014764292878635907, + "loss": 1.4185, + "theoretical_loss": 3.365364535153242, + "tokens_seen": 2817851392 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014762286860581746, + "loss": 1.3713, + "theoretical_loss": 3.365358492035569, + "tokens_seen": 2817916928 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014760280842527582, + "loss": 1.3146, + "theoretical_loss": 3.3653524490977893, + "tokens_seen": 2817982464 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014758274824473418, + "loss": 1.2004, + "theoretical_loss": 3.3653464063398943, + "tokens_seen": 2818048000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014756268806419257, + "loss": 1.2827, + "theoretical_loss": 3.3653403637618737, + "tokens_seen": 2818113536 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014754262788365093, + "loss": 1.4074, + "theoretical_loss": 3.3653343213637186, + "tokens_seen": 2818179072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014752256770310932, + "loss": 1.2958, + "theoretical_loss": 3.365328279145419, + "tokens_seen": 2818244608 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001475025075225677, + "loss": 1.2943, + "theoretical_loss": 3.365322237106966, + "tokens_seen": 2818310144 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001474824473420261, + "loss": 1.3692, + "theoretical_loss": 3.365316195248349, + "tokens_seen": 2818375680 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014746238716148446, + "loss": 1.318, + "theoretical_loss": 3.3653101535695598, + "tokens_seen": 2818441216 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014744232698094285, + "loss": 1.201, + "theoretical_loss": 3.3653041120705875, + "tokens_seen": 2818506752 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014742226680040122, + "loss": 1.1917, + "theoretical_loss": 3.3652980707514235, + "tokens_seen": 2818572288 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001474022066198596, + "loss": 1.223, + "theoretical_loss": 3.365292029612058, + "tokens_seen": 2818637824 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014738214643931797, + "loss": 1.2914, + "theoretical_loss": 3.3652859886524813, + "tokens_seen": 2818703360 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014736208625877633, + "loss": 1.3589, + "theoretical_loss": 3.3652799478726845, + "tokens_seen": 2818768896 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014734202607823472, + "loss": 1.3577, + "theoretical_loss": 3.365273907272657, + "tokens_seen": 2818834432 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014732196589769308, + "loss": 1.2074, + "theoretical_loss": 3.36526786685239, + "tokens_seen": 2818899968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014730190571715147, + "loss": 1.2743, + "theoretical_loss": 3.3652618266118743, + "tokens_seen": 2818965504 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014728184553660983, + "loss": 1.3328, + "theoretical_loss": 3.3652557865510992, + "tokens_seen": 2819031040 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014726178535606822, + "loss": 1.2639, + "theoretical_loss": 3.3652497466700564, + "tokens_seen": 2819096576 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014724172517552658, + "loss": 1.2576, + "theoretical_loss": 3.3652437069687355, + "tokens_seen": 2819162112 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014722166499498497, + "loss": 1.2791, + "theoretical_loss": 3.365237667447128, + "tokens_seen": 2819227648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014720160481444334, + "loss": 1.3342, + "theoretical_loss": 3.365231628105223, + "tokens_seen": 2819293184 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001471815446339017, + "loss": 1.3723, + "theoretical_loss": 3.3652255889430123, + "tokens_seen": 2819358720 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001471614844533601, + "loss": 1.3382, + "theoretical_loss": 3.365219549960485, + "tokens_seen": 2819424256 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014714142427281845, + "loss": 1.2768, + "theoretical_loss": 3.365213511157633, + "tokens_seen": 2819489792 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014712136409227684, + "loss": 1.2425, + "theoretical_loss": 3.3652074725344456, + "tokens_seen": 2819555328 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001471013039117352, + "loss": 1.3266, + "theoretical_loss": 3.365201434090914, + "tokens_seen": 2819620864 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001470812437311936, + "loss": 1.2581, + "theoretical_loss": 3.365195395827029, + "tokens_seen": 2819686400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014706118355065195, + "loss": 1.2493, + "theoretical_loss": 3.36518935774278, + "tokens_seen": 2819751936 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1024583, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.230042815208435, + "objective/train/theoretical_loss": 3.3651833198381578, + "objective/train/tokens_used": 1190276576, + "theoretical_loss": 3.3651833198381578, + "tokens_seen": 2819817472 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014704112337011034, + "loss": 1.2822, + "theoretical_loss": 3.3651833198381578, + "tokens_seen": 2819817472 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001470210631895687, + "loss": 1.2733, + "theoretical_loss": 3.3651772821131534, + "tokens_seen": 2819883008 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014700100300902707, + "loss": 1.3366, + "theoretical_loss": 3.3651712445677573, + "tokens_seen": 2819948544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014698094282848546, + "loss": 1.2138, + "theoretical_loss": 3.3651652072019593, + "tokens_seen": 2820014080 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014696088264794382, + "loss": 1.2268, + "theoretical_loss": 3.3651591700157506, + "tokens_seen": 2820079616 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001469408224674022, + "loss": 1.2365, + "theoretical_loss": 3.365153133009121, + "tokens_seen": 2820145152 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014692076228686057, + "loss": 1.3833, + "theoretical_loss": 3.3651470961820618, + "tokens_seen": 2820210688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014690070210631896, + "loss": 1.2667, + "theoretical_loss": 3.3651410595345626, + "tokens_seen": 2820276224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014688064192577732, + "loss": 1.3036, + "theoretical_loss": 3.365135023066614, + "tokens_seen": 2820341760 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001468605817452357, + "loss": 1.2481, + "theoretical_loss": 3.3651289867782075, + "tokens_seen": 2820407296 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014684052156469407, + "loss": 1.2153, + "theoretical_loss": 3.3651229506693325, + "tokens_seen": 2820472832 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014682046138415243, + "loss": 1.2854, + "theoretical_loss": 3.36511691473998, + "tokens_seen": 2820538368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014680040120361082, + "loss": 1.2934, + "theoretical_loss": 3.3651108789901403, + "tokens_seen": 2820603904 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001467803410230692, + "loss": 1.3354, + "theoretical_loss": 3.365104843419804, + "tokens_seen": 2820669440 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001467602808425276, + "loss": 1.3075, + "theoretical_loss": 3.3650988080289617, + "tokens_seen": 2820734976 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014674022066198597, + "loss": 1.2173, + "theoretical_loss": 3.3650927728176034, + "tokens_seen": 2820800512 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014672016048144435, + "loss": 1.3617, + "theoretical_loss": 3.3650867377857203, + "tokens_seen": 2820866048 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014670010030090272, + "loss": 1.2539, + "theoretical_loss": 3.365080702933302, + "tokens_seen": 2820931584 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001466800401203611, + "loss": 1.2408, + "theoretical_loss": 3.3650746682603403, + "tokens_seen": 2820997120 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014665997993981947, + "loss": 1.237, + "theoretical_loss": 3.3650686337668243, + "tokens_seen": 2821062656 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014663991975927786, + "loss": 1.2972, + "theoretical_loss": 3.365062599452745, + "tokens_seen": 2821128192 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014661985957873622, + "loss": 1.2806, + "theoretical_loss": 3.3650565653180937, + "tokens_seen": 2821193728 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014659979939819458, + "loss": 1.3165, + "theoretical_loss": 3.3650505313628596, + "tokens_seen": 2821259264 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014657973921765297, + "loss": 1.1982, + "theoretical_loss": 3.365044497587034, + "tokens_seen": 2821324800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014655967903711133, + "loss": 1.2624, + "theoretical_loss": 3.365038463990607, + "tokens_seen": 2821390336 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014653961885656972, + "loss": 1.2847, + "theoretical_loss": 3.36503243057357, + "tokens_seen": 2821455872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014651955867602809, + "loss": 1.3052, + "theoretical_loss": 3.365026397335912, + "tokens_seen": 2821521408 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014649949849548647, + "loss": 1.3114, + "theoretical_loss": 3.3650203642776244, + "tokens_seen": 2821586944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014647943831494484, + "loss": 1.3226, + "theoretical_loss": 3.3650143313986978, + "tokens_seen": 2821652480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014645937813440323, + "loss": 1.2237, + "theoretical_loss": 3.3650082986991228, + "tokens_seen": 2821718016 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001464393179538616, + "loss": 1.2157, + "theoretical_loss": 3.365002266178889, + "tokens_seen": 2821783552 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014641925777331995, + "loss": 1.2968, + "theoretical_loss": 3.3649962338379877, + "tokens_seen": 2821849088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014639919759277834, + "loss": 1.3028, + "theoretical_loss": 3.3649902016764095, + "tokens_seen": 2821914624 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001463791374122367, + "loss": 1.3197, + "theoretical_loss": 3.3649841696941443, + "tokens_seen": 2821980160 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001463590772316951, + "loss": 1.3784, + "theoretical_loss": 3.364978137891183, + "tokens_seen": 2822045696 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014633901705115345, + "loss": 1.2514, + "theoretical_loss": 3.3649721062675164, + "tokens_seen": 2822111232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014631895687061184, + "loss": 1.3369, + "theoretical_loss": 3.3649660748231343, + "tokens_seen": 2822176768 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001462988966900702, + "loss": 1.3616, + "theoretical_loss": 3.3649600435580274, + "tokens_seen": 2822242304 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001462788365095286, + "loss": 1.2211, + "theoretical_loss": 3.3649540124721864, + "tokens_seen": 2822307840 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014625877632898696, + "loss": 1.2585, + "theoretical_loss": 3.364947981565602, + "tokens_seen": 2822373376 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014623871614844535, + "loss": 1.3647, + "theoretical_loss": 3.364941950838264, + "tokens_seen": 2822438912 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001462186559679037, + "loss": 1.3117, + "theoretical_loss": 3.364935920290164, + "tokens_seen": 2822504448 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014619859578736207, + "loss": 1.2697, + "theoretical_loss": 3.3649298899212914, + "tokens_seen": 2822569984 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014617853560682046, + "loss": 1.2733, + "theoretical_loss": 3.3649238597316375, + "tokens_seen": 2822635520 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014615847542627882, + "loss": 1.3767, + "theoretical_loss": 3.3649178297211924, + "tokens_seen": 2822701056 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001461384152457372, + "loss": 1.1376, + "theoretical_loss": 3.3649117998899465, + "tokens_seen": 2822766592 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014611835506519557, + "loss": 1.3238, + "theoretical_loss": 3.364905770237891, + "tokens_seen": 2822832128 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014609829488465396, + "loss": 1.2654, + "theoretical_loss": 3.364899740765016, + "tokens_seen": 2822897664 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014607823470411233, + "loss": 1.3341, + "theoretical_loss": 3.3648937114713116, + "tokens_seen": 2822963200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014605817452357072, + "loss": 1.4008, + "theoretical_loss": 3.364887682356769, + "tokens_seen": 2823028736 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1025771, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2943767309188843, + "objective/train/theoretical_loss": 3.364881653421378, + "objective/train/tokens_used": 1193553376, + "theoretical_loss": 3.364881653421378, + "tokens_seen": 2823094272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014603811434302908, + "loss": 1.3036, + "theoretical_loss": 3.364881653421378, + "tokens_seen": 2823094272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014601805416248747, + "loss": 1.4194, + "theoretical_loss": 3.36487562466513, + "tokens_seen": 2823159808 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014599799398194586, + "loss": 1.2723, + "theoretical_loss": 3.364869596088014, + "tokens_seen": 2823225344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014597793380140422, + "loss": 1.29, + "theoretical_loss": 3.3648635676900227, + "tokens_seen": 2823290880 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001459578736208626, + "loss": 1.3217, + "theoretical_loss": 3.3648575394711453, + "tokens_seen": 2823356416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014593781344032097, + "loss": 1.2406, + "theoretical_loss": 3.364851511431372, + "tokens_seen": 2823421952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014591775325977936, + "loss": 1.392, + "theoretical_loss": 3.3648454835706945, + "tokens_seen": 2823487488 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014589769307923772, + "loss": 1.333, + "theoretical_loss": 3.3648394558891024, + "tokens_seen": 2823553024 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001458776328986961, + "loss": 1.3446, + "theoretical_loss": 3.3648334283865866, + "tokens_seen": 2823618560 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014585757271815447, + "loss": 1.3201, + "theoretical_loss": 3.364827401063137, + "tokens_seen": 2823684096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014583751253761284, + "loss": 1.3109, + "theoretical_loss": 3.364821373918745, + "tokens_seen": 2823749632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014581745235707122, + "loss": 1.1853, + "theoretical_loss": 3.364815346953401, + "tokens_seen": 2823815168 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001457973921765296, + "loss": 1.2411, + "theoretical_loss": 3.3648093201670948, + "tokens_seen": 2823880704 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014577733199598798, + "loss": 1.2904, + "theoretical_loss": 3.3648032935598176, + "tokens_seen": 2823946240 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014575727181544634, + "loss": 1.2688, + "theoretical_loss": 3.3647972671315594, + "tokens_seen": 2824011776 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014573721163490473, + "loss": 1.3251, + "theoretical_loss": 3.3647912408823113, + "tokens_seen": 2824077312 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001457171514543631, + "loss": 1.3927, + "theoretical_loss": 3.3647852148120636, + "tokens_seen": 2824142848 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014569709127382148, + "loss": 1.277, + "theoretical_loss": 3.3647791889208065, + "tokens_seen": 2824208384 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014567703109327984, + "loss": 1.341, + "theoretical_loss": 3.3647731632085316, + "tokens_seen": 2824273920 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014565697091273823, + "loss": 1.2855, + "theoretical_loss": 3.364767137675228, + "tokens_seen": 2824339456 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001456369107321966, + "loss": 1.3014, + "theoretical_loss": 3.3647611123208874, + "tokens_seen": 2824404992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014561685055165496, + "loss": 1.2117, + "theoretical_loss": 3.3647550871454994, + "tokens_seen": 2824470528 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014559679037111334, + "loss": 1.2314, + "theoretical_loss": 3.364749062149055, + "tokens_seen": 2824536064 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001455767301905717, + "loss": 1.2678, + "theoretical_loss": 3.364743037331545, + "tokens_seen": 2824601600 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001455566700100301, + "loss": 1.3055, + "theoretical_loss": 3.3647370126929594, + "tokens_seen": 2824667136 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014553660982948846, + "loss": 1.1507, + "theoretical_loss": 3.364730988233289, + "tokens_seen": 2824732672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014551654964894685, + "loss": 1.2466, + "theoretical_loss": 3.3647249639525243, + "tokens_seen": 2824798208 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001454964894684052, + "loss": 1.3325, + "theoretical_loss": 3.364718939850656, + "tokens_seen": 2824863744 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001454764292878636, + "loss": 1.3071, + "theoretical_loss": 3.3647129159276745, + "tokens_seen": 2824929280 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014545636910732196, + "loss": 1.2456, + "theoretical_loss": 3.36470689218357, + "tokens_seen": 2824994816 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014543630892678032, + "loss": 1.3039, + "theoretical_loss": 3.3647008686183337, + "tokens_seen": 2825060352 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001454162487462387, + "loss": 1.2025, + "theoretical_loss": 3.3646948452319556, + "tokens_seen": 2825125888 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014539618856569708, + "loss": 1.2729, + "theoretical_loss": 3.364688822024427, + "tokens_seen": 2825191424 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014537612838515546, + "loss": 1.252, + "theoretical_loss": 3.3646827989957373, + "tokens_seen": 2825256960 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014535606820461383, + "loss": 1.1788, + "theoretical_loss": 3.3646767761458776, + "tokens_seen": 2825322496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014533600802407222, + "loss": 1.2315, + "theoretical_loss": 3.3646707534748383, + "tokens_seen": 2825388032 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014531594784353058, + "loss": 1.4156, + "theoretical_loss": 3.3646647309826108, + "tokens_seen": 2825453568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014529588766298897, + "loss": 1.2846, + "theoretical_loss": 3.364658708669184, + "tokens_seen": 2825519104 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014527582748244733, + "loss": 1.2878, + "theoretical_loss": 3.3646526865345505, + "tokens_seen": 2825584640 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014525576730190575, + "loss": 1.3465, + "theoretical_loss": 3.3646466645786988, + "tokens_seen": 2825650176 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001452357071213641, + "loss": 1.3168, + "theoretical_loss": 3.364640642801621, + "tokens_seen": 2825715712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014521564694082247, + "loss": 1.2529, + "theoretical_loss": 3.3646346212033067, + "tokens_seen": 2825781248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014519558676028086, + "loss": 1.2339, + "theoretical_loss": 3.364628599783747, + "tokens_seen": 2825846784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014517552657973922, + "loss": 1.2838, + "theoretical_loss": 3.364622578542932, + "tokens_seen": 2825912320 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001451554663991976, + "loss": 1.3679, + "theoretical_loss": 3.3646165574808524, + "tokens_seen": 2825977856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014513540621865597, + "loss": 1.201, + "theoretical_loss": 3.364610536597499, + "tokens_seen": 2826043392 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014511534603811436, + "loss": 1.3193, + "theoretical_loss": 3.3646045158928626, + "tokens_seen": 2826108928 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014509528585757273, + "loss": 1.1942, + "theoretical_loss": 3.364598495366933, + "tokens_seen": 2826174464 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014507522567703112, + "loss": 1.2649, + "theoretical_loss": 3.364592475019701, + "tokens_seen": 2826240000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014505516549648948, + "loss": 1.2474, + "theoretical_loss": 3.364586454851157, + "tokens_seen": 2826305536 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1027020, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1658639907836914, + "objective/train/theoretical_loss": 3.364580434861292, + "objective/train/tokens_used": 1196830176, + "theoretical_loss": 3.364580434861292, + "tokens_seen": 2826371072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014503510531594784, + "loss": 1.2868, + "theoretical_loss": 3.364580434861292, + "tokens_seen": 2826371072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014501504513540623, + "loss": 1.1793, + "theoretical_loss": 3.3645744150500967, + "tokens_seen": 2826436608 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001449949849548646, + "loss": 1.3026, + "theoretical_loss": 3.3645683954175607, + "tokens_seen": 2826502144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014497492477432298, + "loss": 1.2663, + "theoretical_loss": 3.3645623759636756, + "tokens_seen": 2826567680 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014495486459378134, + "loss": 1.3111, + "theoretical_loss": 3.364556356688431, + "tokens_seen": 2826633216 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014493480441323973, + "loss": 1.2384, + "theoretical_loss": 3.3645503375918184, + "tokens_seen": 2826698752 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001449147442326981, + "loss": 1.4291, + "theoretical_loss": 3.3645443186738277, + "tokens_seen": 2826764288 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014489468405215648, + "loss": 1.2627, + "theoretical_loss": 3.36453829993445, + "tokens_seen": 2826829824 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014487462387161485, + "loss": 1.2481, + "theoretical_loss": 3.364532281373675, + "tokens_seen": 2826895360 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001448545636910732, + "loss": 1.2689, + "theoretical_loss": 3.3645262629914945, + "tokens_seen": 2826960896 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001448345035105316, + "loss": 1.208, + "theoretical_loss": 3.364520244787898, + "tokens_seen": 2827026432 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014481444332998996, + "loss": 1.2726, + "theoretical_loss": 3.3645142267628763, + "tokens_seen": 2827091968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014479438314944835, + "loss": 1.2645, + "theoretical_loss": 3.36450820891642, + "tokens_seen": 2827157504 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001447743229689067, + "loss": 1.234, + "theoretical_loss": 3.36450219124852, + "tokens_seen": 2827223040 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001447542627883651, + "loss": 1.3298, + "theoretical_loss": 3.364496173759167, + "tokens_seen": 2827288576 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014473420260782346, + "loss": 1.2756, + "theoretical_loss": 3.3644901564483503, + "tokens_seen": 2827354112 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014471414242728185, + "loss": 1.2192, + "theoretical_loss": 3.364484139316062, + "tokens_seen": 2827419648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014469408224674021, + "loss": 1.2172, + "theoretical_loss": 3.3644781223622915, + "tokens_seen": 2827485184 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001446740220661986, + "loss": 1.3347, + "theoretical_loss": 3.3644721055870304, + "tokens_seen": 2827550720 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014465396188565697, + "loss": 1.3454, + "theoretical_loss": 3.3644660889902682, + "tokens_seen": 2827616256 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014463390170511533, + "loss": 1.3022, + "theoretical_loss": 3.364460072571996, + "tokens_seen": 2827681792 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014461384152457372, + "loss": 1.2448, + "theoretical_loss": 3.364454056332205, + "tokens_seen": 2827747328 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014459378134403208, + "loss": 1.2399, + "theoretical_loss": 3.3644480402708847, + "tokens_seen": 2827812864 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014457372116349047, + "loss": 1.3387, + "theoretical_loss": 3.364442024388026, + "tokens_seen": 2827878400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014455366098294883, + "loss": 1.3847, + "theoretical_loss": 3.36443600868362, + "tokens_seen": 2827943936 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014453360080240722, + "loss": 1.309, + "theoretical_loss": 3.3644299931576565, + "tokens_seen": 2828009472 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001445135406218656, + "loss": 1.2764, + "theoretical_loss": 3.3644239778101266, + "tokens_seen": 2828075008 + }, + { + "epoch": 0.71, + "learning_rate": 0.000144493480441324, + "loss": 1.3503, + "theoretical_loss": 3.3644179626410207, + "tokens_seen": 2828140544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014447342026078236, + "loss": 1.2589, + "theoretical_loss": 3.3644119476503294, + "tokens_seen": 2828206080 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014445336008024072, + "loss": 1.249, + "theoretical_loss": 3.364405932838043, + "tokens_seen": 2828271616 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014443329989969911, + "loss": 1.4068, + "theoretical_loss": 3.3643999182041524, + "tokens_seen": 2828337152 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014441323971915748, + "loss": 1.2353, + "theoretical_loss": 3.364393903748648, + "tokens_seen": 2828402688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014439317953861587, + "loss": 1.2533, + "theoretical_loss": 3.3643878894715207, + "tokens_seen": 2828468224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014437311935807423, + "loss": 1.2319, + "theoretical_loss": 3.364381875372761, + "tokens_seen": 2828533760 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014435305917753262, + "loss": 1.2365, + "theoretical_loss": 3.364375861452359, + "tokens_seen": 2828599296 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014433299899699098, + "loss": 1.2301, + "theoretical_loss": 3.3643698477103055, + "tokens_seen": 2828664832 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014431293881644937, + "loss": 1.2235, + "theoretical_loss": 3.3643638341465913, + "tokens_seen": 2828730368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014429287863590773, + "loss": 1.2787, + "theoretical_loss": 3.364357820761207, + "tokens_seen": 2828795904 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001442728184553661, + "loss": 1.2708, + "theoretical_loss": 3.3643518075541428, + "tokens_seen": 2828861440 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014425275827482448, + "loss": 1.3504, + "theoretical_loss": 3.3643457945253896, + "tokens_seen": 2828926976 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014423269809428284, + "loss": 1.2737, + "theoretical_loss": 3.3643397816749383, + "tokens_seen": 2828992512 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014421263791374123, + "loss": 1.3276, + "theoretical_loss": 3.3643337690027786, + "tokens_seen": 2829058048 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001441925777331996, + "loss": 1.3435, + "theoretical_loss": 3.364327756508902, + "tokens_seen": 2829123584 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014417251755265799, + "loss": 1.2439, + "theoretical_loss": 3.3643217441932984, + "tokens_seen": 2829189120 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014415245737211635, + "loss": 1.4018, + "theoretical_loss": 3.3643157320559585, + "tokens_seen": 2829254656 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014413239719157474, + "loss": 1.2549, + "theoretical_loss": 3.3643097200968732, + "tokens_seen": 2829320192 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001441123370110331, + "loss": 1.2949, + "theoretical_loss": 3.364303708316033, + "tokens_seen": 2829385728 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001440922768304915, + "loss": 1.2714, + "theoretical_loss": 3.364297696713428, + "tokens_seen": 2829451264 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014407221664994985, + "loss": 1.2752, + "theoretical_loss": 3.3642916852890496, + "tokens_seen": 2829516800 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001440521564694082, + "loss": 1.3084, + "theoretical_loss": 3.364285674042888, + "tokens_seen": 2829582336 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1028281, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2568520307540894, + "objective/train/theoretical_loss": 3.3642796629749334, + "objective/train/tokens_used": 1200106976, + "theoretical_loss": 3.3642796629749334, + "tokens_seen": 2829647872 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001440320962888666, + "loss": 1.2582, + "theoretical_loss": 3.3642796629749334, + "tokens_seen": 2829647872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014401203610832496, + "loss": 1.2644, + "theoretical_loss": 3.364273652085177, + "tokens_seen": 2829713408 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014399197592778335, + "loss": 1.1729, + "theoretical_loss": 3.364267641373609, + "tokens_seen": 2829778944 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014397191574724172, + "loss": 1.3068, + "theoretical_loss": 3.36426163084022, + "tokens_seen": 2829844480 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001439518555667001, + "loss": 1.3008, + "theoretical_loss": 3.364255620485001, + "tokens_seen": 2829910016 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014393179538615847, + "loss": 1.3146, + "theoretical_loss": 3.364249610307942, + "tokens_seen": 2829975552 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014391173520561686, + "loss": 1.3191, + "theoretical_loss": 3.3642436003090346, + "tokens_seen": 2830041088 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014389167502507522, + "loss": 1.3142, + "theoretical_loss": 3.364237590488268, + "tokens_seen": 2830106624 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014387161484453358, + "loss": 1.2097, + "theoretical_loss": 3.364231580845634, + "tokens_seen": 2830172160 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014385155466399197, + "loss": 1.3364, + "theoretical_loss": 3.364225571381122, + "tokens_seen": 2830237696 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014383149448345033, + "loss": 1.281, + "theoretical_loss": 3.364219562094724, + "tokens_seen": 2830303232 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014381143430290872, + "loss": 1.3632, + "theoretical_loss": 3.3642135529864294, + "tokens_seen": 2830368768 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014379137412236708, + "loss": 1.3048, + "theoretical_loss": 3.36420754405623, + "tokens_seen": 2830434304 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014377131394182547, + "loss": 1.2389, + "theoretical_loss": 3.364201535304115, + "tokens_seen": 2830499840 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014375125376128386, + "loss": 1.2113, + "theoretical_loss": 3.364195526730076, + "tokens_seen": 2830565376 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014373119358074225, + "loss": 1.2114, + "theoretical_loss": 3.364189518334103, + "tokens_seen": 2830630912 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014371113340020062, + "loss": 1.2416, + "theoretical_loss": 3.364183510116187, + "tokens_seen": 2830696448 + }, + { + "epoch": 0.72, + "learning_rate": 0.000143691073219659, + "loss": 1.2644, + "theoretical_loss": 3.3641775020763185, + "tokens_seen": 2830761984 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014367101303911737, + "loss": 1.3088, + "theoretical_loss": 3.3641714942144882, + "tokens_seen": 2830827520 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014365095285857573, + "loss": 1.214, + "theoretical_loss": 3.3641654865306867, + "tokens_seen": 2830893056 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014363089267803412, + "loss": 1.2747, + "theoretical_loss": 3.364159479024904, + "tokens_seen": 2830958592 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014361083249749248, + "loss": 1.2038, + "theoretical_loss": 3.3641534716971315, + "tokens_seen": 2831024128 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014359077231695087, + "loss": 1.2341, + "theoretical_loss": 3.3641474645473597, + "tokens_seen": 2831089664 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014357071213640923, + "loss": 1.2822, + "theoretical_loss": 3.364141457575579, + "tokens_seen": 2831155200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014355065195586762, + "loss": 1.4534, + "theoretical_loss": 3.3641354507817796, + "tokens_seen": 2831220736 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014353059177532598, + "loss": 1.3247, + "theoretical_loss": 3.3641294441659526, + "tokens_seen": 2831286272 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014351053159478437, + "loss": 1.3032, + "theoretical_loss": 3.3641234377280886, + "tokens_seen": 2831351808 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014349047141424274, + "loss": 1.3182, + "theoretical_loss": 3.3641174314681783, + "tokens_seen": 2831417344 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001434704112337011, + "loss": 1.3308, + "theoretical_loss": 3.364111425386212, + "tokens_seen": 2831482880 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001434503510531595, + "loss": 1.3181, + "theoretical_loss": 3.3641054194821804, + "tokens_seen": 2831548416 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014343029087261785, + "loss": 1.2895, + "theoretical_loss": 3.3640994137560742, + "tokens_seen": 2831613952 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014341023069207624, + "loss": 1.3253, + "theoretical_loss": 3.364093408207884, + "tokens_seen": 2831679488 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001433901705115346, + "loss": 1.3764, + "theoretical_loss": 3.3640874028376007, + "tokens_seen": 2831745024 + }, + { + "epoch": 0.72, + "learning_rate": 0.000143370110330993, + "loss": 1.3683, + "theoretical_loss": 3.364081397645214, + "tokens_seen": 2831810560 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014335005015045135, + "loss": 1.2616, + "theoretical_loss": 3.3640753926307156, + "tokens_seen": 2831876096 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014332998996990974, + "loss": 1.2399, + "theoretical_loss": 3.3640693877940957, + "tokens_seen": 2831941632 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001433099297893681, + "loss": 1.2846, + "theoretical_loss": 3.3640633831353446, + "tokens_seen": 2832007168 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014328986960882647, + "loss": 1.3498, + "theoretical_loss": 3.364057378654453, + "tokens_seen": 2832072704 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014326980942828486, + "loss": 1.3158, + "theoretical_loss": 3.364051374351412, + "tokens_seen": 2832138240 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014324974924774322, + "loss": 1.4228, + "theoretical_loss": 3.3640453702262114, + "tokens_seen": 2832203776 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001432296890672016, + "loss": 1.2375, + "theoretical_loss": 3.364039366278843, + "tokens_seen": 2832269312 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014320962888665997, + "loss": 1.3199, + "theoretical_loss": 3.3640333625092964, + "tokens_seen": 2832334848 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014318956870611836, + "loss": 1.2364, + "theoretical_loss": 3.3640273589175624, + "tokens_seen": 2832400384 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014316950852557672, + "loss": 1.3034, + "theoretical_loss": 3.364021355503632, + "tokens_seen": 2832465920 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001431494483450351, + "loss": 1.2495, + "theoretical_loss": 3.3640153522674954, + "tokens_seen": 2832531456 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014312938816449347, + "loss": 1.255, + "theoretical_loss": 3.3640093492091436, + "tokens_seen": 2832596992 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014310932798395183, + "loss": 1.3292, + "theoretical_loss": 3.3640033463285666, + "tokens_seen": 2832662528 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014308926780341022, + "loss": 1.2612, + "theoretical_loss": 3.3639973436257558, + "tokens_seen": 2832728064 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014306920762286859, + "loss": 1.2908, + "theoretical_loss": 3.3639913411007014, + "tokens_seen": 2832793600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014304914744232698, + "loss": 1.2409, + "theoretical_loss": 3.3639853387533942, + "tokens_seen": 2832859136 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1029660, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4082143306732178, + "objective/train/theoretical_loss": 3.3639793365838244, + "objective/train/tokens_used": 1203383776, + "theoretical_loss": 3.3639793365838244, + "tokens_seen": 2832924672 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014302908726178534, + "loss": 1.3257, + "theoretical_loss": 3.3639793365838244, + "tokens_seen": 2832924672 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014300902708124375, + "loss": 1.2782, + "theoretical_loss": 3.363973334591983, + "tokens_seen": 2832990208 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014298896690070212, + "loss": 1.3576, + "theoretical_loss": 3.363967332777861, + "tokens_seen": 2833055744 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001429689067201605, + "loss": 1.3947, + "theoretical_loss": 3.3639613311414482, + "tokens_seen": 2833121280 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014294884653961887, + "loss": 1.2626, + "theoretical_loss": 3.363955329682736, + "tokens_seen": 2833186816 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014292878635907726, + "loss": 1.2876, + "theoretical_loss": 3.3639493284017146, + "tokens_seen": 2833252352 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014290872617853562, + "loss": 1.2542, + "theoretical_loss": 3.363943327298374, + "tokens_seen": 2833317888 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014288866599799398, + "loss": 1.3092, + "theoretical_loss": 3.3639373263727066, + "tokens_seen": 2833383424 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014286860581745237, + "loss": 1.2629, + "theoretical_loss": 3.363931325624701, + "tokens_seen": 2833448960 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014284854563691073, + "loss": 1.228, + "theoretical_loss": 3.3639253250543493, + "tokens_seen": 2833514496 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014282848545636912, + "loss": 1.3005, + "theoretical_loss": 3.3639193246616417, + "tokens_seen": 2833580032 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014280842527582749, + "loss": 1.2397, + "theoretical_loss": 3.3639133244465684, + "tokens_seen": 2833645568 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014278836509528587, + "loss": 1.2603, + "theoretical_loss": 3.36390732440912, + "tokens_seen": 2833711104 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014276830491474424, + "loss": 1.3287, + "theoretical_loss": 3.3639013245492886, + "tokens_seen": 2833776640 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014274824473420263, + "loss": 1.3638, + "theoretical_loss": 3.363895324867063, + "tokens_seen": 2833842176 + }, + { + "epoch": 0.72, + "learning_rate": 0.000142728184553661, + "loss": 1.3131, + "theoretical_loss": 3.3638893253624347, + "tokens_seen": 2833907712 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014270812437311935, + "loss": 1.2498, + "theoretical_loss": 3.363883326035394, + "tokens_seen": 2833973248 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014268806419257774, + "loss": 1.2757, + "theoretical_loss": 3.363877326885932, + "tokens_seen": 2834038784 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001426680040120361, + "loss": 1.3788, + "theoretical_loss": 3.3638713279140395, + "tokens_seen": 2834104320 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001426479438314945, + "loss": 1.2572, + "theoretical_loss": 3.3638653291197063, + "tokens_seen": 2834169856 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014262788365095285, + "loss": 1.3608, + "theoretical_loss": 3.3638593305029234, + "tokens_seen": 2834235392 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014260782347041124, + "loss": 1.3693, + "theoretical_loss": 3.363853332063681, + "tokens_seen": 2834300928 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001425877632898696, + "loss": 1.3047, + "theoretical_loss": 3.363847333801971, + "tokens_seen": 2834366464 + }, + { + "epoch": 0.72, + "learning_rate": 0.000142567703109328, + "loss": 1.3688, + "theoretical_loss": 3.363841335717783, + "tokens_seen": 2834432000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014254764292878636, + "loss": 1.3609, + "theoretical_loss": 3.3638353378111083, + "tokens_seen": 2834497536 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014252758274824475, + "loss": 1.3068, + "theoretical_loss": 3.363829340081937, + "tokens_seen": 2834563072 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001425075225677031, + "loss": 1.3109, + "theoretical_loss": 3.3638233425302597, + "tokens_seen": 2834628608 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014248746238716147, + "loss": 1.2613, + "theoretical_loss": 3.3638173451560673, + "tokens_seen": 2834694144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014246740220661986, + "loss": 1.3885, + "theoretical_loss": 3.3638113479593508, + "tokens_seen": 2834759680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014244734202607822, + "loss": 1.3265, + "theoretical_loss": 3.3638053509401002, + "tokens_seen": 2834825216 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001424272818455366, + "loss": 1.3495, + "theoretical_loss": 3.363799354098306, + "tokens_seen": 2834890752 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014240722166499497, + "loss": 1.321, + "theoretical_loss": 3.3637933574339596, + "tokens_seen": 2834956288 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014238716148445336, + "loss": 1.2629, + "theoretical_loss": 3.363787360947051, + "tokens_seen": 2835021824 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014236710130391173, + "loss": 1.3567, + "theoretical_loss": 3.3637813646375716, + "tokens_seen": 2835087360 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014234704112337011, + "loss": 1.3325, + "theoretical_loss": 3.3637753685055114, + "tokens_seen": 2835152896 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014232698094282848, + "loss": 1.2738, + "theoretical_loss": 3.3637693725508613, + "tokens_seen": 2835218432 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014230692076228684, + "loss": 1.3108, + "theoretical_loss": 3.363763376773612, + "tokens_seen": 2835283968 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014228686058174523, + "loss": 1.3203, + "theoretical_loss": 3.3637573811737536, + "tokens_seen": 2835349504 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014226680040120362, + "loss": 1.3103, + "theoretical_loss": 3.3637513857512773, + "tokens_seen": 2835415040 + }, + { + "epoch": 0.72, + "learning_rate": 0.000142246740220662, + "loss": 1.3049, + "theoretical_loss": 3.363745390506174, + "tokens_seen": 2835480576 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014222668004012037, + "loss": 1.2377, + "theoretical_loss": 3.363739395438434, + "tokens_seen": 2835546112 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014220661985957876, + "loss": 1.286, + "theoretical_loss": 3.363733400548047, + "tokens_seen": 2835611648 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014218655967903712, + "loss": 1.4105, + "theoretical_loss": 3.3637274058350055, + "tokens_seen": 2835677184 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001421664994984955, + "loss": 1.1648, + "theoretical_loss": 3.363721411299299, + "tokens_seen": 2835742720 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014214643931795387, + "loss": 1.2669, + "theoretical_loss": 3.3637154169409187, + "tokens_seen": 2835808256 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014212637913741223, + "loss": 1.257, + "theoretical_loss": 3.3637094227598547, + "tokens_seen": 2835873792 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014210631895687062, + "loss": 1.3676, + "theoretical_loss": 3.363703428756098, + "tokens_seen": 2835939328 + }, + { + "epoch": 0.72, + "learning_rate": 0.000142086258776329, + "loss": 1.2888, + "theoretical_loss": 3.363697434929639, + "tokens_seen": 2836004864 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014206619859578738, + "loss": 1.2788, + "theoretical_loss": 3.3636914412804684, + "tokens_seen": 2836070400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014204613841524574, + "loss": 1.2499, + "theoretical_loss": 3.3636854478085776, + "tokens_seen": 2836135936 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1030811, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4940600395202637, + "objective/train/theoretical_loss": 3.3636794545139566, + "objective/train/tokens_used": 1206660576, + "theoretical_loss": 3.3636794545139566, + "tokens_seen": 2836201472 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014202607823470413, + "loss": 1.4282, + "theoretical_loss": 3.3636794545139566, + "tokens_seen": 2836201472 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001420060180541625, + "loss": 1.2166, + "theoretical_loss": 3.3636734613965955, + "tokens_seen": 2836267008 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014198595787362088, + "loss": 1.2767, + "theoretical_loss": 3.363667468456486, + "tokens_seen": 2836332544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014196589769307924, + "loss": 1.3173, + "theoretical_loss": 3.3636614756936183, + "tokens_seen": 2836398080 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014194583751253763, + "loss": 1.2247, + "theoretical_loss": 3.363655483107983, + "tokens_seen": 2836463616 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141925777331996, + "loss": 1.2578, + "theoretical_loss": 3.3636494906995713, + "tokens_seen": 2836529152 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014190571715145436, + "loss": 1.2593, + "theoretical_loss": 3.363643498468373, + "tokens_seen": 2836594688 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014188565697091274, + "loss": 1.3135, + "theoretical_loss": 3.363637506414379, + "tokens_seen": 2836660224 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001418655967903711, + "loss": 1.3631, + "theoretical_loss": 3.363631514537581, + "tokens_seen": 2836725760 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001418455366098295, + "loss": 1.4396, + "theoretical_loss": 3.363625522837968, + "tokens_seen": 2836791296 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014182547642928786, + "loss": 1.3117, + "theoretical_loss": 3.363619531315532, + "tokens_seen": 2836856832 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014180541624874625, + "loss": 1.3365, + "theoretical_loss": 3.363613539970263, + "tokens_seen": 2836922368 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001417853560682046, + "loss": 1.2605, + "theoretical_loss": 3.3636075488021517, + "tokens_seen": 2836987904 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141765295887663, + "loss": 1.3549, + "theoretical_loss": 3.3636015578111893, + "tokens_seen": 2837053440 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014174523570712136, + "loss": 1.3072, + "theoretical_loss": 3.3635955669973656, + "tokens_seen": 2837118976 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014172517552657972, + "loss": 1.1947, + "theoretical_loss": 3.3635895763606722, + "tokens_seen": 2837184512 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001417051153460381, + "loss": 1.3025, + "theoretical_loss": 3.363583585901099, + "tokens_seen": 2837250048 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014168505516549648, + "loss": 1.3814, + "theoretical_loss": 3.363577595618637, + "tokens_seen": 2837315584 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014166499498495486, + "loss": 1.2895, + "theoretical_loss": 3.363571605513277, + "tokens_seen": 2837381120 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014164493480441323, + "loss": 1.2411, + "theoretical_loss": 3.3635656155850095, + "tokens_seen": 2837446656 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014162487462387162, + "loss": 1.2486, + "theoretical_loss": 3.363559625833825, + "tokens_seen": 2837512192 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014160481444332998, + "loss": 1.2915, + "theoretical_loss": 3.3635536362597147, + "tokens_seen": 2837577728 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014158475426278837, + "loss": 1.3183, + "theoretical_loss": 3.363547646862669, + "tokens_seen": 2837643264 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014156469408224673, + "loss": 1.2194, + "theoretical_loss": 3.3635416576426787, + "tokens_seen": 2837708800 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001415446339017051, + "loss": 1.2664, + "theoretical_loss": 3.3635356685997335, + "tokens_seen": 2837774336 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014152457372116348, + "loss": 1.3235, + "theoretical_loss": 3.3635296797338254, + "tokens_seen": 2837839872 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014150451354062187, + "loss": 1.4048, + "theoretical_loss": 3.3635236910449446, + "tokens_seen": 2837905408 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014148445336008026, + "loss": 1.2698, + "theoretical_loss": 3.3635177025330822, + "tokens_seen": 2837970944 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014146439317953862, + "loss": 1.3681, + "theoretical_loss": 3.3635117141982276, + "tokens_seen": 2838036480 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141444332998997, + "loss": 1.292, + "theoretical_loss": 3.3635057260403727, + "tokens_seen": 2838102016 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014142427281845537, + "loss": 1.3044, + "theoretical_loss": 3.363499738059508, + "tokens_seen": 2838167552 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014140421263791376, + "loss": 1.1697, + "theoretical_loss": 3.363493750255624, + "tokens_seen": 2838233088 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014138415245737213, + "loss": 1.2295, + "theoretical_loss": 3.363487762628711, + "tokens_seen": 2838298624 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014136409227683052, + "loss": 1.2287, + "theoretical_loss": 3.36348177517876, + "tokens_seen": 2838364160 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014134403209628888, + "loss": 1.292, + "theoretical_loss": 3.363475787905762, + "tokens_seen": 2838429696 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014132397191574724, + "loss": 1.1975, + "theoretical_loss": 3.3634698008097077, + "tokens_seen": 2838495232 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014130391173520563, + "loss": 1.3534, + "theoretical_loss": 3.363463813890587, + "tokens_seen": 2838560768 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141283851554664, + "loss": 1.3025, + "theoretical_loss": 3.363457827148391, + "tokens_seen": 2838626304 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014126379137412238, + "loss": 1.1783, + "theoretical_loss": 3.3634518405831106, + "tokens_seen": 2838691840 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014124373119358074, + "loss": 1.1886, + "theoretical_loss": 3.3634458541947367, + "tokens_seen": 2838757376 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014122367101303913, + "loss": 1.2712, + "theoretical_loss": 3.363439867983259, + "tokens_seen": 2838822912 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001412036108324975, + "loss": 1.2046, + "theoretical_loss": 3.3634338819486693, + "tokens_seen": 2838888448 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014118355065195588, + "loss": 1.2057, + "theoretical_loss": 3.363427896090958, + "tokens_seen": 2838953984 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014116349047141425, + "loss": 1.3167, + "theoretical_loss": 3.3634219104101155, + "tokens_seen": 2839019520 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001411434302908726, + "loss": 1.2536, + "theoretical_loss": 3.3634159249061324, + "tokens_seen": 2839085056 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141123370110331, + "loss": 1.3797, + "theoretical_loss": 3.363409939579, + "tokens_seen": 2839150592 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014110330992978936, + "loss": 1.2651, + "theoretical_loss": 3.363403954428708, + "tokens_seen": 2839216128 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014108324974924775, + "loss": 1.2573, + "theoretical_loss": 3.363397969455248, + "tokens_seen": 2839281664 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001410631895687061, + "loss": 1.2693, + "theoretical_loss": 3.3633919846586102, + "tokens_seen": 2839347200 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001410431293881645, + "loss": 1.3165, + "theoretical_loss": 3.3633860000387856, + "tokens_seen": 2839412736 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1031989, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3026928901672363, + "objective/train/theoretical_loss": 3.363380015595765, + "objective/train/tokens_used": 1209937376, + "theoretical_loss": 3.363380015595765, + "tokens_seen": 2839478272 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014102306920762286, + "loss": 1.3013, + "theoretical_loss": 3.363380015595765, + "tokens_seen": 2839478272 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014100300902708125, + "loss": 1.2961, + "theoretical_loss": 3.3633740313295384, + "tokens_seen": 2839543808 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014098294884653961, + "loss": 1.3192, + "theoretical_loss": 3.3633680472400975, + "tokens_seen": 2839609344 + }, + { + "epoch": 0.72, + "learning_rate": 0.000140962888665998, + "loss": 1.2157, + "theoretical_loss": 3.363362063327432, + "tokens_seen": 2839674880 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014094282848545637, + "loss": 1.2511, + "theoretical_loss": 3.3633560795915334, + "tokens_seen": 2839740416 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014092276830491473, + "loss": 1.3161, + "theoretical_loss": 3.363350096032392, + "tokens_seen": 2839805952 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014090270812437312, + "loss": 1.2354, + "theoretical_loss": 3.3633441126499983, + "tokens_seen": 2839871488 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014088264794383148, + "loss": 1.1827, + "theoretical_loss": 3.3633381294443434, + "tokens_seen": 2839937024 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014086258776328987, + "loss": 1.2672, + "theoretical_loss": 3.3633321464154182, + "tokens_seen": 2840002560 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014084252758274823, + "loss": 1.2759, + "theoretical_loss": 3.3633261635632126, + "tokens_seen": 2840068096 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014082246740220662, + "loss": 1.4819, + "theoretical_loss": 3.363320180887718, + "tokens_seen": 2840133632 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014080240722166498, + "loss": 1.2415, + "theoretical_loss": 3.3633141983889248, + "tokens_seen": 2840199168 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014078234704112337, + "loss": 1.3209, + "theoretical_loss": 3.363308216066824, + "tokens_seen": 2840264704 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014076228686058176, + "loss": 1.3605, + "theoretical_loss": 3.363302233921406, + "tokens_seen": 2840330240 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014074222668004012, + "loss": 1.3092, + "theoretical_loss": 3.363296251952661, + "tokens_seen": 2840395776 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001407221664994985, + "loss": 1.2424, + "theoretical_loss": 3.3632902701605807, + "tokens_seen": 2840461312 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014070210631895688, + "loss": 1.2125, + "theoretical_loss": 3.3632842885451555, + "tokens_seen": 2840526848 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014068204613841526, + "loss": 1.3118, + "theoretical_loss": 3.363278307106376, + "tokens_seen": 2840592384 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014066198595787363, + "loss": 1.2292, + "theoretical_loss": 3.3632723258442327, + "tokens_seen": 2840657920 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014064192577733202, + "loss": 1.3743, + "theoretical_loss": 3.3632663447587166, + "tokens_seen": 2840723456 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014062186559679038, + "loss": 1.2437, + "theoretical_loss": 3.363260363849818, + "tokens_seen": 2840788992 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014060180541624877, + "loss": 1.2272, + "theoretical_loss": 3.363254383117529, + "tokens_seen": 2840854528 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014058174523570713, + "loss": 1.2499, + "theoretical_loss": 3.3632484025618385, + "tokens_seen": 2840920064 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001405616850551655, + "loss": 1.2195, + "theoretical_loss": 3.3632424221827377, + "tokens_seen": 2840985600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014054162487462388, + "loss": 1.2881, + "theoretical_loss": 3.363236441980218, + "tokens_seen": 2841051136 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014052156469408224, + "loss": 1.2667, + "theoretical_loss": 3.3632304619542697, + "tokens_seen": 2841116672 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014050150451354063, + "loss": 1.3998, + "theoretical_loss": 3.3632244821048833, + "tokens_seen": 2841182208 + }, + { + "epoch": 0.72, + "learning_rate": 0.000140481444332999, + "loss": 1.3785, + "theoretical_loss": 3.36321850243205, + "tokens_seen": 2841247744 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014046138415245739, + "loss": 1.2861, + "theoretical_loss": 3.36321252293576, + "tokens_seen": 2841313280 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014044132397191575, + "loss": 1.3191, + "theoretical_loss": 3.363206543616004, + "tokens_seen": 2841378816 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014042126379137414, + "loss": 1.3239, + "theoretical_loss": 3.3632005644727734, + "tokens_seen": 2841444352 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001404012036108325, + "loss": 1.3063, + "theoretical_loss": 3.3631945855060583, + "tokens_seen": 2841509888 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001403811434302909, + "loss": 1.3731, + "theoretical_loss": 3.3631886067158496, + "tokens_seen": 2841575424 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014036108324974925, + "loss": 1.2601, + "theoretical_loss": 3.363182628102138, + "tokens_seen": 2841640960 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001403410230692076, + "loss": 1.1639, + "theoretical_loss": 3.3631766496649145, + "tokens_seen": 2841706496 + }, + { + "epoch": 0.72, + "learning_rate": 0.000140320962888666, + "loss": 1.3155, + "theoretical_loss": 3.3631706714041694, + "tokens_seen": 2841772032 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014030090270812436, + "loss": 1.298, + "theoretical_loss": 3.3631646933198933, + "tokens_seen": 2841837568 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014028084252758275, + "loss": 1.2927, + "theoretical_loss": 3.3631587154120774, + "tokens_seen": 2841903104 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014026078234704112, + "loss": 1.241, + "theoretical_loss": 3.3631527376807124, + "tokens_seen": 2841968640 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001402407221664995, + "loss": 1.2656, + "theoretical_loss": 3.3631467601257885, + "tokens_seen": 2842034176 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014022066198595787, + "loss": 1.2658, + "theoretical_loss": 3.3631407827472968, + "tokens_seen": 2842099712 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014020060180541626, + "loss": 1.3889, + "theoretical_loss": 3.3631348055452284, + "tokens_seen": 2842165248 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014018054162487462, + "loss": 1.2931, + "theoretical_loss": 3.3631288285195735, + "tokens_seen": 2842230784 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014016048144433298, + "loss": 1.3352, + "theoretical_loss": 3.3631228516703224, + "tokens_seen": 2842296320 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014014042126379137, + "loss": 1.2629, + "theoretical_loss": 3.363116874997467, + "tokens_seen": 2842361856 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014012036108324973, + "loss": 1.3733, + "theoretical_loss": 3.363110898500997, + "tokens_seen": 2842427392 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014010030090270812, + "loss": 1.3488, + "theoretical_loss": 3.363104922180904, + "tokens_seen": 2842492928 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014008024072216648, + "loss": 1.2476, + "theoretical_loss": 3.363098946037178, + "tokens_seen": 2842558464 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014006018054162487, + "loss": 1.3065, + "theoretical_loss": 3.36309297006981, + "tokens_seen": 2842624000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014004012036108324, + "loss": 1.254, + "theoretical_loss": 3.3630869942787904, + "tokens_seen": 2842689536 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1033097, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.215433120727539, + "objective/train/theoretical_loss": 3.3630810186641105, + "objective/train/tokens_used": 1213214176, + "theoretical_loss": 3.3630810186641105, + "tokens_seen": 2842755072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014002006018054163, + "loss": 1.27, + "theoretical_loss": 3.3630810186641105, + "tokens_seen": 2842755072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014000000000000001, + "loss": 1.2988, + "theoretical_loss": 3.363075043225761, + "tokens_seen": 2842820608 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013997993981945838, + "loss": 1.2765, + "theoretical_loss": 3.363069067963732, + "tokens_seen": 2842886144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013995987963891677, + "loss": 1.2565, + "theoretical_loss": 3.363063092878015, + "tokens_seen": 2842951680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013993981945837513, + "loss": 1.3236, + "theoretical_loss": 3.3630571179685997, + "tokens_seen": 2843017216 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013991975927783352, + "loss": 1.3084, + "theoretical_loss": 3.363051143235478, + "tokens_seen": 2843082752 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013989969909729188, + "loss": 1.1973, + "theoretical_loss": 3.3630451686786405, + "tokens_seen": 2843148288 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013987963891675027, + "loss": 1.2498, + "theoretical_loss": 3.363039194298077, + "tokens_seen": 2843213824 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013985957873620863, + "loss": 1.3285, + "theoretical_loss": 3.363033220093779, + "tokens_seen": 2843279360 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013983951855566702, + "loss": 1.2088, + "theoretical_loss": 3.363027246065737, + "tokens_seen": 2843344896 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013981945837512538, + "loss": 1.3011, + "theoretical_loss": 3.363021272213942, + "tokens_seen": 2843410432 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013979939819458377, + "loss": 1.2459, + "theoretical_loss": 3.363015298538384, + "tokens_seen": 2843475968 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013977933801404214, + "loss": 1.3271, + "theoretical_loss": 3.3630093250390547, + "tokens_seen": 2843541504 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001397592778335005, + "loss": 1.2324, + "theoretical_loss": 3.3630033517159443, + "tokens_seen": 2843607040 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001397392176529589, + "loss": 1.368, + "theoretical_loss": 3.3629973785690437, + "tokens_seen": 2843672576 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013971915747241725, + "loss": 1.244, + "theoretical_loss": 3.362991405598344, + "tokens_seen": 2843738112 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013969909729187564, + "loss": 1.2566, + "theoretical_loss": 3.3629854328038347, + "tokens_seen": 2843803648 + }, + { + "epoch": 0.72, + "learning_rate": 0.000139679037111334, + "loss": 1.1893, + "theoretical_loss": 3.3629794601855076, + "tokens_seen": 2843869184 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001396589769307924, + "loss": 1.2513, + "theoretical_loss": 3.3629734877433535, + "tokens_seen": 2843934720 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013963891675025075, + "loss": 1.2606, + "theoretical_loss": 3.362967515477363, + "tokens_seen": 2844000256 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013961885656970914, + "loss": 1.3295, + "theoretical_loss": 3.3629615433875264, + "tokens_seen": 2844065792 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001395987963891675, + "loss": 1.2431, + "theoretical_loss": 3.362955571473835, + "tokens_seen": 2844131328 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013957873620862587, + "loss": 1.324, + "theoretical_loss": 3.362949599736279, + "tokens_seen": 2844196864 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013955867602808426, + "loss": 1.3429, + "theoretical_loss": 3.3629436281748495, + "tokens_seen": 2844262400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013953861584754262, + "loss": 1.2445, + "theoretical_loss": 3.362937656789537, + "tokens_seen": 2844327936 + }, + { + "epoch": 0.72, + "learning_rate": 0.000139518555667001, + "loss": 1.3079, + "theoretical_loss": 3.3629316855803326, + "tokens_seen": 2844393472 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013949849548645937, + "loss": 1.3129, + "theoretical_loss": 3.362925714547227, + "tokens_seen": 2844459008 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013947843530591776, + "loss": 1.2524, + "theoretical_loss": 3.3629197436902105, + "tokens_seen": 2844524544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013945837512537612, + "loss": 1.2287, + "theoretical_loss": 3.3629137730092746, + "tokens_seen": 2844590080 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001394383149448345, + "loss": 1.3457, + "theoretical_loss": 3.3629078025044095, + "tokens_seen": 2844655616 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013941825476429287, + "loss": 1.2748, + "theoretical_loss": 3.362901832175606, + "tokens_seen": 2844721152 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013939819458375123, + "loss": 1.2505, + "theoretical_loss": 3.362895862022855, + "tokens_seen": 2844786688 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013937813440320962, + "loss": 1.2279, + "theoretical_loss": 3.362889892046147, + "tokens_seen": 2844852224 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013935807422266799, + "loss": 1.2297, + "theoretical_loss": 3.3628839222454734, + "tokens_seen": 2844917760 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013933801404212638, + "loss": 1.3033, + "theoretical_loss": 3.362877952620824, + "tokens_seen": 2844983296 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013931795386158474, + "loss": 1.2289, + "theoretical_loss": 3.3628719831721905, + "tokens_seen": 2845048832 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013929789368104313, + "loss": 1.3884, + "theoretical_loss": 3.3628660138995627, + "tokens_seen": 2845114368 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001392778335005015, + "loss": 1.301, + "theoretical_loss": 3.3628600448029324, + "tokens_seen": 2845179904 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001392577733199599, + "loss": 1.293, + "theoretical_loss": 3.3628540758822894, + "tokens_seen": 2845245440 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013923771313941827, + "loss": 1.2973, + "theoretical_loss": 3.3628481071376255, + "tokens_seen": 2845310976 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013921765295887666, + "loss": 1.1985, + "theoretical_loss": 3.36284213856893, + "tokens_seen": 2845376512 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013919759277833502, + "loss": 1.261, + "theoretical_loss": 3.362836170176195, + "tokens_seen": 2845442048 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013917753259779338, + "loss": 1.2936, + "theoretical_loss": 3.3628302019594107, + "tokens_seen": 2845507584 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013915747241725177, + "loss": 1.2859, + "theoretical_loss": 3.362824233918568, + "tokens_seen": 2845573120 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013913741223671013, + "loss": 1.2863, + "theoretical_loss": 3.362818266053657, + "tokens_seen": 2845638656 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013911735205616852, + "loss": 1.2593, + "theoretical_loss": 3.3628122983646698, + "tokens_seen": 2845704192 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013909729187562688, + "loss": 1.2767, + "theoretical_loss": 3.3628063308515963, + "tokens_seen": 2845769728 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013907723169508527, + "loss": 1.2823, + "theoretical_loss": 3.362800363514427, + "tokens_seen": 2845835264 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013905717151454364, + "loss": 1.2502, + "theoretical_loss": 3.3627943963531535, + "tokens_seen": 2845900800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013903711133400203, + "loss": 1.2711, + "theoretical_loss": 3.3627884293677655, + "tokens_seen": 2845966336 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1034283, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.356002926826477, + "objective/train/theoretical_loss": 3.362782462558255, + "objective/train/tokens_used": 1216490976, + "theoretical_loss": 3.362782462558255, + "tokens_seen": 2846031872 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001390170511534604, + "loss": 1.382, + "theoretical_loss": 3.362782462558255, + "tokens_seen": 2846031872 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013899699097291875, + "loss": 1.2543, + "theoretical_loss": 3.362776495924612, + "tokens_seen": 2846097408 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013897693079237714, + "loss": 1.3108, + "theoretical_loss": 3.362770529466827, + "tokens_seen": 2846162944 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001389568706118355, + "loss": 1.1802, + "theoretical_loss": 3.362764563184891, + "tokens_seen": 2846228480 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001389368104312939, + "loss": 1.2515, + "theoretical_loss": 3.3627585970787957, + "tokens_seen": 2846294016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013891675025075225, + "loss": 1.277, + "theoretical_loss": 3.3627526311485307, + "tokens_seen": 2846359552 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013889669007021064, + "loss": 1.312, + "theoretical_loss": 3.362746665394087, + "tokens_seen": 2846425088 + }, + { + "epoch": 0.73, + "learning_rate": 0.000138876629889669, + "loss": 1.2044, + "theoretical_loss": 3.362740699815456, + "tokens_seen": 2846490624 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001388565697091274, + "loss": 1.2223, + "theoretical_loss": 3.3627347344126277, + "tokens_seen": 2846556160 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013883650952858576, + "loss": 1.3408, + "theoretical_loss": 3.362728769185593, + "tokens_seen": 2846621696 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013881644934804415, + "loss": 1.2945, + "theoretical_loss": 3.3627228041343433, + "tokens_seen": 2846687232 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001387963891675025, + "loss": 1.302, + "theoretical_loss": 3.3627168392588684, + "tokens_seen": 2846752768 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013877632898696087, + "loss": 1.2981, + "theoretical_loss": 3.36271087455916, + "tokens_seen": 2846818304 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013875626880641926, + "loss": 1.2763, + "theoretical_loss": 3.362704910035209, + "tokens_seen": 2846883840 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013873620862587762, + "loss": 1.2566, + "theoretical_loss": 3.3626989456870047, + "tokens_seen": 2846949376 + }, + { + "epoch": 0.73, + "learning_rate": 0.000138716148445336, + "loss": 1.2129, + "theoretical_loss": 3.362692981514539, + "tokens_seen": 2847014912 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013869608826479437, + "loss": 1.3388, + "theoretical_loss": 3.362687017517803, + "tokens_seen": 2847080448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013867602808425276, + "loss": 1.2517, + "theoretical_loss": 3.362681053696787, + "tokens_seen": 2847145984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013865596790371113, + "loss": 1.1954, + "theoretical_loss": 3.3626750900514812, + "tokens_seen": 2847211520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013863590772316951, + "loss": 1.2782, + "theoretical_loss": 3.362669126581877, + "tokens_seen": 2847277056 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013861584754262788, + "loss": 1.3283, + "theoretical_loss": 3.3626631632879653, + "tokens_seen": 2847342592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013859578736208624, + "loss": 1.3451, + "theoretical_loss": 3.362657200169737, + "tokens_seen": 2847408128 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013857572718154463, + "loss": 1.3019, + "theoretical_loss": 3.3626512372271824, + "tokens_seen": 2847473664 + }, + { + "epoch": 0.73, + "learning_rate": 0.000138555667001003, + "loss": 1.3031, + "theoretical_loss": 3.3626452744602924, + "tokens_seen": 2847539200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013853560682046138, + "loss": 1.2594, + "theoretical_loss": 3.3626393118690574, + "tokens_seen": 2847604736 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013851554663991974, + "loss": 1.2217, + "theoretical_loss": 3.3626333494534695, + "tokens_seen": 2847670272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013849548645937816, + "loss": 1.29, + "theoretical_loss": 3.362627387213518, + "tokens_seen": 2847735808 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013847542627883652, + "loss": 1.2655, + "theoretical_loss": 3.3626214251491944, + "tokens_seen": 2847801344 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001384553660982949, + "loss": 1.2121, + "theoretical_loss": 3.3626154632604894, + "tokens_seen": 2847866880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013843530591775327, + "loss": 1.265, + "theoretical_loss": 3.362609501547394, + "tokens_seen": 2847932416 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013841524573721163, + "loss": 1.257, + "theoretical_loss": 3.3626035400098986, + "tokens_seen": 2847997952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013839518555667002, + "loss": 1.2857, + "theoretical_loss": 3.362597578647994, + "tokens_seen": 2848063488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013837512537612839, + "loss": 1.2495, + "theoretical_loss": 3.3625916174616712, + "tokens_seen": 2848129024 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013835506519558678, + "loss": 1.2006, + "theoretical_loss": 3.362585656450921, + "tokens_seen": 2848194560 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013833500501504514, + "loss": 1.1878, + "theoretical_loss": 3.362579695615734, + "tokens_seen": 2848260096 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013831494483450353, + "loss": 1.2493, + "theoretical_loss": 3.3625737349561016, + "tokens_seen": 2848325632 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001382948846539619, + "loss": 1.218, + "theoretical_loss": 3.3625677744720135, + "tokens_seen": 2848391168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013827482447342028, + "loss": 1.3153, + "theoretical_loss": 3.362561814163461, + "tokens_seen": 2848456704 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013825476429287864, + "loss": 1.2687, + "theoretical_loss": 3.362555854030435, + "tokens_seen": 2848522240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013823470411233703, + "loss": 1.2474, + "theoretical_loss": 3.362549894072927, + "tokens_seen": 2848587776 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001382146439317954, + "loss": 1.4171, + "theoretical_loss": 3.3625439342909265, + "tokens_seen": 2848653312 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013819458375125375, + "loss": 1.324, + "theoretical_loss": 3.362537974684425, + "tokens_seen": 2848718848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013817452357071214, + "loss": 1.3141, + "theoretical_loss": 3.3625320152534126, + "tokens_seen": 2848784384 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001381544633901705, + "loss": 1.279, + "theoretical_loss": 3.3625260559978813, + "tokens_seen": 2848849920 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001381344032096289, + "loss": 1.3671, + "theoretical_loss": 3.3625200969178213, + "tokens_seen": 2848915456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013811434302908726, + "loss": 1.2466, + "theoretical_loss": 3.362514138013223, + "tokens_seen": 2848980992 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013809428284854565, + "loss": 1.267, + "theoretical_loss": 3.3625081792840774, + "tokens_seen": 2849046528 + }, + { + "epoch": 0.73, + "learning_rate": 0.000138074222668004, + "loss": 1.2559, + "theoretical_loss": 3.362502220730376, + "tokens_seen": 2849112064 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001380541624874624, + "loss": 1.2957, + "theoretical_loss": 3.362496262352108, + "tokens_seen": 2849177600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013803410230692076, + "loss": 1.3104, + "theoretical_loss": 3.3624903041492664, + "tokens_seen": 2849243136 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1035566, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.132224678993225, + "objective/train/theoretical_loss": 3.3624843461218403, + "objective/train/tokens_used": 1219767776, + "theoretical_loss": 3.3624843461218403, + "tokens_seen": 2849308672 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013801404212637912, + "loss": 1.2302, + "theoretical_loss": 3.3624843461218403, + "tokens_seen": 2849308672 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001379939819458375, + "loss": 1.2984, + "theoretical_loss": 3.362478388269821, + "tokens_seen": 2849374208 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013797392176529588, + "loss": 1.2868, + "theoretical_loss": 3.3624724305931992, + "tokens_seen": 2849439744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013795386158475426, + "loss": 1.2623, + "theoretical_loss": 3.362466473091966, + "tokens_seen": 2849505280 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013793380140421263, + "loss": 1.3412, + "theoretical_loss": 3.3624605157661125, + "tokens_seen": 2849570816 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013791374122367102, + "loss": 1.2942, + "theoretical_loss": 3.3624545586156285, + "tokens_seen": 2849636352 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013789368104312938, + "loss": 1.2638, + "theoretical_loss": 3.3624486016405055, + "tokens_seen": 2849701888 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013787362086258777, + "loss": 1.2946, + "theoretical_loss": 3.362442644840734, + "tokens_seen": 2849767424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013785356068204613, + "loss": 1.1851, + "theoretical_loss": 3.362436688216305, + "tokens_seen": 2849832960 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001378335005015045, + "loss": 1.4534, + "theoretical_loss": 3.3624307317672093, + "tokens_seen": 2849898496 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013781344032096288, + "loss": 1.2632, + "theoretical_loss": 3.3624247754934378, + "tokens_seen": 2849964032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013779338014042124, + "loss": 1.2205, + "theoretical_loss": 3.362418819394981, + "tokens_seen": 2850029568 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013777331995987963, + "loss": 1.2862, + "theoretical_loss": 3.36241286347183, + "tokens_seen": 2850095104 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013775325977933802, + "loss": 1.2347, + "theoretical_loss": 3.3624069077239755, + "tokens_seen": 2850160640 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001377331995987964, + "loss": 1.3118, + "theoretical_loss": 3.3624009521514084, + "tokens_seen": 2850226176 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013771313941825477, + "loss": 1.2523, + "theoretical_loss": 3.3623949967541193, + "tokens_seen": 2850291712 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013769307923771316, + "loss": 1.2328, + "theoretical_loss": 3.362389041532099, + "tokens_seen": 2850357248 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013767301905717153, + "loss": 1.3733, + "theoretical_loss": 3.362383086485339, + "tokens_seen": 2850422784 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013765295887662991, + "loss": 1.2517, + "theoretical_loss": 3.3623771316138287, + "tokens_seen": 2850488320 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013763289869608828, + "loss": 1.333, + "theoretical_loss": 3.36237117691756, + "tokens_seen": 2850553856 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013761283851554664, + "loss": 1.294, + "theoretical_loss": 3.362365222396524, + "tokens_seen": 2850619392 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013759277833500503, + "loss": 1.3312, + "theoretical_loss": 3.3623592680507106, + "tokens_seen": 2850684928 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001375727181544634, + "loss": 1.3046, + "theoretical_loss": 3.3623533138801114, + "tokens_seen": 2850750464 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013755265797392178, + "loss": 1.3821, + "theoretical_loss": 3.3623473598847164, + "tokens_seen": 2850816000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013753259779338014, + "loss": 1.2815, + "theoretical_loss": 3.362341406064517, + "tokens_seen": 2850881536 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013751253761283853, + "loss": 1.3055, + "theoretical_loss": 3.362335452419504, + "tokens_seen": 2850947072 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001374924774322969, + "loss": 1.2355, + "theoretical_loss": 3.3623294989496677, + "tokens_seen": 2851012608 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013747241725175528, + "loss": 1.2259, + "theoretical_loss": 3.362323545655, + "tokens_seen": 2851078144 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013745235707121365, + "loss": 1.2028, + "theoretical_loss": 3.36231759253549, + "tokens_seen": 2851143680 + }, + { + "epoch": 0.73, + "learning_rate": 0.000137432296890672, + "loss": 1.3734, + "theoretical_loss": 3.3623116395911303, + "tokens_seen": 2851209216 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001374122367101304, + "loss": 1.2446, + "theoretical_loss": 3.362305686821911, + "tokens_seen": 2851274752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013739217652958876, + "loss": 1.3479, + "theoretical_loss": 3.3622997342278227, + "tokens_seen": 2851340288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013737211634904715, + "loss": 1.3305, + "theoretical_loss": 3.362293781808856, + "tokens_seen": 2851405824 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001373520561685055, + "loss": 1.2962, + "theoretical_loss": 3.3622878295650027, + "tokens_seen": 2851471360 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001373319959879639, + "loss": 1.4244, + "theoretical_loss": 3.362281877496253, + "tokens_seen": 2851536896 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013731193580742226, + "loss": 1.3285, + "theoretical_loss": 3.3622759256025976, + "tokens_seen": 2851602432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013729187562688065, + "loss": 1.3128, + "theoretical_loss": 3.362269973884027, + "tokens_seen": 2851667968 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013727181544633901, + "loss": 1.251, + "theoretical_loss": 3.3622640223405336, + "tokens_seen": 2851733504 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013725175526579738, + "loss": 1.1932, + "theoretical_loss": 3.3622580709721066, + "tokens_seen": 2851799040 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013723169508525577, + "loss": 1.1893, + "theoretical_loss": 3.3622521197787374, + "tokens_seen": 2851864576 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013721163490471413, + "loss": 1.3322, + "theoretical_loss": 3.3622461687604166, + "tokens_seen": 2851930112 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013719157472417252, + "loss": 1.2787, + "theoretical_loss": 3.362240217917136, + "tokens_seen": 2851995648 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013717151454363088, + "loss": 1.3186, + "theoretical_loss": 3.3622342672488847, + "tokens_seen": 2852061184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013715145436308927, + "loss": 1.3197, + "theoretical_loss": 3.362228316755655, + "tokens_seen": 2852126720 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013713139418254763, + "loss": 1.3807, + "theoretical_loss": 3.3622223664374373, + "tokens_seen": 2852192256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013711133400200602, + "loss": 1.3125, + "theoretical_loss": 3.3622164162942223, + "tokens_seen": 2852257792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013709127382146438, + "loss": 1.1432, + "theoretical_loss": 3.3622104663260006, + "tokens_seen": 2852323328 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013707121364092277, + "loss": 1.2479, + "theoretical_loss": 3.3622045165327634, + "tokens_seen": 2852388864 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013705115346038113, + "loss": 1.2258, + "theoretical_loss": 3.362198566914502, + "tokens_seen": 2852454400 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001370310932798395, + "loss": 1.3217, + "theoretical_loss": 3.362192617471206, + "tokens_seen": 2852519936 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1036772, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2533670663833618, + "objective/train/theoretical_loss": 3.3621866682028676, + "objective/train/tokens_used": 1223044576, + "theoretical_loss": 3.3621866682028676, + "tokens_seen": 2852585472 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001370110330992979, + "loss": 1.2242, + "theoretical_loss": 3.3621866682028676, + "tokens_seen": 2852585472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013699097291875628, + "loss": 1.2711, + "theoretical_loss": 3.3621807191094764, + "tokens_seen": 2852651008 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013697091273821466, + "loss": 1.2958, + "theoretical_loss": 3.3621747701910243, + "tokens_seen": 2852716544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013695085255767303, + "loss": 1.3542, + "theoretical_loss": 3.362168821447501, + "tokens_seen": 2852782080 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013693079237713142, + "loss": 1.3016, + "theoretical_loss": 3.3621628728788986, + "tokens_seen": 2852847616 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013691073219658978, + "loss": 1.3225, + "theoretical_loss": 3.362156924485207, + "tokens_seen": 2852913152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013689067201604817, + "loss": 1.3364, + "theoretical_loss": 3.3621509762664177, + "tokens_seen": 2852978688 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013687061183550653, + "loss": 1.3183, + "theoretical_loss": 3.3621450282225207, + "tokens_seen": 2853044224 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001368505516549649, + "loss": 1.3261, + "theoretical_loss": 3.362139080353508, + "tokens_seen": 2853109760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013683049147442328, + "loss": 1.2838, + "theoretical_loss": 3.362133132659369, + "tokens_seen": 2853175296 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013681043129388164, + "loss": 1.2428, + "theoretical_loss": 3.362127185140096, + "tokens_seen": 2853240832 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013679037111334003, + "loss": 1.1676, + "theoretical_loss": 3.3621212377956784, + "tokens_seen": 2853306368 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001367703109327984, + "loss": 1.3305, + "theoretical_loss": 3.3621152906261083, + "tokens_seen": 2853371904 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013675025075225678, + "loss": 1.3409, + "theoretical_loss": 3.362109343631376, + "tokens_seen": 2853437440 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013673019057171515, + "loss": 1.1933, + "theoretical_loss": 3.3621033968114724, + "tokens_seen": 2853502976 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013671013039117354, + "loss": 1.2475, + "theoretical_loss": 3.362097450166388, + "tokens_seen": 2853568512 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001366900702106319, + "loss": 1.3327, + "theoretical_loss": 3.3620915036961145, + "tokens_seen": 2853634048 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001366700100300903, + "loss": 1.2484, + "theoretical_loss": 3.362085557400642, + "tokens_seen": 2853699584 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013664994984954865, + "loss": 1.2361, + "theoretical_loss": 3.362079611279962, + "tokens_seen": 2853765120 + }, + { + "epoch": 0.73, + "learning_rate": 0.000136629889669007, + "loss": 1.233, + "theoretical_loss": 3.3620736653340644, + "tokens_seen": 2853830656 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001366098294884654, + "loss": 1.2689, + "theoretical_loss": 3.3620677195629405, + "tokens_seen": 2853896192 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013658976930792376, + "loss": 1.2188, + "theoretical_loss": 3.3620617739665817, + "tokens_seen": 2853961728 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013656970912738215, + "loss": 1.1131, + "theoretical_loss": 3.362055828544978, + "tokens_seen": 2854027264 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013654964894684052, + "loss": 1.3007, + "theoretical_loss": 3.3620498832981207, + "tokens_seen": 2854092800 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001365295887662989, + "loss": 1.3021, + "theoretical_loss": 3.362043938226001, + "tokens_seen": 2854158336 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013650952858575727, + "loss": 1.3598, + "theoretical_loss": 3.3620379933286086, + "tokens_seen": 2854223872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013648946840521566, + "loss": 1.2669, + "theoretical_loss": 3.3620320486059354, + "tokens_seen": 2854289408 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013646940822467402, + "loss": 1.3017, + "theoretical_loss": 3.362026104057972, + "tokens_seen": 2854354944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013644934804413238, + "loss": 1.2837, + "theoretical_loss": 3.3620201596847092, + "tokens_seen": 2854420480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013642928786359077, + "loss": 1.3427, + "theoretical_loss": 3.362014215486138, + "tokens_seen": 2854486016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013640922768304913, + "loss": 1.3239, + "theoretical_loss": 3.362008271462249, + "tokens_seen": 2854551552 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013638916750250752, + "loss": 1.3163, + "theoretical_loss": 3.362002327613033, + "tokens_seen": 2854617088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013636910732196588, + "loss": 1.3888, + "theoretical_loss": 3.361996383938481, + "tokens_seen": 2854682624 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013634904714142427, + "loss": 1.1838, + "theoretical_loss": 3.361990440438584, + "tokens_seen": 2854748160 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013632898696088264, + "loss": 1.2234, + "theoretical_loss": 3.361984497113333, + "tokens_seen": 2854813696 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013630892678034103, + "loss": 1.2601, + "theoretical_loss": 3.361978553962718, + "tokens_seen": 2854879232 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001362888665997994, + "loss": 1.2467, + "theoretical_loss": 3.3619726109867307, + "tokens_seen": 2854944768 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013626880641925775, + "loss": 1.2645, + "theoretical_loss": 3.361966668185362, + "tokens_seen": 2855010304 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013624874623871617, + "loss": 1.3255, + "theoretical_loss": 3.3619607255586024, + "tokens_seen": 2855075840 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013622868605817453, + "loss": 1.2271, + "theoretical_loss": 3.3619547831064427, + "tokens_seen": 2855141376 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013620862587763292, + "loss": 1.3389, + "theoretical_loss": 3.3619488408288736, + "tokens_seen": 2855206912 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013618856569709128, + "loss": 1.3132, + "theoretical_loss": 3.3619428987258866, + "tokens_seen": 2855272448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013616850551654967, + "loss": 1.2938, + "theoretical_loss": 3.361936956797472, + "tokens_seen": 2855337984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013614844533600803, + "loss": 1.3458, + "theoretical_loss": 3.3619310150436212, + "tokens_seen": 2855403520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013612838515546642, + "loss": 1.2606, + "theoretical_loss": 3.3619250734643247, + "tokens_seen": 2855469056 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013610832497492478, + "loss": 1.3885, + "theoretical_loss": 3.3619191320595734, + "tokens_seen": 2855534592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013608826479438317, + "loss": 1.335, + "theoretical_loss": 3.361913190829358, + "tokens_seen": 2855600128 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013606820461384153, + "loss": 1.3151, + "theoretical_loss": 3.3619072497736697, + "tokens_seen": 2855665664 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001360481444332999, + "loss": 1.1934, + "theoretical_loss": 3.361901308892499, + "tokens_seen": 2855731200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013602808425275829, + "loss": 1.2572, + "theoretical_loss": 3.361895368185837, + "tokens_seen": 2855796736 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1037860, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2703384160995483, + "objective/train/theoretical_loss": 3.361889427653675, + "objective/train/tokens_used": 1226321376, + "theoretical_loss": 3.361889427653675, + "tokens_seen": 2855862272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013600802407221665, + "loss": 1.3591, + "theoretical_loss": 3.361889427653675, + "tokens_seen": 2855862272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013598796389167504, + "loss": 1.3093, + "theoretical_loss": 3.3618834872960033, + "tokens_seen": 2855927808 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001359679037111334, + "loss": 1.3253, + "theoretical_loss": 3.3618775471128126, + "tokens_seen": 2855993344 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001359478435305918, + "loss": 1.3939, + "theoretical_loss": 3.3618716071040944, + "tokens_seen": 2856058880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013592778335005015, + "loss": 1.3135, + "theoretical_loss": 3.361865667269839, + "tokens_seen": 2856124416 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013590772316950854, + "loss": 1.2988, + "theoretical_loss": 3.3618597276100375, + "tokens_seen": 2856189952 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001358876629889669, + "loss": 1.3361, + "theoretical_loss": 3.361853788124681, + "tokens_seen": 2856255488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013586760280842527, + "loss": 1.2995, + "theoretical_loss": 3.3618478488137598, + "tokens_seen": 2856321024 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013584754262788365, + "loss": 1.3154, + "theoretical_loss": 3.3618419096772656, + "tokens_seen": 2856386560 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013582748244734202, + "loss": 1.359, + "theoretical_loss": 3.3618359707151884, + "tokens_seen": 2856452096 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001358074222668004, + "loss": 1.2401, + "theoretical_loss": 3.3618300319275196, + "tokens_seen": 2856517632 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013578736208625877, + "loss": 1.3276, + "theoretical_loss": 3.36182409331425, + "tokens_seen": 2856583168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013576730190571716, + "loss": 1.2698, + "theoretical_loss": 3.3618181548753707, + "tokens_seen": 2856648704 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013574724172517552, + "loss": 1.2522, + "theoretical_loss": 3.3618122166108724, + "tokens_seen": 2856714240 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001357271815446339, + "loss": 1.3587, + "theoretical_loss": 3.3618062785207456, + "tokens_seen": 2856779776 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013570712136409227, + "loss": 1.2658, + "theoretical_loss": 3.3618003406049817, + "tokens_seen": 2856845312 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013568706118355063, + "loss": 1.2813, + "theoretical_loss": 3.361794402863571, + "tokens_seen": 2856910848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013566700100300902, + "loss": 1.3108, + "theoretical_loss": 3.361788465296505, + "tokens_seen": 2856976384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013564694082246739, + "loss": 1.3942, + "theoretical_loss": 3.3617825279037743, + "tokens_seen": 2857041920 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013562688064192578, + "loss": 1.3227, + "theoretical_loss": 3.36177659068537, + "tokens_seen": 2857107456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013560682046138414, + "loss": 1.3799, + "theoretical_loss": 3.3617706536412824, + "tokens_seen": 2857172992 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013558676028084253, + "loss": 1.3857, + "theoretical_loss": 3.361764716771503, + "tokens_seen": 2857238528 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001355667001003009, + "loss": 1.2706, + "theoretical_loss": 3.3617587800760225, + "tokens_seen": 2857304064 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013554663991975928, + "loss": 1.2951, + "theoretical_loss": 3.3617528435548314, + "tokens_seen": 2857369600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013552657973921764, + "loss": 1.2593, + "theoretical_loss": 3.3617469072079214, + "tokens_seen": 2857435136 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013550651955867606, + "loss": 1.2879, + "theoretical_loss": 3.3617409710352826, + "tokens_seen": 2857500672 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013548645937813442, + "loss": 1.1621, + "theoretical_loss": 3.361735035036906, + "tokens_seen": 2857566208 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013546639919759278, + "loss": 1.1571, + "theoretical_loss": 3.3617290992127833, + "tokens_seen": 2857631744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013544633901705117, + "loss": 1.2811, + "theoretical_loss": 3.3617231635629046, + "tokens_seen": 2857697280 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013542627883650953, + "loss": 1.2069, + "theoretical_loss": 3.3617172280872607, + "tokens_seen": 2857762816 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013540621865596792, + "loss": 1.3711, + "theoretical_loss": 3.3617112927858432, + "tokens_seen": 2857828352 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013538615847542628, + "loss": 1.249, + "theoretical_loss": 3.361705357658642, + "tokens_seen": 2857893888 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013536609829488467, + "loss": 1.2551, + "theoretical_loss": 3.361699422705649, + "tokens_seen": 2857959424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013534603811434304, + "loss": 1.3344, + "theoretical_loss": 3.3616934879268543, + "tokens_seen": 2858024960 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013532597793380143, + "loss": 1.2654, + "theoretical_loss": 3.3616875533222492, + "tokens_seen": 2858090496 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001353059177532598, + "loss": 1.2144, + "theoretical_loss": 3.361681618891825, + "tokens_seen": 2858156032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013528585757271815, + "loss": 1.2414, + "theoretical_loss": 3.3616756846355718, + "tokens_seen": 2858221568 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013526579739217654, + "loss": 1.2314, + "theoretical_loss": 3.3616697505534807, + "tokens_seen": 2858287104 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001352457372116349, + "loss": 1.2107, + "theoretical_loss": 3.3616638166455424, + "tokens_seen": 2858352640 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001352256770310933, + "loss": 1.2958, + "theoretical_loss": 3.3616578829117487, + "tokens_seen": 2858418176 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013520561685055165, + "loss": 1.2691, + "theoretical_loss": 3.3616519493520896, + "tokens_seen": 2858483712 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013518555667001004, + "loss": 1.3637, + "theoretical_loss": 3.3616460159665564, + "tokens_seen": 2858549248 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001351654964894684, + "loss": 1.2776, + "theoretical_loss": 3.36164008275514, + "tokens_seen": 2858614784 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001351454363089268, + "loss": 1.2661, + "theoretical_loss": 3.361634149717831, + "tokens_seen": 2858680320 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013512537612838516, + "loss": 1.2631, + "theoretical_loss": 3.3616282168546205, + "tokens_seen": 2858745856 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013510531594784355, + "loss": 1.2287, + "theoretical_loss": 3.3616222841654992, + "tokens_seen": 2858811392 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001350852557673019, + "loss": 1.2792, + "theoretical_loss": 3.3616163516504587, + "tokens_seen": 2858876928 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013506519558676027, + "loss": 1.2777, + "theoretical_loss": 3.361610419309489, + "tokens_seen": 2858942464 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013504513540621866, + "loss": 1.252, + "theoretical_loss": 3.3616044871425816, + "tokens_seen": 2859008000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013502507522567702, + "loss": 1.287, + "theoretical_loss": 3.361598555149727, + "tokens_seen": 2859073536 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1039109, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.074881911277771, + "objective/train/theoretical_loss": 3.3615926233309166, + "objective/train/tokens_used": 1229598176, + "theoretical_loss": 3.3615926233309166, + "tokens_seen": 2859139072 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001350050150451354, + "loss": 1.2146, + "theoretical_loss": 3.3615926233309166, + "tokens_seen": 2859139072 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013498495486459377, + "loss": 1.2912, + "theoretical_loss": 3.3615866916861408, + "tokens_seen": 2859204608 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013496489468405216, + "loss": 1.2436, + "theoretical_loss": 3.361580760215391, + "tokens_seen": 2859270144 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013494483450351052, + "loss": 1.3489, + "theoretical_loss": 3.3615748289186573, + "tokens_seen": 2859335680 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013492477432296891, + "loss": 1.2693, + "theoretical_loss": 3.3615688977959315, + "tokens_seen": 2859401216 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013490471414242728, + "loss": 1.2564, + "theoretical_loss": 3.3615629668472042, + "tokens_seen": 2859466752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013488465396188564, + "loss": 1.177, + "theoretical_loss": 3.361557036072466, + "tokens_seen": 2859532288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013486459378134403, + "loss": 1.2082, + "theoretical_loss": 3.361551105471708, + "tokens_seen": 2859597824 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001348445336008024, + "loss": 1.322, + "theoretical_loss": 3.3615451750449212, + "tokens_seen": 2859663360 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013482447342026078, + "loss": 1.3585, + "theoretical_loss": 3.361539244792097, + "tokens_seen": 2859728896 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013480441323971914, + "loss": 1.358, + "theoretical_loss": 3.361533314713225, + "tokens_seen": 2859794432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013478435305917753, + "loss": 1.314, + "theoretical_loss": 3.3615273848082974, + "tokens_seen": 2859859968 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001347642928786359, + "loss": 1.2707, + "theoretical_loss": 3.3615214550773045, + "tokens_seen": 2859925504 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001347442326980943, + "loss": 1.3291, + "theoretical_loss": 3.3615155255202374, + "tokens_seen": 2859991040 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013472417251755267, + "loss": 1.3199, + "theoretical_loss": 3.3615095961370867, + "tokens_seen": 2860056576 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013470411233701103, + "loss": 1.2956, + "theoretical_loss": 3.3615036669278435, + "tokens_seen": 2860122112 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013468405215646942, + "loss": 1.3672, + "theoretical_loss": 3.361497737892499, + "tokens_seen": 2860187648 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013466399197592779, + "loss": 1.3036, + "theoretical_loss": 3.361491809031044, + "tokens_seen": 2860253184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013464393179538618, + "loss": 1.3685, + "theoretical_loss": 3.361485880343469, + "tokens_seen": 2860318720 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013462387161484454, + "loss": 1.2023, + "theoretical_loss": 3.3614799518297653, + "tokens_seen": 2860384256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013460381143430293, + "loss": 1.31, + "theoretical_loss": 3.3614740234899236, + "tokens_seen": 2860449792 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001345837512537613, + "loss": 1.2839, + "theoretical_loss": 3.361468095323935, + "tokens_seen": 2860515328 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013456369107321968, + "loss": 1.201, + "theoretical_loss": 3.3614621673317906, + "tokens_seen": 2860580864 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013454363089267804, + "loss": 1.3044, + "theoretical_loss": 3.3614562395134806, + "tokens_seen": 2860646400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013452357071213643, + "loss": 1.1609, + "theoretical_loss": 3.361450311868997, + "tokens_seen": 2860711936 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001345035105315948, + "loss": 1.3288, + "theoretical_loss": 3.3614443843983297, + "tokens_seen": 2860777472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013448345035105315, + "loss": 1.3421, + "theoretical_loss": 3.36143845710147, + "tokens_seen": 2860843008 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013446339017051154, + "loss": 1.3293, + "theoretical_loss": 3.3614325299784094, + "tokens_seen": 2860908544 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001344433299899699, + "loss": 1.2366, + "theoretical_loss": 3.3614266030291375, + "tokens_seen": 2860974080 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001344232698094283, + "loss": 1.2706, + "theoretical_loss": 3.3614206762536467, + "tokens_seen": 2861039616 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013440320962888666, + "loss": 1.2745, + "theoretical_loss": 3.361414749651927, + "tokens_seen": 2861105152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013438314944834505, + "loss": 1.2862, + "theoretical_loss": 3.3614088232239694, + "tokens_seen": 2861170688 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001343630892678034, + "loss": 1.323, + "theoretical_loss": 3.361402896969765, + "tokens_seen": 2861236224 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001343430290872618, + "loss": 1.2055, + "theoretical_loss": 3.361396970889305, + "tokens_seen": 2861301760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013432296890672016, + "loss": 1.3283, + "theoretical_loss": 3.3613910449825797, + "tokens_seen": 2861367296 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013430290872617852, + "loss": 1.299, + "theoretical_loss": 3.3613851192495803, + "tokens_seen": 2861432832 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001342828485456369, + "loss": 1.4004, + "theoretical_loss": 3.361379193690298, + "tokens_seen": 2861498368 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013426278836509527, + "loss": 1.2459, + "theoretical_loss": 3.3613732683047237, + "tokens_seen": 2861563904 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013424272818455366, + "loss": 1.3234, + "theoretical_loss": 3.361367343092848, + "tokens_seen": 2861629440 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013422266800401203, + "loss": 1.2787, + "theoretical_loss": 3.361361418054662, + "tokens_seen": 2861694976 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013420260782347042, + "loss": 1.3121, + "theoretical_loss": 3.361355493190157, + "tokens_seen": 2861760512 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013418254764292878, + "loss": 1.2439, + "theoretical_loss": 3.361349568499323, + "tokens_seen": 2861826048 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013416248746238717, + "loss": 1.2753, + "theoretical_loss": 3.3613436439821514, + "tokens_seen": 2861891584 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013414242728184553, + "loss": 1.2078, + "theoretical_loss": 3.3613377196386334, + "tokens_seen": 2861957120 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001341223671013039, + "loss": 1.2074, + "theoretical_loss": 3.3613317954687596, + "tokens_seen": 2862022656 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013410230692076228, + "loss": 1.2907, + "theoretical_loss": 3.361325871472521, + "tokens_seen": 2862088192 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013408224674022064, + "loss": 1.3058, + "theoretical_loss": 3.361319947649909, + "tokens_seen": 2862153728 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013406218655967903, + "loss": 1.3209, + "theoretical_loss": 3.3613140240009143, + "tokens_seen": 2862219264 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001340421263791374, + "loss": 1.1996, + "theoretical_loss": 3.361308100525527, + "tokens_seen": 2862284800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013402206619859578, + "loss": 1.265, + "theoretical_loss": 3.3613021772237395, + "tokens_seen": 2862350336 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1040389, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3280324935913086, + "objective/train/theoretical_loss": 3.3612962540955413, + "objective/train/tokens_used": 1232874976, + "theoretical_loss": 3.3612962540955413, + "tokens_seen": 2862415872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013400200601805417, + "loss": 1.3278, + "theoretical_loss": 3.3612962540955413, + "tokens_seen": 2862415872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013398194583751256, + "loss": 1.2615, + "theoretical_loss": 3.3612903311409243, + "tokens_seen": 2862481408 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013396188565697093, + "loss": 1.1986, + "theoretical_loss": 3.3612844083598796, + "tokens_seen": 2862546944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013394182547642931, + "loss": 1.2522, + "theoretical_loss": 3.361278485752397, + "tokens_seen": 2862612480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013392176529588768, + "loss": 1.264, + "theoretical_loss": 3.3612725633184684, + "tokens_seen": 2862678016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013390170511534604, + "loss": 1.3042, + "theoretical_loss": 3.3612666410580845, + "tokens_seen": 2862743552 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013388164493480443, + "loss": 1.2564, + "theoretical_loss": 3.361260718971236, + "tokens_seen": 2862809088 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001338615847542628, + "loss": 1.2819, + "theoretical_loss": 3.361254797057914, + "tokens_seen": 2862874624 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013384152457372118, + "loss": 1.259, + "theoretical_loss": 3.36124887531811, + "tokens_seen": 2862940160 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013382146439317954, + "loss": 1.4066, + "theoretical_loss": 3.361242953751814, + "tokens_seen": 2863005696 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013380140421263793, + "loss": 1.2806, + "theoretical_loss": 3.3612370323590173, + "tokens_seen": 2863071232 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001337813440320963, + "loss": 1.3296, + "theoretical_loss": 3.361231111139711, + "tokens_seen": 2863136768 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013376128385155468, + "loss": 1.3041, + "theoretical_loss": 3.3612251900938857, + "tokens_seen": 2863202304 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013374122367101305, + "loss": 1.2624, + "theoretical_loss": 3.3612192692215332, + "tokens_seen": 2863267840 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001337211634904714, + "loss": 1.1185, + "theoretical_loss": 3.3612133485226434, + "tokens_seen": 2863333376 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001337011033099298, + "loss": 1.3558, + "theoretical_loss": 3.361207427997208, + "tokens_seen": 2863398912 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013368104312938816, + "loss": 1.2712, + "theoretical_loss": 3.3612015076452177, + "tokens_seen": 2863464448 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013366098294884655, + "loss": 1.1985, + "theoretical_loss": 3.3611955874666632, + "tokens_seen": 2863529984 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001336409227683049, + "loss": 1.2751, + "theoretical_loss": 3.361189667461536, + "tokens_seen": 2863595520 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001336208625877633, + "loss": 1.3259, + "theoretical_loss": 3.361183747629826, + "tokens_seen": 2863661056 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013360080240722166, + "loss": 1.3511, + "theoretical_loss": 3.3611778279715256, + "tokens_seen": 2863726592 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013358074222668005, + "loss": 1.3354, + "theoretical_loss": 3.3611719084866247, + "tokens_seen": 2863792128 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013356068204613841, + "loss": 1.2503, + "theoretical_loss": 3.361165989175115, + "tokens_seen": 2863857664 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013354062186559678, + "loss": 1.2569, + "theoretical_loss": 3.3611600700369864, + "tokens_seen": 2863923200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013352056168505517, + "loss": 1.2663, + "theoretical_loss": 3.3611541510722307, + "tokens_seen": 2863988736 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013350050150451353, + "loss": 1.2834, + "theoretical_loss": 3.3611482322808386, + "tokens_seen": 2864054272 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013348044132397192, + "loss": 1.337, + "theoretical_loss": 3.361142313662801, + "tokens_seen": 2864119808 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013346038114343028, + "loss": 1.2842, + "theoretical_loss": 3.361136395218109, + "tokens_seen": 2864185344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013344032096288867, + "loss": 1.2324, + "theoretical_loss": 3.3611304769467534, + "tokens_seen": 2864250880 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013342026078234703, + "loss": 1.3692, + "theoretical_loss": 3.3611245588487257, + "tokens_seen": 2864316416 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013340020060180542, + "loss": 1.2172, + "theoretical_loss": 3.361118640924016, + "tokens_seen": 2864381952 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013338014042126378, + "loss": 1.2688, + "theoretical_loss": 3.361112723172616, + "tokens_seen": 2864447488 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013336008024072217, + "loss": 1.3351, + "theoretical_loss": 3.361106805594516, + "tokens_seen": 2864513024 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013334002006018053, + "loss": 1.2518, + "theoretical_loss": 3.361100888189707, + "tokens_seen": 2864578560 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001333199598796389, + "loss": 1.1999, + "theoretical_loss": 3.361094970958181, + "tokens_seen": 2864644096 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013329989969909729, + "loss": 1.3603, + "theoretical_loss": 3.361089053899928, + "tokens_seen": 2864709632 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013327983951855565, + "loss": 1.3838, + "theoretical_loss": 3.3610831370149388, + "tokens_seen": 2864775168 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013325977933801404, + "loss": 1.3036, + "theoretical_loss": 3.361077220303205, + "tokens_seen": 2864840704 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013323971915747243, + "loss": 1.2994, + "theoretical_loss": 3.3610713037647173, + "tokens_seen": 2864906240 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013321965897693082, + "loss": 1.3162, + "theoretical_loss": 3.3610653873994667, + "tokens_seen": 2864971776 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013319959879638918, + "loss": 1.398, + "theoretical_loss": 3.361059471207444, + "tokens_seen": 2865037312 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013317953861584757, + "loss": 1.2654, + "theoretical_loss": 3.3610535551886405, + "tokens_seen": 2865102848 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013315947843530593, + "loss": 1.2778, + "theoretical_loss": 3.361047639343047, + "tokens_seen": 2865168384 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001331394182547643, + "loss": 1.2952, + "theoretical_loss": 3.3610417236706542, + "tokens_seen": 2865233920 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013311935807422268, + "loss": 1.2807, + "theoretical_loss": 3.3610358081714535, + "tokens_seen": 2865299456 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013309929789368104, + "loss": 1.3352, + "theoretical_loss": 3.3610298928454356, + "tokens_seen": 2865364992 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013307923771313943, + "loss": 1.2755, + "theoretical_loss": 3.361023977692591, + "tokens_seen": 2865430528 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001330591775325978, + "loss": 1.164, + "theoretical_loss": 3.361018062712912, + "tokens_seen": 2865496064 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013303911735205618, + "loss": 1.2582, + "theoretical_loss": 3.3610121479063886, + "tokens_seen": 2865561600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013301905717151455, + "loss": 1.2961, + "theoretical_loss": 3.3610062332730117, + "tokens_seen": 2865627136 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1041644, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.493830680847168, + "objective/train/theoretical_loss": 3.3610003188127724, + "objective/train/tokens_used": 1236151776, + "theoretical_loss": 3.3610003188127724, + "tokens_seen": 2865692672 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013299899699097294, + "loss": 1.4425, + "theoretical_loss": 3.3610003188127724, + "tokens_seen": 2865692672 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001329789368104313, + "loss": 1.3133, + "theoretical_loss": 3.360994404525662, + "tokens_seen": 2865758208 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001329588766298897, + "loss": 1.3048, + "theoretical_loss": 3.3609884904116716, + "tokens_seen": 2865823744 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013293881644934805, + "loss": 1.2552, + "theoretical_loss": 3.3609825764707915, + "tokens_seen": 2865889280 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001329187562688064, + "loss": 1.2318, + "theoretical_loss": 3.3609766627030133, + "tokens_seen": 2865954816 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001328986960882648, + "loss": 1.3245, + "theoretical_loss": 3.3609707491083274, + "tokens_seen": 2866020352 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013287863590772316, + "loss": 1.2276, + "theoretical_loss": 3.3609648356867248, + "tokens_seen": 2866085888 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013285857572718155, + "loss": 1.1997, + "theoretical_loss": 3.360958922438197, + "tokens_seen": 2866151424 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013283851554663992, + "loss": 1.3198, + "theoretical_loss": 3.360953009362735, + "tokens_seen": 2866216960 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001328184553660983, + "loss": 1.261, + "theoretical_loss": 3.360947096460329, + "tokens_seen": 2866282496 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013279839518555667, + "loss": 1.1908, + "theoretical_loss": 3.360941183730971, + "tokens_seen": 2866348032 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013277833500501506, + "loss": 1.354, + "theoretical_loss": 3.360935271174651, + "tokens_seen": 2866413568 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013275827482447342, + "loss": 1.2838, + "theoretical_loss": 3.3609293587913607, + "tokens_seen": 2866479104 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013273821464393178, + "loss": 1.248, + "theoretical_loss": 3.360923446581091, + "tokens_seen": 2866544640 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013271815446339017, + "loss": 1.3487, + "theoretical_loss": 3.360917534543832, + "tokens_seen": 2866610176 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013269809428284853, + "loss": 1.4034, + "theoretical_loss": 3.360911622679576, + "tokens_seen": 2866675712 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013267803410230692, + "loss": 1.3126, + "theoretical_loss": 3.3609057109883134, + "tokens_seen": 2866741248 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013265797392176528, + "loss": 1.3419, + "theoretical_loss": 3.3608997994700345, + "tokens_seen": 2866806784 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013263791374122367, + "loss": 1.2162, + "theoretical_loss": 3.3608938881247314, + "tokens_seen": 2866872320 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013261785356068204, + "loss": 1.3526, + "theoretical_loss": 3.3608879769523945, + "tokens_seen": 2866937856 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013259779338014043, + "loss": 1.267, + "theoretical_loss": 3.3608820659530148, + "tokens_seen": 2867003392 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001325777331995988, + "loss": 1.348, + "theoretical_loss": 3.3608761551265838, + "tokens_seen": 2867068928 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013255767301905715, + "loss": 1.1352, + "theoretical_loss": 3.360870244473092, + "tokens_seen": 2867134464 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013253761283851554, + "loss": 1.2705, + "theoretical_loss": 3.36086433399253, + "tokens_seen": 2867200000 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001325175526579739, + "loss": 1.3239, + "theoretical_loss": 3.3608584236848893, + "tokens_seen": 2867265536 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013249749247743232, + "loss": 1.2941, + "theoretical_loss": 3.360852513550161, + "tokens_seen": 2867331072 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013247743229689068, + "loss": 1.3941, + "theoretical_loss": 3.360846603588336, + "tokens_seen": 2867396608 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013245737211634907, + "loss": 1.2477, + "theoretical_loss": 3.360840693799405, + "tokens_seen": 2867462144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013243731193580743, + "loss": 1.208, + "theoretical_loss": 3.3608347841833597, + "tokens_seen": 2867527680 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013241725175526582, + "loss": 1.2796, + "theoretical_loss": 3.36082887474019, + "tokens_seen": 2867593216 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013239719157472418, + "loss": 1.1734, + "theoretical_loss": 3.360822965469888, + "tokens_seen": 2867658752 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013237713139418257, + "loss": 1.3543, + "theoretical_loss": 3.3608170563724435, + "tokens_seen": 2867724288 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013235707121364093, + "loss": 1.1779, + "theoretical_loss": 3.3608111474478486, + "tokens_seen": 2867789824 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001323370110330993, + "loss": 1.2838, + "theoretical_loss": 3.360805238696094, + "tokens_seen": 2867855360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013231695085255769, + "loss": 1.382, + "theoretical_loss": 3.3607993301171706, + "tokens_seen": 2867920896 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013229689067201605, + "loss": 1.3273, + "theoretical_loss": 3.360793421711069, + "tokens_seen": 2867986432 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013227683049147444, + "loss": 1.2484, + "theoretical_loss": 3.360787513477781, + "tokens_seen": 2868051968 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001322567703109328, + "loss": 1.3204, + "theoretical_loss": 3.360781605417297, + "tokens_seen": 2868117504 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001322367101303912, + "loss": 1.2832, + "theoretical_loss": 3.3607756975296077, + "tokens_seen": 2868183040 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013221664994984955, + "loss": 1.3164, + "theoretical_loss": 3.360769789814705, + "tokens_seen": 2868248576 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013219658976930794, + "loss": 1.3658, + "theoretical_loss": 3.3607638822725794, + "tokens_seen": 2868314112 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001321765295887663, + "loss": 1.2019, + "theoretical_loss": 3.3607579749032217, + "tokens_seen": 2868379648 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013215646940822467, + "loss": 1.3563, + "theoretical_loss": 3.360752067706623, + "tokens_seen": 2868445184 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013213640922768305, + "loss": 1.2468, + "theoretical_loss": 3.3607461606827753, + "tokens_seen": 2868510720 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013211634904714142, + "loss": 1.2134, + "theoretical_loss": 3.360740253831668, + "tokens_seen": 2868576256 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001320962888665998, + "loss": 1.2556, + "theoretical_loss": 3.360734347153293, + "tokens_seen": 2868641792 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013207622868605817, + "loss": 1.3016, + "theoretical_loss": 3.360728440647641, + "tokens_seen": 2868707328 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013205616850551656, + "loss": 1.3514, + "theoretical_loss": 3.3607225343147036, + "tokens_seen": 2868772864 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013203610832497492, + "loss": 1.3265, + "theoretical_loss": 3.360716628154471, + "tokens_seen": 2868838400 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001320160481444333, + "loss": 1.3533, + "theoretical_loss": 3.360710722166935, + "tokens_seen": 2868903936 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1042847, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3076987266540527, + "objective/train/theoretical_loss": 3.3607048163520856, + "objective/train/tokens_used": 1239428576, + "theoretical_loss": 3.3607048163520856, + "tokens_seen": 2868969472 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013199598796389167, + "loss": 1.3181, + "theoretical_loss": 3.3607048163520856, + "tokens_seen": 2868969472 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013197592778335003, + "loss": 1.4183, + "theoretical_loss": 3.3606989107099148, + "tokens_seen": 2869035008 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013195586760280842, + "loss": 1.201, + "theoretical_loss": 3.360693005240413, + "tokens_seen": 2869100544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013193580742226679, + "loss": 1.3155, + "theoretical_loss": 3.360687099943571, + "tokens_seen": 2869166080 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013191574724172517, + "loss": 1.2559, + "theoretical_loss": 3.3606811948193807, + "tokens_seen": 2869231616 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013189568706118354, + "loss": 1.2662, + "theoretical_loss": 3.3606752898678325, + "tokens_seen": 2869297152 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013187562688064193, + "loss": 1.2727, + "theoretical_loss": 3.3606693850889178, + "tokens_seen": 2869362688 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001318555667001003, + "loss": 1.298, + "theoretical_loss": 3.3606634804826268, + "tokens_seen": 2869428224 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013183550651955868, + "loss": 1.2553, + "theoretical_loss": 3.3606575760489514, + "tokens_seen": 2869493760 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013181544633901704, + "loss": 1.225, + "theoretical_loss": 3.360651671787882, + "tokens_seen": 2869559296 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013179538615847543, + "loss": 1.3704, + "theoretical_loss": 3.36064576769941, + "tokens_seen": 2869624832 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001317753259779338, + "loss": 1.3302, + "theoretical_loss": 3.3606398637835264, + "tokens_seen": 2869690368 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013175526579739218, + "loss": 1.2347, + "theoretical_loss": 3.360633960040222, + "tokens_seen": 2869755904 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013173520561685057, + "loss": 1.3212, + "theoretical_loss": 3.360628056469488, + "tokens_seen": 2869821440 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013171514543630893, + "loss": 1.2598, + "theoretical_loss": 3.360622153071315, + "tokens_seen": 2869886976 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013169508525576732, + "loss": 1.242, + "theoretical_loss": 3.3606162498456946, + "tokens_seen": 2869952512 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013167502507522568, + "loss": 1.242, + "theoretical_loss": 3.360610346792617, + "tokens_seen": 2870018048 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013165496489468407, + "loss": 1.2927, + "theoretical_loss": 3.3606044439120746, + "tokens_seen": 2870083584 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013163490471414244, + "loss": 1.336, + "theoretical_loss": 3.360598541204057, + "tokens_seen": 2870149120 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013161484453360083, + "loss": 1.3318, + "theoretical_loss": 3.3605926386685563, + "tokens_seen": 2870214656 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001315947843530592, + "loss": 1.2307, + "theoretical_loss": 3.360586736305563, + "tokens_seen": 2870280192 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013157472417251755, + "loss": 1.2576, + "theoretical_loss": 3.3605808341150674, + "tokens_seen": 2870345728 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013155466399197594, + "loss": 1.2808, + "theoretical_loss": 3.360574932097062, + "tokens_seen": 2870411264 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001315346038114343, + "loss": 1.2832, + "theoretical_loss": 3.3605690302515367, + "tokens_seen": 2870476800 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001315145436308927, + "loss": 1.3916, + "theoretical_loss": 3.360563128578483, + "tokens_seen": 2870542336 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013149448345035105, + "loss": 1.3497, + "theoretical_loss": 3.3605572270778916, + "tokens_seen": 2870607872 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013147442326980944, + "loss": 1.3053, + "theoretical_loss": 3.3605513257497543, + "tokens_seen": 2870673408 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001314543630892678, + "loss": 1.261, + "theoretical_loss": 3.3605454245940614, + "tokens_seen": 2870738944 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001314343029087262, + "loss": 1.2618, + "theoretical_loss": 3.3605395236108038, + "tokens_seen": 2870804480 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013141424272818456, + "loss": 1.3406, + "theoretical_loss": 3.3605336227999727, + "tokens_seen": 2870870016 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013139418254764295, + "loss": 1.1237, + "theoretical_loss": 3.3605277221615597, + "tokens_seen": 2870935552 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001313741223671013, + "loss": 1.3064, + "theoretical_loss": 3.3605218216955555, + "tokens_seen": 2871001088 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013135406218655967, + "loss": 1.1355, + "theoretical_loss": 3.3605159214019507, + "tokens_seen": 2871066624 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013133400200601806, + "loss": 1.263, + "theoretical_loss": 3.3605100212807364, + "tokens_seen": 2871132160 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013131394182547642, + "loss": 1.3038, + "theoretical_loss": 3.3605041213319042, + "tokens_seen": 2871197696 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001312938816449348, + "loss": 1.3152, + "theoretical_loss": 3.360498221555445, + "tokens_seen": 2871263232 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013127382146439317, + "loss": 1.3616, + "theoretical_loss": 3.360492321951349, + "tokens_seen": 2871328768 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013125376128385156, + "loss": 1.2023, + "theoretical_loss": 3.360486422519608, + "tokens_seen": 2871394304 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013123370110330992, + "loss": 1.2863, + "theoretical_loss": 3.3604805232602133, + "tokens_seen": 2871459840 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013121364092276831, + "loss": 1.2776, + "theoretical_loss": 3.360474624173155, + "tokens_seen": 2871525376 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013119358074222668, + "loss": 1.2298, + "theoretical_loss": 3.360468725258425, + "tokens_seen": 2871590912 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013117352056168504, + "loss": 1.2521, + "theoretical_loss": 3.360462826516014, + "tokens_seen": 2871656448 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013115346038114343, + "loss": 1.1979, + "theoretical_loss": 3.360456927945913, + "tokens_seen": 2871721984 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001311334002006018, + "loss": 1.2906, + "theoretical_loss": 3.3604510295481127, + "tokens_seen": 2871787520 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013111334002006018, + "loss": 1.3014, + "theoretical_loss": 3.3604451313226047, + "tokens_seen": 2871853056 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013109327983951854, + "loss": 1.2654, + "theoretical_loss": 3.3604392332693798, + "tokens_seen": 2871918592 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013107321965897693, + "loss": 1.3539, + "theoretical_loss": 3.360433335388429, + "tokens_seen": 2871984128 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001310531594784353, + "loss": 1.307, + "theoretical_loss": 3.3604274376797436, + "tokens_seen": 2872049664 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013103309929789368, + "loss": 1.2816, + "theoretical_loss": 3.360421540143314, + "tokens_seen": 2872115200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013101303911735204, + "loss": 1.2879, + "theoretical_loss": 3.3604156427791323, + "tokens_seen": 2872180736 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1044056, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.30495285987854, + "objective/train/theoretical_loss": 3.360409745587188, + "objective/train/tokens_used": 1242705376, + "theoretical_loss": 3.360409745587188, + "tokens_seen": 2872246272 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013099297893681043, + "loss": 1.2925, + "theoretical_loss": 3.360409745587188, + "tokens_seen": 2872246272 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013097291875626882, + "loss": 1.2028, + "theoretical_loss": 3.360403848567474, + "tokens_seen": 2872311808 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013095285857572719, + "loss": 1.2174, + "theoretical_loss": 3.36039795171998, + "tokens_seen": 2872377344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013093279839518558, + "loss": 1.2567, + "theoretical_loss": 3.3603920550446977, + "tokens_seen": 2872442880 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013091273821464394, + "loss": 1.2407, + "theoretical_loss": 3.3603861585416173, + "tokens_seen": 2872508416 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013089267803410233, + "loss": 1.2635, + "theoretical_loss": 3.3603802622107306, + "tokens_seen": 2872573952 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001308726178535607, + "loss": 1.1757, + "theoretical_loss": 3.360374366052029, + "tokens_seen": 2872639488 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013085255767301908, + "loss": 1.2838, + "theoretical_loss": 3.3603684700655023, + "tokens_seen": 2872705024 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013083249749247744, + "loss": 1.3039, + "theoretical_loss": 3.3603625742511425, + "tokens_seen": 2872770560 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013081243731193583, + "loss": 1.306, + "theoretical_loss": 3.36035667860894, + "tokens_seen": 2872836096 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001307923771313942, + "loss": 1.2423, + "theoretical_loss": 3.3603507831388866, + "tokens_seen": 2872901632 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013077231695085255, + "loss": 1.2316, + "theoretical_loss": 3.360344887840973, + "tokens_seen": 2872967168 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013075225677031094, + "loss": 1.2444, + "theoretical_loss": 3.3603389927151905, + "tokens_seen": 2873032704 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001307321965897693, + "loss": 1.2335, + "theoretical_loss": 3.3603330977615293, + "tokens_seen": 2873098240 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001307121364092277, + "loss": 1.3172, + "theoretical_loss": 3.3603272029799816, + "tokens_seen": 2873163776 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013069207622868606, + "loss": 1.2737, + "theoretical_loss": 3.3603213083705374, + "tokens_seen": 2873229312 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013067201604814445, + "loss": 1.2664, + "theoretical_loss": 3.3603154139331886, + "tokens_seen": 2873294848 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001306519558676028, + "loss": 1.1685, + "theoretical_loss": 3.3603095196679256, + "tokens_seen": 2873360384 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001306318956870612, + "loss": 1.2288, + "theoretical_loss": 3.3603036255747396, + "tokens_seen": 2873425920 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013061183550651956, + "loss": 1.3427, + "theoretical_loss": 3.360297731653622, + "tokens_seen": 2873491456 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013059177532597792, + "loss": 1.3497, + "theoretical_loss": 3.3602918379045636, + "tokens_seen": 2873556992 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001305717151454363, + "loss": 1.2567, + "theoretical_loss": 3.3602859443275555, + "tokens_seen": 2873622528 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013055165496489467, + "loss": 1.382, + "theoretical_loss": 3.3602800509225887, + "tokens_seen": 2873688064 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013053159478435306, + "loss": 1.2403, + "theoretical_loss": 3.360274157689654, + "tokens_seen": 2873753600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013051153460381143, + "loss": 1.2919, + "theoretical_loss": 3.3602682646287434, + "tokens_seen": 2873819136 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013049147442326982, + "loss": 1.21, + "theoretical_loss": 3.360262371739847, + "tokens_seen": 2873884672 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013047141424272818, + "loss": 1.2831, + "theoretical_loss": 3.360256479022956, + "tokens_seen": 2873950208 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013045135406218657, + "loss": 1.3281, + "theoretical_loss": 3.360250586478062, + "tokens_seen": 2874015744 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013043129388164493, + "loss": 1.3184, + "theoretical_loss": 3.3602446941051554, + "tokens_seen": 2874081280 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001304112337011033, + "loss": 1.2069, + "theoretical_loss": 3.3602388019042273, + "tokens_seen": 2874146816 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013039117352056168, + "loss": 1.2442, + "theoretical_loss": 3.3602329098752692, + "tokens_seen": 2874212352 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013037111334002004, + "loss": 1.3352, + "theoretical_loss": 3.360227018018272, + "tokens_seen": 2874277888 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013035105315947843, + "loss": 1.2521, + "theoretical_loss": 3.3602211263332267, + "tokens_seen": 2874343424 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001303309929789368, + "loss": 1.3301, + "theoretical_loss": 3.3602152348201244, + "tokens_seen": 2874408960 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013031093279839518, + "loss": 1.282, + "theoretical_loss": 3.3602093434789557, + "tokens_seen": 2874474496 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013029087261785355, + "loss": 1.1713, + "theoretical_loss": 3.3602034523097126, + "tokens_seen": 2874540032 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013027081243731194, + "loss": 1.21, + "theoretical_loss": 3.3601975613123853, + "tokens_seen": 2874605568 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013025075225677033, + "loss": 1.1898, + "theoretical_loss": 3.360191670486966, + "tokens_seen": 2874671104 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013023069207622871, + "loss": 1.3336, + "theoretical_loss": 3.3601857798334445, + "tokens_seen": 2874736640 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013021063189568708, + "loss": 1.272, + "theoretical_loss": 3.360179889351812, + "tokens_seen": 2874802176 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013019057171514544, + "loss": 1.3136, + "theoretical_loss": 3.3601739990420603, + "tokens_seen": 2874867712 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013017051153460383, + "loss": 1.3086, + "theoretical_loss": 3.36016810890418, + "tokens_seen": 2874933248 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001301504513540622, + "loss": 1.2509, + "theoretical_loss": 3.360162218938162, + "tokens_seen": 2874998784 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013013039117352058, + "loss": 1.4024, + "theoretical_loss": 3.360156329143998, + "tokens_seen": 2875064320 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013011033099297894, + "loss": 1.3123, + "theoretical_loss": 3.3601504395216786, + "tokens_seen": 2875129856 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013009027081243733, + "loss": 1.4079, + "theoretical_loss": 3.360144550071195, + "tokens_seen": 2875195392 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001300702106318957, + "loss": 1.297, + "theoretical_loss": 3.360138660792538, + "tokens_seen": 2875260928 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013005015045135408, + "loss": 1.4151, + "theoretical_loss": 3.3601327716856995, + "tokens_seen": 2875326464 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013003009027081245, + "loss": 1.2381, + "theoretical_loss": 3.360126882750669, + "tokens_seen": 2875392000 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001300100300902708, + "loss": 1.2952, + "theoretical_loss": 3.3601209939874392, + "tokens_seen": 2875457536 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1045259, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1272075176239014, + "objective/train/theoretical_loss": 3.360115105396001, + "objective/train/tokens_used": 1245982176, + "theoretical_loss": 3.360115105396001, + "tokens_seen": 2875523072 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001299899699097292, + "loss": 1.1347, + "theoretical_loss": 3.360115105396001, + "tokens_seen": 2875523072 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012996990972918756, + "loss": 1.325, + "theoretical_loss": 3.360109216976344, + "tokens_seen": 2875588608 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012994984954864595, + "loss": 1.2679, + "theoretical_loss": 3.3601033287284605, + "tokens_seen": 2875654144 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001299297893681043, + "loss": 1.3538, + "theoretical_loss": 3.360097440652342, + "tokens_seen": 2875719680 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001299097291875627, + "loss": 1.2612, + "theoretical_loss": 3.3600915527479778, + "tokens_seen": 2875785216 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012988966900702106, + "loss": 1.2171, + "theoretical_loss": 3.3600856650153608, + "tokens_seen": 2875850752 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012986960882647945, + "loss": 1.3469, + "theoretical_loss": 3.360079777454481, + "tokens_seen": 2875916288 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012984954864593781, + "loss": 1.3091, + "theoretical_loss": 3.3600738900653306, + "tokens_seen": 2875981824 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012982948846539618, + "loss": 1.2796, + "theoretical_loss": 3.3600680028478993, + "tokens_seen": 2876047360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012980942828485457, + "loss": 1.3176, + "theoretical_loss": 3.3600621158021786, + "tokens_seen": 2876112896 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012978936810431293, + "loss": 1.3179, + "theoretical_loss": 3.36005622892816, + "tokens_seen": 2876178432 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012976930792377132, + "loss": 1.3048, + "theoretical_loss": 3.3600503422258345, + "tokens_seen": 2876243968 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012974924774322968, + "loss": 1.3129, + "theoretical_loss": 3.360044455695193, + "tokens_seen": 2876309504 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012972918756268807, + "loss": 1.2316, + "theoretical_loss": 3.360038569336227, + "tokens_seen": 2876375040 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012970912738214643, + "loss": 1.3501, + "theoretical_loss": 3.3600326831489267, + "tokens_seen": 2876440576 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012968906720160482, + "loss": 1.3354, + "theoretical_loss": 3.3600267971332833, + "tokens_seen": 2876506112 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012966900702106318, + "loss": 1.3307, + "theoretical_loss": 3.360020911289289, + "tokens_seen": 2876571648 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012964894684052157, + "loss": 1.2026, + "theoretical_loss": 3.360015025616934, + "tokens_seen": 2876637184 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012962888665997993, + "loss": 1.2982, + "theoretical_loss": 3.3600091401162095, + "tokens_seen": 2876702720 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001296088264794383, + "loss": 1.3225, + "theoretical_loss": 3.360003254787107, + "tokens_seen": 2876768256 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012958876629889669, + "loss": 1.2878, + "theoretical_loss": 3.3599973696296166, + "tokens_seen": 2876833792 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012956870611835505, + "loss": 1.2253, + "theoretical_loss": 3.35999148464373, + "tokens_seen": 2876899328 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012954864593781344, + "loss": 1.3596, + "theoretical_loss": 3.3599855998294386, + "tokens_seen": 2876964864 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001295285857572718, + "loss": 1.3773, + "theoretical_loss": 3.359979715186733, + "tokens_seen": 2877030400 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001295085255767302, + "loss": 1.3419, + "theoretical_loss": 3.3599738307156044, + "tokens_seen": 2877095936 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012948846539618858, + "loss": 1.2522, + "theoretical_loss": 3.359967946416044, + "tokens_seen": 2877161472 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012946840521564697, + "loss": 1.2733, + "theoretical_loss": 3.3599620622880426, + "tokens_seen": 2877227008 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012944834503510533, + "loss": 1.3679, + "theoretical_loss": 3.3599561783315917, + "tokens_seen": 2877292544 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001294282848545637, + "loss": 1.2515, + "theoretical_loss": 3.3599502945466826, + "tokens_seen": 2877358080 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012940822467402208, + "loss": 1.2765, + "theoretical_loss": 3.3599444109333056, + "tokens_seen": 2877423616 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012938816449348044, + "loss": 1.2555, + "theoretical_loss": 3.3599385274914524, + "tokens_seen": 2877489152 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012936810431293883, + "loss": 1.2391, + "theoretical_loss": 3.359932644221114, + "tokens_seen": 2877554688 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001293480441323972, + "loss": 1.3326, + "theoretical_loss": 3.359926761122281, + "tokens_seen": 2877620224 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012932798395185558, + "loss": 1.2456, + "theoretical_loss": 3.359920878194945, + "tokens_seen": 2877685760 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012930792377131395, + "loss": 1.3433, + "theoretical_loss": 3.359914995439097, + "tokens_seen": 2877751296 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012928786359077234, + "loss": 1.2622, + "theoretical_loss": 3.3599091128547283, + "tokens_seen": 2877816832 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001292678034102307, + "loss": 1.2421, + "theoretical_loss": 3.3599032304418297, + "tokens_seen": 2877882368 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001292477432296891, + "loss": 1.3695, + "theoretical_loss": 3.359897348200392, + "tokens_seen": 2877947904 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012922768304914745, + "loss": 1.2896, + "theoretical_loss": 3.3598914661304073, + "tokens_seen": 2878013440 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001292076228686058, + "loss": 1.3571, + "theoretical_loss": 3.359885584231866, + "tokens_seen": 2878078976 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001291875626880642, + "loss": 1.2737, + "theoretical_loss": 3.3598797025047586, + "tokens_seen": 2878144512 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012916750250752256, + "loss": 1.2156, + "theoretical_loss": 3.3598738209490775, + "tokens_seen": 2878210048 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012914744232698095, + "loss": 1.4239, + "theoretical_loss": 3.359867939564813, + "tokens_seen": 2878275584 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012912738214643932, + "loss": 1.3553, + "theoretical_loss": 3.359862058351956, + "tokens_seen": 2878341120 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001291073219658977, + "loss": 1.2511, + "theoretical_loss": 3.3598561773104985, + "tokens_seen": 2878406656 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012908726178535607, + "loss": 1.239, + "theoretical_loss": 3.3598502964404307, + "tokens_seen": 2878472192 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012906720160481446, + "loss": 1.2741, + "theoretical_loss": 3.3598444157417444, + "tokens_seen": 2878537728 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012904714142427282, + "loss": 1.2905, + "theoretical_loss": 3.35983853521443, + "tokens_seen": 2878603264 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012902708124373118, + "loss": 1.3805, + "theoretical_loss": 3.3598326548584794, + "tokens_seen": 2878668800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012900702106318957, + "loss": 1.3362, + "theoretical_loss": 3.359826774673883, + "tokens_seen": 2878734336 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1046332, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2998217344284058, + "objective/train/theoretical_loss": 3.3598208946606323, + "objective/train/tokens_used": 1249258976, + "theoretical_loss": 3.3598208946606323, + "tokens_seen": 2878799872 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012898696088264793, + "loss": 1.2947, + "theoretical_loss": 3.3598208946606323, + "tokens_seen": 2878799872 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012896690070210632, + "loss": 1.1581, + "theoretical_loss": 3.359815014818718, + "tokens_seen": 2878865408 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012894684052156468, + "loss": 1.2567, + "theoretical_loss": 3.359809135148132, + "tokens_seen": 2878930944 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012892678034102307, + "loss": 1.1676, + "theoretical_loss": 3.3598032556488646, + "tokens_seen": 2878996480 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012890672016048144, + "loss": 1.3338, + "theoretical_loss": 3.3597973763209072, + "tokens_seen": 2879062016 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012888665997993982, + "loss": 1.3064, + "theoretical_loss": 3.3597914971642515, + "tokens_seen": 2879127552 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001288665997993982, + "loss": 1.3314, + "theoretical_loss": 3.359785618178887, + "tokens_seen": 2879193088 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012884653961885655, + "loss": 1.3627, + "theoretical_loss": 3.359779739364807, + "tokens_seen": 2879258624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012882647943831494, + "loss": 1.1772, + "theoretical_loss": 3.359773860722001, + "tokens_seen": 2879324160 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001288064192577733, + "loss": 1.2653, + "theoretical_loss": 3.3597679822504602, + "tokens_seen": 2879389696 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001287863590772317, + "loss": 1.3859, + "theoretical_loss": 3.3597621039501764, + "tokens_seen": 2879455232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012876629889669005, + "loss": 1.1782, + "theoretical_loss": 3.3597562258211404, + "tokens_seen": 2879520768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012874623871614847, + "loss": 1.3524, + "theoretical_loss": 3.3597503478633435, + "tokens_seen": 2879586304 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012872617853560683, + "loss": 1.2798, + "theoretical_loss": 3.3597444700767762, + "tokens_seen": 2879651840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012870611835506522, + "loss": 1.1491, + "theoretical_loss": 3.3597385924614303, + "tokens_seen": 2879717376 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012868605817452358, + "loss": 1.2291, + "theoretical_loss": 3.3597327150172966, + "tokens_seen": 2879782912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012866599799398197, + "loss": 1.3303, + "theoretical_loss": 3.3597268377443665, + "tokens_seen": 2879848448 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012864593781344033, + "loss": 1.2701, + "theoretical_loss": 3.3597209606426306, + "tokens_seen": 2879913984 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001286258776328987, + "loss": 1.3429, + "theoretical_loss": 3.3597150837120804, + "tokens_seen": 2879979520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012860581745235709, + "loss": 1.3296, + "theoretical_loss": 3.359709206952707, + "tokens_seen": 2880045056 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012858575727181545, + "loss": 1.2772, + "theoretical_loss": 3.3597033303645016, + "tokens_seen": 2880110592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012856569709127384, + "loss": 1.2677, + "theoretical_loss": 3.3596974539474553, + "tokens_seen": 2880176128 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001285456369107322, + "loss": 1.2368, + "theoretical_loss": 3.3596915777015584, + "tokens_seen": 2880241664 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001285255767301906, + "loss": 1.2463, + "theoretical_loss": 3.3596857016268036, + "tokens_seen": 2880307200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012850551654964895, + "loss": 1.3352, + "theoretical_loss": 3.3596798257231804, + "tokens_seen": 2880372736 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012848545636910734, + "loss": 1.3348, + "theoretical_loss": 3.3596739499906807, + "tokens_seen": 2880438272 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001284653961885657, + "loss": 1.2882, + "theoretical_loss": 3.3596680744292957, + "tokens_seen": 2880503808 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012844533600802407, + "loss": 1.2548, + "theoretical_loss": 3.3596621990390165, + "tokens_seen": 2880569344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012842527582748245, + "loss": 1.3124, + "theoretical_loss": 3.3596563238198343, + "tokens_seen": 2880634880 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012840521564694082, + "loss": 1.1857, + "theoretical_loss": 3.35965044877174, + "tokens_seen": 2880700416 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001283851554663992, + "loss": 1.2723, + "theoretical_loss": 3.359644573894725, + "tokens_seen": 2880765952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012836509528585757, + "loss": 1.176, + "theoretical_loss": 3.3596386991887797, + "tokens_seen": 2880831488 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012834503510531596, + "loss": 1.3352, + "theoretical_loss": 3.359632824653896, + "tokens_seen": 2880897024 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012832497492477432, + "loss": 1.1698, + "theoretical_loss": 3.3596269502900644, + "tokens_seen": 2880962560 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001283049147442327, + "loss": 1.2782, + "theoretical_loss": 3.3596210760972767, + "tokens_seen": 2881028096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012828485456369107, + "loss": 1.2754, + "theoretical_loss": 3.3596152020755237, + "tokens_seen": 2881093632 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012826479438314943, + "loss": 1.2992, + "theoretical_loss": 3.3596093282247965, + "tokens_seen": 2881159168 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012824473420260782, + "loss": 1.2769, + "theoretical_loss": 3.3596034545450864, + "tokens_seen": 2881224704 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012822467402206619, + "loss": 1.376, + "theoretical_loss": 3.3595975810363843, + "tokens_seen": 2881290240 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012820461384152457, + "loss": 1.3276, + "theoretical_loss": 3.3595917076986814, + "tokens_seen": 2881355776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012818455366098294, + "loss": 1.2317, + "theoretical_loss": 3.359585834531969, + "tokens_seen": 2881421312 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012816449348044133, + "loss": 1.2515, + "theoretical_loss": 3.359579961536238, + "tokens_seen": 2881486848 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001281444332998997, + "loss": 1.311, + "theoretical_loss": 3.3595740887114798, + "tokens_seen": 2881552384 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012812437311935808, + "loss": 1.3302, + "theoretical_loss": 3.359568216057685, + "tokens_seen": 2881617920 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012810431293881644, + "loss": 1.2785, + "theoretical_loss": 3.3595623435748454, + "tokens_seen": 2881683456 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012808425275827483, + "loss": 1.1957, + "theoretical_loss": 3.3595564712629518, + "tokens_seen": 2881748992 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001280641925777332, + "loss": 1.3127, + "theoretical_loss": 3.3595505991219956, + "tokens_seen": 2881814528 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012804413239719155, + "loss": 1.3311, + "theoretical_loss": 3.3595447271519676, + "tokens_seen": 2881880064 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012802407221664994, + "loss": 1.237, + "theoretical_loss": 3.359538855352859, + "tokens_seen": 2881945600 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001280040120361083, + "loss": 1.1972, + "theoretical_loss": 3.3595329837246606, + "tokens_seen": 2882011136 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1047629, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.259054183959961, + "objective/train/theoretical_loss": 3.3595271122673642, + "objective/train/tokens_used": 1252535776, + "theoretical_loss": 3.3595271122673642, + "tokens_seen": 2882076672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012798395185556672, + "loss": 1.2636, + "theoretical_loss": 3.3595271122673642, + "tokens_seen": 2882076672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012796389167502508, + "loss": 1.2657, + "theoretical_loss": 3.359521240980961, + "tokens_seen": 2882142208 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012794383149448347, + "loss": 1.2269, + "theoretical_loss": 3.359515369865442, + "tokens_seen": 2882207744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012792377131394184, + "loss": 1.3754, + "theoretical_loss": 3.359509498920797, + "tokens_seen": 2882273280 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012790371113340023, + "loss": 1.3001, + "theoretical_loss": 3.3595036281470194, + "tokens_seen": 2882338816 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001278836509528586, + "loss": 1.1938, + "theoretical_loss": 3.3594977575440987, + "tokens_seen": 2882404352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012786359077231695, + "loss": 1.1626, + "theoretical_loss": 3.359491887112027, + "tokens_seen": 2882469888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012784353059177534, + "loss": 1.3039, + "theoretical_loss": 3.359486016850795, + "tokens_seen": 2882535424 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001278234704112337, + "loss": 1.2614, + "theoretical_loss": 3.3594801467603936, + "tokens_seen": 2882600960 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001278034102306921, + "loss": 1.3059, + "theoretical_loss": 3.3594742768408143, + "tokens_seen": 2882666496 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012778335005015045, + "loss": 1.2943, + "theoretical_loss": 3.359468407092048, + "tokens_seen": 2882732032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012776328986960884, + "loss": 1.3141, + "theoretical_loss": 3.3594625375140863, + "tokens_seen": 2882797568 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001277432296890672, + "loss": 1.354, + "theoretical_loss": 3.35945666810692, + "tokens_seen": 2882863104 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001277231695085256, + "loss": 1.3261, + "theoretical_loss": 3.35945079887054, + "tokens_seen": 2882928640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012770310932798396, + "loss": 1.2708, + "theoretical_loss": 3.359444929804938, + "tokens_seen": 2882994176 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012768304914744232, + "loss": 1.2833, + "theoretical_loss": 3.359439060910105, + "tokens_seen": 2883059712 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001276629889669007, + "loss": 1.273, + "theoretical_loss": 3.359433192186032, + "tokens_seen": 2883125248 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012764292878635907, + "loss": 1.3292, + "theoretical_loss": 3.35942732363271, + "tokens_seen": 2883190784 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012762286860581746, + "loss": 1.2661, + "theoretical_loss": 3.3594214552501307, + "tokens_seen": 2883256320 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012760280842527582, + "loss": 1.2378, + "theoretical_loss": 3.3594155870382845, + "tokens_seen": 2883321856 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001275827482447342, + "loss": 1.1999, + "theoretical_loss": 3.359409718997163, + "tokens_seen": 2883387392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012756268806419257, + "loss": 1.3329, + "theoretical_loss": 3.359403851126758, + "tokens_seen": 2883452928 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012754262788365096, + "loss": 1.3805, + "theoretical_loss": 3.359397983427059, + "tokens_seen": 2883518464 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012752256770310932, + "loss": 1.2621, + "theoretical_loss": 3.3593921158980584, + "tokens_seen": 2883584000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012750250752256771, + "loss": 1.2705, + "theoretical_loss": 3.359386248539747, + "tokens_seen": 2883649536 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012748244734202608, + "loss": 1.2912, + "theoretical_loss": 3.3593803813521164, + "tokens_seen": 2883715072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012746238716148444, + "loss": 1.3411, + "theoretical_loss": 3.3593745143351574, + "tokens_seen": 2883780608 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012744232698094283, + "loss": 1.1657, + "theoretical_loss": 3.3593686474888607, + "tokens_seen": 2883846144 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001274222668004012, + "loss": 1.3279, + "theoretical_loss": 3.359362780813218, + "tokens_seen": 2883911680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012740220661985958, + "loss": 1.2878, + "theoretical_loss": 3.3593569143082207, + "tokens_seen": 2883977216 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012738214643931794, + "loss": 1.2543, + "theoretical_loss": 3.3593510479738597, + "tokens_seen": 2884042752 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012736208625877633, + "loss": 1.3126, + "theoretical_loss": 3.3593451818101254, + "tokens_seen": 2884108288 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001273420260782347, + "loss": 1.2738, + "theoretical_loss": 3.35933931581701, + "tokens_seen": 2884173824 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012732196589769308, + "loss": 1.3123, + "theoretical_loss": 3.3593334499945042, + "tokens_seen": 2884239360 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012730190571715144, + "loss": 1.3151, + "theoretical_loss": 3.3593275843425996, + "tokens_seen": 2884304896 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001272818455366098, + "loss": 1.2079, + "theoretical_loss": 3.3593217188612865, + "tokens_seen": 2884370432 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001272617853560682, + "loss": 1.289, + "theoretical_loss": 3.359315853550557, + "tokens_seen": 2884435968 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012724172517552659, + "loss": 1.25, + "theoretical_loss": 3.3593099884104016, + "tokens_seen": 2884501504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012722166499498497, + "loss": 1.2274, + "theoretical_loss": 3.359304123440812, + "tokens_seen": 2884567040 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012720160481444334, + "loss": 1.2939, + "theoretical_loss": 3.359298258641779, + "tokens_seen": 2884632576 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012718154463390173, + "loss": 1.2804, + "theoretical_loss": 3.3592923940132935, + "tokens_seen": 2884698112 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001271614844533601, + "loss": 1.3013, + "theoretical_loss": 3.359286529555347, + "tokens_seen": 2884763648 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012714142427281848, + "loss": 1.2153, + "theoretical_loss": 3.359280665267931, + "tokens_seen": 2884829184 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012712136409227684, + "loss": 1.3348, + "theoretical_loss": 3.3592748011510363, + "tokens_seen": 2884894720 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012710130391173523, + "loss": 1.177, + "theoretical_loss": 3.359268937204654, + "tokens_seen": 2884960256 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001270812437311936, + "loss": 1.1492, + "theoretical_loss": 3.3592630734287754, + "tokens_seen": 2885025792 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012706118355065195, + "loss": 1.1847, + "theoretical_loss": 3.359257209823392, + "tokens_seen": 2885091328 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012704112337011034, + "loss": 1.2752, + "theoretical_loss": 3.359251346388494, + "tokens_seen": 2885156864 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001270210631895687, + "loss": 1.2747, + "theoretical_loss": 3.3592454831240732, + "tokens_seen": 2885222400 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001270010030090271, + "loss": 1.3102, + "theoretical_loss": 3.359239620030121, + "tokens_seen": 2885287936 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1048861, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4479341506958008, + "objective/train/theoretical_loss": 3.3592337571066286, + "objective/train/tokens_used": 1255812576, + "theoretical_loss": 3.3592337571066286, + "tokens_seen": 2885353472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012698094282848546, + "loss": 1.3971, + "theoretical_loss": 3.3592337571066286, + "tokens_seen": 2885353472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012696088264794385, + "loss": 1.3006, + "theoretical_loss": 3.3592278943535865, + "tokens_seen": 2885419008 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001269408224674022, + "loss": 1.2556, + "theoretical_loss": 3.3592220317709867, + "tokens_seen": 2885484544 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001269207622868606, + "loss": 1.3132, + "theoretical_loss": 3.3592161693588194, + "tokens_seen": 2885550080 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012690070210631896, + "loss": 1.258, + "theoretical_loss": 3.3592103071170767, + "tokens_seen": 2885615616 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012688064192577732, + "loss": 1.1862, + "theoretical_loss": 3.359204445045749, + "tokens_seen": 2885681152 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001268605817452357, + "loss": 1.359, + "theoretical_loss": 3.359198583144828, + "tokens_seen": 2885746688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012684052156469407, + "loss": 1.3561, + "theoretical_loss": 3.3591927214143054, + "tokens_seen": 2885812224 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012682046138415246, + "loss": 1.2621, + "theoretical_loss": 3.359186859854171, + "tokens_seen": 2885877760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012680040120361083, + "loss": 1.1891, + "theoretical_loss": 3.3591809984644168, + "tokens_seen": 2885943296 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012678034102306922, + "loss": 1.2165, + "theoretical_loss": 3.359175137245034, + "tokens_seen": 2886008832 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012676028084252758, + "loss": 1.2729, + "theoretical_loss": 3.3591692761960137, + "tokens_seen": 2886074368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012674022066198597, + "loss": 1.3222, + "theoretical_loss": 3.359163415317347, + "tokens_seen": 2886139904 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012672016048144433, + "loss": 1.174, + "theoretical_loss": 3.359157554609025, + "tokens_seen": 2886205440 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001267001003009027, + "loss": 1.3416, + "theoretical_loss": 3.3591516940710395, + "tokens_seen": 2886270976 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012668004012036108, + "loss": 1.2618, + "theoretical_loss": 3.359145833703381, + "tokens_seen": 2886336512 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012665997993981944, + "loss": 1.3255, + "theoretical_loss": 3.35913997350604, + "tokens_seen": 2886402048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012663991975927783, + "loss": 1.2621, + "theoretical_loss": 3.3591341134790094, + "tokens_seen": 2886467584 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001266198595787362, + "loss": 1.2575, + "theoretical_loss": 3.3591282536222797, + "tokens_seen": 2886533120 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012659979939819458, + "loss": 1.2627, + "theoretical_loss": 3.3591223939358414, + "tokens_seen": 2886598656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012657973921765295, + "loss": 1.246, + "theoretical_loss": 3.3591165344196865, + "tokens_seen": 2886664192 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012655967903711134, + "loss": 1.2892, + "theoretical_loss": 3.3591106750738056, + "tokens_seen": 2886729728 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001265396188565697, + "loss": 1.2543, + "theoretical_loss": 3.3591048158981907, + "tokens_seen": 2886795264 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001265195586760281, + "loss": 1.3226, + "theoretical_loss": 3.359098956892832, + "tokens_seen": 2886860800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012649949849548648, + "loss": 1.3302, + "theoretical_loss": 3.359093098057721, + "tokens_seen": 2886926336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012647943831494484, + "loss": 1.3453, + "theoretical_loss": 3.3590872393928493, + "tokens_seen": 2886991872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012645937813440323, + "loss": 1.2373, + "theoretical_loss": 3.359081380898208, + "tokens_seen": 2887057408 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001264393179538616, + "loss": 1.2703, + "theoretical_loss": 3.359075522573788, + "tokens_seen": 2887122944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012641925777331998, + "loss": 1.2318, + "theoretical_loss": 3.3590696644195805, + "tokens_seen": 2887188480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012639919759277834, + "loss": 1.2219, + "theoretical_loss": 3.359063806435577, + "tokens_seen": 2887254016 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012637913741223673, + "loss": 1.27, + "theoretical_loss": 3.3590579486217687, + "tokens_seen": 2887319552 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001263590772316951, + "loss": 1.2922, + "theoretical_loss": 3.359052090978146, + "tokens_seen": 2887385088 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012633901705115348, + "loss": 1.2746, + "theoretical_loss": 3.359046233504701, + "tokens_seen": 2887450624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012631895687061185, + "loss": 1.3157, + "theoretical_loss": 3.3590403762014245, + "tokens_seen": 2887516160 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001262988966900702, + "loss": 1.2406, + "theoretical_loss": 3.3590345190683077, + "tokens_seen": 2887581696 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001262788365095286, + "loss": 1.3354, + "theoretical_loss": 3.359028662105342, + "tokens_seen": 2887647232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012625877632898696, + "loss": 1.224, + "theoretical_loss": 3.3590228053125184, + "tokens_seen": 2887712768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012623871614844535, + "loss": 1.2338, + "theoretical_loss": 3.3590169486898285, + "tokens_seen": 2887778304 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001262186559679037, + "loss": 1.2563, + "theoretical_loss": 3.3590110922372625, + "tokens_seen": 2887843840 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001261985957873621, + "loss": 1.3234, + "theoretical_loss": 3.359005235954813, + "tokens_seen": 2887909376 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012617853560682046, + "loss": 1.2603, + "theoretical_loss": 3.3589993798424693, + "tokens_seen": 2887974912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012615847542627885, + "loss": 1.2571, + "theoretical_loss": 3.3589935239002244, + "tokens_seen": 2888040448 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012613841524573721, + "loss": 1.2066, + "theoretical_loss": 3.3589876681280693, + "tokens_seen": 2888105984 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012611835506519558, + "loss": 1.3657, + "theoretical_loss": 3.358981812525994, + "tokens_seen": 2888171520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012609829488465397, + "loss": 1.3804, + "theoretical_loss": 3.358975957093991, + "tokens_seen": 2888237056 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012607823470411233, + "loss": 1.2582, + "theoretical_loss": 3.3589701018320506, + "tokens_seen": 2888302592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012605817452357072, + "loss": 1.2406, + "theoretical_loss": 3.3589642467401646, + "tokens_seen": 2888368128 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012603811434302908, + "loss": 1.2726, + "theoretical_loss": 3.358958391818324, + "tokens_seen": 2888433664 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012601805416248747, + "loss": 1.3003, + "theoretical_loss": 3.3589525370665196, + "tokens_seen": 2888499200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012599799398194583, + "loss": 1.2988, + "theoretical_loss": 3.358946682484743, + "tokens_seen": 2888564736 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1050131, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3088003396987915, + "objective/train/theoretical_loss": 3.3589408280729858, + "objective/train/tokens_used": 1259089376, + "theoretical_loss": 3.3589408280729858, + "tokens_seen": 2888630272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012597793380140422, + "loss": 1.351, + "theoretical_loss": 3.3589408280729858, + "tokens_seen": 2888630272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012595787362086258, + "loss": 1.3932, + "theoretical_loss": 3.3589349738312384, + "tokens_seen": 2888695808 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012593781344032097, + "loss": 1.292, + "theoretical_loss": 3.3589291197594924, + "tokens_seen": 2888761344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012591775325977933, + "loss": 1.3416, + "theoretical_loss": 3.3589232658577393, + "tokens_seen": 2888826880 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001258976930792377, + "loss": 1.3498, + "theoretical_loss": 3.3589174121259693, + "tokens_seen": 2888892416 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012587763289869609, + "loss": 1.2653, + "theoretical_loss": 3.358911558564175, + "tokens_seen": 2888957952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012585757271815445, + "loss": 1.2262, + "theoretical_loss": 3.3589057051723463, + "tokens_seen": 2889023488 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012583751253761284, + "loss": 1.2866, + "theoretical_loss": 3.3588998519504756, + "tokens_seen": 2889089024 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001258174523570712, + "loss": 1.3262, + "theoretical_loss": 3.3588939988985533, + "tokens_seen": 2889154560 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001257973921765296, + "loss": 1.358, + "theoretical_loss": 3.3588881460165707, + "tokens_seen": 2889220096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012577733199598795, + "loss": 1.2262, + "theoretical_loss": 3.3588822933045193, + "tokens_seen": 2889285632 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012575727181544634, + "loss": 1.2981, + "theoretical_loss": 3.35887644076239, + "tokens_seen": 2889351168 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012573721163490473, + "loss": 1.2124, + "theoretical_loss": 3.3588705883901744, + "tokens_seen": 2889416704 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001257171514543631, + "loss": 1.2657, + "theoretical_loss": 3.358864736187863, + "tokens_seen": 2889482240 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012569709127382148, + "loss": 1.2874, + "theoretical_loss": 3.358858884155448, + "tokens_seen": 2889547776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012567703109327984, + "loss": 1.1942, + "theoretical_loss": 3.35885303229292, + "tokens_seen": 2889613312 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012565697091273823, + "loss": 1.3924, + "theoretical_loss": 3.3588471806002707, + "tokens_seen": 2889678848 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001256369107321966, + "loss": 1.342, + "theoretical_loss": 3.3588413290774906, + "tokens_seen": 2889744384 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012561685055165498, + "loss": 1.1956, + "theoretical_loss": 3.358835477724571, + "tokens_seen": 2889809920 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012559679037111335, + "loss": 1.3283, + "theoretical_loss": 3.3588296265415036, + "tokens_seen": 2889875456 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012557673019057174, + "loss": 1.1962, + "theoretical_loss": 3.3588237755282795, + "tokens_seen": 2889940992 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001255566700100301, + "loss": 1.2736, + "theoretical_loss": 3.3588179246848897, + "tokens_seen": 2890006528 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001255366098294885, + "loss": 1.2906, + "theoretical_loss": 3.3588120740113254, + "tokens_seen": 2890072064 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012551654964894685, + "loss": 1.2743, + "theoretical_loss": 3.358806223507578, + "tokens_seen": 2890137600 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001254964894684052, + "loss": 1.2443, + "theoretical_loss": 3.358800373173639, + "tokens_seen": 2890203136 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001254764292878636, + "loss": 1.2607, + "theoretical_loss": 3.358794523009499, + "tokens_seen": 2890268672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012545636910732196, + "loss": 1.24, + "theoretical_loss": 3.35878867301515, + "tokens_seen": 2890334208 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012543630892678035, + "loss": 1.3611, + "theoretical_loss": 3.358782823190582, + "tokens_seen": 2890399744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012541624874623872, + "loss": 1.237, + "theoretical_loss": 3.3587769735357873, + "tokens_seen": 2890465280 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001253961885656971, + "loss": 1.2598, + "theoretical_loss": 3.3587711240507567, + "tokens_seen": 2890530816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012537612838515547, + "loss": 1.2704, + "theoretical_loss": 3.3587652747354815, + "tokens_seen": 2890596352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012535606820461386, + "loss": 1.248, + "theoretical_loss": 3.358759425589953, + "tokens_seen": 2890661888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012533600802407222, + "loss": 1.3579, + "theoretical_loss": 3.3587535766141627, + "tokens_seen": 2890727424 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012531594784353058, + "loss": 1.2317, + "theoretical_loss": 3.358747727808101, + "tokens_seen": 2890792960 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012529588766298897, + "loss": 1.1502, + "theoretical_loss": 3.35874187917176, + "tokens_seen": 2890858496 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012527582748244733, + "loss": 1.3642, + "theoretical_loss": 3.3587360307051304, + "tokens_seen": 2890924032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012525576730190572, + "loss": 1.3189, + "theoretical_loss": 3.3587301824082036, + "tokens_seen": 2890989568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012523570712136408, + "loss": 1.2503, + "theoretical_loss": 3.3587243342809705, + "tokens_seen": 2891055104 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012521564694082247, + "loss": 1.2277, + "theoretical_loss": 3.358718486323423, + "tokens_seen": 2891120640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012519558676028084, + "loss": 1.238, + "theoretical_loss": 3.3587126385355517, + "tokens_seen": 2891186176 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012517552657973922, + "loss": 1.2468, + "theoretical_loss": 3.3587067909173483, + "tokens_seen": 2891251712 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001251554663991976, + "loss": 1.3207, + "theoretical_loss": 3.3587009434688033, + "tokens_seen": 2891317248 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012513540621865595, + "loss": 1.242, + "theoretical_loss": 3.3586950961899094, + "tokens_seen": 2891382784 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012511534603811434, + "loss": 1.2189, + "theoretical_loss": 3.358689249080656, + "tokens_seen": 2891448320 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001250952858575727, + "loss": 1.3497, + "theoretical_loss": 3.3586834021410357, + "tokens_seen": 2891513856 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001250752256770311, + "loss": 1.3459, + "theoretical_loss": 3.358677555371039, + "tokens_seen": 2891579392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012505516549648945, + "loss": 1.2288, + "theoretical_loss": 3.3586717087706575, + "tokens_seen": 2891644928 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012503510531594784, + "loss": 1.2225, + "theoretical_loss": 3.3586658623398824, + "tokens_seen": 2891710464 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001250150451354062, + "loss": 1.2826, + "theoretical_loss": 3.358660016078705, + "tokens_seen": 2891776000 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001249949849548646, + "loss": 1.3092, + "theoretical_loss": 3.3586541699871164, + "tokens_seen": 2891841536 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1051420, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3007147312164307, + "objective/train/theoretical_loss": 3.3586483240651077, + "objective/train/tokens_used": 1262366176, + "theoretical_loss": 3.3586483240651077, + "tokens_seen": 2891907072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012497492477432298, + "loss": 1.2745, + "theoretical_loss": 3.3586483240651077, + "tokens_seen": 2891907072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012495486459378134, + "loss": 1.2569, + "theoretical_loss": 3.35864247831267, + "tokens_seen": 2891972608 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001249348044132397, + "loss": 1.2302, + "theoretical_loss": 3.3586366327297954, + "tokens_seen": 2892038144 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001249147442326981, + "loss": 1.2995, + "theoretical_loss": 3.358630787316474, + "tokens_seen": 2892103680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012489468405215646, + "loss": 1.2435, + "theoretical_loss": 3.358624942072698, + "tokens_seen": 2892169216 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012487462387161485, + "loss": 1.3936, + "theoretical_loss": 3.3586190969984577, + "tokens_seen": 2892234752 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001248545636910732, + "loss": 1.1891, + "theoretical_loss": 3.358613252093745, + "tokens_seen": 2892300288 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001248345035105316, + "loss": 1.2804, + "theoretical_loss": 3.3586074073585515, + "tokens_seen": 2892365824 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012481444332998996, + "loss": 1.2798, + "theoretical_loss": 3.3586015627928676, + "tokens_seen": 2892431360 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012479438314944835, + "loss": 1.2063, + "theoretical_loss": 3.358595718396685, + "tokens_seen": 2892496896 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012477432296890674, + "loss": 1.3339, + "theoretical_loss": 3.358589874169995, + "tokens_seen": 2892562432 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001247542627883651, + "loss": 1.3072, + "theoretical_loss": 3.3585840301127883, + "tokens_seen": 2892627968 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012473420260782346, + "loss": 1.2976, + "theoretical_loss": 3.3585781862250568, + "tokens_seen": 2892693504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012471414242728185, + "loss": 1.297, + "theoretical_loss": 3.3585723425067915, + "tokens_seen": 2892759040 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012469408224674022, + "loss": 1.319, + "theoretical_loss": 3.3585664989579835, + "tokens_seen": 2892824576 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001246740220661986, + "loss": 1.3235, + "theoretical_loss": 3.358560655578624, + "tokens_seen": 2892890112 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012465396188565697, + "loss": 1.3441, + "theoretical_loss": 3.358554812368705, + "tokens_seen": 2892955648 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012463390170511536, + "loss": 1.182, + "theoretical_loss": 3.3585489693282167, + "tokens_seen": 2893021184 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012461384152457372, + "loss": 1.2126, + "theoretical_loss": 3.358543126457151, + "tokens_seen": 2893086720 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001245937813440321, + "loss": 1.3363, + "theoretical_loss": 3.358537283755499, + "tokens_seen": 2893152256 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012457372116349047, + "loss": 1.2319, + "theoretical_loss": 3.358531441223252, + "tokens_seen": 2893217792 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012455366098294883, + "loss": 1.4171, + "theoretical_loss": 3.358525598860401, + "tokens_seen": 2893283328 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012453360080240722, + "loss": 1.2531, + "theoretical_loss": 3.3585197566669374, + "tokens_seen": 2893348864 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012451354062186559, + "loss": 1.3336, + "theoretical_loss": 3.3585139146428524, + "tokens_seen": 2893414400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012449348044132397, + "loss": 1.3165, + "theoretical_loss": 3.358508072788138, + "tokens_seen": 2893479936 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012447342026078234, + "loss": 1.2905, + "theoretical_loss": 3.3585022311027837, + "tokens_seen": 2893545472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012445336008024073, + "loss": 1.2053, + "theoretical_loss": 3.3584963895867825, + "tokens_seen": 2893611008 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012443329989969912, + "loss": 1.2622, + "theoretical_loss": 3.358490548240125, + "tokens_seen": 2893676544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012441323971915748, + "loss": 1.2656, + "theoretical_loss": 3.3584847070628023, + "tokens_seen": 2893742080 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012439317953861587, + "loss": 1.3273, + "theoretical_loss": 3.358478866054806, + "tokens_seen": 2893807616 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012437311935807423, + "loss": 1.2308, + "theoretical_loss": 3.358473025216127, + "tokens_seen": 2893873152 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001243530591775326, + "loss": 1.2575, + "theoretical_loss": 3.3584671845467566, + "tokens_seen": 2893938688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012433299899699098, + "loss": 1.1865, + "theoretical_loss": 3.3584613440466864, + "tokens_seen": 2894004224 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012431293881644934, + "loss": 1.2262, + "theoretical_loss": 3.3584555037159074, + "tokens_seen": 2894069760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012429287863590773, + "loss": 1.2379, + "theoretical_loss": 3.3584496635544108, + "tokens_seen": 2894135296 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001242728184553661, + "loss": 1.2788, + "theoretical_loss": 3.3584438235621876, + "tokens_seen": 2894200832 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012425275827482448, + "loss": 1.1976, + "theoretical_loss": 3.35843798373923, + "tokens_seen": 2894266368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012423269809428285, + "loss": 1.357, + "theoretical_loss": 3.3584321440855285, + "tokens_seen": 2894331904 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012421263791374124, + "loss": 1.2663, + "theoretical_loss": 3.3584263046010747, + "tokens_seen": 2894397440 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001241925777331996, + "loss": 1.316, + "theoretical_loss": 3.3584204652858594, + "tokens_seen": 2894462976 + }, + { + "epoch": 0.75, + "learning_rate": 0.000124172517552658, + "loss": 1.3264, + "theoretical_loss": 3.3584146261398744, + "tokens_seen": 2894528512 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012415245737211635, + "loss": 1.2514, + "theoretical_loss": 3.3584087871631105, + "tokens_seen": 2894594048 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001241323971915747, + "loss": 1.3369, + "theoretical_loss": 3.3584029483555593, + "tokens_seen": 2894659584 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001241123370110331, + "loss": 1.2765, + "theoretical_loss": 3.3583971097172123, + "tokens_seen": 2894725120 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012409227683049146, + "loss": 1.3588, + "theoretical_loss": 3.3583912712480597, + "tokens_seen": 2894790656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012407221664994985, + "loss": 1.2898, + "theoretical_loss": 3.358385432948094, + "tokens_seen": 2894856192 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012405215646940824, + "loss": 1.3736, + "theoretical_loss": 3.358379594817306, + "tokens_seen": 2894921728 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001240320962888666, + "loss": 1.2924, + "theoretical_loss": 3.358373756855687, + "tokens_seen": 2894987264 + }, + { + "epoch": 0.75, + "learning_rate": 0.000124012036108325, + "loss": 1.2636, + "theoretical_loss": 3.3583679190632276, + "tokens_seen": 2895052800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012399197592778336, + "loss": 1.2431, + "theoretical_loss": 3.35836208143992, + "tokens_seen": 2895118336 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1052611, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3063397407531738, + "objective/train/theoretical_loss": 3.3583562439857557, + "objective/train/tokens_used": 1265642976, + "theoretical_loss": 3.3583562439857557, + "tokens_seen": 2895183872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012397191574724172, + "loss": 1.2549, + "theoretical_loss": 3.3583562439857557, + "tokens_seen": 2895183872 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001239518555667001, + "loss": 1.2484, + "theoretical_loss": 3.358350406700725, + "tokens_seen": 2895249408 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012393179538615847, + "loss": 1.2705, + "theoretical_loss": 3.358344569584819, + "tokens_seen": 2895314944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012391173520561686, + "loss": 1.2912, + "theoretical_loss": 3.35833873263803, + "tokens_seen": 2895380480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012389167502507522, + "loss": 1.2337, + "theoretical_loss": 3.3583328958603493, + "tokens_seen": 2895446016 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001238716148445336, + "loss": 1.2566, + "theoretical_loss": 3.358327059251767, + "tokens_seen": 2895511552 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012385155466399197, + "loss": 1.3419, + "theoretical_loss": 3.3583212228122754, + "tokens_seen": 2895577088 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012383149448345036, + "loss": 1.2042, + "theoretical_loss": 3.3583153865418653, + "tokens_seen": 2895642624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012381143430290872, + "loss": 1.3001, + "theoretical_loss": 3.3583095504405285, + "tokens_seen": 2895708160 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012379137412236711, + "loss": 1.2776, + "theoretical_loss": 3.3583037145082555, + "tokens_seen": 2895773696 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012377131394182548, + "loss": 1.1903, + "theoretical_loss": 3.358297878745038, + "tokens_seen": 2895839232 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012375125376128384, + "loss": 1.3128, + "theoretical_loss": 3.3582920431508674, + "tokens_seen": 2895904768 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012373119358074223, + "loss": 1.2044, + "theoretical_loss": 3.358286207725735, + "tokens_seen": 2895970304 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001237111334002006, + "loss": 1.2816, + "theoretical_loss": 3.3582803724696313, + "tokens_seen": 2896035840 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012369107321965898, + "loss": 1.2737, + "theoretical_loss": 3.358274537382549, + "tokens_seen": 2896101376 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012367101303911737, + "loss": 1.2983, + "theoretical_loss": 3.3582687024644775, + "tokens_seen": 2896166912 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012365095285857573, + "loss": 1.2941, + "theoretical_loss": 3.35826286771541, + "tokens_seen": 2896232448 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012363089267803412, + "loss": 1.3383, + "theoretical_loss": 3.3582570331353367, + "tokens_seen": 2896297984 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012361083249749248, + "loss": 1.218, + "theoretical_loss": 3.358251198724249, + "tokens_seen": 2896363520 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012359077231695087, + "loss": 1.3084, + "theoretical_loss": 3.3582453644821384, + "tokens_seen": 2896429056 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012357071213640923, + "loss": 1.199, + "theoretical_loss": 3.358239530408996, + "tokens_seen": 2896494592 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001235506519558676, + "loss": 1.2717, + "theoretical_loss": 3.358233696504813, + "tokens_seen": 2896560128 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012353059177532599, + "loss": 1.3929, + "theoretical_loss": 3.3582278627695814, + "tokens_seen": 2896625664 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012351053159478435, + "loss": 1.2284, + "theoretical_loss": 3.358222029203292, + "tokens_seen": 2896691200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012349047141424274, + "loss": 1.3018, + "theoretical_loss": 3.358216195805935, + "tokens_seen": 2896756736 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001234704112337011, + "loss": 1.2322, + "theoretical_loss": 3.3582103625775037, + "tokens_seen": 2896822272 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001234503510531595, + "loss": 1.3656, + "theoretical_loss": 3.358204529517988, + "tokens_seen": 2896887808 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012343029087261785, + "loss": 1.1616, + "theoretical_loss": 3.358198696627379, + "tokens_seen": 2896953344 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012341023069207624, + "loss": 1.2544, + "theoretical_loss": 3.3581928639056695, + "tokens_seen": 2897018880 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001233901705115346, + "loss": 1.3367, + "theoretical_loss": 3.3581870313528492, + "tokens_seen": 2897084416 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012337011033099296, + "loss": 1.3122, + "theoretical_loss": 3.3581811989689103, + "tokens_seen": 2897149952 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012335005015045135, + "loss": 1.2067, + "theoretical_loss": 3.358175366753844, + "tokens_seen": 2897215488 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012332998996990972, + "loss": 1.2107, + "theoretical_loss": 3.358169534707641, + "tokens_seen": 2897281024 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001233099297893681, + "loss": 1.2981, + "theoretical_loss": 3.3581637028302933, + "tokens_seen": 2897346560 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001232898696088265, + "loss": 1.2868, + "theoretical_loss": 3.358157871121792, + "tokens_seen": 2897412096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012326980942828486, + "loss": 1.2071, + "theoretical_loss": 3.358152039582128, + "tokens_seen": 2897477632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012324974924774325, + "loss": 1.2432, + "theoretical_loss": 3.3581462082112927, + "tokens_seen": 2897543168 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001232296890672016, + "loss": 1.2332, + "theoretical_loss": 3.358140377009278, + "tokens_seen": 2897608704 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012320962888666, + "loss": 1.2381, + "theoretical_loss": 3.3581345459760747, + "tokens_seen": 2897674240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012318956870611836, + "loss": 1.2103, + "theoretical_loss": 3.358128715111674, + "tokens_seen": 2897739776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012316950852557672, + "loss": 1.2493, + "theoretical_loss": 3.3581228844160678, + "tokens_seen": 2897805312 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001231494483450351, + "loss": 1.2953, + "theoretical_loss": 3.3581170538892464, + "tokens_seen": 2897870848 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012312938816449347, + "loss": 1.2338, + "theoretical_loss": 3.358111223531202, + "tokens_seen": 2897936384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012310932798395186, + "loss": 1.2654, + "theoretical_loss": 3.3581053933419254, + "tokens_seen": 2898001920 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012308926780341023, + "loss": 1.3701, + "theoretical_loss": 3.3580995633214084, + "tokens_seen": 2898067456 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012306920762286862, + "loss": 1.2994, + "theoretical_loss": 3.3580937334696417, + "tokens_seen": 2898132992 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012304914744232698, + "loss": 1.2217, + "theoretical_loss": 3.3580879037866165, + "tokens_seen": 2898198528 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012302908726178537, + "loss": 1.2893, + "theoretical_loss": 3.3580820742723247, + "tokens_seen": 2898264064 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012300902708124373, + "loss": 1.3858, + "theoretical_loss": 3.3580762449267576, + "tokens_seen": 2898329600 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001229889669007021, + "loss": 1.3045, + "theoretical_loss": 3.358070415749906, + "tokens_seen": 2898395136 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1053920, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2260544300079346, + "objective/train/theoretical_loss": 3.3580645867417616, + "objective/train/tokens_used": 1268919776, + "theoretical_loss": 3.3580645867417616, + "tokens_seen": 2898460672 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012296890672016048, + "loss": 1.2255, + "theoretical_loss": 3.3580645867417616, + "tokens_seen": 2898460672 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012294884653961884, + "loss": 1.2847, + "theoretical_loss": 3.358058757902316, + "tokens_seen": 2898526208 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012292878635907723, + "loss": 1.1922, + "theoretical_loss": 3.358052929231559, + "tokens_seen": 2898591744 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012290872617853562, + "loss": 1.2579, + "theoretical_loss": 3.358047100729484, + "tokens_seen": 2898657280 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012288866599799398, + "loss": 1.2243, + "theoretical_loss": 3.358041272396081, + "tokens_seen": 2898722816 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012286860581745237, + "loss": 1.2643, + "theoretical_loss": 3.358035444231341, + "tokens_seen": 2898788352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012284854563691074, + "loss": 1.2704, + "theoretical_loss": 3.3580296162352563, + "tokens_seen": 2898853888 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012282848545636912, + "loss": 1.3148, + "theoretical_loss": 3.358023788407818, + "tokens_seen": 2898919424 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001228084252758275, + "loss": 1.1846, + "theoretical_loss": 3.3580179607490166, + "tokens_seen": 2898984960 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012278836509528585, + "loss": 1.3181, + "theoretical_loss": 3.358012133258845, + "tokens_seen": 2899050496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012276830491474424, + "loss": 1.3236, + "theoretical_loss": 3.358006305937293, + "tokens_seen": 2899116032 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001227482447342026, + "loss": 1.2018, + "theoretical_loss": 3.358000478784352, + "tokens_seen": 2899181568 + }, + { + "epoch": 0.76, + "learning_rate": 0.000122728184553661, + "loss": 1.3174, + "theoretical_loss": 3.357994651800014, + "tokens_seen": 2899247104 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012270812437311935, + "loss": 1.2147, + "theoretical_loss": 3.35798882498427, + "tokens_seen": 2899312640 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012268806419257774, + "loss": 1.1632, + "theoretical_loss": 3.3579829983371114, + "tokens_seen": 2899378176 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001226680040120361, + "loss": 1.3318, + "theoretical_loss": 3.35797717185853, + "tokens_seen": 2899443712 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001226479438314945, + "loss": 1.3189, + "theoretical_loss": 3.357971345548516, + "tokens_seen": 2899509248 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012262788365095286, + "loss": 1.2445, + "theoretical_loss": 3.3579655194070614, + "tokens_seen": 2899574784 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012260782347041122, + "loss": 1.2511, + "theoretical_loss": 3.3579596934341573, + "tokens_seen": 2899640320 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001225877632898696, + "loss": 1.2532, + "theoretical_loss": 3.3579538676297953, + "tokens_seen": 2899705856 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012256770310932797, + "loss": 1.2349, + "theoretical_loss": 3.357948041993967, + "tokens_seen": 2899771392 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012254764292878636, + "loss": 1.2493, + "theoretical_loss": 3.3579422165266624, + "tokens_seen": 2899836928 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012252758274824475, + "loss": 1.3252, + "theoretical_loss": 3.357936391227874, + "tokens_seen": 2899902464 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001225075225677031, + "loss": 1.3464, + "theoretical_loss": 3.357930566097593, + "tokens_seen": 2899968000 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001224874623871615, + "loss": 1.3642, + "theoretical_loss": 3.35792474113581, + "tokens_seen": 2900033536 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012246740220661986, + "loss": 1.2585, + "theoretical_loss": 3.357918916342517, + "tokens_seen": 2900099072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012244734202607825, + "loss": 1.2647, + "theoretical_loss": 3.357913091717706, + "tokens_seen": 2900164608 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001224272818455366, + "loss": 1.3259, + "theoretical_loss": 3.3579072672613663, + "tokens_seen": 2900230144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012240722166499498, + "loss": 1.2925, + "theoretical_loss": 3.357901442973491, + "tokens_seen": 2900295680 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012238716148445336, + "loss": 1.2456, + "theoretical_loss": 3.3578956188540707, + "tokens_seen": 2900361216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012236710130391173, + "loss": 1.3255, + "theoretical_loss": 3.357889794903097, + "tokens_seen": 2900426752 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012234704112337012, + "loss": 1.2737, + "theoretical_loss": 3.3578839711205606, + "tokens_seen": 2900492288 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012232698094282848, + "loss": 1.2088, + "theoretical_loss": 3.3578781475064536, + "tokens_seen": 2900557824 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012230692076228687, + "loss": 1.339, + "theoretical_loss": 3.3578723240607666, + "tokens_seen": 2900623360 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012228686058174523, + "loss": 1.2755, + "theoretical_loss": 3.3578665007834916, + "tokens_seen": 2900688896 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012226680040120362, + "loss": 1.3068, + "theoretical_loss": 3.35786067767462, + "tokens_seen": 2900754432 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012224674022066198, + "loss": 1.301, + "theoretical_loss": 3.3578548547341422, + "tokens_seen": 2900819968 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012222668004012037, + "loss": 1.3002, + "theoretical_loss": 3.3578490319620506, + "tokens_seen": 2900885504 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012220661985957873, + "loss": 1.2695, + "theoretical_loss": 3.3578432093583355, + "tokens_seen": 2900951040 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001221865596790371, + "loss": 1.2566, + "theoretical_loss": 3.357837386922989, + "tokens_seen": 2901016576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012216649949849549, + "loss": 1.2543, + "theoretical_loss": 3.3578315646560024, + "tokens_seen": 2901082112 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012214643931795387, + "loss": 1.145, + "theoretical_loss": 3.3578257425573663, + "tokens_seen": 2901147648 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012212637913741224, + "loss": 1.2466, + "theoretical_loss": 3.357819920627073, + "tokens_seen": 2901213184 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012210631895687063, + "loss": 1.3696, + "theoretical_loss": 3.3578140988651133, + "tokens_seen": 2901278720 + }, + { + "epoch": 0.76, + "learning_rate": 0.000122086258776329, + "loss": 1.3178, + "theoretical_loss": 3.357808277271478, + "tokens_seen": 2901344256 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012206619859578736, + "loss": 1.2602, + "theoretical_loss": 3.35780245584616, + "tokens_seen": 2901409792 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012204613841524574, + "loss": 1.3121, + "theoretical_loss": 3.357796634589149, + "tokens_seen": 2901475328 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012202607823470412, + "loss": 1.2096, + "theoretical_loss": 3.3577908135004373, + "tokens_seen": 2901540864 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012200601805416249, + "loss": 1.2788, + "theoretical_loss": 3.3577849925800156, + "tokens_seen": 2901606400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012198595787362087, + "loss": 1.2508, + "theoretical_loss": 3.3577791718278758, + "tokens_seen": 2901671936 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1055165, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.303365707397461, + "objective/train/theoretical_loss": 3.357773351244009, + "objective/train/tokens_used": 1272196576, + "theoretical_loss": 3.357773351244009, + "tokens_seen": 2901737472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012196589769307924, + "loss": 1.3423, + "theoretical_loss": 3.357773351244009, + "tokens_seen": 2901737472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012194583751253762, + "loss": 1.3355, + "theoretical_loss": 3.3577675308284065, + "tokens_seen": 2901803008 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012192577733199598, + "loss": 1.1717, + "theoretical_loss": 3.3577617105810598, + "tokens_seen": 2901868544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012190571715145436, + "loss": 1.4403, + "theoretical_loss": 3.3577558905019598, + "tokens_seen": 2901934080 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012188565697091273, + "loss": 1.3367, + "theoretical_loss": 3.357750070591098, + "tokens_seen": 2901999616 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012186559679037111, + "loss": 1.3513, + "theoretical_loss": 3.3577442508484663, + "tokens_seen": 2902065152 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012184553660982948, + "loss": 1.2988, + "theoretical_loss": 3.3577384312740555, + "tokens_seen": 2902130688 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012182547642928786, + "loss": 1.2557, + "theoretical_loss": 3.357732611867857, + "tokens_seen": 2902196224 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012180541624874625, + "loss": 1.2659, + "theoretical_loss": 3.357726792629862, + "tokens_seen": 2902261760 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012178535606820463, + "loss": 1.37, + "theoretical_loss": 3.3577209735600624, + "tokens_seen": 2902327296 + }, + { + "epoch": 0.76, + "learning_rate": 0.000121765295887663, + "loss": 1.3019, + "theoretical_loss": 3.357715154658449, + "tokens_seen": 2902392832 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012174523570712138, + "loss": 1.401, + "theoretical_loss": 3.3577093359250134, + "tokens_seen": 2902458368 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012172517552657974, + "loss": 1.3092, + "theoretical_loss": 3.357703517359747, + "tokens_seen": 2902523904 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012170511534603811, + "loss": 1.2745, + "theoretical_loss": 3.3576976989626406, + "tokens_seen": 2902589440 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012168505516549649, + "loss": 1.2987, + "theoretical_loss": 3.357691880733686, + "tokens_seen": 2902654976 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012166499498495487, + "loss": 1.2132, + "theoretical_loss": 3.3576860626728746, + "tokens_seen": 2902720512 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012164493480441324, + "loss": 1.2113, + "theoretical_loss": 3.3576802447801977, + "tokens_seen": 2902786048 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012162487462387162, + "loss": 1.2508, + "theoretical_loss": 3.3576744270556462, + "tokens_seen": 2902851584 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012160481444333, + "loss": 1.2411, + "theoretical_loss": 3.3576686094992123, + "tokens_seen": 2902917120 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012158475426278837, + "loss": 1.258, + "theoretical_loss": 3.3576627921108866, + "tokens_seen": 2902982656 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012156469408224675, + "loss": 1.3663, + "theoretical_loss": 3.357656974890661, + "tokens_seen": 2903048192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012154463390170512, + "loss": 1.2618, + "theoretical_loss": 3.3576511578385264, + "tokens_seen": 2903113728 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012152457372116348, + "loss": 1.3469, + "theoretical_loss": 3.357645340954474, + "tokens_seen": 2903179264 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012150451354062186, + "loss": 1.2503, + "theoretical_loss": 3.357639524238496, + "tokens_seen": 2903244800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012148445336008024, + "loss": 1.2973, + "theoretical_loss": 3.357633707690583, + "tokens_seen": 2903310336 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012146439317953861, + "loss": 1.1646, + "theoretical_loss": 3.3576278913107265, + "tokens_seen": 2903375872 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012144433299899699, + "loss": 1.234, + "theoretical_loss": 3.357622075098918, + "tokens_seen": 2903441408 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012142427281845538, + "loss": 1.3465, + "theoretical_loss": 3.3576162590551486, + "tokens_seen": 2903506944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012140421263791375, + "loss": 1.2033, + "theoretical_loss": 3.35761044317941, + "tokens_seen": 2903572480 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012138415245737213, + "loss": 1.2084, + "theoretical_loss": 3.3576046274716935, + "tokens_seen": 2903638016 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001213640922768305, + "loss": 1.3632, + "theoretical_loss": 3.35759881193199, + "tokens_seen": 2903703552 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012134403209628888, + "loss": 1.3129, + "theoretical_loss": 3.357592996560291, + "tokens_seen": 2903769088 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012132397191574724, + "loss": 1.286, + "theoretical_loss": 3.3575871813565885, + "tokens_seen": 2903834624 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012130391173520562, + "loss": 1.1585, + "theoretical_loss": 3.3575813663208733, + "tokens_seen": 2903900160 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012128385155466399, + "loss": 1.3248, + "theoretical_loss": 3.3575755514531367, + "tokens_seen": 2903965696 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012126379137412237, + "loss": 1.3052, + "theoretical_loss": 3.35756973675337, + "tokens_seen": 2904031232 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012124373119358074, + "loss": 1.3088, + "theoretical_loss": 3.3575639222215656, + "tokens_seen": 2904096768 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012122367101303912, + "loss": 1.2514, + "theoretical_loss": 3.357558107857713, + "tokens_seen": 2904162304 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001212036108324975, + "loss": 1.3064, + "theoretical_loss": 3.357552293661805, + "tokens_seen": 2904227840 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012118355065195587, + "loss": 1.1858, + "theoretical_loss": 3.357546479633833, + "tokens_seen": 2904293376 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012116349047141425, + "loss": 1.3208, + "theoretical_loss": 3.3575406657737874, + "tokens_seen": 2904358912 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012114343029087261, + "loss": 1.3063, + "theoretical_loss": 3.3575348520816597, + "tokens_seen": 2904424448 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012112337011033099, + "loss": 1.2641, + "theoretical_loss": 3.357529038557442, + "tokens_seen": 2904489984 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012110330992978936, + "loss": 1.2615, + "theoretical_loss": 3.357523225201126, + "tokens_seen": 2904555520 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012108324974924774, + "loss": 1.254, + "theoretical_loss": 3.3575174120127014, + "tokens_seen": 2904621056 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012106318956870611, + "loss": 1.2646, + "theoretical_loss": 3.3575115989921605, + "tokens_seen": 2904686592 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001210431293881645, + "loss": 1.2277, + "theoretical_loss": 3.357505786139495, + "tokens_seen": 2904752128 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012102306920762288, + "loss": 1.2648, + "theoretical_loss": 3.357499973454696, + "tokens_seen": 2904817664 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012100300902708125, + "loss": 1.2993, + "theoretical_loss": 3.3574941609377547, + "tokens_seen": 2904883200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012098294884653963, + "loss": 1.2185, + "theoretical_loss": 3.3574883485886624, + "tokens_seen": 2904948736 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1056274, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4536802768707275, + "objective/train/theoretical_loss": 3.357482536407411, + "objective/train/tokens_used": 1275473376, + "theoretical_loss": 3.357482536407411, + "tokens_seen": 2905014272 + }, + { + "epoch": 0.76, + "learning_rate": 0.000120962888665998, + "loss": 1.3944, + "theoretical_loss": 3.357482536407411, + "tokens_seen": 2905014272 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012094282848545637, + "loss": 1.2635, + "theoretical_loss": 3.357476724393991, + "tokens_seen": 2905079808 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012092276830491474, + "loss": 1.2549, + "theoretical_loss": 3.357470912548395, + "tokens_seen": 2905145344 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012090270812437312, + "loss": 1.3087, + "theoretical_loss": 3.3574651008706127, + "tokens_seen": 2905210880 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001208826479438315, + "loss": 1.2233, + "theoretical_loss": 3.357459289360637, + "tokens_seen": 2905276416 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012086258776328987, + "loss": 1.2891, + "theoretical_loss": 3.3574534780184586, + "tokens_seen": 2905341952 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012084252758274825, + "loss": 1.2403, + "theoretical_loss": 3.357447666844069, + "tokens_seen": 2905407488 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012082246740220662, + "loss": 1.2183, + "theoretical_loss": 3.3574418558374592, + "tokens_seen": 2905473024 + }, + { + "epoch": 0.76, + "learning_rate": 0.000120802407221665, + "loss": 1.3268, + "theoretical_loss": 3.357436044998621, + "tokens_seen": 2905538560 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012078234704112337, + "loss": 1.2874, + "theoretical_loss": 3.3574302343275457, + "tokens_seen": 2905604096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012076228686058175, + "loss": 1.2693, + "theoretical_loss": 3.357424423824225, + "tokens_seen": 2905669632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012074222668004011, + "loss": 1.3742, + "theoretical_loss": 3.3574186134886492, + "tokens_seen": 2905735168 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012072216649949849, + "loss": 1.2258, + "theoretical_loss": 3.357412803320811, + "tokens_seen": 2905800704 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012070210631895686, + "loss": 1.2762, + "theoretical_loss": 3.357406993320701, + "tokens_seen": 2905866240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012068204613841525, + "loss": 1.2559, + "theoretical_loss": 3.3574011834883106, + "tokens_seen": 2905931776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012066198595787363, + "loss": 1.3421, + "theoretical_loss": 3.3573953738236315, + "tokens_seen": 2905997312 + }, + { + "epoch": 0.76, + "learning_rate": 0.000120641925777332, + "loss": 1.2924, + "theoretical_loss": 3.3573895643266547, + "tokens_seen": 2906062848 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012062186559679038, + "loss": 1.3278, + "theoretical_loss": 3.3573837549973717, + "tokens_seen": 2906128384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012060180541624876, + "loss": 1.2662, + "theoretical_loss": 3.357377945835774, + "tokens_seen": 2906193920 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012058174523570713, + "loss": 1.2327, + "theoretical_loss": 3.357372136841853, + "tokens_seen": 2906259456 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012056168505516551, + "loss": 1.2646, + "theoretical_loss": 3.3573663280156, + "tokens_seen": 2906324992 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012054162487462387, + "loss": 1.1859, + "theoretical_loss": 3.357360519357006, + "tokens_seen": 2906390528 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012052156469408225, + "loss": 1.3437, + "theoretical_loss": 3.3573547108660633, + "tokens_seen": 2906456064 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012050150451354062, + "loss": 1.2724, + "theoretical_loss": 3.3573489025427623, + "tokens_seen": 2906521600 + }, + { + "epoch": 0.76, + "learning_rate": 0.000120481444332999, + "loss": 1.2344, + "theoretical_loss": 3.357343094387095, + "tokens_seen": 2906587136 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012046138415245737, + "loss": 1.3557, + "theoretical_loss": 3.357337286399053, + "tokens_seen": 2906652672 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012044132397191575, + "loss": 1.2315, + "theoretical_loss": 3.357331478578627, + "tokens_seen": 2906718208 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012042126379137412, + "loss": 1.3091, + "theoretical_loss": 3.3573256709258086, + "tokens_seen": 2906783744 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001204012036108325, + "loss": 1.2848, + "theoretical_loss": 3.3573198634405887, + "tokens_seen": 2906849280 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012038114343029088, + "loss": 1.358, + "theoretical_loss": 3.35731405612296, + "tokens_seen": 2906914816 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012036108324974924, + "loss": 1.2356, + "theoretical_loss": 3.357308248972913, + "tokens_seen": 2906980352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012034102306920761, + "loss": 1.1913, + "theoretical_loss": 3.357302441990439, + "tokens_seen": 2907045888 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012032096288866599, + "loss": 1.2853, + "theoretical_loss": 3.35729663517553, + "tokens_seen": 2907111424 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012030090270812438, + "loss": 1.2886, + "theoretical_loss": 3.3572908285281766, + "tokens_seen": 2907176960 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012028084252758276, + "loss": 1.2432, + "theoretical_loss": 3.3572850220483703, + "tokens_seen": 2907242496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012026078234704113, + "loss": 1.1851, + "theoretical_loss": 3.3572792157361033, + "tokens_seen": 2907308032 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012024072216649951, + "loss": 1.2722, + "theoretical_loss": 3.3572734095913663, + "tokens_seen": 2907373568 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012022066198595788, + "loss": 1.3336, + "theoretical_loss": 3.357267603614151, + "tokens_seen": 2907439104 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012020060180541626, + "loss": 1.2288, + "theoretical_loss": 3.357261797804448, + "tokens_seen": 2907504640 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012018054162487463, + "loss": 1.3346, + "theoretical_loss": 3.35725599216225, + "tokens_seen": 2907570176 + }, + { + "epoch": 0.76, + "learning_rate": 0.000120160481444333, + "loss": 1.1662, + "theoretical_loss": 3.3572501866875477, + "tokens_seen": 2907635712 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012014042126379137, + "loss": 1.297, + "theoretical_loss": 3.357244381380332, + "tokens_seen": 2907701248 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012012036108324975, + "loss": 1.287, + "theoretical_loss": 3.357238576240595, + "tokens_seen": 2907766784 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012010030090270812, + "loss": 1.1871, + "theoretical_loss": 3.357232771268328, + "tokens_seen": 2907832320 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001200802407221665, + "loss": 1.3915, + "theoretical_loss": 3.3572269664635224, + "tokens_seen": 2907897856 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012006018054162488, + "loss": 1.3082, + "theoretical_loss": 3.3572211618261694, + "tokens_seen": 2907963392 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012004012036108325, + "loss": 1.1708, + "theoretical_loss": 3.35721535735626, + "tokens_seen": 2908028928 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012002006018054163, + "loss": 1.1656, + "theoretical_loss": 3.357209553053787, + "tokens_seen": 2908094464 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012, + "loss": 1.293, + "theoretical_loss": 3.35720374891874, + "tokens_seen": 2908160000 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011997993981945838, + "loss": 1.2081, + "theoretical_loss": 3.357197944951112, + "tokens_seen": 2908225536 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1057433, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.301594614982605, + "objective/train/theoretical_loss": 3.357192141150893, + "objective/train/tokens_used": 1278750176, + "theoretical_loss": 3.357192141150893, + "tokens_seen": 2908291072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011995987963891674, + "loss": 1.2957, + "theoretical_loss": 3.357192141150893, + "tokens_seen": 2908291072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011993981945837512, + "loss": 1.263, + "theoretical_loss": 3.3571863375180753, + "tokens_seen": 2908356608 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001199197592778335, + "loss": 1.3379, + "theoretical_loss": 3.35718053405265, + "tokens_seen": 2908422144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011989969909729188, + "loss": 1.2291, + "theoretical_loss": 3.357174730754609, + "tokens_seen": 2908487680 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011987963891675026, + "loss": 1.2519, + "theoretical_loss": 3.3571689276239427, + "tokens_seen": 2908553216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011985957873620863, + "loss": 1.2557, + "theoretical_loss": 3.3571631246606435, + "tokens_seen": 2908618752 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011983951855566701, + "loss": 1.2438, + "theoretical_loss": 3.357157321864702, + "tokens_seen": 2908684288 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011981945837512539, + "loss": 1.2389, + "theoretical_loss": 3.35715151923611, + "tokens_seen": 2908749824 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011979939819458376, + "loss": 1.2059, + "theoretical_loss": 3.3571457167748595, + "tokens_seen": 2908815360 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011977933801404214, + "loss": 1.2509, + "theoretical_loss": 3.357139914480941, + "tokens_seen": 2908880896 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001197592778335005, + "loss": 1.2672, + "theoretical_loss": 3.3571341123543457, + "tokens_seen": 2908946432 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011973921765295887, + "loss": 1.0964, + "theoretical_loss": 3.357128310395066, + "tokens_seen": 2909011968 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011971915747241725, + "loss": 1.2918, + "theoretical_loss": 3.3571225086030925, + "tokens_seen": 2909077504 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011969909729187563, + "loss": 1.3304, + "theoretical_loss": 3.357116706978417, + "tokens_seen": 2909143040 + }, + { + "epoch": 0.76, + "learning_rate": 0.000119679037111334, + "loss": 1.3241, + "theoretical_loss": 3.357110905521031, + "tokens_seen": 2909208576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011965897693079238, + "loss": 1.2552, + "theoretical_loss": 3.3571051042309255, + "tokens_seen": 2909274112 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011963891675025075, + "loss": 1.349, + "theoretical_loss": 3.3570993031080922, + "tokens_seen": 2909339648 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011961885656970913, + "loss": 1.1997, + "theoretical_loss": 3.357093502152522, + "tokens_seen": 2909405184 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001195987963891675, + "loss": 1.2821, + "theoretical_loss": 3.3570877013642075, + "tokens_seen": 2909470720 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011957873620862587, + "loss": 1.2409, + "theoretical_loss": 3.3570819007431387, + "tokens_seen": 2909536256 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011955867602808424, + "loss": 1.3237, + "theoretical_loss": 3.357076100289308, + "tokens_seen": 2909601792 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011953861584754263, + "loss": 1.3533, + "theoretical_loss": 3.3570703000027065, + "tokens_seen": 2909667328 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011951855566700101, + "loss": 1.3026, + "theoretical_loss": 3.3570644998833257, + "tokens_seen": 2909732864 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011949849548645938, + "loss": 1.1834, + "theoretical_loss": 3.357058699931157, + "tokens_seen": 2909798400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011947843530591776, + "loss": 1.2628, + "theoretical_loss": 3.3570529001461913, + "tokens_seen": 2909863936 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011945837512537614, + "loss": 1.2491, + "theoretical_loss": 3.357047100528421, + "tokens_seen": 2909929472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011943831494483451, + "loss": 1.1878, + "theoretical_loss": 3.357041301077836, + "tokens_seen": 2909995008 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011941825476429289, + "loss": 1.3373, + "theoretical_loss": 3.3570355017944293, + "tokens_seen": 2910060544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011939819458375126, + "loss": 1.3134, + "theoretical_loss": 3.357029702678192, + "tokens_seen": 2910126080 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011937813440320963, + "loss": 1.2403, + "theoretical_loss": 3.3570239037291145, + "tokens_seen": 2910191616 + }, + { + "epoch": 0.76, + "learning_rate": 0.000119358074222668, + "loss": 1.1522, + "theoretical_loss": 3.357018104947189, + "tokens_seen": 2910257152 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011933801404212638, + "loss": 1.3349, + "theoretical_loss": 3.357012306332407, + "tokens_seen": 2910322688 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011931795386158475, + "loss": 1.4379, + "theoretical_loss": 3.35700650788476, + "tokens_seen": 2910388224 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011929789368104313, + "loss": 1.2491, + "theoretical_loss": 3.3570007096042387, + "tokens_seen": 2910453760 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001192778335005015, + "loss": 1.183, + "theoretical_loss": 3.3569949114908355, + "tokens_seen": 2910519296 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011925777331995988, + "loss": 1.1454, + "theoretical_loss": 3.3569891135445413, + "tokens_seen": 2910584832 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011923771313941826, + "loss": 1.3843, + "theoretical_loss": 3.356983315765347, + "tokens_seen": 2910650368 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011921765295887663, + "loss": 1.2253, + "theoretical_loss": 3.356977518153245, + "tokens_seen": 2910715904 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011919759277833501, + "loss": 1.3764, + "theoretical_loss": 3.356971720708226, + "tokens_seen": 2910781440 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011917753259779338, + "loss": 1.3867, + "theoretical_loss": 3.3569659234302818, + "tokens_seen": 2910846976 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011915747241725176, + "loss": 1.2544, + "theoretical_loss": 3.3569601263194038, + "tokens_seen": 2910912512 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011913741223671014, + "loss": 1.3672, + "theoretical_loss": 3.3569543293755832, + "tokens_seen": 2910978048 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011911735205616851, + "loss": 1.3181, + "theoretical_loss": 3.3569485325988113, + "tokens_seen": 2911043584 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011909729187562689, + "loss": 1.2403, + "theoretical_loss": 3.3569427359890804, + "tokens_seen": 2911109120 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011907723169508526, + "loss": 1.4067, + "theoretical_loss": 3.3569369395463813, + "tokens_seen": 2911174656 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011905717151454364, + "loss": 1.2597, + "theoretical_loss": 3.356931143270705, + "tokens_seen": 2911240192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011903711133400201, + "loss": 1.2838, + "theoretical_loss": 3.3569253471620435, + "tokens_seen": 2911305728 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011901705115346039, + "loss": 1.2984, + "theoretical_loss": 3.356919551220388, + "tokens_seen": 2911371264 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011899699097291877, + "loss": 1.2265, + "theoretical_loss": 3.3569137554457305, + "tokens_seen": 2911436800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011897693079237713, + "loss": 1.2851, + "theoretical_loss": 3.3569079598380616, + "tokens_seen": 2911502336 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1058588, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3200011253356934, + "objective/train/theoretical_loss": 3.3569021643973733, + "objective/train/tokens_used": 1282026976, + "theoretical_loss": 3.3569021643973733, + "tokens_seen": 2911567872 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001189568706118355, + "loss": 1.2851, + "theoretical_loss": 3.3569021643973733, + "tokens_seen": 2911567872 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011893681043129388, + "loss": 1.2325, + "theoretical_loss": 3.356896369123657, + "tokens_seen": 2911633408 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011891675025075226, + "loss": 1.1928, + "theoretical_loss": 3.3568905740169033, + "tokens_seen": 2911698944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011889669007021063, + "loss": 1.2467, + "theoretical_loss": 3.3568847790771046, + "tokens_seen": 2911764480 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011887662988966901, + "loss": 1.2583, + "theoretical_loss": 3.3568789843042524, + "tokens_seen": 2911830016 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011885656970912738, + "loss": 1.2635, + "theoretical_loss": 3.3568731896983373, + "tokens_seen": 2911895552 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011883650952858576, + "loss": 1.2502, + "theoretical_loss": 3.3568673952593513, + "tokens_seen": 2911961088 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011881644934804413, + "loss": 1.2589, + "theoretical_loss": 3.3568616009872856, + "tokens_seen": 2912026624 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011879638916750251, + "loss": 1.3119, + "theoretical_loss": 3.356855806882132, + "tokens_seen": 2912092160 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011877632898696089, + "loss": 1.3859, + "theoretical_loss": 3.3568500129438816, + "tokens_seen": 2912157696 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011875626880641926, + "loss": 1.3816, + "theoretical_loss": 3.3568442191725256, + "tokens_seen": 2912223232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011873620862587764, + "loss": 1.2385, + "theoretical_loss": 3.356838425568056, + "tokens_seen": 2912288768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011871614844533601, + "loss": 1.3177, + "theoretical_loss": 3.3568326321304642, + "tokens_seen": 2912354304 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011869608826479439, + "loss": 1.3196, + "theoretical_loss": 3.3568268388597415, + "tokens_seen": 2912419840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011867602808425276, + "loss": 1.2776, + "theoretical_loss": 3.356821045755879, + "tokens_seen": 2912485376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011865596790371114, + "loss": 1.2592, + "theoretical_loss": 3.3568152528188686, + "tokens_seen": 2912550912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011863590772316952, + "loss": 1.3153, + "theoretical_loss": 3.3568094600487015, + "tokens_seen": 2912616448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011861584754262789, + "loss": 1.1542, + "theoretical_loss": 3.356803667445369, + "tokens_seen": 2912681984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011859578736208625, + "loss": 1.331, + "theoretical_loss": 3.356797875008863, + "tokens_seen": 2912747520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011857572718154463, + "loss": 1.2723, + "theoretical_loss": 3.3567920827391746, + "tokens_seen": 2912813056 + }, + { + "epoch": 0.77, + "learning_rate": 0.000118555667001003, + "loss": 1.3059, + "theoretical_loss": 3.3567862906362955, + "tokens_seen": 2912878592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011853560682046138, + "loss": 1.2813, + "theoretical_loss": 3.3567804987002168, + "tokens_seen": 2912944128 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011851554663991976, + "loss": 1.1765, + "theoretical_loss": 3.35677470693093, + "tokens_seen": 2913009664 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011849548645937813, + "loss": 1.1466, + "theoretical_loss": 3.3567689153284266, + "tokens_seen": 2913075200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011847542627883651, + "loss": 1.2231, + "theoretical_loss": 3.3567631238926983, + "tokens_seen": 2913140736 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011845536609829488, + "loss": 1.3583, + "theoretical_loss": 3.3567573326237365, + "tokens_seen": 2913206272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011843530591775326, + "loss": 1.2452, + "theoretical_loss": 3.3567515415215325, + "tokens_seen": 2913271808 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011841524573721165, + "loss": 1.3854, + "theoretical_loss": 3.3567457505860774, + "tokens_seen": 2913337344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011839518555667001, + "loss": 1.2305, + "theoretical_loss": 3.356739959817363, + "tokens_seen": 2913402880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011837512537612839, + "loss": 1.2568, + "theoretical_loss": 3.356734169215381, + "tokens_seen": 2913468416 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011835506519558676, + "loss": 1.2091, + "theoretical_loss": 3.3567283787801223, + "tokens_seen": 2913533952 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011833500501504514, + "loss": 1.3707, + "theoretical_loss": 3.356722588511579, + "tokens_seen": 2913599488 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011831494483450352, + "loss": 1.3394, + "theoretical_loss": 3.356716798409742, + "tokens_seen": 2913665024 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011829488465396189, + "loss": 1.2965, + "theoretical_loss": 3.356711008474603, + "tokens_seen": 2913730560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011827482447342027, + "loss": 1.2505, + "theoretical_loss": 3.3567052187061535, + "tokens_seen": 2913796096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011825476429287864, + "loss": 1.2279, + "theoretical_loss": 3.3566994291043843, + "tokens_seen": 2913861632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011823470411233702, + "loss": 1.2859, + "theoretical_loss": 3.356693639669288, + "tokens_seen": 2913927168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011821464393179538, + "loss": 1.1989, + "theoretical_loss": 3.356687850400855, + "tokens_seen": 2913992704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011819458375125376, + "loss": 1.2564, + "theoretical_loss": 3.3566820612990775, + "tokens_seen": 2914058240 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011817452357071213, + "loss": 1.2986, + "theoretical_loss": 3.3566762723639467, + "tokens_seen": 2914123776 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011815446339017051, + "loss": 1.2156, + "theoretical_loss": 3.3566704835954537, + "tokens_seen": 2914189312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011813440320962888, + "loss": 1.2844, + "theoretical_loss": 3.3566646949935905, + "tokens_seen": 2914254848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011811434302908726, + "loss": 1.2067, + "theoretical_loss": 3.3566589065583483, + "tokens_seen": 2914320384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011809428284854564, + "loss": 1.2674, + "theoretical_loss": 3.3566531182897186, + "tokens_seen": 2914385920 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011807422266800401, + "loss": 1.2731, + "theoretical_loss": 3.356647330187693, + "tokens_seen": 2914451456 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001180541624874624, + "loss": 1.3727, + "theoretical_loss": 3.3566415422522624, + "tokens_seen": 2914516992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011803410230692078, + "loss": 1.2784, + "theoretical_loss": 3.356635754483419, + "tokens_seen": 2914582528 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011801404212637914, + "loss": 1.2426, + "theoretical_loss": 3.3566299668811537, + "tokens_seen": 2914648064 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011799398194583751, + "loss": 1.1966, + "theoretical_loss": 3.3566241794454585, + "tokens_seen": 2914713600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011797392176529589, + "loss": 1.2821, + "theoretical_loss": 3.356618392176324, + "tokens_seen": 2914779136 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1059748, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2119574546813965, + "objective/train/theoretical_loss": 3.3566126050737424, + "objective/train/tokens_used": 1285303776, + "theoretical_loss": 3.3566126050737424, + "tokens_seen": 2914844672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011795386158475427, + "loss": 1.332, + "theoretical_loss": 3.3566126050737424, + "tokens_seen": 2914844672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011793380140421264, + "loss": 1.2718, + "theoretical_loss": 3.3566068181377053, + "tokens_seen": 2914910208 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011791374122367102, + "loss": 1.3486, + "theoretical_loss": 3.3566010313682035, + "tokens_seen": 2914975744 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001178936810431294, + "loss": 1.2795, + "theoretical_loss": 3.356595244765229, + "tokens_seen": 2915041280 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011787362086258777, + "loss": 1.2299, + "theoretical_loss": 3.3565894583287728, + "tokens_seen": 2915106816 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011785356068204615, + "loss": 1.187, + "theoretical_loss": 3.356583672058827, + "tokens_seen": 2915172352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011783350050150452, + "loss": 1.2874, + "theoretical_loss": 3.356577885955382, + "tokens_seen": 2915237888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011781344032096288, + "loss": 1.3674, + "theoretical_loss": 3.3565721000184308, + "tokens_seen": 2915303424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011779338014042126, + "loss": 1.4094, + "theoretical_loss": 3.3565663142479636, + "tokens_seen": 2915368960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011777331995987963, + "loss": 1.2934, + "theoretical_loss": 3.356560528643972, + "tokens_seen": 2915434496 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011775325977933801, + "loss": 1.314, + "theoretical_loss": 3.3565547432064484, + "tokens_seen": 2915500032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011773319959879639, + "loss": 1.3932, + "theoretical_loss": 3.356548957935383, + "tokens_seen": 2915565568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011771313941825476, + "loss": 1.2776, + "theoretical_loss": 3.356543172830768, + "tokens_seen": 2915631104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011769307923771314, + "loss": 1.3302, + "theoretical_loss": 3.356537387892595, + "tokens_seen": 2915696640 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011767301905717153, + "loss": 1.2146, + "theoretical_loss": 3.356531603120855, + "tokens_seen": 2915762176 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001176529588766299, + "loss": 1.3351, + "theoretical_loss": 3.35652581851554, + "tokens_seen": 2915827712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011763289869608828, + "loss": 1.3367, + "theoretical_loss": 3.356520034076641, + "tokens_seen": 2915893248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011761283851554664, + "loss": 1.2307, + "theoretical_loss": 3.3565142498041496, + "tokens_seen": 2915958784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011759277833500502, + "loss": 1.1699, + "theoretical_loss": 3.3565084656980573, + "tokens_seen": 2916024320 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011757271815446339, + "loss": 1.3098, + "theoretical_loss": 3.356502681758356, + "tokens_seen": 2916089856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011755265797392177, + "loss": 1.2805, + "theoretical_loss": 3.356496897985036, + "tokens_seen": 2916155392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011753259779338014, + "loss": 1.2671, + "theoretical_loss": 3.3564911143780902, + "tokens_seen": 2916220928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011751253761283852, + "loss": 1.2808, + "theoretical_loss": 3.356485330937509, + "tokens_seen": 2916286464 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001174924774322969, + "loss": 1.2282, + "theoretical_loss": 3.3564795476632847, + "tokens_seen": 2916352000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011747241725175527, + "loss": 1.1891, + "theoretical_loss": 3.356473764555408, + "tokens_seen": 2916417536 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011745235707121365, + "loss": 1.2068, + "theoretical_loss": 3.356467981613871, + "tokens_seen": 2916483072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011743229689067201, + "loss": 1.2595, + "theoretical_loss": 3.3564621988386647, + "tokens_seen": 2916548608 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011741223671013039, + "loss": 1.2834, + "theoretical_loss": 3.356456416229781, + "tokens_seen": 2916614144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011739217652958876, + "loss": 1.2401, + "theoretical_loss": 3.356450633787211, + "tokens_seen": 2916679680 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011737211634904714, + "loss": 1.3473, + "theoretical_loss": 3.3564448515109464, + "tokens_seen": 2916745216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011735205616850551, + "loss": 1.3066, + "theoretical_loss": 3.3564390694009787, + "tokens_seen": 2916810752 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011733199598796389, + "loss": 1.2135, + "theoretical_loss": 3.3564332874572993, + "tokens_seen": 2916876288 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011731193580742226, + "loss": 1.2235, + "theoretical_loss": 3.3564275056799, + "tokens_seen": 2916941824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011729187562688065, + "loss": 1.3134, + "theoretical_loss": 3.3564217240687713, + "tokens_seen": 2917007360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011727181544633903, + "loss": 1.2554, + "theoretical_loss": 3.3564159426239057, + "tokens_seen": 2917072896 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001172517552657974, + "loss": 1.2583, + "theoretical_loss": 3.3564101613452944, + "tokens_seen": 2917138432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011723169508525577, + "loss": 1.3428, + "theoretical_loss": 3.356404380232929, + "tokens_seen": 2917203968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011721163490471414, + "loss": 1.2906, + "theoretical_loss": 3.3563985992868006, + "tokens_seen": 2917269504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011719157472417252, + "loss": 1.2445, + "theoretical_loss": 3.356392818506901, + "tokens_seen": 2917335040 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001171715145436309, + "loss": 1.3416, + "theoretical_loss": 3.3563870378932217, + "tokens_seen": 2917400576 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011715145436308927, + "loss": 1.208, + "theoretical_loss": 3.3563812574457534, + "tokens_seen": 2917466112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011713139418254765, + "loss": 1.3958, + "theoretical_loss": 3.356375477164489, + "tokens_seen": 2917531648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011711133400200602, + "loss": 1.2348, + "theoretical_loss": 3.356369697049419, + "tokens_seen": 2917597184 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001170912738214644, + "loss": 1.2404, + "theoretical_loss": 3.3563639171005355, + "tokens_seen": 2917662720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011707121364092277, + "loss": 1.2669, + "theoretical_loss": 3.3563581373178293, + "tokens_seen": 2917728256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011705115346038115, + "loss": 1.3331, + "theoretical_loss": 3.3563523577012924, + "tokens_seen": 2917793792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011703109327983951, + "loss": 1.2615, + "theoretical_loss": 3.356346578250916, + "tokens_seen": 2917859328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011701103309929789, + "loss": 1.3063, + "theoretical_loss": 3.3563407989666914, + "tokens_seen": 2917924864 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011699097291875626, + "loss": 1.2499, + "theoretical_loss": 3.356335019848611, + "tokens_seen": 2917990400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011697091273821464, + "loss": 1.2496, + "theoretical_loss": 3.356329240896665, + "tokens_seen": 2918055936 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1060857, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3790392875671387, + "objective/train/theoretical_loss": 3.356323462110846, + "objective/train/tokens_used": 1288580576, + "theoretical_loss": 3.356323462110846, + "tokens_seen": 2918121472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011695085255767302, + "loss": 1.3866, + "theoretical_loss": 3.356323462110846, + "tokens_seen": 2918121472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011693079237713139, + "loss": 1.2707, + "theoretical_loss": 3.3563176834911452, + "tokens_seen": 2918187008 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011691073219658978, + "loss": 1.3591, + "theoretical_loss": 3.3563119050375536, + "tokens_seen": 2918252544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011689067201604816, + "loss": 1.1274, + "theoretical_loss": 3.3563061267500633, + "tokens_seen": 2918318080 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011687061183550653, + "loss": 1.3603, + "theoretical_loss": 3.3563003486286656, + "tokens_seen": 2918383616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011685055165496491, + "loss": 1.3425, + "theoretical_loss": 3.356294570673352, + "tokens_seen": 2918449152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011683049147442327, + "loss": 1.2642, + "theoretical_loss": 3.3562887928841136, + "tokens_seen": 2918514688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011681043129388165, + "loss": 1.2044, + "theoretical_loss": 3.356283015260942, + "tokens_seen": 2918580224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011679037111334002, + "loss": 1.3268, + "theoretical_loss": 3.35627723780383, + "tokens_seen": 2918645760 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001167703109327984, + "loss": 1.3613, + "theoretical_loss": 3.3562714605127675, + "tokens_seen": 2918711296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011675025075225677, + "loss": 1.2259, + "theoretical_loss": 3.3562656833877464, + "tokens_seen": 2918776832 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011673019057171515, + "loss": 1.3234, + "theoretical_loss": 3.3562599064287584, + "tokens_seen": 2918842368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011671013039117352, + "loss": 1.3335, + "theoretical_loss": 3.3562541296357953, + "tokens_seen": 2918907904 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001166900702106319, + "loss": 1.2858, + "theoretical_loss": 3.356248353008848, + "tokens_seen": 2918973440 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011667001003009028, + "loss": 1.196, + "theoretical_loss": 3.356242576547908, + "tokens_seen": 2919038976 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011664994984954864, + "loss": 1.2403, + "theoretical_loss": 3.356236800252967, + "tokens_seen": 2919104512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011662988966900701, + "loss": 1.3344, + "theoretical_loss": 3.3562310241240167, + "tokens_seen": 2919170048 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011660982948846539, + "loss": 1.3802, + "theoretical_loss": 3.356225248161049, + "tokens_seen": 2919235584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011658976930792377, + "loss": 1.296, + "theoretical_loss": 3.3562194723640544, + "tokens_seen": 2919301120 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011656970912738214, + "loss": 1.3802, + "theoretical_loss": 3.356213696733025, + "tokens_seen": 2919366656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011654964894684053, + "loss": 1.1916, + "theoretical_loss": 3.356207921267952, + "tokens_seen": 2919432192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011652958876629891, + "loss": 1.349, + "theoretical_loss": 3.356202145968827, + "tokens_seen": 2919497728 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011650952858575728, + "loss": 1.2874, + "theoretical_loss": 3.356196370835642, + "tokens_seen": 2919563264 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011648946840521566, + "loss": 1.3145, + "theoretical_loss": 3.3561905958683877, + "tokens_seen": 2919628800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011646940822467403, + "loss": 1.2463, + "theoretical_loss": 3.356184821067056, + "tokens_seen": 2919694336 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001164493480441324, + "loss": 1.3217, + "theoretical_loss": 3.3561790464316386, + "tokens_seen": 2919759872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011642928786359077, + "loss": 1.27, + "theoretical_loss": 3.3561732719621267, + "tokens_seen": 2919825408 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011640922768304915, + "loss": 1.2888, + "theoretical_loss": 3.356167497658512, + "tokens_seen": 2919890944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011638916750250752, + "loss": 1.2051, + "theoretical_loss": 3.356161723520786, + "tokens_seen": 2919956480 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001163691073219659, + "loss": 1.2705, + "theoretical_loss": 3.35615594954894, + "tokens_seen": 2920022016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011634904714142428, + "loss": 1.2671, + "theoretical_loss": 3.356150175742966, + "tokens_seen": 2920087552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011632898696088265, + "loss": 1.3574, + "theoretical_loss": 3.3561444021028546, + "tokens_seen": 2920153088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011630892678034103, + "loss": 1.2741, + "theoretical_loss": 3.3561386286285986, + "tokens_seen": 2920218624 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001162888665997994, + "loss": 1.3651, + "theoretical_loss": 3.356132855320188, + "tokens_seen": 2920284160 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011626880641925778, + "loss": 1.3317, + "theoretical_loss": 3.3561270821776157, + "tokens_seen": 2920349696 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011624874623871614, + "loss": 1.3371, + "theoretical_loss": 3.3561213092008724, + "tokens_seen": 2920415232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011622868605817452, + "loss": 1.3, + "theoretical_loss": 3.35611553638995, + "tokens_seen": 2920480768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011620862587763289, + "loss": 1.3815, + "theoretical_loss": 3.35610976374484, + "tokens_seen": 2920546304 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011618856569709127, + "loss": 1.2739, + "theoretical_loss": 3.3561039912655333, + "tokens_seen": 2920611840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011616850551654966, + "loss": 1.3136, + "theoretical_loss": 3.356098218952022, + "tokens_seen": 2920677376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011614844533600803, + "loss": 1.2431, + "theoretical_loss": 3.356092446804298, + "tokens_seen": 2920742912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011612838515546641, + "loss": 1.2444, + "theoretical_loss": 3.3560866748223517, + "tokens_seen": 2920808448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011610832497492478, + "loss": 1.3174, + "theoretical_loss": 3.3560809030061756, + "tokens_seen": 2920873984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011608826479438316, + "loss": 1.2236, + "theoretical_loss": 3.3560751313557606, + "tokens_seen": 2920939520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011606820461384154, + "loss": 1.3986, + "theoretical_loss": 3.356069359871099, + "tokens_seen": 2921005056 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001160481444332999, + "loss": 1.2207, + "theoretical_loss": 3.3560635885521815, + "tokens_seen": 2921070592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011602808425275827, + "loss": 1.3446, + "theoretical_loss": 3.356057817399, + "tokens_seen": 2921136128 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011600802407221665, + "loss": 1.2191, + "theoretical_loss": 3.3560520464115458, + "tokens_seen": 2921201664 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011598796389167503, + "loss": 1.2412, + "theoretical_loss": 3.3560462755898106, + "tokens_seen": 2921267200 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001159679037111334, + "loss": 1.23, + "theoretical_loss": 3.3560405049337865, + "tokens_seen": 2921332736 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1062024, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.446655511856079, + "objective/train/theoretical_loss": 3.3560347344434636, + "objective/train/tokens_used": 1291857376, + "theoretical_loss": 3.3560347344434636, + "tokens_seen": 2921398272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011594784353059178, + "loss": 1.4145, + "theoretical_loss": 3.3560347344434636, + "tokens_seen": 2921398272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011592778335005015, + "loss": 1.2557, + "theoretical_loss": 3.356028964118835, + "tokens_seen": 2921463808 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011590772316950853, + "loss": 1.3249, + "theoretical_loss": 3.356023193959891, + "tokens_seen": 2921529344 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001158876629889669, + "loss": 1.237, + "theoretical_loss": 3.356017423966623, + "tokens_seen": 2921594880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011586760280842527, + "loss": 1.2585, + "theoretical_loss": 3.3560116541390244, + "tokens_seen": 2921660416 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011584754262788364, + "loss": 1.1971, + "theoretical_loss": 3.3560058844770846, + "tokens_seen": 2921725952 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011582748244734202, + "loss": 1.3269, + "theoretical_loss": 3.3560001149807963, + "tokens_seen": 2921791488 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001158074222668004, + "loss": 1.2063, + "theoretical_loss": 3.3559943456501506, + "tokens_seen": 2921857024 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011578736208625878, + "loss": 1.3635, + "theoretical_loss": 3.3559885764851396, + "tokens_seen": 2921922560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011576730190571716, + "loss": 1.284, + "theoretical_loss": 3.355982807485754, + "tokens_seen": 2921988096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011574724172517554, + "loss": 1.3149, + "theoretical_loss": 3.3559770386519854, + "tokens_seen": 2922053632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011572718154463391, + "loss": 1.2923, + "theoretical_loss": 3.3559712699838258, + "tokens_seen": 2922119168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011570712136409229, + "loss": 1.2851, + "theoretical_loss": 3.3559655014812666, + "tokens_seen": 2922184704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011568706118355066, + "loss": 1.2962, + "theoretical_loss": 3.3559597331443, + "tokens_seen": 2922250240 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011566700100300903, + "loss": 1.3059, + "theoretical_loss": 3.355953964972916, + "tokens_seen": 2922315776 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001156469408224674, + "loss": 1.2163, + "theoretical_loss": 3.3559481969671072, + "tokens_seen": 2922381312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011562688064192578, + "loss": 1.1857, + "theoretical_loss": 3.355942429126865, + "tokens_seen": 2922446848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011560682046138415, + "loss": 1.3108, + "theoretical_loss": 3.3559366614521804, + "tokens_seen": 2922512384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011558676028084253, + "loss": 1.368, + "theoretical_loss": 3.355930893943046, + "tokens_seen": 2922577920 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001155667001003009, + "loss": 1.3746, + "theoretical_loss": 3.3559251265994523, + "tokens_seen": 2922643456 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011554663991975928, + "loss": 1.2374, + "theoretical_loss": 3.3559193594213914, + "tokens_seen": 2922708992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011552657973921766, + "loss": 1.3339, + "theoretical_loss": 3.3559135924088546, + "tokens_seen": 2922774528 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011550651955867603, + "loss": 1.3011, + "theoretical_loss": 3.3559078255618338, + "tokens_seen": 2922840064 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011548645937813441, + "loss": 1.2958, + "theoretical_loss": 3.3559020588803197, + "tokens_seen": 2922905600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011546639919759277, + "loss": 1.2747, + "theoretical_loss": 3.3558962923643048, + "tokens_seen": 2922971136 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011544633901705115, + "loss": 1.2239, + "theoretical_loss": 3.3558905260137806, + "tokens_seen": 2923036672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011542627883650953, + "loss": 1.2791, + "theoretical_loss": 3.3558847598287374, + "tokens_seen": 2923102208 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011540621865596791, + "loss": 1.2826, + "theoretical_loss": 3.355878993809168, + "tokens_seen": 2923167744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011538615847542629, + "loss": 1.2651, + "theoretical_loss": 3.355873227955064, + "tokens_seen": 2923233280 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011536609829488466, + "loss": 1.2447, + "theoretical_loss": 3.355867462266416, + "tokens_seen": 2923298816 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011534603811434304, + "loss": 1.3747, + "theoretical_loss": 3.355861696743216, + "tokens_seen": 2923364352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011532597793380141, + "loss": 1.2445, + "theoretical_loss": 3.355855931385456, + "tokens_seen": 2923429888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011530591775325979, + "loss": 1.2848, + "theoretical_loss": 3.355850166193127, + "tokens_seen": 2923495424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011528585757271815, + "loss": 1.2731, + "theoretical_loss": 3.3558444011662205, + "tokens_seen": 2923560960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011526579739217653, + "loss": 1.286, + "theoretical_loss": 3.3558386363047283, + "tokens_seen": 2923626496 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001152457372116349, + "loss": 1.3196, + "theoretical_loss": 3.3558328716086416, + "tokens_seen": 2923692032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011522567703109328, + "loss": 1.3804, + "theoretical_loss": 3.355827107077953, + "tokens_seen": 2923757568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011520561685055165, + "loss": 1.3095, + "theoretical_loss": 3.3558213427126526, + "tokens_seen": 2923823104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011518555667001003, + "loss": 1.2644, + "theoretical_loss": 3.3558155785127326, + "tokens_seen": 2923888640 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001151654964894684, + "loss": 1.2042, + "theoretical_loss": 3.355809814478185, + "tokens_seen": 2923954176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011514543630892678, + "loss": 1.2809, + "theoretical_loss": 3.3558040506090006, + "tokens_seen": 2924019712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011512537612838516, + "loss": 1.2034, + "theoretical_loss": 3.3557982869051712, + "tokens_seen": 2924085248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011510531594784353, + "loss": 1.3033, + "theoretical_loss": 3.3557925233666888, + "tokens_seen": 2924150784 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001150852557673019, + "loss": 1.3918, + "theoretical_loss": 3.3557867599935443, + "tokens_seen": 2924216320 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011506519558676027, + "loss": 1.3332, + "theoretical_loss": 3.3557809967857293, + "tokens_seen": 2924281856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011504513540621866, + "loss": 1.3195, + "theoretical_loss": 3.355775233743236, + "tokens_seen": 2924347392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011502507522567704, + "loss": 1.3215, + "theoretical_loss": 3.355769470866055, + "tokens_seen": 2924412928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011500501504513541, + "loss": 1.4775, + "theoretical_loss": 3.355763708154179, + "tokens_seen": 2924478464 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011498495486459379, + "loss": 1.369, + "theoretical_loss": 3.355757945607598, + "tokens_seen": 2924544000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011498495486459379, + "loss": 1.2242, + "theoretical_loss": 3.3557521832263055, + "tokens_seen": 2924609536 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1063229, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2917735576629639, + "objective/train/theoretical_loss": 3.3557464210102914, + "objective/train/tokens_used": 1295134176, + "theoretical_loss": 3.3557464210102914, + "tokens_seen": 2924675072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011496489468405216, + "loss": 1.3079, + "theoretical_loss": 3.3557464210102914, + "tokens_seen": 2924675072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011494483450351054, + "loss": 1.298, + "theoretical_loss": 3.355740658959548, + "tokens_seen": 2924740608 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011492477432296892, + "loss": 1.3317, + "theoretical_loss": 3.355734897074067, + "tokens_seen": 2924806144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011490471414242729, + "loss": 1.3744, + "theoretical_loss": 3.3557291353538394, + "tokens_seen": 2924871680 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011488465396188565, + "loss": 1.3396, + "theoretical_loss": 3.3557233737988574, + "tokens_seen": 2924937216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011486459378134403, + "loss": 1.2951, + "theoretical_loss": 3.355717612409112, + "tokens_seen": 2925002752 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011486459378134403, + "loss": 1.4221, + "theoretical_loss": 3.355711851184595, + "tokens_seen": 2925068288 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001148445336008024, + "loss": 1.4226, + "theoretical_loss": 3.355706090125298, + "tokens_seen": 2925133824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011482447342026078, + "loss": 1.2511, + "theoretical_loss": 3.3557003292312126, + "tokens_seen": 2925199360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011480441323971916, + "loss": 1.2314, + "theoretical_loss": 3.3556945685023303, + "tokens_seen": 2925264896 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011478435305917753, + "loss": 1.228, + "theoretical_loss": 3.355688807938642, + "tokens_seen": 2925330432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011476429287863591, + "loss": 1.2571, + "theoretical_loss": 3.3556830475401407, + "tokens_seen": 2925395968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011474423269809428, + "loss": 1.2227, + "theoretical_loss": 3.3556772873068166, + "tokens_seen": 2925461504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011472417251755266, + "loss": 1.3631, + "theoretical_loss": 3.355671527238662, + "tokens_seen": 2925527040 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011470411233701104, + "loss": 1.328, + "theoretical_loss": 3.3556657673356685, + "tokens_seen": 2925592576 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001146840521564694, + "loss": 1.3338, + "theoretical_loss": 3.355660007597827, + "tokens_seen": 2925658112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011466399197592779, + "loss": 1.3326, + "theoretical_loss": 3.3556542480251297, + "tokens_seen": 2925723648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011464393179538616, + "loss": 1.3686, + "theoretical_loss": 3.355648488617568, + "tokens_seen": 2925789184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011462387161484454, + "loss": 1.3599, + "theoretical_loss": 3.3556427293751336, + "tokens_seen": 2925854720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011460381143430292, + "loss": 1.2965, + "theoretical_loss": 3.3556369702978177, + "tokens_seen": 2925920256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011458375125376129, + "loss": 1.3105, + "theoretical_loss": 3.355631211385612, + "tokens_seen": 2925985792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011456369107321967, + "loss": 1.2652, + "theoretical_loss": 3.3556254526385083, + "tokens_seen": 2926051328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011454363089267804, + "loss": 1.2803, + "theoretical_loss": 3.355619694056498, + "tokens_seen": 2926116864 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011452357071213642, + "loss": 1.3198, + "theoretical_loss": 3.355613935639573, + "tokens_seen": 2926182400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011450351053159478, + "loss": 1.1843, + "theoretical_loss": 3.3556081773877238, + "tokens_seen": 2926247936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011448345035105316, + "loss": 1.316, + "theoretical_loss": 3.355602419300943, + "tokens_seen": 2926313472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011446339017051153, + "loss": 1.329, + "theoretical_loss": 3.3555966613792223, + "tokens_seen": 2926379008 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011444332998996991, + "loss": 1.149, + "theoretical_loss": 3.3555909036225526, + "tokens_seen": 2926444544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011442326980942828, + "loss": 1.2594, + "theoretical_loss": 3.3555851460309256, + "tokens_seen": 2926510080 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011440320962888666, + "loss": 1.3694, + "theoretical_loss": 3.3555793886043332, + "tokens_seen": 2926575616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011438314944834504, + "loss": 1.314, + "theoretical_loss": 3.3555736313427666, + "tokens_seen": 2926641152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011436308926780341, + "loss": 1.2509, + "theoretical_loss": 3.355567874246218, + "tokens_seen": 2926706688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011434302908726179, + "loss": 1.2499, + "theoretical_loss": 3.355562117314678, + "tokens_seen": 2926772224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011432296890672016, + "loss": 1.317, + "theoretical_loss": 3.355556360548139, + "tokens_seen": 2926837760 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011430290872617853, + "loss": 1.2198, + "theoretical_loss": 3.3555506039465923, + "tokens_seen": 2926903296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011428284854563691, + "loss": 1.3026, + "theoretical_loss": 3.3555448475100293, + "tokens_seen": 2926968832 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011426278836509529, + "loss": 1.2603, + "theoretical_loss": 3.355539091238442, + "tokens_seen": 2927034368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011424272818455367, + "loss": 1.2875, + "theoretical_loss": 3.3555333351318213, + "tokens_seen": 2927099904 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011422266800401204, + "loss": 1.2543, + "theoretical_loss": 3.3555275791901593, + "tokens_seen": 2927165440 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011420260782347042, + "loss": 1.2982, + "theoretical_loss": 3.3555218234134476, + "tokens_seen": 2927230976 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001141825476429288, + "loss": 1.3203, + "theoretical_loss": 3.3555160678016778, + "tokens_seen": 2927296512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011416248746238717, + "loss": 1.3163, + "theoretical_loss": 3.3555103123548413, + "tokens_seen": 2927362048 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011414242728184554, + "loss": 1.2206, + "theoretical_loss": 3.35550455707293, + "tokens_seen": 2927427584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011412236710130392, + "loss": 1.1722, + "theoretical_loss": 3.3554988019559344, + "tokens_seen": 2927493120 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011410230692076228, + "loss": 1.3449, + "theoretical_loss": 3.3554930470038475, + "tokens_seen": 2927558656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011408224674022066, + "loss": 1.2625, + "theoretical_loss": 3.35548729221666, + "tokens_seen": 2927624192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011406218655967903, + "loss": 1.3185, + "theoretical_loss": 3.355481537594364, + "tokens_seen": 2927689728 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011404212637913741, + "loss": 1.2693, + "theoretical_loss": 3.3554757831369506, + "tokens_seen": 2927755264 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011402206619859579, + "loss": 1.1886, + "theoretical_loss": 3.3554700288444117, + "tokens_seen": 2927820800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011400200601805416, + "loss": 1.2836, + "theoretical_loss": 3.3554642747167387, + "tokens_seen": 2927886336 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1064246, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3533320426940918, + "objective/train/theoretical_loss": 3.3554585207539236, + "objective/train/tokens_used": 1298410976, + "theoretical_loss": 3.3554585207539236, + "tokens_seen": 2927951872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011398194583751254, + "loss": 1.2954, + "theoretical_loss": 3.3554585207539236, + "tokens_seen": 2927951872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011396188565697091, + "loss": 1.3291, + "theoretical_loss": 3.3554527669559575, + "tokens_seen": 2928017408 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011394182547642929, + "loss": 1.2692, + "theoretical_loss": 3.355447013322832, + "tokens_seen": 2928082944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011392176529588768, + "loss": 1.1988, + "theoretical_loss": 3.355441259854539, + "tokens_seen": 2928148480 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011390170511534604, + "loss": 1.4107, + "theoretical_loss": 3.3554355065510704, + "tokens_seen": 2928214016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011388164493480442, + "loss": 1.2076, + "theoretical_loss": 3.355429753412417, + "tokens_seen": 2928279552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011386158475426279, + "loss": 1.1879, + "theoretical_loss": 3.3554240004385703, + "tokens_seen": 2928345088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011384152457372117, + "loss": 1.271, + "theoretical_loss": 3.3554182476295225, + "tokens_seen": 2928410624 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011382146439317954, + "loss": 1.305, + "theoretical_loss": 3.3554124949852655, + "tokens_seen": 2928476160 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011380140421263792, + "loss": 1.2371, + "theoretical_loss": 3.3554067425057896, + "tokens_seen": 2928541696 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001137813440320963, + "loss": 1.3585, + "theoretical_loss": 3.3554009901910877, + "tokens_seen": 2928607232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011376128385155467, + "loss": 1.2836, + "theoretical_loss": 3.355395238041151, + "tokens_seen": 2928672768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011374122367101305, + "loss": 1.2538, + "theoretical_loss": 3.3553894860559708, + "tokens_seen": 2928738304 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011372116349047141, + "loss": 1.2818, + "theoretical_loss": 3.355383734235539, + "tokens_seen": 2928803840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011370110330992979, + "loss": 1.33, + "theoretical_loss": 3.355377982579847, + "tokens_seen": 2928869376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011368104312938816, + "loss": 1.2647, + "theoretical_loss": 3.3553722310888863, + "tokens_seen": 2928934912 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011366098294884654, + "loss": 1.3394, + "theoretical_loss": 3.355366479762649, + "tokens_seen": 2929000448 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011364092276830491, + "loss": 1.391, + "theoretical_loss": 3.355360728601126, + "tokens_seen": 2929065984 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011362086258776329, + "loss": 1.2536, + "theoretical_loss": 3.3553549776043097, + "tokens_seen": 2929131520 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011360080240722166, + "loss": 1.2874, + "theoretical_loss": 3.3553492267721907, + "tokens_seen": 2929197056 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011358074222668004, + "loss": 1.1702, + "theoretical_loss": 3.3553434761047614, + "tokens_seen": 2929262592 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011356068204613842, + "loss": 1.3316, + "theoretical_loss": 3.355337725602013, + "tokens_seen": 2929328128 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001135406218655968, + "loss": 1.237, + "theoretical_loss": 3.3553319752639377, + "tokens_seen": 2929393664 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011352056168505517, + "loss": 1.282, + "theoretical_loss": 3.355326225090526, + "tokens_seen": 2929459200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011350050150451354, + "loss": 1.3515, + "theoretical_loss": 3.3553204750817707, + "tokens_seen": 2929524736 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011348044132397192, + "loss": 1.192, + "theoretical_loss": 3.3553147252376627, + "tokens_seen": 2929590272 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001134603811434303, + "loss": 1.2338, + "theoretical_loss": 3.3553089755581937, + "tokens_seen": 2929655808 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011344032096288867, + "loss": 1.2625, + "theoretical_loss": 3.3553032260433553, + "tokens_seen": 2929721344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011342026078234705, + "loss": 1.3583, + "theoretical_loss": 3.3552974766931394, + "tokens_seen": 2929786880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011340020060180542, + "loss": 1.263, + "theoretical_loss": 3.355291727507537, + "tokens_seen": 2929852416 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001133801404212638, + "loss": 1.2492, + "theoretical_loss": 3.35528597848654, + "tokens_seen": 2929917952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011336008024072217, + "loss": 1.1665, + "theoretical_loss": 3.3552802296301407, + "tokens_seen": 2929983488 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011334002006018055, + "loss": 1.3263, + "theoretical_loss": 3.3552744809383297, + "tokens_seen": 2930049024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011331995987963891, + "loss": 1.326, + "theoretical_loss": 3.355268732411099, + "tokens_seen": 2930114560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011329989969909729, + "loss": 1.2541, + "theoretical_loss": 3.3552629840484403, + "tokens_seen": 2930180096 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011327983951855566, + "loss": 1.3078, + "theoretical_loss": 3.355257235850345, + "tokens_seen": 2930245632 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011325977933801404, + "loss": 1.1916, + "theoretical_loss": 3.3552514878168047, + "tokens_seen": 2930311168 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011323971915747241, + "loss": 1.2518, + "theoretical_loss": 3.3552457399478115, + "tokens_seen": 2930376704 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011321965897693079, + "loss": 1.2269, + "theoretical_loss": 3.355239992243356, + "tokens_seen": 2930442240 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011319959879638917, + "loss": 1.231, + "theoretical_loss": 3.355234244703431, + "tokens_seen": 2930507776 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011317953861584754, + "loss": 1.395, + "theoretical_loss": 3.3552284973280275, + "tokens_seen": 2930573312 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011315947843530593, + "loss": 1.1969, + "theoretical_loss": 3.355222750117137, + "tokens_seen": 2930638848 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011313941825476431, + "loss": 1.3898, + "theoretical_loss": 3.3552170030707513, + "tokens_seen": 2930704384 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011311935807422267, + "loss": 1.3245, + "theoretical_loss": 3.3552112561888623, + "tokens_seen": 2930769920 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011309929789368105, + "loss": 1.2349, + "theoretical_loss": 3.355205509471461, + "tokens_seen": 2930835456 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011307923771313942, + "loss": 1.3437, + "theoretical_loss": 3.3551997629185393, + "tokens_seen": 2930900992 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001130591775325978, + "loss": 1.271, + "theoretical_loss": 3.355194016530089, + "tokens_seen": 2930966528 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011303911735205617, + "loss": 1.2776, + "theoretical_loss": 3.355188270306101, + "tokens_seen": 2931032064 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011301905717151455, + "loss": 1.2051, + "theoretical_loss": 3.355182524246568, + "tokens_seen": 2931097600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011299899699097292, + "loss": 1.3442, + "theoretical_loss": 3.355176778351481, + "tokens_seen": 2931163136 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1065435, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4551390409469604, + "objective/train/theoretical_loss": 3.3551710326208317, + "objective/train/tokens_used": 1301687776, + "theoretical_loss": 3.3551710326208317, + "tokens_seen": 2931228672 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001129789368104313, + "loss": 1.4143, + "theoretical_loss": 3.3551710326208317, + "tokens_seen": 2931228672 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011295887662988968, + "loss": 1.248, + "theoretical_loss": 3.3551652870546116, + "tokens_seen": 2931294208 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011293881644934804, + "loss": 1.3355, + "theoretical_loss": 3.355159541652813, + "tokens_seen": 2931359744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011291875626880641, + "loss": 1.2533, + "theoretical_loss": 3.355153796415426, + "tokens_seen": 2931425280 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011289869608826479, + "loss": 1.3328, + "theoretical_loss": 3.355148051342444, + "tokens_seen": 2931490816 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011287863590772317, + "loss": 1.2277, + "theoretical_loss": 3.3551423064338572, + "tokens_seen": 2931556352 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011285857572718154, + "loss": 1.1976, + "theoretical_loss": 3.3551365616896582, + "tokens_seen": 2931621888 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011283851554663992, + "loss": 1.3851, + "theoretical_loss": 3.355130817109838, + "tokens_seen": 2931687424 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011281845536609829, + "loss": 1.2478, + "theoretical_loss": 3.3551250726943884, + "tokens_seen": 2931752960 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011279839518555668, + "loss": 1.2291, + "theoretical_loss": 3.355119328443301, + "tokens_seen": 2931818496 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011277833500501506, + "loss": 1.289, + "theoretical_loss": 3.3551135843565683, + "tokens_seen": 2931884032 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011275827482447343, + "loss": 1.3137, + "theoretical_loss": 3.3551078404341803, + "tokens_seen": 2931949568 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001127382146439318, + "loss": 1.395, + "theoretical_loss": 3.35510209667613, + "tokens_seen": 2932015104 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011271815446339017, + "loss": 1.1337, + "theoretical_loss": 3.3550963530824083, + "tokens_seen": 2932080640 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011269809428284855, + "loss": 1.2596, + "theoretical_loss": 3.355090609653007, + "tokens_seen": 2932146176 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011267803410230692, + "loss": 1.2674, + "theoretical_loss": 3.355084866387917, + "tokens_seen": 2932211712 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001126579739217653, + "loss": 1.1698, + "theoretical_loss": 3.3550791232871315, + "tokens_seen": 2932277248 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011263791374122368, + "loss": 1.2975, + "theoretical_loss": 3.355073380350641, + "tokens_seen": 2932342784 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011261785356068205, + "loss": 1.2116, + "theoretical_loss": 3.3550676375784376, + "tokens_seen": 2932408320 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011259779338014043, + "loss": 1.2861, + "theoretical_loss": 3.3550618949705124, + "tokens_seen": 2932473856 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001125777331995988, + "loss": 1.1947, + "theoretical_loss": 3.3550561525268576, + "tokens_seen": 2932539392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011255767301905718, + "loss": 1.2127, + "theoretical_loss": 3.3550504102474648, + "tokens_seen": 2932604928 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011253761283851554, + "loss": 1.2556, + "theoretical_loss": 3.355044668132325, + "tokens_seen": 2932670464 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011251755265797392, + "loss": 1.3877, + "theoretical_loss": 3.3550389261814306, + "tokens_seen": 2932736000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011249749247743229, + "loss": 1.3004, + "theoretical_loss": 3.355033184394773, + "tokens_seen": 2932801536 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011247743229689067, + "loss": 1.3006, + "theoretical_loss": 3.355027442772343, + "tokens_seen": 2932867072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011245737211634904, + "loss": 1.1752, + "theoretical_loss": 3.3550217013141337, + "tokens_seen": 2932932608 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011243731193580742, + "loss": 1.2681, + "theoretical_loss": 3.3550159600201357, + "tokens_seen": 2932998144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011241725175526581, + "loss": 1.1922, + "theoretical_loss": 3.355010218890341, + "tokens_seen": 2933063680 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011239719157472418, + "loss": 1.273, + "theoretical_loss": 3.355004477924741, + "tokens_seen": 2933129216 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011237713139418256, + "loss": 1.2449, + "theoretical_loss": 3.354998737123328, + "tokens_seen": 2933194752 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011235707121364094, + "loss": 1.2818, + "theoretical_loss": 3.3549929964860925, + "tokens_seen": 2933260288 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001123370110330993, + "loss": 1.1807, + "theoretical_loss": 3.354987256013027, + "tokens_seen": 2933325824 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011231695085255767, + "loss": 1.3404, + "theoretical_loss": 3.354981515704123, + "tokens_seen": 2933391360 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011229689067201605, + "loss": 1.3541, + "theoretical_loss": 3.354975775559372, + "tokens_seen": 2933456896 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011227683049147443, + "loss": 1.2335, + "theoretical_loss": 3.3549700355787655, + "tokens_seen": 2933522432 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001122567703109328, + "loss": 1.348, + "theoretical_loss": 3.3549642957622954, + "tokens_seen": 2933587968 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011223671013039118, + "loss": 1.2824, + "theoretical_loss": 3.3549585561099535, + "tokens_seen": 2933653504 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011221664994984955, + "loss": 1.2805, + "theoretical_loss": 3.354952816621731, + "tokens_seen": 2933719040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011219658976930793, + "loss": 1.2578, + "theoretical_loss": 3.35494707729762, + "tokens_seen": 2933784576 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001121765295887663, + "loss": 1.2495, + "theoretical_loss": 3.3549413381376114, + "tokens_seen": 2933850112 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011215646940822467, + "loss": 1.2604, + "theoretical_loss": 3.354935599141698, + "tokens_seen": 2933915648 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011213640922768304, + "loss": 1.2886, + "theoretical_loss": 3.3549298603098703, + "tokens_seen": 2933981184 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011211634904714142, + "loss": 1.2876, + "theoretical_loss": 3.3549241216421204, + "tokens_seen": 2934046720 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001120962888665998, + "loss": 1.3899, + "theoretical_loss": 3.35491838313844, + "tokens_seen": 2934112256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011207622868605817, + "loss": 1.3004, + "theoretical_loss": 3.354912644798821, + "tokens_seen": 2934177792 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011205616850551655, + "loss": 1.3066, + "theoretical_loss": 3.3549069066232544, + "tokens_seen": 2934243328 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011203610832497494, + "loss": 1.2423, + "theoretical_loss": 3.3549011686117325, + "tokens_seen": 2934308864 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011201604814443331, + "loss": 1.2383, + "theoretical_loss": 3.3548954307642465, + "tokens_seen": 2934374400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011199598796389169, + "loss": 1.2403, + "theoretical_loss": 3.3548896930807883, + "tokens_seen": 2934439936 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1066769, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3130096197128296, + "objective/train/theoretical_loss": 3.354883955561349, + "objective/train/tokens_used": 1304964576, + "theoretical_loss": 3.354883955561349, + "tokens_seen": 2934505472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011197592778335006, + "loss": 1.328, + "theoretical_loss": 3.354883955561349, + "tokens_seen": 2934505472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011195586760280843, + "loss": 1.3026, + "theoretical_loss": 3.354878218205921, + "tokens_seen": 2934571008 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001119358074222668, + "loss": 1.1792, + "theoretical_loss": 3.3548724810144956, + "tokens_seen": 2934636544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011191574724172518, + "loss": 1.2153, + "theoretical_loss": 3.3548667439870643, + "tokens_seen": 2934702080 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011189568706118355, + "loss": 1.2689, + "theoretical_loss": 3.3548610071236196, + "tokens_seen": 2934767616 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011187562688064193, + "loss": 1.3382, + "theoretical_loss": 3.3548552704241517, + "tokens_seen": 2934833152 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001118555667001003, + "loss": 1.2594, + "theoretical_loss": 3.3548495338886535, + "tokens_seen": 2934898688 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011183550651955868, + "loss": 1.354, + "theoretical_loss": 3.354843797517116, + "tokens_seen": 2934964224 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011181544633901706, + "loss": 1.2339, + "theoretical_loss": 3.354838061309531, + "tokens_seen": 2935029760 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011179538615847543, + "loss": 1.2885, + "theoretical_loss": 3.3548323252658903, + "tokens_seen": 2935095296 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011177532597793381, + "loss": 1.2015, + "theoretical_loss": 3.3548265893861853, + "tokens_seen": 2935160832 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011175526579739217, + "loss": 1.1984, + "theoretical_loss": 3.3548208536704083, + "tokens_seen": 2935226368 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011173520561685055, + "loss": 1.3061, + "theoretical_loss": 3.35481511811855, + "tokens_seen": 2935291904 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011171514543630892, + "loss": 1.2024, + "theoretical_loss": 3.3548093827306027, + "tokens_seen": 2935357440 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001116950852557673, + "loss": 1.2201, + "theoretical_loss": 3.3548036475065577, + "tokens_seen": 2935422976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011167502507522567, + "loss": 1.1942, + "theoretical_loss": 3.354797912446407, + "tokens_seen": 2935488512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011165496489468406, + "loss": 1.1872, + "theoretical_loss": 3.354792177550142, + "tokens_seen": 2935554048 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011163490471414244, + "loss": 1.456, + "theoretical_loss": 3.3547864428177543, + "tokens_seen": 2935619584 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011161484453360081, + "loss": 1.2579, + "theoretical_loss": 3.354780708249236, + "tokens_seen": 2935685120 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011159478435305919, + "loss": 1.3248, + "theoretical_loss": 3.3547749738445782, + "tokens_seen": 2935750656 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011157472417251755, + "loss": 1.3145, + "theoretical_loss": 3.354769239603773, + "tokens_seen": 2935816192 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011155466399197593, + "loss": 1.2613, + "theoretical_loss": 3.3547635055268117, + "tokens_seen": 2935881728 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001115346038114343, + "loss": 1.2289, + "theoretical_loss": 3.354757771613686, + "tokens_seen": 2935947264 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011151454363089268, + "loss": 1.1793, + "theoretical_loss": 3.354752037864388, + "tokens_seen": 2936012800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011149448345035105, + "loss": 1.205, + "theoretical_loss": 3.354746304278909, + "tokens_seen": 2936078336 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011147442326980943, + "loss": 1.2304, + "theoretical_loss": 3.354740570857241, + "tokens_seen": 2936143872 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001114543630892678, + "loss": 1.1876, + "theoretical_loss": 3.354734837599375, + "tokens_seen": 2936209408 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011143430290872618, + "loss": 1.2994, + "theoretical_loss": 3.3547291045053034, + "tokens_seen": 2936274944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011141424272818456, + "loss": 1.2276, + "theoretical_loss": 3.3547233715750173, + "tokens_seen": 2936340480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011139418254764293, + "loss": 1.1639, + "theoretical_loss": 3.3547176388085083, + "tokens_seen": 2936406016 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001113741223671013, + "loss": 1.2459, + "theoretical_loss": 3.354711906205769, + "tokens_seen": 2936471552 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011135406218655967, + "loss": 1.1971, + "theoretical_loss": 3.35470617376679, + "tokens_seen": 2936537088 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011133400200601805, + "loss": 1.2492, + "theoretical_loss": 3.3547004414915635, + "tokens_seen": 2936602624 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011131394182547642, + "loss": 1.3323, + "theoretical_loss": 3.354694709380081, + "tokens_seen": 2936668160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011129388164493481, + "loss": 1.2684, + "theoretical_loss": 3.3546889774323345, + "tokens_seen": 2936733696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011127382146439319, + "loss": 1.2434, + "theoretical_loss": 3.3546832456483147, + "tokens_seen": 2936799232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011125376128385156, + "loss": 1.2675, + "theoretical_loss": 3.3546775140280145, + "tokens_seen": 2936864768 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011123370110330994, + "loss": 1.1926, + "theoretical_loss": 3.354671782571425, + "tokens_seen": 2936930304 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011121364092276832, + "loss": 1.2278, + "theoretical_loss": 3.354666051278538, + "tokens_seen": 2936995840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011119358074222669, + "loss": 1.2411, + "theoretical_loss": 3.3546603201493452, + "tokens_seen": 2937061376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011117352056168505, + "loss": 1.3097, + "theoretical_loss": 3.3546545891838377, + "tokens_seen": 2937126912 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011115346038114343, + "loss": 1.2361, + "theoretical_loss": 3.3546488583820078, + "tokens_seen": 2937192448 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001111334002006018, + "loss": 1.2464, + "theoretical_loss": 3.354643127743847, + "tokens_seen": 2937257984 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011111334002006018, + "loss": 1.2934, + "theoretical_loss": 3.354637397269347, + "tokens_seen": 2937323520 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011109327983951856, + "loss": 1.3192, + "theoretical_loss": 3.3546316669584995, + "tokens_seen": 2937389056 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011107321965897693, + "loss": 1.2517, + "theoretical_loss": 3.354625936811296, + "tokens_seen": 2937454592 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011105315947843531, + "loss": 1.3322, + "theoretical_loss": 3.3546202068277284, + "tokens_seen": 2937520128 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011103309929789368, + "loss": 1.3894, + "theoretical_loss": 3.3546144770077886, + "tokens_seen": 2937585664 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011101303911735206, + "loss": 1.2307, + "theoretical_loss": 3.3546087473514676, + "tokens_seen": 2937651200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011099297893681042, + "loss": 1.2067, + "theoretical_loss": 3.354603017858757, + "tokens_seen": 2937716736 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1068027, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3222405910491943, + "objective/train/theoretical_loss": 3.3545972885296496, + "objective/train/tokens_used": 1308241376, + "theoretical_loss": 3.3545972885296496, + "tokens_seen": 2937782272 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001109729187562688, + "loss": 1.288, + "theoretical_loss": 3.3545972885296496, + "tokens_seen": 2937782272 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011095285857572717, + "loss": 1.3147, + "theoretical_loss": 3.354591559364136, + "tokens_seen": 2937847808 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011093279839518555, + "loss": 1.1894, + "theoretical_loss": 3.3545858303622085, + "tokens_seen": 2937913344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011091273821464394, + "loss": 1.2842, + "theoretical_loss": 3.354580101523858, + "tokens_seen": 2937978880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011089267803410231, + "loss": 1.2746, + "theoretical_loss": 3.3545743728490773, + "tokens_seen": 2938044416 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011087261785356069, + "loss": 1.2047, + "theoretical_loss": 3.3545686443378573, + "tokens_seen": 2938109952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011085255767301907, + "loss": 1.3801, + "theoretical_loss": 3.35456291599019, + "tokens_seen": 2938175488 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011083249749247744, + "loss": 1.2663, + "theoretical_loss": 3.354557187806067, + "tokens_seen": 2938241024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011081243731193582, + "loss": 1.4094, + "theoretical_loss": 3.3545514597854798, + "tokens_seen": 2938306560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011079237713139418, + "loss": 1.2556, + "theoretical_loss": 3.3545457319284204, + "tokens_seen": 2938372096 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011077231695085256, + "loss": 1.4124, + "theoretical_loss": 3.35454000423488, + "tokens_seen": 2938437632 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011075225677031093, + "loss": 1.2328, + "theoretical_loss": 3.354534276704851, + "tokens_seen": 2938503168 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011073219658976931, + "loss": 1.313, + "theoretical_loss": 3.354528549338325, + "tokens_seen": 2938568704 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011071213640922768, + "loss": 1.2572, + "theoretical_loss": 3.3545228221352925, + "tokens_seen": 2938634240 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011069207622868606, + "loss": 1.2647, + "theoretical_loss": 3.354517095095747, + "tokens_seen": 2938699776 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011067201604814444, + "loss": 1.3265, + "theoretical_loss": 3.3545113682196783, + "tokens_seen": 2938765312 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011065195586760281, + "loss": 1.2034, + "theoretical_loss": 3.3545056415070795, + "tokens_seen": 2938830848 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011063189568706119, + "loss": 1.2738, + "theoretical_loss": 3.354499914957942, + "tokens_seen": 2938896384 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011061183550651956, + "loss": 1.2061, + "theoretical_loss": 3.354494188572257, + "tokens_seen": 2938961920 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011059177532597792, + "loss": 1.3495, + "theoretical_loss": 3.354488462350017, + "tokens_seen": 2939027456 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001105717151454363, + "loss": 1.2654, + "theoretical_loss": 3.3544827362912124, + "tokens_seen": 2939092992 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011055165496489468, + "loss": 1.2972, + "theoretical_loss": 3.3544770103958363, + "tokens_seen": 2939158528 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011053159478435307, + "loss": 1.1829, + "theoretical_loss": 3.3544712846638793, + "tokens_seen": 2939224064 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011051153460381144, + "loss": 1.2808, + "theoretical_loss": 3.354465559095334, + "tokens_seen": 2939289600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011049147442326982, + "loss": 1.3403, + "theoretical_loss": 3.354459833690192, + "tokens_seen": 2939355136 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011047141424272819, + "loss": 1.2992, + "theoretical_loss": 3.3544541084484436, + "tokens_seen": 2939420672 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011045135406218657, + "loss": 1.3119, + "theoretical_loss": 3.3544483833700824, + "tokens_seen": 2939486208 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011043129388164494, + "loss": 1.2986, + "theoretical_loss": 3.354442658455099, + "tokens_seen": 2939551744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011041123370110332, + "loss": 1.2654, + "theoretical_loss": 3.3544369337034854, + "tokens_seen": 2939617280 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011039117352056168, + "loss": 1.3601, + "theoretical_loss": 3.354431209115233, + "tokens_seen": 2939682816 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011037111334002006, + "loss": 1.3127, + "theoretical_loss": 3.354425484690334, + "tokens_seen": 2939748352 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011035105315947843, + "loss": 1.2018, + "theoretical_loss": 3.35441976042878, + "tokens_seen": 2939813888 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011033099297893681, + "loss": 1.3327, + "theoretical_loss": 3.3544140363305623, + "tokens_seen": 2939879424 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011031093279839519, + "loss": 1.2339, + "theoretical_loss": 3.354408312395673, + "tokens_seen": 2939944960 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011029087261785356, + "loss": 1.302, + "theoretical_loss": 3.3544025886241036, + "tokens_seen": 2940010496 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011027081243731194, + "loss": 1.3366, + "theoretical_loss": 3.3543968650158456, + "tokens_seen": 2940076032 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011025075225677031, + "loss": 1.1958, + "theoretical_loss": 3.354391141570891, + "tokens_seen": 2940141568 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011023069207622869, + "loss": 1.2889, + "theoretical_loss": 3.3543854182892314, + "tokens_seen": 2940207104 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011021063189568705, + "loss": 1.2625, + "theoretical_loss": 3.354379695170859, + "tokens_seen": 2940272640 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011019057171514543, + "loss": 1.1784, + "theoretical_loss": 3.3543739722157646, + "tokens_seen": 2940338176 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011017051153460382, + "loss": 1.3156, + "theoretical_loss": 3.35436824942394, + "tokens_seen": 2940403712 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011015045135406219, + "loss": 1.2898, + "theoretical_loss": 3.354362526795378, + "tokens_seen": 2940469248 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011013039117352057, + "loss": 1.1606, + "theoretical_loss": 3.3543568043300693, + "tokens_seen": 2940534784 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011011033099297894, + "loss": 1.2644, + "theoretical_loss": 3.3543510820280056, + "tokens_seen": 2940600320 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011009027081243732, + "loss": 1.2664, + "theoretical_loss": 3.354345359889179, + "tokens_seen": 2940665856 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001100702106318957, + "loss": 1.2643, + "theoretical_loss": 3.354339637913581, + "tokens_seen": 2940731392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011005015045135407, + "loss": 1.2751, + "theoretical_loss": 3.3543339161012034, + "tokens_seen": 2940796928 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011003009027081245, + "loss": 1.2449, + "theoretical_loss": 3.354328194452038, + "tokens_seen": 2940862464 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011001003009027081, + "loss": 1.2916, + "theoretical_loss": 3.354322472966076, + "tokens_seen": 2940928000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010998996990972919, + "loss": 1.2894, + "theoretical_loss": 3.35431675164331, + "tokens_seen": 2940993536 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1069310, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2102913856506348, + "objective/train/theoretical_loss": 3.354311030483731, + "objective/train/tokens_used": 1311518176, + "theoretical_loss": 3.354311030483731, + "tokens_seen": 2941059072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010996990972918756, + "loss": 1.2982, + "theoretical_loss": 3.354311030483731, + "tokens_seen": 2941059072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010994984954864594, + "loss": 1.2014, + "theoretical_loss": 3.354305309487331, + "tokens_seen": 2941124608 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010992978936810431, + "loss": 1.232, + "theoretical_loss": 3.354299588654101, + "tokens_seen": 2941190144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010990972918756269, + "loss": 1.3297, + "theoretical_loss": 3.354293867984034, + "tokens_seen": 2941255680 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010988966900702106, + "loss": 1.2724, + "theoretical_loss": 3.3542881474771207, + "tokens_seen": 2941321216 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010986960882647944, + "loss": 1.2637, + "theoretical_loss": 3.3542824271333536, + "tokens_seen": 2941386752 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010984954864593782, + "loss": 1.2578, + "theoretical_loss": 3.3542767069527235, + "tokens_seen": 2941452288 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010982948846539619, + "loss": 1.3669, + "theoretical_loss": 3.354270986935223, + "tokens_seen": 2941517824 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010980942828485455, + "loss": 1.2827, + "theoretical_loss": 3.354265267080843, + "tokens_seen": 2941583360 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010978936810431294, + "loss": 1.3365, + "theoretical_loss": 3.3542595473895758, + "tokens_seen": 2941648896 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010976930792377132, + "loss": 1.255, + "theoretical_loss": 3.3542538278614127, + "tokens_seen": 2941714432 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001097492477432297, + "loss": 1.2034, + "theoretical_loss": 3.354248108496346, + "tokens_seen": 2941779968 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010972918756268807, + "loss": 1.1882, + "theoretical_loss": 3.3542423892943667, + "tokens_seen": 2941845504 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010970912738214645, + "loss": 1.2152, + "theoretical_loss": 3.354236670255467, + "tokens_seen": 2941911040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010968906720160482, + "loss": 1.3105, + "theoretical_loss": 3.3542309513796384, + "tokens_seen": 2941976576 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001096690070210632, + "loss": 1.3013, + "theoretical_loss": 3.354225232666873, + "tokens_seen": 2942042112 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010964894684052157, + "loss": 1.3088, + "theoretical_loss": 3.354219514117162, + "tokens_seen": 2942107648 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010962888665997995, + "loss": 1.3586, + "theoretical_loss": 3.354213795730497, + "tokens_seen": 2942173184 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010960882647943831, + "loss": 1.3207, + "theoretical_loss": 3.3542080775068706, + "tokens_seen": 2942238720 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010958876629889669, + "loss": 1.2336, + "theoretical_loss": 3.354202359446274, + "tokens_seen": 2942304256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010956870611835506, + "loss": 1.3287, + "theoretical_loss": 3.3541966415486986, + "tokens_seen": 2942369792 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010954864593781344, + "loss": 1.1451, + "theoretical_loss": 3.3541909238141363, + "tokens_seen": 2942435328 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010952858575727181, + "loss": 1.251, + "theoretical_loss": 3.354185206242579, + "tokens_seen": 2942500864 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010950852557673019, + "loss": 1.2135, + "theoretical_loss": 3.354179488834019, + "tokens_seen": 2942566400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010948846539618857, + "loss": 1.2855, + "theoretical_loss": 3.3541737715884468, + "tokens_seen": 2942631936 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010946840521564694, + "loss": 1.3289, + "theoretical_loss": 3.3541680545058545, + "tokens_seen": 2942697472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010944834503510532, + "loss": 1.237, + "theoretical_loss": 3.3541623375862346, + "tokens_seen": 2942763008 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010942828485456368, + "loss": 1.302, + "theoretical_loss": 3.354156620829578, + "tokens_seen": 2942828544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010940822467402207, + "loss": 1.2203, + "theoretical_loss": 3.3541509042358766, + "tokens_seen": 2942894080 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010938816449348045, + "loss": 1.2499, + "theoretical_loss": 3.3541451878051225, + "tokens_seen": 2942959616 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010936810431293882, + "loss": 1.183, + "theoretical_loss": 3.354139471537307, + "tokens_seen": 2943025152 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001093480441323972, + "loss": 1.2639, + "theoretical_loss": 3.354133755432422, + "tokens_seen": 2943090688 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010932798395185557, + "loss": 1.2803, + "theoretical_loss": 3.354128039490459, + "tokens_seen": 2943156224 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010930792377131395, + "loss": 1.2005, + "theoretical_loss": 3.35412232371141, + "tokens_seen": 2943221760 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010928786359077232, + "loss": 1.3162, + "theoretical_loss": 3.354116608095267, + "tokens_seen": 2943287296 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001092678034102307, + "loss": 1.4113, + "theoretical_loss": 3.3541108926420207, + "tokens_seen": 2943352832 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010924774322968908, + "loss": 1.2496, + "theoretical_loss": 3.354105177351664, + "tokens_seen": 2943418368 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010922768304914744, + "loss": 1.201, + "theoretical_loss": 3.354099462224188, + "tokens_seen": 2943483904 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010920762286860581, + "loss": 1.3392, + "theoretical_loss": 3.3540937472595846, + "tokens_seen": 2943549440 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010918756268806419, + "loss": 1.3501, + "theoretical_loss": 3.3540880324578453, + "tokens_seen": 2943614976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010916750250752257, + "loss": 1.2198, + "theoretical_loss": 3.3540823178189623, + "tokens_seen": 2943680512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010914744232698094, + "loss": 1.2893, + "theoretical_loss": 3.3540766033429272, + "tokens_seen": 2943746048 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010912738214643932, + "loss": 1.235, + "theoretical_loss": 3.3540708890297317, + "tokens_seen": 2943811584 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010910732196589769, + "loss": 1.2933, + "theoretical_loss": 3.3540651748793673, + "tokens_seen": 2943877120 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010908726178535607, + "loss": 1.3418, + "theoretical_loss": 3.3540594608918255, + "tokens_seen": 2943942656 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010906720160481444, + "loss": 1.4109, + "theoretical_loss": 3.354053747067099, + "tokens_seen": 2944008192 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010904714142427282, + "loss": 1.3483, + "theoretical_loss": 3.3540480334051783, + "tokens_seen": 2944073728 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001090270812437312, + "loss": 1.3063, + "theoretical_loss": 3.354042319906056, + "tokens_seen": 2944139264 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010900702106318957, + "loss": 1.2426, + "theoretical_loss": 3.354036606569724, + "tokens_seen": 2944204800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010898696088264795, + "loss": 1.281, + "theoretical_loss": 3.3540308933961733, + "tokens_seen": 2944270336 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1070538, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4315128326416016, + "objective/train/theoretical_loss": 3.3540251803853964, + "objective/train/tokens_used": 1314794976, + "theoretical_loss": 3.3540251803853964, + "tokens_seen": 2944335872 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010896690070210632, + "loss": 1.4296, + "theoretical_loss": 3.3540251803853964, + "tokens_seen": 2944335872 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001089468405215647, + "loss": 1.3117, + "theoretical_loss": 3.354019467537384, + "tokens_seen": 2944401408 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010892678034102307, + "loss": 1.3662, + "theoretical_loss": 3.354013754852129, + "tokens_seen": 2944466944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010890672016048145, + "loss": 1.4459, + "theoretical_loss": 3.354008042329623, + "tokens_seen": 2944532480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010888665997993983, + "loss": 1.2789, + "theoretical_loss": 3.3540023299698563, + "tokens_seen": 2944598016 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001088665997993982, + "loss": 1.2728, + "theoretical_loss": 3.353996617772822, + "tokens_seen": 2944663552 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010884653961885658, + "loss": 1.371, + "theoretical_loss": 3.353990905738512, + "tokens_seen": 2944729088 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010882647943831494, + "loss": 1.2207, + "theoretical_loss": 3.353985193866918, + "tokens_seen": 2944794624 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010880641925777332, + "loss": 1.2757, + "theoretical_loss": 3.3539794821580307, + "tokens_seen": 2944860160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010878635907723169, + "loss": 1.3555, + "theoretical_loss": 3.353973770611842, + "tokens_seen": 2944925696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010876629889669007, + "loss": 1.218, + "theoretical_loss": 3.353968059228345, + "tokens_seen": 2944991232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010874623871614844, + "loss": 1.2394, + "theoretical_loss": 3.35396234800753, + "tokens_seen": 2945056768 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010872617853560682, + "loss": 1.3426, + "theoretical_loss": 3.3539566369493894, + "tokens_seen": 2945122304 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001087061183550652, + "loss": 1.303, + "theoretical_loss": 3.3539509260539155, + "tokens_seen": 2945187840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010868605817452357, + "loss": 1.3722, + "theoretical_loss": 3.3539452153210987, + "tokens_seen": 2945253376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010866599799398196, + "loss": 1.3092, + "theoretical_loss": 3.3539395047509317, + "tokens_seen": 2945318912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010864593781344032, + "loss": 1.3246, + "theoretical_loss": 3.3539337943434058, + "tokens_seen": 2945384448 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001086258776328987, + "loss": 1.2342, + "theoretical_loss": 3.3539280840985133, + "tokens_seen": 2945449984 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010860581745235707, + "loss": 1.2874, + "theoretical_loss": 3.3539223740162454, + "tokens_seen": 2945515520 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010858575727181545, + "loss": 1.2455, + "theoretical_loss": 3.353916664096594, + "tokens_seen": 2945581056 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010856569709127383, + "loss": 1.2538, + "theoretical_loss": 3.3539109543395513, + "tokens_seen": 2945646592 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001085456369107322, + "loss": 1.3126, + "theoretical_loss": 3.3539052447451083, + "tokens_seen": 2945712128 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010852557673019058, + "loss": 1.2985, + "theoretical_loss": 3.353899535313257, + "tokens_seen": 2945777664 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010850551654964895, + "loss": 1.1916, + "theoretical_loss": 3.35389382604399, + "tokens_seen": 2945843200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010848545636910733, + "loss": 1.179, + "theoretical_loss": 3.3538881169372976, + "tokens_seen": 2945908736 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001084653961885657, + "loss": 1.2687, + "theoretical_loss": 3.353882407993172, + "tokens_seen": 2945974272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010844533600802407, + "loss": 1.3906, + "theoretical_loss": 3.353876699211606, + "tokens_seen": 2946039808 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010842527582748244, + "loss": 1.2713, + "theoretical_loss": 3.3538709905925903, + "tokens_seen": 2946105344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010840521564694082, + "loss": 1.2755, + "theoretical_loss": 3.353865282136117, + "tokens_seen": 2946170880 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001083851554663992, + "loss": 1.2639, + "theoretical_loss": 3.3538595738421777, + "tokens_seen": 2946236416 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010836509528585757, + "loss": 1.3521, + "theoretical_loss": 3.3538538657107644, + "tokens_seen": 2946301952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010834503510531595, + "loss": 1.2594, + "theoretical_loss": 3.3538481577418686, + "tokens_seen": 2946367488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010832497492477432, + "loss": 1.3191, + "theoretical_loss": 3.353842449935482, + "tokens_seen": 2946433024 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001083049147442327, + "loss": 1.2137, + "theoretical_loss": 3.3538367422915965, + "tokens_seen": 2946498560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010828485456369109, + "loss": 1.247, + "theoretical_loss": 3.353831034810204, + "tokens_seen": 2946564096 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010826479438314946, + "loss": 1.2901, + "theoretical_loss": 3.3538253274912964, + "tokens_seen": 2946629632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010824473420260782, + "loss": 1.2819, + "theoretical_loss": 3.353819620334865, + "tokens_seen": 2946695168 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001082246740220662, + "loss": 1.3852, + "theoretical_loss": 3.3538139133409017, + "tokens_seen": 2946760704 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010820461384152458, + "loss": 1.2125, + "theoretical_loss": 3.3538082065093984, + "tokens_seen": 2946826240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010818455366098295, + "loss": 1.3135, + "theoretical_loss": 3.353802499840347, + "tokens_seen": 2946891776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010816449348044133, + "loss": 1.1829, + "theoretical_loss": 3.3537967933337387, + "tokens_seen": 2946957312 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001081444332998997, + "loss": 1.3341, + "theoretical_loss": 3.353791086989566, + "tokens_seen": 2947022848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010812437311935808, + "loss": 1.2486, + "theoretical_loss": 3.3537853808078197, + "tokens_seen": 2947088384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010810431293881646, + "loss": 1.2929, + "theoretical_loss": 3.353779674788493, + "tokens_seen": 2947153920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010808425275827483, + "loss": 1.2557, + "theoretical_loss": 3.353773968931576, + "tokens_seen": 2947219456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010806419257773321, + "loss": 1.1678, + "theoretical_loss": 3.3537682632370616, + "tokens_seen": 2947284992 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010804413239719157, + "loss": 1.2282, + "theoretical_loss": 3.3537625577049415, + "tokens_seen": 2947350528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010802407221664995, + "loss": 1.2543, + "theoretical_loss": 3.3537568523352066, + "tokens_seen": 2947416064 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010800401203610832, + "loss": 1.2732, + "theoretical_loss": 3.35375114712785, + "tokens_seen": 2947481600 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001079839518555667, + "loss": 1.2233, + "theoretical_loss": 3.353745442082862, + "tokens_seen": 2947547136 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1071710, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.390841007232666, + "objective/train/theoretical_loss": 3.3537397372002355, + "objective/train/tokens_used": 1318071776, + "theoretical_loss": 3.3537397372002355, + "tokens_seen": 2947612672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010796389167502507, + "loss": 1.374, + "theoretical_loss": 3.3537397372002355, + "tokens_seen": 2947612672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010794383149448345, + "loss": 1.1638, + "theoretical_loss": 3.353734032479962, + "tokens_seen": 2947678208 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010792377131394182, + "loss": 1.2187, + "theoretical_loss": 3.353728327922033, + "tokens_seen": 2947743744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010790371113340021, + "loss": 1.2204, + "theoretical_loss": 3.3537226235264406, + "tokens_seen": 2947809280 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010788365095285859, + "loss": 1.3467, + "theoretical_loss": 3.353716919293176, + "tokens_seen": 2947874816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010786359077231695, + "loss": 1.2073, + "theoretical_loss": 3.3537112152222317, + "tokens_seen": 2947940352 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010784353059177533, + "loss": 1.2159, + "theoretical_loss": 3.353705511313599, + "tokens_seen": 2948005888 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001078234704112337, + "loss": 1.285, + "theoretical_loss": 3.35369980756727, + "tokens_seen": 2948071424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010780341023069208, + "loss": 1.2636, + "theoretical_loss": 3.3536941039832366, + "tokens_seen": 2948136960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010778335005015045, + "loss": 1.3616, + "theoretical_loss": 3.3536884005614898, + "tokens_seen": 2948202496 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010776328986960883, + "loss": 1.2919, + "theoretical_loss": 3.353682697302022, + "tokens_seen": 2948268032 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001077432296890672, + "loss": 1.3484, + "theoretical_loss": 3.3536769942048243, + "tokens_seen": 2948333568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010772316950852558, + "loss": 1.2979, + "theoretical_loss": 3.35367129126989, + "tokens_seen": 2948399104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010770310932798396, + "loss": 1.2565, + "theoretical_loss": 3.353665588497209, + "tokens_seen": 2948464640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010768304914744233, + "loss": 1.2197, + "theoretical_loss": 3.353659885886774, + "tokens_seen": 2948530176 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001076629889669007, + "loss": 1.3412, + "theoretical_loss": 3.3536541834385774, + "tokens_seen": 2948595712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010764292878635907, + "loss": 1.2059, + "theoretical_loss": 3.35364848115261, + "tokens_seen": 2948661248 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010762286860581745, + "loss": 1.2645, + "theoretical_loss": 3.3536427790288634, + "tokens_seen": 2948726784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010760280842527582, + "loss": 1.2753, + "theoretical_loss": 3.3536370770673303, + "tokens_seen": 2948792320 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001075827482447342, + "loss": 1.2142, + "theoretical_loss": 3.353631375268002, + "tokens_seen": 2948857856 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010756268806419257, + "loss": 1.1472, + "theoretical_loss": 3.3536256736308703, + "tokens_seen": 2948923392 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010754262788365096, + "loss": 1.3431, + "theoretical_loss": 3.353619972155927, + "tokens_seen": 2948988928 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010752256770310934, + "loss": 1.2154, + "theoretical_loss": 3.3536142708431638, + "tokens_seen": 2949054464 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010750250752256772, + "loss": 1.5054, + "theoretical_loss": 3.3536085696925726, + "tokens_seen": 2949120000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010748244734202609, + "loss": 1.3101, + "theoretical_loss": 3.3536028687041455, + "tokens_seen": 2949185536 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010746238716148445, + "loss": 1.2374, + "theoretical_loss": 3.3535971678778735, + "tokens_seen": 2949251072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010744232698094283, + "loss": 1.1507, + "theoretical_loss": 3.3535914672137492, + "tokens_seen": 2949316608 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001074222668004012, + "loss": 1.38, + "theoretical_loss": 3.3535857667117637, + "tokens_seen": 2949382144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010740220661985958, + "loss": 1.2136, + "theoretical_loss": 3.353580066371909, + "tokens_seen": 2949447680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010738214643931796, + "loss": 1.192, + "theoretical_loss": 3.3535743661941773, + "tokens_seen": 2949513216 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010736208625877633, + "loss": 1.3322, + "theoretical_loss": 3.35356866617856, + "tokens_seen": 2949578752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010734202607823471, + "loss": 1.1165, + "theoretical_loss": 3.3535629663250486, + "tokens_seen": 2949644288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010732196589769308, + "loss": 1.3117, + "theoretical_loss": 3.3535572666336355, + "tokens_seen": 2949709824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010730190571715146, + "loss": 1.3075, + "theoretical_loss": 3.3535515671043123, + "tokens_seen": 2949775360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010728184553660982, + "loss": 1.3242, + "theoretical_loss": 3.353545867737071, + "tokens_seen": 2949840896 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001072617853560682, + "loss": 1.1797, + "theoretical_loss": 3.353540168531903, + "tokens_seen": 2949906432 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010724172517552657, + "loss": 1.2302, + "theoretical_loss": 3.3535344694887996, + "tokens_seen": 2949971968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010722166499498495, + "loss": 1.2498, + "theoretical_loss": 3.353528770607754, + "tokens_seen": 2950037504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010720160481444333, + "loss": 1.1837, + "theoretical_loss": 3.3535230718887563, + "tokens_seen": 2950103040 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001071815446339017, + "loss": 1.1967, + "theoretical_loss": 3.3535173733318, + "tokens_seen": 2950168576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010716148445336009, + "loss": 1.2354, + "theoretical_loss": 3.3535116749368754, + "tokens_seen": 2950234112 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010714142427281847, + "loss": 1.3625, + "theoretical_loss": 3.3535059767039757, + "tokens_seen": 2950299648 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010712136409227684, + "loss": 1.243, + "theoretical_loss": 3.353500278633091, + "tokens_seen": 2950365184 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010710130391173522, + "loss": 1.2154, + "theoretical_loss": 3.353494580724215, + "tokens_seen": 2950430720 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010708124373119358, + "loss": 1.2859, + "theoretical_loss": 3.353488882977338, + "tokens_seen": 2950496256 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010706118355065196, + "loss": 1.3003, + "theoretical_loss": 3.3534831853924527, + "tokens_seen": 2950561792 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010704112337011033, + "loss": 1.2969, + "theoretical_loss": 3.3534774879695504, + "tokens_seen": 2950627328 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010702106318956871, + "loss": 1.2291, + "theoretical_loss": 3.353471790708623, + "tokens_seen": 2950692864 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010700100300902708, + "loss": 1.2578, + "theoretical_loss": 3.3534660936096623, + "tokens_seen": 2950758400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010698094282848546, + "loss": 1.2862, + "theoretical_loss": 3.35346039667266, + "tokens_seen": 2950823936 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1072835, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2957708835601807, + "objective/train/theoretical_loss": 3.3534546998976085, + "objective/train/tokens_used": 1321348576, + "theoretical_loss": 3.3534546998976085, + "tokens_seen": 2950889472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010696088264794383, + "loss": 1.3347, + "theoretical_loss": 3.3534546998976085, + "tokens_seen": 2950889472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010694082246740221, + "loss": 1.2847, + "theoretical_loss": 3.3534490032844984, + "tokens_seen": 2950955008 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010692076228686059, + "loss": 1.2933, + "theoretical_loss": 3.3534433068333227, + "tokens_seen": 2951020544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010690070210631896, + "loss": 1.2119, + "theoretical_loss": 3.3534376105440726, + "tokens_seen": 2951086080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010688064192577732, + "loss": 1.3079, + "theoretical_loss": 3.35343191441674, + "tokens_seen": 2951151616 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001068605817452357, + "loss": 1.2319, + "theoretical_loss": 3.3534262184513173, + "tokens_seen": 2951217152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010684052156469408, + "loss": 1.2377, + "theoretical_loss": 3.353420522647795, + "tokens_seen": 2951282688 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010682046138415245, + "loss": 1.311, + "theoretical_loss": 3.353414827006166, + "tokens_seen": 2951348224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010680040120361083, + "loss": 1.3401, + "theoretical_loss": 3.353409131526422, + "tokens_seen": 2951413760 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010678034102306922, + "loss": 1.2281, + "theoretical_loss": 3.353403436208554, + "tokens_seen": 2951479296 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010676028084252759, + "loss": 1.2802, + "theoretical_loss": 3.3533977410525546, + "tokens_seen": 2951544832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010674022066198597, + "loss": 1.2435, + "theoretical_loss": 3.3533920460584152, + "tokens_seen": 2951610368 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010672016048144434, + "loss": 1.3664, + "theoretical_loss": 3.353386351226128, + "tokens_seen": 2951675904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010670010030090272, + "loss": 1.2067, + "theoretical_loss": 3.3533806565556845, + "tokens_seen": 2951741440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010668004012036108, + "loss": 1.2811, + "theoretical_loss": 3.3533749620470767, + "tokens_seen": 2951806976 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010665997993981946, + "loss": 1.2928, + "theoretical_loss": 3.3533692677002964, + "tokens_seen": 2951872512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010663991975927783, + "loss": 1.2573, + "theoretical_loss": 3.3533635735153347, + "tokens_seen": 2951938048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010661985957873621, + "loss": 1.2763, + "theoretical_loss": 3.3533578794921843, + "tokens_seen": 2952003584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010659979939819459, + "loss": 1.3431, + "theoretical_loss": 3.353352185630837, + "tokens_seen": 2952069120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010657973921765296, + "loss": 1.2606, + "theoretical_loss": 3.3533464919312843, + "tokens_seen": 2952134656 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010655967903711134, + "loss": 1.203, + "theoretical_loss": 3.3533407983935177, + "tokens_seen": 2952200192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010653961885656971, + "loss": 1.3478, + "theoretical_loss": 3.3533351050175293, + "tokens_seen": 2952265728 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010651955867602809, + "loss": 1.3264, + "theoretical_loss": 3.3533294118033115, + "tokens_seen": 2952331264 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010649949849548645, + "loss": 1.2222, + "theoretical_loss": 3.353323718750855, + "tokens_seen": 2952396800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010647943831494483, + "loss": 1.188, + "theoretical_loss": 3.3533180258601525, + "tokens_seen": 2952462336 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001064593781344032, + "loss": 1.2573, + "theoretical_loss": 3.3533123331311954, + "tokens_seen": 2952527872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010643931795386158, + "loss": 1.3127, + "theoretical_loss": 3.3533066405639755, + "tokens_seen": 2952593408 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010641925777331995, + "loss": 1.2041, + "theoretical_loss": 3.353300948158485, + "tokens_seen": 2952658944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010639919759277834, + "loss": 1.2584, + "theoretical_loss": 3.3532952559147153, + "tokens_seen": 2952724480 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010637913741223672, + "loss": 1.3646, + "theoretical_loss": 3.3532895638326585, + "tokens_seen": 2952790016 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001063590772316951, + "loss": 1.3156, + "theoretical_loss": 3.353283871912306, + "tokens_seen": 2952855552 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010633901705115347, + "loss": 1.2683, + "theoretical_loss": 3.35327818015365, + "tokens_seen": 2952921088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010631895687061185, + "loss": 1.3719, + "theoretical_loss": 3.3532724885566827, + "tokens_seen": 2952986624 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010629889669007021, + "loss": 1.3105, + "theoretical_loss": 3.3532667971213947, + "tokens_seen": 2953052160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010627883650952858, + "loss": 1.3171, + "theoretical_loss": 3.353261105847779, + "tokens_seen": 2953117696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010625877632898696, + "loss": 1.1772, + "theoretical_loss": 3.353255414735827, + "tokens_seen": 2953183232 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010623871614844534, + "loss": 1.2061, + "theoretical_loss": 3.35324972378553, + "tokens_seen": 2953248768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010621865596790371, + "loss": 1.2777, + "theoretical_loss": 3.353244032996881, + "tokens_seen": 2953314304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010619859578736209, + "loss": 1.3758, + "theoretical_loss": 3.3532383423698704, + "tokens_seen": 2953379840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010617853560682046, + "loss": 1.155, + "theoretical_loss": 3.3532326519044915, + "tokens_seen": 2953445376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010615847542627884, + "loss": 1.3682, + "theoretical_loss": 3.353226961600735, + "tokens_seen": 2953510912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010613841524573722, + "loss": 1.2182, + "theoretical_loss": 3.353221271458593, + "tokens_seen": 2953576448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010611835506519559, + "loss": 1.3615, + "theoretical_loss": 3.3532155814780573, + "tokens_seen": 2953641984 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010609829488465395, + "loss": 1.2202, + "theoretical_loss": 3.3532098916591204, + "tokens_seen": 2953707520 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010607823470411233, + "loss": 1.3027, + "theoretical_loss": 3.353204202001773, + "tokens_seen": 2953773056 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001060581745235707, + "loss": 1.2408, + "theoretical_loss": 3.3531985125060078, + "tokens_seen": 2953838592 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001060381143430291, + "loss": 1.426, + "theoretical_loss": 3.3531928231718164, + "tokens_seen": 2953904128 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010601805416248747, + "loss": 1.3704, + "theoretical_loss": 3.3531871339991906, + "tokens_seen": 2953969664 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010599799398194585, + "loss": 1.3166, + "theoretical_loss": 3.353181444988122, + "tokens_seen": 2954035200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010597793380140422, + "loss": 1.285, + "theoretical_loss": 3.3531757561386026, + "tokens_seen": 2954100736 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1074062, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.253406047821045, + "objective/train/theoretical_loss": 3.353170067450624, + "objective/train/tokens_used": 1324625376, + "theoretical_loss": 3.353170067450624, + "tokens_seen": 2954166272 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001059578736208626, + "loss": 1.2778, + "theoretical_loss": 3.353170067450624, + "tokens_seen": 2954166272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010593781344032097, + "loss": 1.2449, + "theoretical_loss": 3.3531643789241787, + "tokens_seen": 2954231808 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010591775325977935, + "loss": 1.2636, + "theoretical_loss": 3.3531586905592583, + "tokens_seen": 2954297344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010589769307923771, + "loss": 1.2372, + "theoretical_loss": 3.353153002355854, + "tokens_seen": 2954362880 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010587763289869609, + "loss": 1.2658, + "theoretical_loss": 3.3531473143139583, + "tokens_seen": 2954428416 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010585757271815446, + "loss": 1.3216, + "theoretical_loss": 3.3531416264335627, + "tokens_seen": 2954493952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010583751253761284, + "loss": 1.1897, + "theoretical_loss": 3.353135938714659, + "tokens_seen": 2954559488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010581745235707121, + "loss": 1.3416, + "theoretical_loss": 3.353130251157239, + "tokens_seen": 2954625024 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010579739217652959, + "loss": 1.2818, + "theoretical_loss": 3.353124563761295, + "tokens_seen": 2954690560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010577733199598797, + "loss": 1.3515, + "theoretical_loss": 3.3531188765268185, + "tokens_seen": 2954756096 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010575727181544634, + "loss": 1.2602, + "theoretical_loss": 3.3531131894538015, + "tokens_seen": 2954821632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010573721163490472, + "loss": 1.3204, + "theoretical_loss": 3.353107502542236, + "tokens_seen": 2954887168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010571715145436308, + "loss": 1.2992, + "theoretical_loss": 3.3531018157921126, + "tokens_seen": 2954952704 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010569709127382146, + "loss": 1.2965, + "theoretical_loss": 3.3530961292034247, + "tokens_seen": 2955018240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010567703109327983, + "loss": 1.2707, + "theoretical_loss": 3.353090442776163, + "tokens_seen": 2955083776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010565697091273822, + "loss": 1.2556, + "theoretical_loss": 3.3530847565103206, + "tokens_seen": 2955149312 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001056369107321966, + "loss": 1.332, + "theoretical_loss": 3.3530790704058884, + "tokens_seen": 2955214848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010561685055165497, + "loss": 1.1985, + "theoretical_loss": 3.3530733844628577, + "tokens_seen": 2955280384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010559679037111335, + "loss": 1.3327, + "theoretical_loss": 3.3530676986812216, + "tokens_seen": 2955345920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010557673019057172, + "loss": 1.2746, + "theoretical_loss": 3.3530620130609714, + "tokens_seen": 2955411456 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001055566700100301, + "loss": 1.2918, + "theoretical_loss": 3.353056327602099, + "tokens_seen": 2955476992 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010553660982948848, + "loss": 1.1558, + "theoretical_loss": 3.3530506423045963, + "tokens_seen": 2955542528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010551654964894684, + "loss": 1.2668, + "theoretical_loss": 3.3530449571684544, + "tokens_seen": 2955608064 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010549648946840521, + "loss": 1.222, + "theoretical_loss": 3.3530392721936666, + "tokens_seen": 2955673600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010547642928786359, + "loss": 1.2035, + "theoretical_loss": 3.3530335873802235, + "tokens_seen": 2955739136 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010545636910732197, + "loss": 1.2112, + "theoretical_loss": 3.353027902728117, + "tokens_seen": 2955804672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010543630892678034, + "loss": 1.2836, + "theoretical_loss": 3.3530222182373395, + "tokens_seen": 2955870208 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010541624874623872, + "loss": 1.2345, + "theoretical_loss": 3.353016533907883, + "tokens_seen": 2955935744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010539618856569709, + "loss": 1.2494, + "theoretical_loss": 3.353010849739739, + "tokens_seen": 2956001280 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010537612838515547, + "loss": 1.2144, + "theoretical_loss": 3.353005165732899, + "tokens_seen": 2956066816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010535606820461384, + "loss": 1.3161, + "theoretical_loss": 3.3529994818873554, + "tokens_seen": 2956132352 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010533600802407222, + "loss": 1.1748, + "theoretical_loss": 3.3529937982030997, + "tokens_seen": 2956197888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010531594784353058, + "loss": 1.2701, + "theoretical_loss": 3.352988114680124, + "tokens_seen": 2956263424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010529588766298896, + "loss": 1.2717, + "theoretical_loss": 3.3529824313184196, + "tokens_seen": 2956328960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010527582748244735, + "loss": 1.4392, + "theoretical_loss": 3.352976748117979, + "tokens_seen": 2956394496 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010525576730190572, + "loss": 1.3403, + "theoretical_loss": 3.352971065078794, + "tokens_seen": 2956460032 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001052357071213641, + "loss": 1.326, + "theoretical_loss": 3.352965382200856, + "tokens_seen": 2956525568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010521564694082247, + "loss": 1.4231, + "theoretical_loss": 3.3529596994841575, + "tokens_seen": 2956591104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010519558676028085, + "loss": 1.1832, + "theoretical_loss": 3.35295401692869, + "tokens_seen": 2956656640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010517552657973923, + "loss": 1.21, + "theoretical_loss": 3.352948334534445, + "tokens_seen": 2956722176 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001051554663991976, + "loss": 1.3016, + "theoretical_loss": 3.3529426523014147, + "tokens_seen": 2956787712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010513540621865598, + "loss": 1.2884, + "theoretical_loss": 3.352936970229591, + "tokens_seen": 2956853248 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010511534603811434, + "loss": 1.3146, + "theoretical_loss": 3.3529312883189655, + "tokens_seen": 2956918784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010509528585757272, + "loss": 1.2065, + "theoretical_loss": 3.3529256065695305, + "tokens_seen": 2956984320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010507522567703109, + "loss": 1.3611, + "theoretical_loss": 3.3529199249812778, + "tokens_seen": 2957049856 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010505516549648947, + "loss": 1.2852, + "theoretical_loss": 3.3529142435541988, + "tokens_seen": 2957115392 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010503510531594784, + "loss": 1.3212, + "theoretical_loss": 3.3529085622882855, + "tokens_seen": 2957180928 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010501504513540622, + "loss": 1.3165, + "theoretical_loss": 3.35290288118353, + "tokens_seen": 2957246464 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001049949849548646, + "loss": 1.3091, + "theoretical_loss": 3.3528972002399238, + "tokens_seen": 2957312000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010497492477432297, + "loss": 1.3333, + "theoretical_loss": 3.3528915194574593, + "tokens_seen": 2957377536 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1075270, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3097165822982788, + "objective/train/theoretical_loss": 3.352885838836128, + "objective/train/tokens_used": 1327902176, + "theoretical_loss": 3.352885838836128, + "tokens_seen": 2957443072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010495486459378135, + "loss": 1.2758, + "theoretical_loss": 3.352885838836128, + "tokens_seen": 2957443072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010493480441323971, + "loss": 1.2561, + "theoretical_loss": 3.3528801583759216, + "tokens_seen": 2957508608 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001049147442326981, + "loss": 1.267, + "theoretical_loss": 3.3528744780768323, + "tokens_seen": 2957574144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010489468405215647, + "loss": 1.2733, + "theoretical_loss": 3.352868797938852, + "tokens_seen": 2957639680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010487462387161485, + "loss": 1.2519, + "theoretical_loss": 3.352863117961972, + "tokens_seen": 2957705216 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010485456369107323, + "loss": 1.2237, + "theoretical_loss": 3.3528574381461844, + "tokens_seen": 2957770752 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001048345035105316, + "loss": 1.2137, + "theoretical_loss": 3.3528517584914814, + "tokens_seen": 2957836288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010481444332998998, + "loss": 1.2308, + "theoretical_loss": 3.352846078997855, + "tokens_seen": 2957901824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010479438314944835, + "loss": 1.3492, + "theoretical_loss": 3.352840399665296, + "tokens_seen": 2957967360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010477432296890673, + "loss": 1.2196, + "theoretical_loss": 3.3528347204937976, + "tokens_seen": 2958032896 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001047542627883651, + "loss": 1.1902, + "theoretical_loss": 3.352829041483351, + "tokens_seen": 2958098432 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010473420260782347, + "loss": 1.2267, + "theoretical_loss": 3.352823362633948, + "tokens_seen": 2958163968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010471414242728184, + "loss": 1.2216, + "theoretical_loss": 3.3528176839455806, + "tokens_seen": 2958229504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010469408224674022, + "loss": 1.1906, + "theoretical_loss": 3.3528120054182406, + "tokens_seen": 2958295040 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001046740220661986, + "loss": 1.1864, + "theoretical_loss": 3.35280632705192, + "tokens_seen": 2958360576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010465396188565697, + "loss": 1.3177, + "theoretical_loss": 3.3528006488466104, + "tokens_seen": 2958426112 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010463390170511535, + "loss": 1.308, + "theoretical_loss": 3.352794970802304, + "tokens_seen": 2958491648 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010461384152457372, + "loss": 1.3224, + "theoretical_loss": 3.3527892929189926, + "tokens_seen": 2958557184 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001045937813440321, + "loss": 1.2478, + "theoretical_loss": 3.352783615196668, + "tokens_seen": 2958622720 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010457372116349047, + "loss": 1.3254, + "theoretical_loss": 3.352777937635322, + "tokens_seen": 2958688256 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010455366098294885, + "loss": 1.3037, + "theoretical_loss": 3.3527722602349463, + "tokens_seen": 2958753792 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010453360080240722, + "loss": 1.2391, + "theoretical_loss": 3.3527665829955335, + "tokens_seen": 2958819328 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001045135406218656, + "loss": 1.2978, + "theoretical_loss": 3.3527609059170747, + "tokens_seen": 2958884864 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010449348044132398, + "loss": 1.3058, + "theoretical_loss": 3.3527552289995617, + "tokens_seen": 2958950400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010447342026078235, + "loss": 1.2384, + "theoretical_loss": 3.3527495522429875, + "tokens_seen": 2959015936 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010445336008024073, + "loss": 1.2911, + "theoretical_loss": 3.3527438756473424, + "tokens_seen": 2959081472 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001044332998996991, + "loss": 1.1744, + "theoretical_loss": 3.3527381992126193, + "tokens_seen": 2959147008 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010441323971915748, + "loss": 1.3145, + "theoretical_loss": 3.35273252293881, + "tokens_seen": 2959212544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010439317953861586, + "loss": 1.2959, + "theoretical_loss": 3.3527268468259064, + "tokens_seen": 2959278080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010437311935807423, + "loss": 1.3442, + "theoretical_loss": 3.3527211708738998, + "tokens_seen": 2959343616 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010435305917753259, + "loss": 1.193, + "theoretical_loss": 3.3527154950827827, + "tokens_seen": 2959409152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010433299899699097, + "loss": 1.3113, + "theoretical_loss": 3.3527098194525466, + "tokens_seen": 2959474688 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010431293881644934, + "loss": 1.3615, + "theoretical_loss": 3.3527041439831833, + "tokens_seen": 2959540224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010429287863590772, + "loss": 1.2596, + "theoretical_loss": 3.352698468674685, + "tokens_seen": 2959605760 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001042728184553661, + "loss": 1.265, + "theoretical_loss": 3.352692793527044, + "tokens_seen": 2959671296 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010425275827482447, + "loss": 1.2134, + "theoretical_loss": 3.352687118540251, + "tokens_seen": 2959736832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010423269809428285, + "loss": 1.2859, + "theoretical_loss": 3.352681443714299, + "tokens_seen": 2959802368 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010421263791374122, + "loss": 1.3616, + "theoretical_loss": 3.352675769049179, + "tokens_seen": 2959867904 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001041925777331996, + "loss": 1.4126, + "theoretical_loss": 3.3526700945448837, + "tokens_seen": 2959933440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010417251755265798, + "loss": 1.2584, + "theoretical_loss": 3.352664420201404, + "tokens_seen": 2959998976 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010415245737211635, + "loss": 1.3204, + "theoretical_loss": 3.352658746018733, + "tokens_seen": 2960064512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010413239719157473, + "loss": 1.1653, + "theoretical_loss": 3.352653071996862, + "tokens_seen": 2960130048 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001041123370110331, + "loss": 1.3276, + "theoretical_loss": 3.3526473981357823, + "tokens_seen": 2960195584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010409227683049148, + "loss": 1.3512, + "theoretical_loss": 3.3526417244354865, + "tokens_seen": 2960261120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010407221664994985, + "loss": 1.2048, + "theoretical_loss": 3.352636050895966, + "tokens_seen": 2960326656 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010405215646940823, + "loss": 1.3013, + "theoretical_loss": 3.3526303775172135, + "tokens_seen": 2960392192 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001040320962888666, + "loss": 1.2994, + "theoretical_loss": 3.3526247042992203, + "tokens_seen": 2960457728 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010401203610832498, + "loss": 1.2428, + "theoretical_loss": 3.352619031241978, + "tokens_seen": 2960523264 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010399197592778336, + "loss": 1.2342, + "theoretical_loss": 3.3526133583454794, + "tokens_seen": 2960588800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010397191574724173, + "loss": 1.278, + "theoretical_loss": 3.352607685609715, + "tokens_seen": 2960654336 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.7304384858452192, + "debugging/entropy-1-grams": 4.9022521807961565, + "debugging/length": 488.7142857142857, + "debugging/num_segments": 7, + "debugging/score": 0.0068462501065870455, + "debugging/score_std": 0.004935555558817252, + "epoch": 0.79, + "objective/train/docs_used": 1076444, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2278602123260498, + "objective/train/theoretical_loss": 3.3526020130346783, + "objective/train/tokens_used": 1331178976, + "theoretical_loss": 3.3526020130346783, + "tokens_seen": 2960719872 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001039518555667001, + "loss": 1.2271, + "theoretical_loss": 3.3526020130346783, + "tokens_seen": 2960719872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010393179538615847, + "loss": 1.3717, + "theoretical_loss": 3.3525963406203596, + "tokens_seen": 2960785408 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010391173520561685, + "loss": 1.162, + "theoretical_loss": 3.3525906683667523, + "tokens_seen": 2960850944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010389167502507522, + "loss": 1.2339, + "theoretical_loss": 3.352584996273847, + "tokens_seen": 2960916480 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001038716148445336, + "loss": 1.3635, + "theoretical_loss": 3.3525793243416366, + "tokens_seen": 2960982016 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010385155466399197, + "loss": 1.2894, + "theoretical_loss": 3.3525736525701126, + "tokens_seen": 2961047552 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010383149448345035, + "loss": 1.3101, + "theoretical_loss": 3.352567980959267, + "tokens_seen": 2961113088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010381143430290873, + "loss": 1.3295, + "theoretical_loss": 3.352562309509091, + "tokens_seen": 2961178624 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001037913741223671, + "loss": 1.2459, + "theoretical_loss": 3.3525566382195775, + "tokens_seen": 2961244160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010377131394182549, + "loss": 1.1554, + "theoretical_loss": 3.3525509670907176, + "tokens_seen": 2961309696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010375125376128385, + "loss": 1.1917, + "theoretical_loss": 3.3525452961225035, + "tokens_seen": 2961375232 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010373119358074223, + "loss": 1.3334, + "theoretical_loss": 3.352539625314927, + "tokens_seen": 2961440768 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001037111334002006, + "loss": 1.3269, + "theoretical_loss": 3.352533954667981, + "tokens_seen": 2961506304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010369107321965898, + "loss": 1.2182, + "theoretical_loss": 3.3525282841816555, + "tokens_seen": 2961571840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010367101303911736, + "loss": 1.2387, + "theoretical_loss": 3.352522613855944, + "tokens_seen": 2961637376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010365095285857573, + "loss": 1.2952, + "theoretical_loss": 3.3525169436908375, + "tokens_seen": 2961702912 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010363089267803411, + "loss": 1.3007, + "theoretical_loss": 3.352511273686328, + "tokens_seen": 2961768448 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010361083249749248, + "loss": 1.2958, + "theoretical_loss": 3.352505603842408, + "tokens_seen": 2961833984 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010359077231695086, + "loss": 1.1777, + "theoretical_loss": 3.352499934159069, + "tokens_seen": 2961899520 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010357071213640922, + "loss": 1.3482, + "theoretical_loss": 3.3524942646363027, + "tokens_seen": 2961965056 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001035506519558676, + "loss": 1.2846, + "theoretical_loss": 3.3524885952741013, + "tokens_seen": 2962030592 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010353059177532597, + "loss": 1.3025, + "theoretical_loss": 3.3524829260724562, + "tokens_seen": 2962096128 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010351053159478435, + "loss": 1.3466, + "theoretical_loss": 3.35247725703136, + "tokens_seen": 2962161664 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010349047141424273, + "loss": 1.2362, + "theoretical_loss": 3.3524715881508045, + "tokens_seen": 2962227200 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001034704112337011, + "loss": 1.1246, + "theoretical_loss": 3.3524659194307813, + "tokens_seen": 2962292736 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010345035105315948, + "loss": 1.265, + "theoretical_loss": 3.352460250871282, + "tokens_seen": 2962358272 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010343029087261785, + "loss": 1.4261, + "theoretical_loss": 3.3524545824722995, + "tokens_seen": 2962423808 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010341023069207624, + "loss": 1.2877, + "theoretical_loss": 3.352448914233825, + "tokens_seen": 2962489344 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010339017051153462, + "loss": 1.1853, + "theoretical_loss": 3.3524432461558504, + "tokens_seen": 2962554880 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010337011033099298, + "loss": 1.1856, + "theoretical_loss": 3.3524375782383675, + "tokens_seen": 2962620416 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010335005015045136, + "loss": 1.3495, + "theoretical_loss": 3.3524319104813687, + "tokens_seen": 2962685952 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010332998996990973, + "loss": 1.19, + "theoretical_loss": 3.3524262428848455, + "tokens_seen": 2962751488 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010330992978936811, + "loss": 1.1854, + "theoretical_loss": 3.35242057544879, + "tokens_seen": 2962817024 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010328986960882648, + "loss": 1.2579, + "theoretical_loss": 3.352414908173194, + "tokens_seen": 2962882560 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010326980942828486, + "loss": 1.27, + "theoretical_loss": 3.3524092410580493, + "tokens_seen": 2962948096 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010324974924774323, + "loss": 1.2207, + "theoretical_loss": 3.352403574103348, + "tokens_seen": 2963013632 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010322968906720161, + "loss": 1.1163, + "theoretical_loss": 3.352397907309082, + "tokens_seen": 2963079168 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010320962888665999, + "loss": 1.2509, + "theoretical_loss": 3.3523922406752433, + "tokens_seen": 2963144704 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010318956870611836, + "loss": 1.146, + "theoretical_loss": 3.352386574201824, + "tokens_seen": 2963210240 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010316950852557672, + "loss": 1.2404, + "theoretical_loss": 3.352380907888815, + "tokens_seen": 2963275776 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001031494483450351, + "loss": 1.2701, + "theoretical_loss": 3.352375241736209, + "tokens_seen": 2963341312 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010312938816449348, + "loss": 1.2068, + "theoretical_loss": 3.3523695757439986, + "tokens_seen": 2963406848 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010310932798395185, + "loss": 1.3703, + "theoretical_loss": 3.3523639099121745, + "tokens_seen": 2963472384 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010308926780341023, + "loss": 1.1609, + "theoretical_loss": 3.3523582442407287, + "tokens_seen": 2963537920 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001030692076228686, + "loss": 1.299, + "theoretical_loss": 3.3523525787296538, + "tokens_seen": 2963603456 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010304914744232698, + "loss": 1.33, + "theoretical_loss": 3.352346913378941, + "tokens_seen": 2963668992 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010302908726178537, + "loss": 1.1599, + "theoretical_loss": 3.352341248188583, + "tokens_seen": 2963734528 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010300902708124374, + "loss": 1.1693, + "theoretical_loss": 3.3523355831585713, + "tokens_seen": 2963800064 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010298896690070212, + "loss": 1.3254, + "theoretical_loss": 3.3523299182888975, + "tokens_seen": 2963865600 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010296890672016048, + "loss": 1.3193, + "theoretical_loss": 3.352324253579554, + "tokens_seen": 2963931136 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1077722, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1816346645355225, + "objective/train/theoretical_loss": 3.3523185890305327, + "objective/train/tokens_used": 1334455776, + "theoretical_loss": 3.3523185890305327, + "tokens_seen": 2963996672 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010294884653961886, + "loss": 1.1716, + "theoretical_loss": 3.3523185890305327, + "tokens_seen": 2963996672 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010292878635907723, + "loss": 1.2886, + "theoretical_loss": 3.3523129246418253, + "tokens_seen": 2964062208 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010290872617853561, + "loss": 1.2381, + "theoretical_loss": 3.352307260413424, + "tokens_seen": 2964127744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010288866599799399, + "loss": 1.288, + "theoretical_loss": 3.3523015963453204, + "tokens_seen": 2964193280 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010286860581745236, + "loss": 1.2356, + "theoretical_loss": 3.352295932437506, + "tokens_seen": 2964258816 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010284854563691074, + "loss": 1.3147, + "theoretical_loss": 3.352290268689974, + "tokens_seen": 2964324352 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010282848545636911, + "loss": 1.3789, + "theoretical_loss": 3.352284605102715, + "tokens_seen": 2964389888 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010280842527582749, + "loss": 1.1977, + "theoretical_loss": 3.352278941675722, + "tokens_seen": 2964455424 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010278836509528585, + "loss": 1.2529, + "theoretical_loss": 3.352273278408986, + "tokens_seen": 2964520960 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010276830491474423, + "loss": 1.2755, + "theoretical_loss": 3.3522676153024995, + "tokens_seen": 2964586496 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001027482447342026, + "loss": 1.2476, + "theoretical_loss": 3.3522619523562542, + "tokens_seen": 2964652032 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010272818455366098, + "loss": 1.2611, + "theoretical_loss": 3.352256289570242, + "tokens_seen": 2964717568 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010270812437311935, + "loss": 1.2286, + "theoretical_loss": 3.352250626944455, + "tokens_seen": 2964783104 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010268806419257773, + "loss": 1.2784, + "theoretical_loss": 3.3522449644788854, + "tokens_seen": 2964848640 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001026680040120361, + "loss": 1.1412, + "theoretical_loss": 3.3522393021735244, + "tokens_seen": 2964914176 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001026479438314945, + "loss": 1.2137, + "theoretical_loss": 3.352233640028364, + "tokens_seen": 2964979712 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010262788365095287, + "loss": 1.2401, + "theoretical_loss": 3.352227978043397, + "tokens_seen": 2965045248 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010260782347041125, + "loss": 1.32, + "theoretical_loss": 3.3522223162186147, + "tokens_seen": 2965110784 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010258776328986961, + "loss": 1.3425, + "theoretical_loss": 3.3522166545540086, + "tokens_seen": 2965176320 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010256770310932798, + "loss": 1.2181, + "theoretical_loss": 3.3522109930495714, + "tokens_seen": 2965241856 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010254764292878636, + "loss": 1.2091, + "theoretical_loss": 3.352205331705295, + "tokens_seen": 2965307392 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010252758274824474, + "loss": 1.1886, + "theoretical_loss": 3.3521996705211707, + "tokens_seen": 2965372928 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010250752256770311, + "loss": 1.1221, + "theoretical_loss": 3.3521940094971905, + "tokens_seen": 2965438464 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010248746238716149, + "loss": 1.2789, + "theoretical_loss": 3.352188348633347, + "tokens_seen": 2965504000 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010246740220661986, + "loss": 1.2697, + "theoretical_loss": 3.3521826879296315, + "tokens_seen": 2965569536 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010244734202607824, + "loss": 1.1643, + "theoretical_loss": 3.3521770273860367, + "tokens_seen": 2965635072 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010242728184553662, + "loss": 1.2152, + "theoretical_loss": 3.3521713670025535, + "tokens_seen": 2965700608 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010240722166499499, + "loss": 1.1597, + "theoretical_loss": 3.3521657067791746, + "tokens_seen": 2965766144 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010238716148445335, + "loss": 1.3076, + "theoretical_loss": 3.3521600467158916, + "tokens_seen": 2965831680 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010236710130391173, + "loss": 1.2866, + "theoretical_loss": 3.3521543868126966, + "tokens_seen": 2965897216 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001023470411233701, + "loss": 1.1956, + "theoretical_loss": 3.3521487270695816, + "tokens_seen": 2965962752 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010232698094282848, + "loss": 1.1166, + "theoretical_loss": 3.352143067486538, + "tokens_seen": 2966028288 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010230692076228686, + "loss": 1.2588, + "theoretical_loss": 3.3521374080635584, + "tokens_seen": 2966093824 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010228686058174525, + "loss": 1.1963, + "theoretical_loss": 3.352131748800634, + "tokens_seen": 2966159360 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010226680040120362, + "loss": 1.2627, + "theoretical_loss": 3.352126089697758, + "tokens_seen": 2966224896 + }, + { + "epoch": 0.8, + "learning_rate": 0.000102246740220662, + "loss": 1.4016, + "theoretical_loss": 3.352120430754921, + "tokens_seen": 2966290432 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010222668004012037, + "loss": 1.3104, + "theoretical_loss": 3.3521147719721154, + "tokens_seen": 2966355968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010220661985957875, + "loss": 1.1939, + "theoretical_loss": 3.3521091133493335, + "tokens_seen": 2966421504 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010218655967903711, + "loss": 1.274, + "theoretical_loss": 3.3521034548865667, + "tokens_seen": 2966487040 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010216649949849549, + "loss": 1.2374, + "theoretical_loss": 3.352097796583807, + "tokens_seen": 2966552576 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010214643931795386, + "loss": 1.2543, + "theoretical_loss": 3.352092138441047, + "tokens_seen": 2966618112 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010212637913741224, + "loss": 1.2639, + "theoretical_loss": 3.352086480458278, + "tokens_seen": 2966683648 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010210631895687061, + "loss": 1.3438, + "theoretical_loss": 3.352080822635492, + "tokens_seen": 2966749184 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010208625877632899, + "loss": 1.2041, + "theoretical_loss": 3.352075164972681, + "tokens_seen": 2966814720 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010206619859578737, + "loss": 1.3666, + "theoretical_loss": 3.3520695074698375, + "tokens_seen": 2966880256 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010204613841524574, + "loss": 1.3108, + "theoretical_loss": 3.352063850126952, + "tokens_seen": 2966945792 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010202607823470412, + "loss": 1.3297, + "theoretical_loss": 3.3520581929440185, + "tokens_seen": 2967011328 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010200601805416248, + "loss": 1.2337, + "theoretical_loss": 3.352052535921027, + "tokens_seen": 2967076864 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010198595787362086, + "loss": 1.3376, + "theoretical_loss": 3.352046879057971, + "tokens_seen": 2967142400 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010196589769307923, + "loss": 1.3452, + "theoretical_loss": 3.352041222354841, + "tokens_seen": 2967207936 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1078969, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3053348064422607, + "objective/train/theoretical_loss": 3.3520355658116303, + "objective/train/tokens_used": 1337732576, + "theoretical_loss": 3.3520355658116303, + "tokens_seen": 2967273472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010194583751253761, + "loss": 1.278, + "theoretical_loss": 3.3520355658116303, + "tokens_seen": 2967273472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010192577733199598, + "loss": 1.2441, + "theoretical_loss": 3.35202990942833, + "tokens_seen": 2967339008 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010190571715145437, + "loss": 1.327, + "theoretical_loss": 3.352024253204932, + "tokens_seen": 2967404544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010188565697091275, + "loss": 1.2416, + "theoretical_loss": 3.3520185971414285, + "tokens_seen": 2967470080 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010186559679037112, + "loss": 1.2221, + "theoretical_loss": 3.3520129412378115, + "tokens_seen": 2967535616 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001018455366098295, + "loss": 1.2241, + "theoretical_loss": 3.3520072854940732, + "tokens_seen": 2967601152 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010182547642928788, + "loss": 1.259, + "theoretical_loss": 3.352001629910205, + "tokens_seen": 2967666688 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010180541624874624, + "loss": 1.1815, + "theoretical_loss": 3.3519959744861993, + "tokens_seen": 2967732224 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010178535606820461, + "loss": 1.2014, + "theoretical_loss": 3.3519903192220477, + "tokens_seen": 2967797760 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010176529588766299, + "loss": 1.2987, + "theoretical_loss": 3.3519846641177424, + "tokens_seen": 2967863296 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010174523570712136, + "loss": 1.318, + "theoretical_loss": 3.3519790091732755, + "tokens_seen": 2967928832 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010172517552657974, + "loss": 1.3367, + "theoretical_loss": 3.3519733543886385, + "tokens_seen": 2967994368 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010170511534603812, + "loss": 1.2238, + "theoretical_loss": 3.351967699763823, + "tokens_seen": 2968059904 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010168505516549649, + "loss": 1.1826, + "theoretical_loss": 3.3519620452988224, + "tokens_seen": 2968125440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010166499498495487, + "loss": 1.2077, + "theoretical_loss": 3.3519563909936276, + "tokens_seen": 2968190976 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010164493480441324, + "loss": 1.2648, + "theoretical_loss": 3.3519507368482304, + "tokens_seen": 2968256512 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010162487462387162, + "loss": 1.2105, + "theoretical_loss": 3.351945082862623, + "tokens_seen": 2968322048 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010160481444332998, + "loss": 1.2881, + "theoretical_loss": 3.351939429036798, + "tokens_seen": 2968387584 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010158475426278836, + "loss": 1.3104, + "theoretical_loss": 3.3519337753707465, + "tokens_seen": 2968453120 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010156469408224673, + "loss": 1.2829, + "theoretical_loss": 3.3519281218644608, + "tokens_seen": 2968518656 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010154463390170511, + "loss": 1.2961, + "theoretical_loss": 3.351922468517933, + "tokens_seen": 2968584192 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001015245737211635, + "loss": 1.295, + "theoretical_loss": 3.3519168153311547, + "tokens_seen": 2968649728 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010150451354062187, + "loss": 1.2108, + "theoretical_loss": 3.351911162304118, + "tokens_seen": 2968715264 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010148445336008025, + "loss": 1.1974, + "theoretical_loss": 3.351905509436815, + "tokens_seen": 2968780800 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010146439317953863, + "loss": 1.2778, + "theoretical_loss": 3.3518998567292373, + "tokens_seen": 2968846336 + }, + { + "epoch": 0.8, + "learning_rate": 0.000101444332998997, + "loss": 1.2704, + "theoretical_loss": 3.3518942041813777, + "tokens_seen": 2968911872 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010142427281845536, + "loss": 1.1382, + "theoretical_loss": 3.351888551793227, + "tokens_seen": 2968977408 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010140421263791374, + "loss": 1.2578, + "theoretical_loss": 3.351882899564778, + "tokens_seen": 2969042944 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010138415245737212, + "loss": 1.2709, + "theoretical_loss": 3.3518772474960223, + "tokens_seen": 2969108480 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010136409227683049, + "loss": 1.2953, + "theoretical_loss": 3.3518715955869522, + "tokens_seen": 2969174016 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010134403209628887, + "loss": 1.2752, + "theoretical_loss": 3.3518659438375593, + "tokens_seen": 2969239552 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010132397191574724, + "loss": 1.2509, + "theoretical_loss": 3.3518602922478355, + "tokens_seen": 2969305088 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010130391173520562, + "loss": 1.2648, + "theoretical_loss": 3.3518546408177734, + "tokens_seen": 2969370624 + }, + { + "epoch": 0.8, + "learning_rate": 0.000101283851554664, + "loss": 1.2878, + "theoretical_loss": 3.3518489895473644, + "tokens_seen": 2969436160 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010126379137412237, + "loss": 1.2142, + "theoretical_loss": 3.3518433384366, + "tokens_seen": 2969501696 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010124373119358075, + "loss": 1.1704, + "theoretical_loss": 3.3518376874854736, + "tokens_seen": 2969567232 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010122367101303911, + "loss": 1.3463, + "theoretical_loss": 3.351832036693976, + "tokens_seen": 2969632768 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010120361083249748, + "loss": 1.2971, + "theoretical_loss": 3.3518263860620996, + "tokens_seen": 2969698304 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010118355065195586, + "loss": 1.2164, + "theoretical_loss": 3.351820735589836, + "tokens_seen": 2969763840 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010116349047141425, + "loss": 1.2387, + "theoretical_loss": 3.3518150852771775, + "tokens_seen": 2969829376 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010114343029087263, + "loss": 1.2313, + "theoretical_loss": 3.351809435124116, + "tokens_seen": 2969894912 + }, + { + "epoch": 0.8, + "learning_rate": 0.000101123370110331, + "loss": 1.3482, + "theoretical_loss": 3.351803785130644, + "tokens_seen": 2969960448 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010110330992978938, + "loss": 1.3213, + "theoretical_loss": 3.351798135296752, + "tokens_seen": 2970025984 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010108324974924775, + "loss": 1.1861, + "theoretical_loss": 3.3517924856224335, + "tokens_seen": 2970091520 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010106318956870613, + "loss": 1.3543, + "theoretical_loss": 3.35178683610768, + "tokens_seen": 2970157056 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001010431293881645, + "loss": 1.2061, + "theoretical_loss": 3.351781186752483, + "tokens_seen": 2970222592 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010102306920762287, + "loss": 1.248, + "theoretical_loss": 3.3517755375568354, + "tokens_seen": 2970288128 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010100300902708124, + "loss": 1.2039, + "theoretical_loss": 3.351769888520728, + "tokens_seen": 2970353664 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010098294884653962, + "loss": 1.2298, + "theoretical_loss": 3.351764239644154, + "tokens_seen": 2970419200 + }, + { + "epoch": 0.8, + "learning_rate": 0.000100962888665998, + "loss": 1.2233, + "theoretical_loss": 3.3517585909271044, + "tokens_seen": 2970484736 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1080058, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3455166816711426, + "objective/train/theoretical_loss": 3.3517529423695716, + "objective/train/tokens_used": 1341009376, + "theoretical_loss": 3.3517529423695716, + "tokens_seen": 2970550272 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010094282848545637, + "loss": 1.2756, + "theoretical_loss": 3.3517529423695716, + "tokens_seen": 2970550272 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010092276830491475, + "loss": 1.2529, + "theoretical_loss": 3.351747293971547, + "tokens_seen": 2970615808 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010090270812437312, + "loss": 1.204, + "theoretical_loss": 3.351741645733024, + "tokens_seen": 2970681344 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001008826479438315, + "loss": 1.2084, + "theoretical_loss": 3.3517359976539933, + "tokens_seen": 2970746880 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010086258776328987, + "loss": 1.2278, + "theoretical_loss": 3.351730349734447, + "tokens_seen": 2970812416 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010084252758274825, + "loss": 1.3062, + "theoretical_loss": 3.3517247019743777, + "tokens_seen": 2970877952 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010082246740220661, + "loss": 1.2295, + "theoretical_loss": 3.3517190543737767, + "tokens_seen": 2970943488 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010080240722166499, + "loss": 1.2044, + "theoretical_loss": 3.351713406932636, + "tokens_seen": 2971009024 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010078234704112338, + "loss": 1.1457, + "theoretical_loss": 3.3517077596509486, + "tokens_seen": 2971074560 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010076228686058175, + "loss": 1.2549, + "theoretical_loss": 3.3517021125287054, + "tokens_seen": 2971140096 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010074222668004013, + "loss": 1.2474, + "theoretical_loss": 3.3516964655658987, + "tokens_seen": 2971205632 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001007221664994985, + "loss": 1.2753, + "theoretical_loss": 3.3516908187625205, + "tokens_seen": 2971271168 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010070210631895688, + "loss": 1.3213, + "theoretical_loss": 3.351685172118563, + "tokens_seen": 2971336704 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010068204613841525, + "loss": 1.2769, + "theoretical_loss": 3.351679525634018, + "tokens_seen": 2971402240 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010066198595787363, + "loss": 1.2987, + "theoretical_loss": 3.351673879308877, + "tokens_seen": 2971467776 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010064192577733199, + "loss": 1.3087, + "theoretical_loss": 3.351668233143133, + "tokens_seen": 2971533312 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010062186559679037, + "loss": 1.3229, + "theoretical_loss": 3.351662587136777, + "tokens_seen": 2971598848 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010060180541624874, + "loss": 1.216, + "theoretical_loss": 3.351656941289802, + "tokens_seen": 2971664384 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010058174523570712, + "loss": 1.258, + "theoretical_loss": 3.3516512956021987, + "tokens_seen": 2971729920 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001005616850551655, + "loss": 1.2629, + "theoretical_loss": 3.35164565007396, + "tokens_seen": 2971795456 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010054162487462387, + "loss": 1.2367, + "theoretical_loss": 3.351640004705078, + "tokens_seen": 2971860992 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010052156469408225, + "loss": 1.1479, + "theoretical_loss": 3.351634359495544, + "tokens_seen": 2971926528 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010050150451354062, + "loss": 1.3501, + "theoretical_loss": 3.3516287144453503, + "tokens_seen": 2971992064 + }, + { + "epoch": 0.8, + "learning_rate": 0.000100481444332999, + "loss": 1.2462, + "theoretical_loss": 3.3516230695544893, + "tokens_seen": 2972057600 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010046138415245738, + "loss": 1.2546, + "theoretical_loss": 3.351617424822952, + "tokens_seen": 2972123136 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010044132397191574, + "loss": 1.2418, + "theoretical_loss": 3.3516117802507317, + "tokens_seen": 2972188672 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010042126379137411, + "loss": 1.1772, + "theoretical_loss": 3.3516061358378195, + "tokens_seen": 2972254208 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001004012036108325, + "loss": 1.3895, + "theoretical_loss": 3.3516004915842075, + "tokens_seen": 2972319744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010038114343029088, + "loss": 1.2457, + "theoretical_loss": 3.3515948474898876, + "tokens_seen": 2972385280 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010036108324974925, + "loss": 1.1557, + "theoretical_loss": 3.3515892035548522, + "tokens_seen": 2972450816 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010034102306920763, + "loss": 1.2511, + "theoretical_loss": 3.351583559779093, + "tokens_seen": 2972516352 + }, + { + "epoch": 0.8, + "learning_rate": 0.000100320962888666, + "loss": 1.2101, + "theoretical_loss": 3.3515779161626025, + "tokens_seen": 2972581888 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010030090270812438, + "loss": 1.3762, + "theoretical_loss": 3.351572272705372, + "tokens_seen": 2972647424 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010028084252758276, + "loss": 1.3009, + "theoretical_loss": 3.351566629407394, + "tokens_seen": 2972712960 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010026078234704113, + "loss": 1.238, + "theoretical_loss": 3.3515609862686597, + "tokens_seen": 2972778496 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001002407221664995, + "loss": 1.3088, + "theoretical_loss": 3.351555343289162, + "tokens_seen": 2972844032 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010022066198595787, + "loss": 1.3399, + "theoretical_loss": 3.351549700468892, + "tokens_seen": 2972909568 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010020060180541625, + "loss": 1.244, + "theoretical_loss": 3.351544057807843, + "tokens_seen": 2972975104 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010018054162487462, + "loss": 1.2284, + "theoretical_loss": 3.351538415306006, + "tokens_seen": 2973040640 + }, + { + "epoch": 0.8, + "learning_rate": 0.000100160481444333, + "loss": 1.1566, + "theoretical_loss": 3.351532772963373, + "tokens_seen": 2973106176 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010014042126379137, + "loss": 1.2468, + "theoretical_loss": 3.3515271307799366, + "tokens_seen": 2973171712 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010012036108324975, + "loss": 1.2895, + "theoretical_loss": 3.3515214887556883, + "tokens_seen": 2973237248 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010010030090270813, + "loss": 1.302, + "theoretical_loss": 3.3515158468906203, + "tokens_seen": 2973302784 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001000802407221665, + "loss": 1.4016, + "theoretical_loss": 3.3515102051847245, + "tokens_seen": 2973368320 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010006018054162486, + "loss": 1.2979, + "theoretical_loss": 3.3515045636379925, + "tokens_seen": 2973433856 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010004012036108324, + "loss": 1.3343, + "theoretical_loss": 3.3514989222504177, + "tokens_seen": 2973499392 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010002006018054163, + "loss": 1.167, + "theoretical_loss": 3.35149328102199, + "tokens_seen": 2973564928 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001, + "loss": 1.1974, + "theoretical_loss": 3.3514876399527034, + "tokens_seen": 2973630464 + }, + { + "epoch": 0.8, + "learning_rate": 9.997993981945838e-05, + "loss": 1.24, + "theoretical_loss": 3.351481999042549, + "tokens_seen": 2973696000 + }, + { + "epoch": 0.8, + "learning_rate": 9.995987963891676e-05, + "loss": 1.3974, + "theoretical_loss": 3.3514763582915186, + "tokens_seen": 2973761536 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1081215, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2280032634735107, + "objective/train/theoretical_loss": 3.3514707176996046, + "objective/train/tokens_used": 1344286176, + "theoretical_loss": 3.3514707176996046, + "tokens_seen": 2973827072 + }, + { + "epoch": 0.8, + "learning_rate": 9.993981945837513e-05, + "loss": 1.2228, + "theoretical_loss": 3.3514707176996046, + "tokens_seen": 2973827072 + }, + { + "epoch": 0.8, + "learning_rate": 9.991975927783351e-05, + "loss": 1.1668, + "theoretical_loss": 3.3514650772667984, + "tokens_seen": 2973892608 + }, + { + "epoch": 0.8, + "learning_rate": 9.989969909729188e-05, + "loss": 1.2275, + "theoretical_loss": 3.351459436993093, + "tokens_seen": 2973958144 + }, + { + "epoch": 0.8, + "learning_rate": 9.987963891675026e-05, + "loss": 1.1917, + "theoretical_loss": 3.35145379687848, + "tokens_seen": 2974023680 + }, + { + "epoch": 0.8, + "learning_rate": 9.985957873620862e-05, + "loss": 1.3146, + "theoretical_loss": 3.3514481569229506, + "tokens_seen": 2974089216 + }, + { + "epoch": 0.8, + "learning_rate": 9.9839518555667e-05, + "loss": 1.3115, + "theoretical_loss": 3.351442517126498, + "tokens_seen": 2974154752 + }, + { + "epoch": 0.8, + "learning_rate": 9.981945837512537e-05, + "loss": 1.2594, + "theoretical_loss": 3.351436877489114, + "tokens_seen": 2974220288 + }, + { + "epoch": 0.8, + "learning_rate": 9.979939819458375e-05, + "loss": 1.2331, + "theoretical_loss": 3.3514312380107896, + "tokens_seen": 2974285824 + }, + { + "epoch": 0.8, + "learning_rate": 9.977933801404212e-05, + "loss": 1.3198, + "theoretical_loss": 3.351425598691518, + "tokens_seen": 2974351360 + }, + { + "epoch": 0.8, + "learning_rate": 9.97592778335005e-05, + "loss": 1.2737, + "theoretical_loss": 3.3514199595312903, + "tokens_seen": 2974416896 + }, + { + "epoch": 0.8, + "learning_rate": 9.973921765295888e-05, + "loss": 1.17, + "theoretical_loss": 3.3514143205300995, + "tokens_seen": 2974482432 + }, + { + "epoch": 0.8, + "learning_rate": 9.971915747241725e-05, + "loss": 1.2878, + "theoretical_loss": 3.3514086816879365, + "tokens_seen": 2974547968 + }, + { + "epoch": 0.8, + "learning_rate": 9.969909729187563e-05, + "loss": 1.2632, + "theoretical_loss": 3.3514030430047943, + "tokens_seen": 2974613504 + }, + { + "epoch": 0.8, + "learning_rate": 9.9679037111334e-05, + "loss": 1.2277, + "theoretical_loss": 3.3513974044806645, + "tokens_seen": 2974679040 + }, + { + "epoch": 0.8, + "learning_rate": 9.965897693079238e-05, + "loss": 1.3057, + "theoretical_loss": 3.3513917661155386, + "tokens_seen": 2974744576 + }, + { + "epoch": 0.8, + "learning_rate": 9.963891675025076e-05, + "loss": 1.2152, + "theoretical_loss": 3.3513861279094095, + "tokens_seen": 2974810112 + }, + { + "epoch": 0.8, + "learning_rate": 9.961885656970913e-05, + "loss": 1.2329, + "theoretical_loss": 3.3513804898622688, + "tokens_seen": 2974875648 + }, + { + "epoch": 0.8, + "learning_rate": 9.959879638916751e-05, + "loss": 1.2704, + "theoretical_loss": 3.3513748519741084, + "tokens_seen": 2974941184 + }, + { + "epoch": 0.8, + "learning_rate": 9.957873620862588e-05, + "loss": 1.1847, + "theoretical_loss": 3.3513692142449205, + "tokens_seen": 2975006720 + }, + { + "epoch": 0.8, + "learning_rate": 9.955867602808426e-05, + "loss": 1.3175, + "theoretical_loss": 3.351363576674697, + "tokens_seen": 2975072256 + }, + { + "epoch": 0.8, + "learning_rate": 9.953861584754263e-05, + "loss": 1.354, + "theoretical_loss": 3.35135793926343, + "tokens_seen": 2975137792 + }, + { + "epoch": 0.8, + "learning_rate": 9.951855566700101e-05, + "loss": 1.3846, + "theoretical_loss": 3.3513523020111116, + "tokens_seen": 2975203328 + }, + { + "epoch": 0.8, + "learning_rate": 9.949849548645939e-05, + "loss": 1.2978, + "theoretical_loss": 3.3513466649177337, + "tokens_seen": 2975268864 + }, + { + "epoch": 0.8, + "learning_rate": 9.947843530591776e-05, + "loss": 1.1916, + "theoretical_loss": 3.3513410279832883, + "tokens_seen": 2975334400 + }, + { + "epoch": 0.8, + "learning_rate": 9.945837512537612e-05, + "loss": 1.136, + "theoretical_loss": 3.3513353912077672, + "tokens_seen": 2975399936 + }, + { + "epoch": 0.8, + "learning_rate": 9.94383149448345e-05, + "loss": 1.2404, + "theoretical_loss": 3.3513297545911627, + "tokens_seen": 2975465472 + }, + { + "epoch": 0.8, + "learning_rate": 9.941825476429288e-05, + "loss": 1.2584, + "theoretical_loss": 3.351324118133467, + "tokens_seen": 2975531008 + }, + { + "epoch": 0.8, + "learning_rate": 9.939819458375125e-05, + "loss": 1.2891, + "theoretical_loss": 3.3513184818346717, + "tokens_seen": 2975596544 + }, + { + "epoch": 0.8, + "learning_rate": 9.937813440320963e-05, + "loss": 1.2435, + "theoretical_loss": 3.3513128456947694, + "tokens_seen": 2975662080 + }, + { + "epoch": 0.8, + "learning_rate": 9.9358074222668e-05, + "loss": 1.2389, + "theoretical_loss": 3.3513072097137515, + "tokens_seen": 2975727616 + }, + { + "epoch": 0.8, + "learning_rate": 9.933801404212638e-05, + "loss": 1.2969, + "theoretical_loss": 3.35130157389161, + "tokens_seen": 2975793152 + }, + { + "epoch": 0.8, + "learning_rate": 9.931795386158475e-05, + "loss": 1.2759, + "theoretical_loss": 3.3512959382283376, + "tokens_seen": 2975858688 + }, + { + "epoch": 0.8, + "learning_rate": 9.929789368104313e-05, + "loss": 1.3555, + "theoretical_loss": 3.351290302723926, + "tokens_seen": 2975924224 + }, + { + "epoch": 0.8, + "learning_rate": 9.927783350050152e-05, + "loss": 1.2865, + "theoretical_loss": 3.351284667378367, + "tokens_seen": 2975989760 + }, + { + "epoch": 0.8, + "learning_rate": 9.925777331995988e-05, + "loss": 1.3238, + "theoretical_loss": 3.3512790321916524, + "tokens_seen": 2976055296 + }, + { + "epoch": 0.8, + "learning_rate": 9.923771313941826e-05, + "loss": 1.2802, + "theoretical_loss": 3.3512733971637747, + "tokens_seen": 2976120832 + }, + { + "epoch": 0.8, + "learning_rate": 9.921765295887663e-05, + "loss": 1.3185, + "theoretical_loss": 3.3512677622947264, + "tokens_seen": 2976186368 + }, + { + "epoch": 0.8, + "learning_rate": 9.919759277833501e-05, + "loss": 1.1753, + "theoretical_loss": 3.3512621275844983, + "tokens_seen": 2976251904 + }, + { + "epoch": 0.8, + "learning_rate": 9.917753259779339e-05, + "loss": 1.2229, + "theoretical_loss": 3.3512564930330835, + "tokens_seen": 2976317440 + }, + { + "epoch": 0.8, + "learning_rate": 9.915747241725176e-05, + "loss": 1.3348, + "theoretical_loss": 3.3512508586404732, + "tokens_seen": 2976382976 + }, + { + "epoch": 0.8, + "learning_rate": 9.913741223671014e-05, + "loss": 1.2053, + "theoretical_loss": 3.35124522440666, + "tokens_seen": 2976448512 + }, + { + "epoch": 0.8, + "learning_rate": 9.911735205616851e-05, + "loss": 1.3329, + "theoretical_loss": 3.3512395903316357, + "tokens_seen": 2976514048 + }, + { + "epoch": 0.8, + "learning_rate": 9.909729187562689e-05, + "loss": 1.2599, + "theoretical_loss": 3.3512339564153923, + "tokens_seen": 2976579584 + }, + { + "epoch": 0.8, + "learning_rate": 9.907723169508525e-05, + "loss": 1.2646, + "theoretical_loss": 3.3512283226579225, + "tokens_seen": 2976645120 + }, + { + "epoch": 0.8, + "learning_rate": 9.905717151454363e-05, + "loss": 1.3008, + "theoretical_loss": 3.3512226890592167, + "tokens_seen": 2976710656 + }, + { + "epoch": 0.8, + "learning_rate": 9.9037111334002e-05, + "loss": 1.1878, + "theoretical_loss": 3.351217055619269, + "tokens_seen": 2976776192 + }, + { + "epoch": 0.8, + "learning_rate": 9.901705115346038e-05, + "loss": 1.3495, + "theoretical_loss": 3.3512114223380696, + "tokens_seen": 2976841728 + }, + { + "epoch": 0.8, + "learning_rate": 9.899699097291875e-05, + "loss": 1.1998, + "theoretical_loss": 3.3512057892156117, + "tokens_seen": 2976907264 + }, + { + "epoch": 0.8, + "learning_rate": 9.897693079237713e-05, + "loss": 1.2351, + "theoretical_loss": 3.3512001562518874, + "tokens_seen": 2976972800 + }, + { + "epoch": 0.8, + "learning_rate": 9.89568706118355e-05, + "loss": 1.2719, + "theoretical_loss": 3.3511945234468876, + "tokens_seen": 2977038336 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1082519, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1863994598388672, + "objective/train/theoretical_loss": 3.3511888908006053, + "objective/train/tokens_used": 1347562976, + "theoretical_loss": 3.3511888908006053, + "tokens_seen": 2977103872 + }, + { + "epoch": 0.8, + "learning_rate": 9.893681043129388e-05, + "loss": 1.2401, + "theoretical_loss": 3.3511888908006053, + "tokens_seen": 2977103872 + }, + { + "epoch": 0.8, + "learning_rate": 9.891675025075226e-05, + "loss": 1.2231, + "theoretical_loss": 3.351183258313032, + "tokens_seen": 2977169408 + }, + { + "epoch": 0.8, + "learning_rate": 9.889669007021065e-05, + "loss": 1.1329, + "theoretical_loss": 3.3511776259841604, + "tokens_seen": 2977234944 + }, + { + "epoch": 0.8, + "learning_rate": 9.887662988966901e-05, + "loss": 1.3585, + "theoretical_loss": 3.3511719938139817, + "tokens_seen": 2977300480 + }, + { + "epoch": 0.8, + "learning_rate": 9.885656970912738e-05, + "loss": 1.2183, + "theoretical_loss": 3.3511663618024885, + "tokens_seen": 2977366016 + }, + { + "epoch": 0.8, + "learning_rate": 9.883650952858576e-05, + "loss": 1.2542, + "theoretical_loss": 3.351160729949673, + "tokens_seen": 2977431552 + }, + { + "epoch": 0.8, + "learning_rate": 9.881644934804414e-05, + "loss": 1.2012, + "theoretical_loss": 3.3511550982555267, + "tokens_seen": 2977497088 + }, + { + "epoch": 0.8, + "learning_rate": 9.879638916750251e-05, + "loss": 1.35, + "theoretical_loss": 3.351149466720042, + "tokens_seen": 2977562624 + }, + { + "epoch": 0.8, + "learning_rate": 9.877632898696089e-05, + "loss": 1.2356, + "theoretical_loss": 3.3511438353432106, + "tokens_seen": 2977628160 + }, + { + "epoch": 0.8, + "learning_rate": 9.875626880641926e-05, + "loss": 1.2489, + "theoretical_loss": 3.351138204125025, + "tokens_seen": 2977693696 + }, + { + "epoch": 0.8, + "learning_rate": 9.873620862587764e-05, + "loss": 1.2599, + "theoretical_loss": 3.351132573065477, + "tokens_seen": 2977759232 + }, + { + "epoch": 0.8, + "learning_rate": 9.871614844533601e-05, + "loss": 1.1962, + "theoretical_loss": 3.3511269421645586, + "tokens_seen": 2977824768 + }, + { + "epoch": 0.8, + "learning_rate": 9.869608826479439e-05, + "loss": 1.2831, + "theoretical_loss": 3.3511213114222618, + "tokens_seen": 2977890304 + }, + { + "epoch": 0.8, + "learning_rate": 9.867602808425275e-05, + "loss": 1.2348, + "theoretical_loss": 3.3511156808385785, + "tokens_seen": 2977955840 + }, + { + "epoch": 0.8, + "learning_rate": 9.865596790371113e-05, + "loss": 1.1939, + "theoretical_loss": 3.351110050413501, + "tokens_seen": 2978021376 + }, + { + "epoch": 0.8, + "learning_rate": 9.86359077231695e-05, + "loss": 1.2191, + "theoretical_loss": 3.351104420147022, + "tokens_seen": 2978086912 + }, + { + "epoch": 0.8, + "learning_rate": 9.861584754262788e-05, + "loss": 1.3502, + "theoretical_loss": 3.3510987900391322, + "tokens_seen": 2978152448 + }, + { + "epoch": 0.8, + "learning_rate": 9.859578736208626e-05, + "loss": 1.1563, + "theoretical_loss": 3.3510931600898246, + "tokens_seen": 2978217984 + }, + { + "epoch": 0.81, + "learning_rate": 9.857572718154463e-05, + "loss": 1.3839, + "theoretical_loss": 3.3510875302990906, + "tokens_seen": 2978283520 + }, + { + "epoch": 0.81, + "learning_rate": 9.855566700100301e-05, + "loss": 1.244, + "theoretical_loss": 3.3510819006669226, + "tokens_seen": 2978349056 + }, + { + "epoch": 0.81, + "learning_rate": 9.85356068204614e-05, + "loss": 1.247, + "theoretical_loss": 3.3510762711933126, + "tokens_seen": 2978414592 + }, + { + "epoch": 0.81, + "learning_rate": 9.851554663991977e-05, + "loss": 1.263, + "theoretical_loss": 3.3510706418782528, + "tokens_seen": 2978480128 + }, + { + "epoch": 0.81, + "learning_rate": 9.849548645937815e-05, + "loss": 1.3059, + "theoretical_loss": 3.3510650127217354, + "tokens_seen": 2978545664 + }, + { + "epoch": 0.81, + "learning_rate": 9.847542627883651e-05, + "loss": 1.2493, + "theoretical_loss": 3.3510593837237517, + "tokens_seen": 2978611200 + }, + { + "epoch": 0.81, + "learning_rate": 9.845536609829489e-05, + "loss": 1.2885, + "theoretical_loss": 3.3510537548842945, + "tokens_seen": 2978676736 + }, + { + "epoch": 0.81, + "learning_rate": 9.843530591775326e-05, + "loss": 1.1418, + "theoretical_loss": 3.3510481262033553, + "tokens_seen": 2978742272 + }, + { + "epoch": 0.81, + "learning_rate": 9.841524573721164e-05, + "loss": 1.3473, + "theoretical_loss": 3.3510424976809263, + "tokens_seen": 2978807808 + }, + { + "epoch": 0.81, + "learning_rate": 9.839518555667001e-05, + "loss": 1.2249, + "theoretical_loss": 3.3510368693170003, + "tokens_seen": 2978873344 + }, + { + "epoch": 0.81, + "learning_rate": 9.837512537612839e-05, + "loss": 1.1512, + "theoretical_loss": 3.351031241111568, + "tokens_seen": 2978938880 + }, + { + "epoch": 0.81, + "learning_rate": 9.835506519558677e-05, + "loss": 1.2328, + "theoretical_loss": 3.3510256130646225, + "tokens_seen": 2979004416 + }, + { + "epoch": 0.81, + "learning_rate": 9.833500501504514e-05, + "loss": 1.3418, + "theoretical_loss": 3.3510199851761553, + "tokens_seen": 2979069952 + }, + { + "epoch": 0.81, + "learning_rate": 9.831494483450352e-05, + "loss": 1.3074, + "theoretical_loss": 3.351014357446159, + "tokens_seen": 2979135488 + }, + { + "epoch": 0.81, + "learning_rate": 9.829488465396188e-05, + "loss": 1.2039, + "theoretical_loss": 3.351008729874625, + "tokens_seen": 2979201024 + }, + { + "epoch": 0.81, + "learning_rate": 9.827482447342026e-05, + "loss": 1.3313, + "theoretical_loss": 3.3510031024615454, + "tokens_seen": 2979266560 + }, + { + "epoch": 0.81, + "learning_rate": 9.825476429287863e-05, + "loss": 1.2903, + "theoretical_loss": 3.350997475206913, + "tokens_seen": 2979332096 + }, + { + "epoch": 0.81, + "learning_rate": 9.823470411233701e-05, + "loss": 1.3071, + "theoretical_loss": 3.350991848110719, + "tokens_seen": 2979397632 + }, + { + "epoch": 0.81, + "learning_rate": 9.821464393179538e-05, + "loss": 1.2009, + "theoretical_loss": 3.350986221172956, + "tokens_seen": 2979463168 + }, + { + "epoch": 0.81, + "learning_rate": 9.819458375125376e-05, + "loss": 1.3254, + "theoretical_loss": 3.350980594393616, + "tokens_seen": 2979528704 + }, + { + "epoch": 0.81, + "learning_rate": 9.817452357071213e-05, + "loss": 1.3119, + "theoretical_loss": 3.3509749677726908, + "tokens_seen": 2979594240 + }, + { + "epoch": 0.81, + "learning_rate": 9.815446339017052e-05, + "loss": 1.1786, + "theoretical_loss": 3.3509693413101727, + "tokens_seen": 2979659776 + }, + { + "epoch": 0.81, + "learning_rate": 9.81344032096289e-05, + "loss": 1.3683, + "theoretical_loss": 3.3509637150060536, + "tokens_seen": 2979725312 + }, + { + "epoch": 0.81, + "learning_rate": 9.811434302908728e-05, + "loss": 1.3236, + "theoretical_loss": 3.3509580888603256, + "tokens_seen": 2979790848 + }, + { + "epoch": 0.81, + "learning_rate": 9.809428284854564e-05, + "loss": 1.3002, + "theoretical_loss": 3.3509524628729803, + "tokens_seen": 2979856384 + }, + { + "epoch": 0.81, + "learning_rate": 9.807422266800401e-05, + "loss": 1.3067, + "theoretical_loss": 3.3509468370440105, + "tokens_seen": 2979921920 + }, + { + "epoch": 0.81, + "learning_rate": 9.805416248746239e-05, + "loss": 1.2925, + "theoretical_loss": 3.3509412113734083, + "tokens_seen": 2979987456 + }, + { + "epoch": 0.81, + "learning_rate": 9.803410230692076e-05, + "loss": 1.3157, + "theoretical_loss": 3.350935585861165, + "tokens_seen": 2980052992 + }, + { + "epoch": 0.81, + "learning_rate": 9.801404212637914e-05, + "loss": 1.2843, + "theoretical_loss": 3.3509299605072735, + "tokens_seen": 2980118528 + }, + { + "epoch": 0.81, + "learning_rate": 9.799398194583752e-05, + "loss": 1.2787, + "theoretical_loss": 3.350924335311725, + "tokens_seen": 2980184064 + }, + { + "epoch": 0.81, + "learning_rate": 9.797392176529589e-05, + "loss": 1.3422, + "theoretical_loss": 3.350918710274512, + "tokens_seen": 2980249600 + }, + { + "epoch": 0.81, + "learning_rate": 9.795386158475427e-05, + "loss": 1.3155, + "theoretical_loss": 3.3509130853956273, + "tokens_seen": 2980315136 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1083727, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2798722982406616, + "objective/train/theoretical_loss": 3.3509074606750615, + "objective/train/tokens_used": 1350839776, + "theoretical_loss": 3.3509074606750615, + "tokens_seen": 2980380672 + }, + { + "epoch": 0.81, + "learning_rate": 9.793380140421264e-05, + "loss": 1.2945, + "theoretical_loss": 3.3509074606750615, + "tokens_seen": 2980380672 + }, + { + "epoch": 0.81, + "learning_rate": 9.791374122367102e-05, + "loss": 1.2086, + "theoretical_loss": 3.350901836112808, + "tokens_seen": 2980446208 + }, + { + "epoch": 0.81, + "learning_rate": 9.789368104312938e-05, + "loss": 1.287, + "theoretical_loss": 3.3508962117088577, + "tokens_seen": 2980511744 + }, + { + "epoch": 0.81, + "learning_rate": 9.787362086258776e-05, + "loss": 1.2899, + "theoretical_loss": 3.350890587463203, + "tokens_seen": 2980577280 + }, + { + "epoch": 0.81, + "learning_rate": 9.785356068204613e-05, + "loss": 1.3216, + "theoretical_loss": 3.350884963375837, + "tokens_seen": 2980642816 + }, + { + "epoch": 0.81, + "learning_rate": 9.783350050150451e-05, + "loss": 1.3078, + "theoretical_loss": 3.3508793394467506, + "tokens_seen": 2980708352 + }, + { + "epoch": 0.81, + "learning_rate": 9.781344032096288e-05, + "loss": 1.3245, + "theoretical_loss": 3.3508737156759363, + "tokens_seen": 2980773888 + }, + { + "epoch": 0.81, + "learning_rate": 9.779338014042126e-05, + "loss": 1.2229, + "theoretical_loss": 3.3508680920633855, + "tokens_seen": 2980839424 + }, + { + "epoch": 0.81, + "learning_rate": 9.777331995987965e-05, + "loss": 1.2024, + "theoretical_loss": 3.3508624686090913, + "tokens_seen": 2980904960 + }, + { + "epoch": 0.81, + "learning_rate": 9.775325977933803e-05, + "loss": 1.1896, + "theoretical_loss": 3.3508568453130456, + "tokens_seen": 2980970496 + }, + { + "epoch": 0.81, + "learning_rate": 9.77331995987964e-05, + "loss": 1.2317, + "theoretical_loss": 3.3508512221752396, + "tokens_seen": 2981036032 + }, + { + "epoch": 0.81, + "learning_rate": 9.771313941825476e-05, + "loss": 1.1943, + "theoretical_loss": 3.3508455991956665, + "tokens_seen": 2981101568 + }, + { + "epoch": 0.81, + "learning_rate": 9.769307923771314e-05, + "loss": 1.195, + "theoretical_loss": 3.3508399763743175, + "tokens_seen": 2981167104 + }, + { + "epoch": 0.81, + "learning_rate": 9.767301905717152e-05, + "loss": 1.2353, + "theoretical_loss": 3.350834353711185, + "tokens_seen": 2981232640 + }, + { + "epoch": 0.81, + "learning_rate": 9.765295887662989e-05, + "loss": 1.3386, + "theoretical_loss": 3.3508287312062612, + "tokens_seen": 2981298176 + }, + { + "epoch": 0.81, + "learning_rate": 9.763289869608827e-05, + "loss": 1.278, + "theoretical_loss": 3.350823108859538, + "tokens_seen": 2981363712 + }, + { + "epoch": 0.81, + "learning_rate": 9.761283851554664e-05, + "loss": 1.332, + "theoretical_loss": 3.3508174866710077, + "tokens_seen": 2981429248 + }, + { + "epoch": 0.81, + "learning_rate": 9.759277833500502e-05, + "loss": 1.2715, + "theoretical_loss": 3.3508118646406615, + "tokens_seen": 2981494784 + }, + { + "epoch": 0.81, + "learning_rate": 9.75727181544634e-05, + "loss": 1.3201, + "theoretical_loss": 3.350806242768493, + "tokens_seen": 2981560320 + }, + { + "epoch": 0.81, + "learning_rate": 9.755265797392177e-05, + "loss": 1.3149, + "theoretical_loss": 3.3508006210544927, + "tokens_seen": 2981625856 + }, + { + "epoch": 0.81, + "learning_rate": 9.753259779338015e-05, + "loss": 1.1822, + "theoretical_loss": 3.350794999498654, + "tokens_seen": 2981691392 + }, + { + "epoch": 0.81, + "learning_rate": 9.751253761283851e-05, + "loss": 1.3545, + "theoretical_loss": 3.350789378100968, + "tokens_seen": 2981756928 + }, + { + "epoch": 0.81, + "learning_rate": 9.749247743229688e-05, + "loss": 1.2418, + "theoretical_loss": 3.350783756861427, + "tokens_seen": 2981822464 + }, + { + "epoch": 0.81, + "learning_rate": 9.747241725175526e-05, + "loss": 1.2736, + "theoretical_loss": 3.3507781357800237, + "tokens_seen": 2981888000 + }, + { + "epoch": 0.81, + "learning_rate": 9.745235707121364e-05, + "loss": 1.2042, + "theoretical_loss": 3.3507725148567493, + "tokens_seen": 2981953536 + }, + { + "epoch": 0.81, + "learning_rate": 9.743229689067201e-05, + "loss": 1.3373, + "theoretical_loss": 3.3507668940915964, + "tokens_seen": 2982019072 + }, + { + "epoch": 0.81, + "learning_rate": 9.741223671013039e-05, + "loss": 1.3524, + "theoretical_loss": 3.350761273484557, + "tokens_seen": 2982084608 + }, + { + "epoch": 0.81, + "learning_rate": 9.739217652958878e-05, + "loss": 1.3226, + "theoretical_loss": 3.3507556530356233, + "tokens_seen": 2982150144 + }, + { + "epoch": 0.81, + "learning_rate": 9.737211634904715e-05, + "loss": 1.311, + "theoretical_loss": 3.350750032744787, + "tokens_seen": 2982215680 + }, + { + "epoch": 0.81, + "learning_rate": 9.735205616850553e-05, + "loss": 1.277, + "theoretical_loss": 3.3507444126120403, + "tokens_seen": 2982281216 + }, + { + "epoch": 0.81, + "learning_rate": 9.73319959879639e-05, + "loss": 1.2577, + "theoretical_loss": 3.3507387926373755, + "tokens_seen": 2982346752 + }, + { + "epoch": 0.81, + "learning_rate": 9.731193580742227e-05, + "loss": 1.1941, + "theoretical_loss": 3.3507331728207843, + "tokens_seen": 2982412288 + }, + { + "epoch": 0.81, + "learning_rate": 9.729187562688064e-05, + "loss": 1.1898, + "theoretical_loss": 3.3507275531622596, + "tokens_seen": 2982477824 + }, + { + "epoch": 0.81, + "learning_rate": 9.727181544633902e-05, + "loss": 1.2584, + "theoretical_loss": 3.3507219336617924, + "tokens_seen": 2982543360 + }, + { + "epoch": 0.81, + "learning_rate": 9.72517552657974e-05, + "loss": 1.2434, + "theoretical_loss": 3.350716314319375, + "tokens_seen": 2982608896 + }, + { + "epoch": 0.81, + "learning_rate": 9.723169508525577e-05, + "loss": 1.3056, + "theoretical_loss": 3.3507106951350005, + "tokens_seen": 2982674432 + }, + { + "epoch": 0.81, + "learning_rate": 9.721163490471415e-05, + "loss": 1.3555, + "theoretical_loss": 3.3507050761086603, + "tokens_seen": 2982739968 + }, + { + "epoch": 0.81, + "learning_rate": 9.719157472417252e-05, + "loss": 1.2167, + "theoretical_loss": 3.3506994572403457, + "tokens_seen": 2982805504 + }, + { + "epoch": 0.81, + "learning_rate": 9.71715145436309e-05, + "loss": 1.2628, + "theoretical_loss": 3.35069383853005, + "tokens_seen": 2982871040 + }, + { + "epoch": 0.81, + "learning_rate": 9.715145436308927e-05, + "loss": 1.1934, + "theoretical_loss": 3.350688219977765, + "tokens_seen": 2982936576 + }, + { + "epoch": 0.81, + "learning_rate": 9.713139418254765e-05, + "loss": 1.2659, + "theoretical_loss": 3.350682601583482, + "tokens_seen": 2983002112 + }, + { + "epoch": 0.81, + "learning_rate": 9.711133400200601e-05, + "loss": 1.1631, + "theoretical_loss": 3.350676983347194, + "tokens_seen": 2983067648 + }, + { + "epoch": 0.81, + "learning_rate": 9.709127382146439e-05, + "loss": 1.3327, + "theoretical_loss": 3.3506713652688926, + "tokens_seen": 2983133184 + }, + { + "epoch": 0.81, + "learning_rate": 9.707121364092276e-05, + "loss": 1.2945, + "theoretical_loss": 3.3506657473485704, + "tokens_seen": 2983198720 + }, + { + "epoch": 0.81, + "learning_rate": 9.705115346038114e-05, + "loss": 1.1561, + "theoretical_loss": 3.3506601295862186, + "tokens_seen": 2983264256 + }, + { + "epoch": 0.81, + "learning_rate": 9.703109327983953e-05, + "loss": 1.2109, + "theoretical_loss": 3.3506545119818303, + "tokens_seen": 2983329792 + }, + { + "epoch": 0.81, + "learning_rate": 9.70110330992979e-05, + "loss": 1.2563, + "theoretical_loss": 3.350648894535397, + "tokens_seen": 2983395328 + }, + { + "epoch": 0.81, + "learning_rate": 9.699097291875628e-05, + "loss": 1.2409, + "theoretical_loss": 3.3506432772469106, + "tokens_seen": 2983460864 + }, + { + "epoch": 0.81, + "learning_rate": 9.697091273821465e-05, + "loss": 1.1304, + "theoretical_loss": 3.350637660116364, + "tokens_seen": 2983526400 + }, + { + "epoch": 0.81, + "learning_rate": 9.695085255767303e-05, + "loss": 1.2487, + "theoretical_loss": 3.350632043143748, + "tokens_seen": 2983591936 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1084846, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2808384895324707, + "objective/train/theoretical_loss": 3.3506264263290566, + "objective/train/tokens_used": 1354116576, + "theoretical_loss": 3.3506264263290566, + "tokens_seen": 2983657472 + }, + { + "epoch": 0.81, + "learning_rate": 9.693079237713139e-05, + "loss": 1.3225, + "theoretical_loss": 3.3506264263290566, + "tokens_seen": 2983657472 + }, + { + "epoch": 0.81, + "learning_rate": 9.691073219658977e-05, + "loss": 1.2182, + "theoretical_loss": 3.35062080967228, + "tokens_seen": 2983723008 + }, + { + "epoch": 0.81, + "learning_rate": 9.689067201604814e-05, + "loss": 1.2261, + "theoretical_loss": 3.350615193173411, + "tokens_seen": 2983788544 + }, + { + "epoch": 0.81, + "learning_rate": 9.687061183550652e-05, + "loss": 1.3611, + "theoretical_loss": 3.3506095768324418, + "tokens_seen": 2983854080 + }, + { + "epoch": 0.81, + "learning_rate": 9.68505516549649e-05, + "loss": 1.2591, + "theoretical_loss": 3.350603960649365, + "tokens_seen": 2983919616 + }, + { + "epoch": 0.81, + "learning_rate": 9.683049147442327e-05, + "loss": 1.207, + "theoretical_loss": 3.3505983446241716, + "tokens_seen": 2983985152 + }, + { + "epoch": 0.81, + "learning_rate": 9.681043129388165e-05, + "loss": 1.2045, + "theoretical_loss": 3.350592728756854, + "tokens_seen": 2984050688 + }, + { + "epoch": 0.81, + "learning_rate": 9.679037111334002e-05, + "loss": 1.2148, + "theoretical_loss": 3.350587113047405, + "tokens_seen": 2984116224 + }, + { + "epoch": 0.81, + "learning_rate": 9.67703109327984e-05, + "loss": 1.2735, + "theoretical_loss": 3.350581497495816, + "tokens_seen": 2984181760 + }, + { + "epoch": 0.81, + "learning_rate": 9.675025075225677e-05, + "loss": 1.2386, + "theoretical_loss": 3.3505758821020795, + "tokens_seen": 2984247296 + }, + { + "epoch": 0.81, + "learning_rate": 9.673019057171514e-05, + "loss": 1.2282, + "theoretical_loss": 3.350570266866187, + "tokens_seen": 2984312832 + }, + { + "epoch": 0.81, + "learning_rate": 9.671013039117351e-05, + "loss": 1.2705, + "theoretical_loss": 3.3505646517881313, + "tokens_seen": 2984378368 + }, + { + "epoch": 0.81, + "learning_rate": 9.669007021063189e-05, + "loss": 1.2562, + "theoretical_loss": 3.3505590368679044, + "tokens_seen": 2984443904 + }, + { + "epoch": 0.81, + "learning_rate": 9.667001003009026e-05, + "loss": 1.2877, + "theoretical_loss": 3.3505534221054982, + "tokens_seen": 2984509440 + }, + { + "epoch": 0.81, + "learning_rate": 9.664994984954865e-05, + "loss": 1.2434, + "theoretical_loss": 3.3505478075009045, + "tokens_seen": 2984574976 + }, + { + "epoch": 0.81, + "learning_rate": 9.662988966900703e-05, + "loss": 1.241, + "theoretical_loss": 3.3505421930541157, + "tokens_seen": 2984640512 + }, + { + "epoch": 0.81, + "learning_rate": 9.66098294884654e-05, + "loss": 1.3194, + "theoretical_loss": 3.350536578765124, + "tokens_seen": 2984706048 + }, + { + "epoch": 0.81, + "learning_rate": 9.658976930792378e-05, + "loss": 1.2329, + "theoretical_loss": 3.350530964633921, + "tokens_seen": 2984771584 + }, + { + "epoch": 0.81, + "learning_rate": 9.656970912738216e-05, + "loss": 1.1831, + "theoretical_loss": 3.3505253506605, + "tokens_seen": 2984837120 + }, + { + "epoch": 0.81, + "learning_rate": 9.654964894684053e-05, + "loss": 1.307, + "theoretical_loss": 3.350519736844852, + "tokens_seen": 2984902656 + }, + { + "epoch": 0.81, + "learning_rate": 9.65295887662989e-05, + "loss": 1.2491, + "theoretical_loss": 3.350514123186969, + "tokens_seen": 2984968192 + }, + { + "epoch": 0.81, + "learning_rate": 9.650952858575727e-05, + "loss": 1.2113, + "theoretical_loss": 3.350508509686844, + "tokens_seen": 2985033728 + }, + { + "epoch": 0.81, + "learning_rate": 9.648946840521565e-05, + "loss": 1.1169, + "theoretical_loss": 3.3505028963444685, + "tokens_seen": 2985099264 + }, + { + "epoch": 0.81, + "learning_rate": 9.646940822467402e-05, + "loss": 1.237, + "theoretical_loss": 3.3504972831598345, + "tokens_seen": 2985164800 + }, + { + "epoch": 0.81, + "learning_rate": 9.64493480441324e-05, + "loss": 1.2115, + "theoretical_loss": 3.3504916701329344, + "tokens_seen": 2985230336 + }, + { + "epoch": 0.81, + "learning_rate": 9.642928786359077e-05, + "loss": 1.179, + "theoretical_loss": 3.3504860572637605, + "tokens_seen": 2985295872 + }, + { + "epoch": 0.81, + "learning_rate": 9.640922768304915e-05, + "loss": 1.3143, + "theoretical_loss": 3.350480444552304, + "tokens_seen": 2985361408 + }, + { + "epoch": 0.81, + "learning_rate": 9.638916750250753e-05, + "loss": 1.3195, + "theoretical_loss": 3.3504748319985582, + "tokens_seen": 2985426944 + }, + { + "epoch": 0.81, + "learning_rate": 9.63691073219659e-05, + "loss": 1.2363, + "theoretical_loss": 3.3504692196025143, + "tokens_seen": 2985492480 + }, + { + "epoch": 0.81, + "learning_rate": 9.634904714142426e-05, + "loss": 1.1735, + "theoretical_loss": 3.350463607364165, + "tokens_seen": 2985558016 + }, + { + "epoch": 0.81, + "learning_rate": 9.632898696088264e-05, + "loss": 1.3224, + "theoretical_loss": 3.3504579952835023, + "tokens_seen": 2985623552 + }, + { + "epoch": 0.81, + "learning_rate": 9.630892678034102e-05, + "loss": 1.1297, + "theoretical_loss": 3.3504523833605178, + "tokens_seen": 2985689088 + }, + { + "epoch": 0.81, + "learning_rate": 9.628886659979939e-05, + "loss": 1.1864, + "theoretical_loss": 3.350446771595204, + "tokens_seen": 2985754624 + }, + { + "epoch": 0.81, + "learning_rate": 9.626880641925778e-05, + "loss": 1.2853, + "theoretical_loss": 3.350441159987553, + "tokens_seen": 2985820160 + }, + { + "epoch": 0.81, + "learning_rate": 9.624874623871616e-05, + "loss": 1.2968, + "theoretical_loss": 3.350435548537557, + "tokens_seen": 2985885696 + }, + { + "epoch": 0.81, + "learning_rate": 9.622868605817453e-05, + "loss": 1.1237, + "theoretical_loss": 3.350429937245208, + "tokens_seen": 2985951232 + }, + { + "epoch": 0.81, + "learning_rate": 9.620862587763291e-05, + "loss": 1.357, + "theoretical_loss": 3.350424326110498, + "tokens_seen": 2986016768 + }, + { + "epoch": 0.81, + "learning_rate": 9.618856569709128e-05, + "loss": 1.19, + "theoretical_loss": 3.3504187151334195, + "tokens_seen": 2986082304 + }, + { + "epoch": 0.81, + "learning_rate": 9.616850551654966e-05, + "loss": 1.2234, + "theoretical_loss": 3.350413104313964, + "tokens_seen": 2986147840 + }, + { + "epoch": 0.81, + "learning_rate": 9.614844533600802e-05, + "loss": 1.3105, + "theoretical_loss": 3.350407493652124, + "tokens_seen": 2986213376 + }, + { + "epoch": 0.81, + "learning_rate": 9.61283851554664e-05, + "loss": 1.2464, + "theoretical_loss": 3.350401883147892, + "tokens_seen": 2986278912 + }, + { + "epoch": 0.81, + "learning_rate": 9.610832497492477e-05, + "loss": 1.297, + "theoretical_loss": 3.3503962728012593, + "tokens_seen": 2986344448 + }, + { + "epoch": 0.81, + "learning_rate": 9.608826479438315e-05, + "loss": 1.1948, + "theoretical_loss": 3.3503906626122184, + "tokens_seen": 2986409984 + }, + { + "epoch": 0.81, + "learning_rate": 9.606820461384152e-05, + "loss": 1.209, + "theoretical_loss": 3.3503850525807612, + "tokens_seen": 2986475520 + }, + { + "epoch": 0.81, + "learning_rate": 9.60481444332999e-05, + "loss": 1.1895, + "theoretical_loss": 3.3503794427068803, + "tokens_seen": 2986541056 + }, + { + "epoch": 0.81, + "learning_rate": 9.602808425275828e-05, + "loss": 1.3712, + "theoretical_loss": 3.350373832990568, + "tokens_seen": 2986606592 + }, + { + "epoch": 0.81, + "learning_rate": 9.600802407221665e-05, + "loss": 1.222, + "theoretical_loss": 3.3503682234318153, + "tokens_seen": 2986672128 + }, + { + "epoch": 0.81, + "learning_rate": 9.598796389167503e-05, + "loss": 1.2072, + "theoretical_loss": 3.3503626140306153, + "tokens_seen": 2986737664 + }, + { + "epoch": 0.81, + "learning_rate": 9.59679037111334e-05, + "loss": 1.2888, + "theoretical_loss": 3.3503570047869595, + "tokens_seen": 2986803200 + }, + { + "epoch": 0.81, + "learning_rate": 9.594784353059177e-05, + "loss": 1.2422, + "theoretical_loss": 3.3503513957008404, + "tokens_seen": 2986868736 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1086010, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2536039352416992, + "objective/train/theoretical_loss": 3.35034578677225, + "objective/train/tokens_used": 1357393376, + "theoretical_loss": 3.35034578677225, + "tokens_seen": 2986934272 + }, + { + "epoch": 0.81, + "learning_rate": 9.592778335005014e-05, + "loss": 1.3204, + "theoretical_loss": 3.35034578677225, + "tokens_seen": 2986934272 + }, + { + "epoch": 0.81, + "learning_rate": 9.590772316950853e-05, + "loss": 1.2518, + "theoretical_loss": 3.3503401780011806, + "tokens_seen": 2986999808 + }, + { + "epoch": 0.81, + "learning_rate": 9.588766298896691e-05, + "loss": 1.2164, + "theoretical_loss": 3.3503345693876243, + "tokens_seen": 2987065344 + }, + { + "epoch": 0.81, + "learning_rate": 9.586760280842528e-05, + "loss": 1.3118, + "theoretical_loss": 3.3503289609315727, + "tokens_seen": 2987130880 + }, + { + "epoch": 0.81, + "learning_rate": 9.584754262788366e-05, + "loss": 1.3303, + "theoretical_loss": 3.3503233526330183, + "tokens_seen": 2987196416 + }, + { + "epoch": 0.81, + "learning_rate": 9.582748244734203e-05, + "loss": 1.3471, + "theoretical_loss": 3.350317744491954, + "tokens_seen": 2987261952 + }, + { + "epoch": 0.81, + "learning_rate": 9.580742226680041e-05, + "loss": 1.1706, + "theoretical_loss": 3.35031213650837, + "tokens_seen": 2987327488 + }, + { + "epoch": 0.81, + "learning_rate": 9.578736208625879e-05, + "loss": 1.1709, + "theoretical_loss": 3.3503065286822604, + "tokens_seen": 2987393024 + }, + { + "epoch": 0.81, + "learning_rate": 9.576730190571716e-05, + "loss": 1.2586, + "theoretical_loss": 3.3503009210136163, + "tokens_seen": 2987458560 + }, + { + "epoch": 0.81, + "learning_rate": 9.574724172517552e-05, + "loss": 1.3649, + "theoretical_loss": 3.35029531350243, + "tokens_seen": 2987524096 + }, + { + "epoch": 0.81, + "learning_rate": 9.57271815446339e-05, + "loss": 1.2442, + "theoretical_loss": 3.3502897061486934, + "tokens_seen": 2987589632 + }, + { + "epoch": 0.81, + "learning_rate": 9.570712136409228e-05, + "loss": 1.2332, + "theoretical_loss": 3.3502840989523994, + "tokens_seen": 2987655168 + }, + { + "epoch": 0.81, + "learning_rate": 9.568706118355065e-05, + "loss": 1.275, + "theoretical_loss": 3.3502784919135395, + "tokens_seen": 2987720704 + }, + { + "epoch": 0.81, + "learning_rate": 9.566700100300903e-05, + "loss": 1.324, + "theoretical_loss": 3.3502728850321057, + "tokens_seen": 2987786240 + }, + { + "epoch": 0.81, + "learning_rate": 9.56469408224674e-05, + "loss": 1.221, + "theoretical_loss": 3.3502672783080905, + "tokens_seen": 2987851776 + }, + { + "epoch": 0.81, + "learning_rate": 9.562688064192578e-05, + "loss": 1.2474, + "theoretical_loss": 3.3502616717414857, + "tokens_seen": 2987917312 + }, + { + "epoch": 0.81, + "learning_rate": 9.560682046138415e-05, + "loss": 1.2294, + "theoretical_loss": 3.3502560653322835, + "tokens_seen": 2987982848 + }, + { + "epoch": 0.81, + "learning_rate": 9.558676028084253e-05, + "loss": 1.2732, + "theoretical_loss": 3.3502504590804767, + "tokens_seen": 2988048384 + }, + { + "epoch": 0.81, + "learning_rate": 9.556670010030089e-05, + "loss": 1.2183, + "theoretical_loss": 3.3502448529860565, + "tokens_seen": 2988113920 + }, + { + "epoch": 0.81, + "learning_rate": 9.554663991975927e-05, + "loss": 1.2175, + "theoretical_loss": 3.350239247049015, + "tokens_seen": 2988179456 + }, + { + "epoch": 0.81, + "learning_rate": 9.552657973921766e-05, + "loss": 1.2108, + "theoretical_loss": 3.3502336412693454, + "tokens_seen": 2988244992 + }, + { + "epoch": 0.81, + "learning_rate": 9.550651955867603e-05, + "loss": 1.3035, + "theoretical_loss": 3.350228035647039, + "tokens_seen": 2988310528 + }, + { + "epoch": 0.81, + "learning_rate": 9.548645937813441e-05, + "loss": 1.306, + "theoretical_loss": 3.350222430182088, + "tokens_seen": 2988376064 + }, + { + "epoch": 0.81, + "learning_rate": 9.546639919759278e-05, + "loss": 1.1677, + "theoretical_loss": 3.350216824874485, + "tokens_seen": 2988441600 + }, + { + "epoch": 0.81, + "learning_rate": 9.544633901705116e-05, + "loss": 1.1502, + "theoretical_loss": 3.3502112197242213, + "tokens_seen": 2988507136 + }, + { + "epoch": 0.81, + "learning_rate": 9.542627883650954e-05, + "loss": 1.2611, + "theoretical_loss": 3.3502056147312897, + "tokens_seen": 2988572672 + }, + { + "epoch": 0.81, + "learning_rate": 9.540621865596791e-05, + "loss": 1.2895, + "theoretical_loss": 3.3502000098956817, + "tokens_seen": 2988638208 + }, + { + "epoch": 0.81, + "learning_rate": 9.538615847542629e-05, + "loss": 1.2964, + "theoretical_loss": 3.3501944052173904, + "tokens_seen": 2988703744 + }, + { + "epoch": 0.81, + "learning_rate": 9.536609829488465e-05, + "loss": 1.1738, + "theoretical_loss": 3.350188800696407, + "tokens_seen": 2988769280 + }, + { + "epoch": 0.81, + "learning_rate": 9.534603811434303e-05, + "loss": 1.3936, + "theoretical_loss": 3.350183196332724, + "tokens_seen": 2988834816 + }, + { + "epoch": 0.81, + "learning_rate": 9.53259779338014e-05, + "loss": 1.1619, + "theoretical_loss": 3.350177592126334, + "tokens_seen": 2988900352 + }, + { + "epoch": 0.81, + "learning_rate": 9.530591775325978e-05, + "loss": 1.1933, + "theoretical_loss": 3.3501719880772285, + "tokens_seen": 2988965888 + }, + { + "epoch": 0.81, + "learning_rate": 9.528585757271815e-05, + "loss": 1.2011, + "theoretical_loss": 3.3501663841854, + "tokens_seen": 2989031424 + }, + { + "epoch": 0.81, + "learning_rate": 9.526579739217653e-05, + "loss": 1.2437, + "theoretical_loss": 3.3501607804508406, + "tokens_seen": 2989096960 + }, + { + "epoch": 0.81, + "learning_rate": 9.52457372116349e-05, + "loss": 1.2139, + "theoretical_loss": 3.3501551768735416, + "tokens_seen": 2989162496 + }, + { + "epoch": 0.81, + "learning_rate": 9.522567703109328e-05, + "loss": 1.2065, + "theoretical_loss": 3.3501495734534963, + "tokens_seen": 2989228032 + }, + { + "epoch": 0.81, + "learning_rate": 9.520561685055166e-05, + "loss": 1.3012, + "theoretical_loss": 3.3501439701906968, + "tokens_seen": 2989293568 + }, + { + "epoch": 0.81, + "learning_rate": 9.518555667001003e-05, + "loss": 1.374, + "theoretical_loss": 3.350138367085134, + "tokens_seen": 2989359104 + }, + { + "epoch": 0.81, + "learning_rate": 9.51654964894684e-05, + "loss": 1.2918, + "theoretical_loss": 3.3501327641368013, + "tokens_seen": 2989424640 + }, + { + "epoch": 0.81, + "learning_rate": 9.514543630892678e-05, + "loss": 1.1764, + "theoretical_loss": 3.3501271613456907, + "tokens_seen": 2989490176 + }, + { + "epoch": 0.81, + "learning_rate": 9.512537612838516e-05, + "loss": 1.1968, + "theoretical_loss": 3.350121558711794, + "tokens_seen": 2989555712 + }, + { + "epoch": 0.81, + "learning_rate": 9.510531594784354e-05, + "loss": 1.2142, + "theoretical_loss": 3.350115956235103, + "tokens_seen": 2989621248 + }, + { + "epoch": 0.81, + "learning_rate": 9.508525576730191e-05, + "loss": 1.343, + "theoretical_loss": 3.3501103539156105, + "tokens_seen": 2989686784 + }, + { + "epoch": 0.81, + "learning_rate": 9.506519558676029e-05, + "loss": 1.3011, + "theoretical_loss": 3.350104751753308, + "tokens_seen": 2989752320 + }, + { + "epoch": 0.81, + "learning_rate": 9.504513540621866e-05, + "loss": 1.2683, + "theoretical_loss": 3.3500991497481887, + "tokens_seen": 2989817856 + }, + { + "epoch": 0.81, + "learning_rate": 9.502507522567704e-05, + "loss": 1.241, + "theoretical_loss": 3.3500935479002436, + "tokens_seen": 2989883392 + }, + { + "epoch": 0.81, + "learning_rate": 9.500501504513541e-05, + "loss": 1.2046, + "theoretical_loss": 3.3500879462094657, + "tokens_seen": 2989948928 + }, + { + "epoch": 0.81, + "learning_rate": 9.498495486459379e-05, + "loss": 1.2391, + "theoretical_loss": 3.3500823446758465, + "tokens_seen": 2990014464 + }, + { + "epoch": 0.81, + "learning_rate": 9.496489468405215e-05, + "loss": 1.246, + "theoretical_loss": 3.350076743299378, + "tokens_seen": 2990080000 + }, + { + "epoch": 0.81, + "learning_rate": 9.494483450351053e-05, + "loss": 1.3428, + "theoretical_loss": 3.3500711420800533, + "tokens_seen": 2990145536 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1087207, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.278800368309021, + "objective/train/theoretical_loss": 3.350065541017864, + "objective/train/tokens_used": 1360670176, + "theoretical_loss": 3.350065541017864, + "tokens_seen": 2990211072 + }, + { + "epoch": 0.81, + "learning_rate": 9.49247743229689e-05, + "loss": 1.3025, + "theoretical_loss": 3.350065541017864, + "tokens_seen": 2990211072 + }, + { + "epoch": 0.81, + "learning_rate": 9.490471414242728e-05, + "loss": 1.2455, + "theoretical_loss": 3.350059940112802, + "tokens_seen": 2990276608 + }, + { + "epoch": 0.81, + "learning_rate": 9.488465396188566e-05, + "loss": 1.247, + "theoretical_loss": 3.35005433936486, + "tokens_seen": 2990342144 + }, + { + "epoch": 0.81, + "learning_rate": 9.486459378134403e-05, + "loss": 1.2122, + "theoretical_loss": 3.3500487387740296, + "tokens_seen": 2990407680 + }, + { + "epoch": 0.81, + "learning_rate": 9.484453360080241e-05, + "loss": 1.3196, + "theoretical_loss": 3.3500431383403035, + "tokens_seen": 2990473216 + }, + { + "epoch": 0.81, + "learning_rate": 9.482447342026078e-05, + "loss": 1.1483, + "theoretical_loss": 3.3500375380636735, + "tokens_seen": 2990538752 + }, + { + "epoch": 0.81, + "learning_rate": 9.480441323971916e-05, + "loss": 1.2013, + "theoretical_loss": 3.3500319379441317, + "tokens_seen": 2990604288 + }, + { + "epoch": 0.81, + "learning_rate": 9.478435305917752e-05, + "loss": 1.3137, + "theoretical_loss": 3.3500263379816704, + "tokens_seen": 2990669824 + }, + { + "epoch": 0.81, + "learning_rate": 9.476429287863591e-05, + "loss": 1.1882, + "theoretical_loss": 3.3500207381762817, + "tokens_seen": 2990735360 + }, + { + "epoch": 0.81, + "learning_rate": 9.474423269809429e-05, + "loss": 1.2482, + "theoretical_loss": 3.3500151385279575, + "tokens_seen": 2990800896 + }, + { + "epoch": 0.81, + "learning_rate": 9.472417251755266e-05, + "loss": 1.4778, + "theoretical_loss": 3.3500095390366904, + "tokens_seen": 2990866432 + }, + { + "epoch": 0.81, + "learning_rate": 9.470411233701104e-05, + "loss": 1.2052, + "theoretical_loss": 3.3500039397024723, + "tokens_seen": 2990931968 + }, + { + "epoch": 0.81, + "learning_rate": 9.468405215646941e-05, + "loss": 1.2904, + "theoretical_loss": 3.3499983405252958, + "tokens_seen": 2990997504 + }, + { + "epoch": 0.81, + "learning_rate": 9.466399197592779e-05, + "loss": 1.2726, + "theoretical_loss": 3.3499927415051522, + "tokens_seen": 2991063040 + }, + { + "epoch": 0.81, + "learning_rate": 9.464393179538617e-05, + "loss": 1.2132, + "theoretical_loss": 3.3499871426420347, + "tokens_seen": 2991128576 + }, + { + "epoch": 0.81, + "learning_rate": 9.462387161484454e-05, + "loss": 1.1422, + "theoretical_loss": 3.349981543935934, + "tokens_seen": 2991194112 + }, + { + "epoch": 0.81, + "learning_rate": 9.460381143430292e-05, + "loss": 1.1107, + "theoretical_loss": 3.349975945386844, + "tokens_seen": 2991259648 + }, + { + "epoch": 0.81, + "learning_rate": 9.458375125376128e-05, + "loss": 1.1999, + "theoretical_loss": 3.3499703469947555, + "tokens_seen": 2991325184 + }, + { + "epoch": 0.81, + "learning_rate": 9.456369107321966e-05, + "loss": 1.2622, + "theoretical_loss": 3.3499647487596613, + "tokens_seen": 2991390720 + }, + { + "epoch": 0.81, + "learning_rate": 9.454363089267803e-05, + "loss": 1.2387, + "theoretical_loss": 3.3499591506815536, + "tokens_seen": 2991456256 + }, + { + "epoch": 0.81, + "learning_rate": 9.45235707121364e-05, + "loss": 1.2891, + "theoretical_loss": 3.3499535527604243, + "tokens_seen": 2991521792 + }, + { + "epoch": 0.81, + "learning_rate": 9.450351053159478e-05, + "loss": 1.2914, + "theoretical_loss": 3.3499479549962654, + "tokens_seen": 2991587328 + }, + { + "epoch": 0.81, + "learning_rate": 9.448345035105316e-05, + "loss": 1.108, + "theoretical_loss": 3.3499423573890694, + "tokens_seen": 2991652864 + }, + { + "epoch": 0.81, + "learning_rate": 9.446339017051153e-05, + "loss": 1.2749, + "theoretical_loss": 3.349936759938829, + "tokens_seen": 2991718400 + }, + { + "epoch": 0.81, + "learning_rate": 9.444332998996991e-05, + "loss": 1.3502, + "theoretical_loss": 3.3499311626455346, + "tokens_seen": 2991783936 + }, + { + "epoch": 0.81, + "learning_rate": 9.442326980942829e-05, + "loss": 1.2798, + "theoretical_loss": 3.34992556550918, + "tokens_seen": 2991849472 + }, + { + "epoch": 0.81, + "learning_rate": 9.440320962888667e-05, + "loss": 1.2301, + "theoretical_loss": 3.3499199685297567, + "tokens_seen": 2991915008 + }, + { + "epoch": 0.81, + "learning_rate": 9.438314944834504e-05, + "loss": 1.1726, + "theoretical_loss": 3.349914371707257, + "tokens_seen": 2991980544 + }, + { + "epoch": 0.81, + "learning_rate": 9.436308926780341e-05, + "loss": 1.2775, + "theoretical_loss": 3.3499087750416736, + "tokens_seen": 2992046080 + }, + { + "epoch": 0.81, + "learning_rate": 9.434302908726179e-05, + "loss": 1.217, + "theoretical_loss": 3.3499031785329976, + "tokens_seen": 2992111616 + }, + { + "epoch": 0.81, + "learning_rate": 9.432296890672016e-05, + "loss": 1.1684, + "theoretical_loss": 3.349897582181222, + "tokens_seen": 2992177152 + }, + { + "epoch": 0.81, + "learning_rate": 9.430290872617854e-05, + "loss": 1.2367, + "theoretical_loss": 3.3498919859863383, + "tokens_seen": 2992242688 + }, + { + "epoch": 0.81, + "learning_rate": 9.428284854563692e-05, + "loss": 1.2296, + "theoretical_loss": 3.349886389948339, + "tokens_seen": 2992308224 + }, + { + "epoch": 0.81, + "learning_rate": 9.426278836509529e-05, + "loss": 1.1951, + "theoretical_loss": 3.349880794067217, + "tokens_seen": 2992373760 + }, + { + "epoch": 0.81, + "learning_rate": 9.424272818455367e-05, + "loss": 1.2533, + "theoretical_loss": 3.3498751983429633, + "tokens_seen": 2992439296 + }, + { + "epoch": 0.81, + "learning_rate": 9.422266800401204e-05, + "loss": 1.1607, + "theoretical_loss": 3.3498696027755703, + "tokens_seen": 2992504832 + }, + { + "epoch": 0.81, + "learning_rate": 9.420260782347042e-05, + "loss": 1.177, + "theoretical_loss": 3.3498640073650305, + "tokens_seen": 2992570368 + }, + { + "epoch": 0.81, + "learning_rate": 9.418254764292878e-05, + "loss": 1.2753, + "theoretical_loss": 3.349858412111336, + "tokens_seen": 2992635904 + }, + { + "epoch": 0.81, + "learning_rate": 9.416248746238716e-05, + "loss": 1.1972, + "theoretical_loss": 3.349852817014479, + "tokens_seen": 2992701440 + }, + { + "epoch": 0.81, + "learning_rate": 9.414242728184553e-05, + "loss": 1.247, + "theoretical_loss": 3.3498472220744517, + "tokens_seen": 2992766976 + }, + { + "epoch": 0.81, + "learning_rate": 9.412236710130391e-05, + "loss": 1.2316, + "theoretical_loss": 3.349841627291246, + "tokens_seen": 2992832512 + }, + { + "epoch": 0.81, + "learning_rate": 9.410230692076228e-05, + "loss": 1.2217, + "theoretical_loss": 3.349836032664854, + "tokens_seen": 2992898048 + }, + { + "epoch": 0.81, + "learning_rate": 9.408224674022066e-05, + "loss": 1.1049, + "theoretical_loss": 3.3498304381952684, + "tokens_seen": 2992963584 + }, + { + "epoch": 0.81, + "learning_rate": 9.406218655967904e-05, + "loss": 1.2731, + "theoretical_loss": 3.349824843882481, + "tokens_seen": 2993029120 + }, + { + "epoch": 0.81, + "learning_rate": 9.404212637913741e-05, + "loss": 1.2789, + "theoretical_loss": 3.3498192497264845, + "tokens_seen": 2993094656 + }, + { + "epoch": 0.81, + "learning_rate": 9.40220661985958e-05, + "loss": 1.1476, + "theoretical_loss": 3.34981365572727, + "tokens_seen": 2993160192 + }, + { + "epoch": 0.81, + "learning_rate": 9.400200601805416e-05, + "loss": 1.2877, + "theoretical_loss": 3.3498080618848305, + "tokens_seen": 2993225728 + }, + { + "epoch": 0.81, + "learning_rate": 9.398194583751254e-05, + "loss": 1.21, + "theoretical_loss": 3.349802468199158, + "tokens_seen": 2993291264 + }, + { + "epoch": 0.81, + "learning_rate": 9.396188565697092e-05, + "loss": 1.196, + "theoretical_loss": 3.349796874670245, + "tokens_seen": 2993356800 + }, + { + "epoch": 0.81, + "learning_rate": 9.394182547642929e-05, + "loss": 1.4338, + "theoretical_loss": 3.349791281298083, + "tokens_seen": 2993422336 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1088408, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.320845365524292, + "objective/train/theoretical_loss": 3.3497856880826644, + "objective/train/tokens_used": 1363946976, + "theoretical_loss": 3.3497856880826644, + "tokens_seen": 2993487872 + }, + { + "epoch": 0.81, + "learning_rate": 9.392176529588767e-05, + "loss": 1.284, + "theoretical_loss": 3.3497856880826644, + "tokens_seen": 2993487872 + }, + { + "epoch": 0.81, + "learning_rate": 9.390170511534604e-05, + "loss": 1.2967, + "theoretical_loss": 3.3497800950239816, + "tokens_seen": 2993553408 + }, + { + "epoch": 0.81, + "learning_rate": 9.388164493480442e-05, + "loss": 1.3187, + "theoretical_loss": 3.3497745021220267, + "tokens_seen": 2993618944 + }, + { + "epoch": 0.81, + "learning_rate": 9.38615847542628e-05, + "loss": 1.3896, + "theoretical_loss": 3.3497689093767917, + "tokens_seen": 2993684480 + }, + { + "epoch": 0.81, + "learning_rate": 9.384152457372117e-05, + "loss": 1.2537, + "theoretical_loss": 3.3497633167882688, + "tokens_seen": 2993750016 + }, + { + "epoch": 0.81, + "learning_rate": 9.382146439317955e-05, + "loss": 1.2819, + "theoretical_loss": 3.3497577243564507, + "tokens_seen": 2993815552 + }, + { + "epoch": 0.81, + "learning_rate": 9.380140421263791e-05, + "loss": 1.2074, + "theoretical_loss": 3.3497521320813286, + "tokens_seen": 2993881088 + }, + { + "epoch": 0.81, + "learning_rate": 9.378134403209628e-05, + "loss": 1.2624, + "theoretical_loss": 3.3497465399628954, + "tokens_seen": 2993946624 + }, + { + "epoch": 0.81, + "learning_rate": 9.376128385155466e-05, + "loss": 1.3431, + "theoretical_loss": 3.3497409480011435, + "tokens_seen": 2994012160 + }, + { + "epoch": 0.81, + "learning_rate": 9.374122367101304e-05, + "loss": 1.1889, + "theoretical_loss": 3.3497353561960646, + "tokens_seen": 2994077696 + }, + { + "epoch": 0.81, + "learning_rate": 9.372116349047141e-05, + "loss": 1.298, + "theoretical_loss": 3.3497297645476505, + "tokens_seen": 2994143232 + }, + { + "epoch": 0.81, + "learning_rate": 9.370110330992979e-05, + "loss": 1.3405, + "theoretical_loss": 3.349724173055894, + "tokens_seen": 2994208768 + }, + { + "epoch": 0.81, + "learning_rate": 9.368104312938816e-05, + "loss": 1.3388, + "theoretical_loss": 3.3497185817207877, + "tokens_seen": 2994274304 + }, + { + "epoch": 0.81, + "learning_rate": 9.366098294884654e-05, + "loss": 1.1593, + "theoretical_loss": 3.349712990542323, + "tokens_seen": 2994339840 + }, + { + "epoch": 0.81, + "learning_rate": 9.364092276830493e-05, + "loss": 1.1546, + "theoretical_loss": 3.3497073995204922, + "tokens_seen": 2994405376 + }, + { + "epoch": 0.81, + "learning_rate": 9.36208625877633e-05, + "loss": 1.287, + "theoretical_loss": 3.3497018086552877, + "tokens_seen": 2994470912 + }, + { + "epoch": 0.81, + "learning_rate": 9.360080240722167e-05, + "loss": 1.2868, + "theoretical_loss": 3.3496962179467014, + "tokens_seen": 2994536448 + }, + { + "epoch": 0.81, + "learning_rate": 9.358074222668004e-05, + "loss": 1.2287, + "theoretical_loss": 3.349690627394726, + "tokens_seen": 2994601984 + }, + { + "epoch": 0.81, + "learning_rate": 9.356068204613842e-05, + "loss": 1.2284, + "theoretical_loss": 3.349685036999353, + "tokens_seen": 2994667520 + }, + { + "epoch": 0.81, + "learning_rate": 9.35406218655968e-05, + "loss": 1.3419, + "theoretical_loss": 3.349679446760575, + "tokens_seen": 2994733056 + }, + { + "epoch": 0.82, + "learning_rate": 9.352056168505517e-05, + "loss": 1.2661, + "theoretical_loss": 3.3496738566783844, + "tokens_seen": 2994798592 + }, + { + "epoch": 0.82, + "learning_rate": 9.350050150451354e-05, + "loss": 1.2759, + "theoretical_loss": 3.3496682667527726, + "tokens_seen": 2994864128 + }, + { + "epoch": 0.82, + "learning_rate": 9.348044132397192e-05, + "loss": 1.2649, + "theoretical_loss": 3.3496626769837325, + "tokens_seen": 2994929664 + }, + { + "epoch": 0.82, + "learning_rate": 9.34603811434303e-05, + "loss": 1.4221, + "theoretical_loss": 3.3496570873712566, + "tokens_seen": 2994995200 + }, + { + "epoch": 0.82, + "learning_rate": 9.344032096288867e-05, + "loss": 1.3019, + "theoretical_loss": 3.3496514979153362, + "tokens_seen": 2995060736 + }, + { + "epoch": 0.82, + "learning_rate": 9.342026078234703e-05, + "loss": 1.1576, + "theoretical_loss": 3.3496459086159636, + "tokens_seen": 2995126272 + }, + { + "epoch": 0.82, + "learning_rate": 9.340020060180541e-05, + "loss": 1.4138, + "theoretical_loss": 3.3496403194731315, + "tokens_seen": 2995191808 + }, + { + "epoch": 0.82, + "learning_rate": 9.338014042126379e-05, + "loss": 1.3926, + "theoretical_loss": 3.349634730486832, + "tokens_seen": 2995257344 + }, + { + "epoch": 0.82, + "learning_rate": 9.336008024072216e-05, + "loss": 1.2614, + "theoretical_loss": 3.349629141657057, + "tokens_seen": 2995322880 + }, + { + "epoch": 0.82, + "learning_rate": 9.334002006018054e-05, + "loss": 1.1885, + "theoretical_loss": 3.349623552983799, + "tokens_seen": 2995388416 + }, + { + "epoch": 0.82, + "learning_rate": 9.331995987963891e-05, + "loss": 1.2733, + "theoretical_loss": 3.3496179644670496, + "tokens_seen": 2995453952 + }, + { + "epoch": 0.82, + "learning_rate": 9.329989969909729e-05, + "loss": 1.2561, + "theoretical_loss": 3.3496123761068017, + "tokens_seen": 2995519488 + }, + { + "epoch": 0.82, + "learning_rate": 9.327983951855568e-05, + "loss": 1.2798, + "theoretical_loss": 3.3496067879030473, + "tokens_seen": 2995585024 + }, + { + "epoch": 0.82, + "learning_rate": 9.325977933801405e-05, + "loss": 1.246, + "theoretical_loss": 3.3496011998557784, + "tokens_seen": 2995650560 + }, + { + "epoch": 0.82, + "learning_rate": 9.323971915747243e-05, + "loss": 1.2428, + "theoretical_loss": 3.3495956119649875, + "tokens_seen": 2995716096 + }, + { + "epoch": 0.82, + "learning_rate": 9.321965897693079e-05, + "loss": 1.2692, + "theoretical_loss": 3.3495900242306664, + "tokens_seen": 2995781632 + }, + { + "epoch": 0.82, + "learning_rate": 9.319959879638917e-05, + "loss": 1.3167, + "theoretical_loss": 3.3495844366528074, + "tokens_seen": 2995847168 + }, + { + "epoch": 0.82, + "learning_rate": 9.317953861584754e-05, + "loss": 1.2095, + "theoretical_loss": 3.349578849231403, + "tokens_seen": 2995912704 + }, + { + "epoch": 0.82, + "learning_rate": 9.315947843530592e-05, + "loss": 1.2499, + "theoretical_loss": 3.3495732619664453, + "tokens_seen": 2995978240 + }, + { + "epoch": 0.82, + "learning_rate": 9.31394182547643e-05, + "loss": 1.2268, + "theoretical_loss": 3.349567674857926, + "tokens_seen": 2996043776 + }, + { + "epoch": 0.82, + "learning_rate": 9.311935807422267e-05, + "loss": 1.2849, + "theoretical_loss": 3.349562087905838, + "tokens_seen": 2996109312 + }, + { + "epoch": 0.82, + "learning_rate": 9.309929789368105e-05, + "loss": 1.2213, + "theoretical_loss": 3.3495565011101727, + "tokens_seen": 2996174848 + }, + { + "epoch": 0.82, + "learning_rate": 9.307923771313942e-05, + "loss": 1.1999, + "theoretical_loss": 3.3495509144709232, + "tokens_seen": 2996240384 + }, + { + "epoch": 0.82, + "learning_rate": 9.30591775325978e-05, + "loss": 1.1939, + "theoretical_loss": 3.349545327988081, + "tokens_seen": 2996305920 + }, + { + "epoch": 0.82, + "learning_rate": 9.303911735205617e-05, + "loss": 1.2167, + "theoretical_loss": 3.3495397416616393, + "tokens_seen": 2996371456 + }, + { + "epoch": 0.82, + "learning_rate": 9.301905717151454e-05, + "loss": 1.3351, + "theoretical_loss": 3.3495341554915887, + "tokens_seen": 2996436992 + }, + { + "epoch": 0.82, + "learning_rate": 9.299899699097291e-05, + "loss": 1.1796, + "theoretical_loss": 3.349528569477923, + "tokens_seen": 2996502528 + }, + { + "epoch": 0.82, + "learning_rate": 9.297893681043129e-05, + "loss": 1.2834, + "theoretical_loss": 3.3495229836206333, + "tokens_seen": 2996568064 + }, + { + "epoch": 0.82, + "learning_rate": 9.295887662988966e-05, + "loss": 1.3152, + "theoretical_loss": 3.349517397919712, + "tokens_seen": 2996633600 + }, + { + "epoch": 0.82, + "learning_rate": 9.293881644934804e-05, + "loss": 1.2479, + "theoretical_loss": 3.3495118123751517, + "tokens_seen": 2996699136 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1089453, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2000184059143066, + "objective/train/theoretical_loss": 3.3495062269869447, + "objective/train/tokens_used": 1367223776, + "theoretical_loss": 3.3495062269869447, + "tokens_seen": 2996764672 + }, + { + "epoch": 0.82, + "learning_rate": 9.291875626880642e-05, + "loss": 1.254, + "theoretical_loss": 3.3495062269869447, + "tokens_seen": 2996764672 + }, + { + "epoch": 0.82, + "learning_rate": 9.28986960882648e-05, + "loss": 1.2589, + "theoretical_loss": 3.3495006417550823, + "tokens_seen": 2996830208 + }, + { + "epoch": 0.82, + "learning_rate": 9.287863590772318e-05, + "loss": 1.4098, + "theoretical_loss": 3.3494950566795576, + "tokens_seen": 2996895744 + }, + { + "epoch": 0.82, + "learning_rate": 9.285857572718156e-05, + "loss": 1.1957, + "theoretical_loss": 3.3494894717603625, + "tokens_seen": 2996961280 + }, + { + "epoch": 0.82, + "learning_rate": 9.283851554663993e-05, + "loss": 1.2338, + "theoretical_loss": 3.3494838869974894, + "tokens_seen": 2997026816 + }, + { + "epoch": 0.82, + "learning_rate": 9.28184553660983e-05, + "loss": 1.2686, + "theoretical_loss": 3.34947830239093, + "tokens_seen": 2997092352 + }, + { + "epoch": 0.82, + "learning_rate": 9.279839518555667e-05, + "loss": 1.187, + "theoretical_loss": 3.349472717940677, + "tokens_seen": 2997157888 + }, + { + "epoch": 0.82, + "learning_rate": 9.277833500501505e-05, + "loss": 1.2107, + "theoretical_loss": 3.3494671336467223, + "tokens_seen": 2997223424 + }, + { + "epoch": 0.82, + "learning_rate": 9.275827482447342e-05, + "loss": 1.315, + "theoretical_loss": 3.3494615495090585, + "tokens_seen": 2997288960 + }, + { + "epoch": 0.82, + "learning_rate": 9.27382146439318e-05, + "loss": 1.3023, + "theoretical_loss": 3.349455965527677, + "tokens_seen": 2997354496 + }, + { + "epoch": 0.82, + "learning_rate": 9.271815446339017e-05, + "loss": 1.3099, + "theoretical_loss": 3.3494503817025714, + "tokens_seen": 2997420032 + }, + { + "epoch": 0.82, + "learning_rate": 9.269809428284855e-05, + "loss": 1.3273, + "theoretical_loss": 3.3494447980337325, + "tokens_seen": 2997485568 + }, + { + "epoch": 0.82, + "learning_rate": 9.267803410230693e-05, + "loss": 1.2116, + "theoretical_loss": 3.349439214521153, + "tokens_seen": 2997551104 + }, + { + "epoch": 0.82, + "learning_rate": 9.26579739217653e-05, + "loss": 1.1808, + "theoretical_loss": 3.3494336311648256, + "tokens_seen": 2997616640 + }, + { + "epoch": 0.82, + "learning_rate": 9.263791374122366e-05, + "loss": 1.2779, + "theoretical_loss": 3.349428047964742, + "tokens_seen": 2997682176 + }, + { + "epoch": 0.82, + "learning_rate": 9.261785356068204e-05, + "loss": 1.3094, + "theoretical_loss": 3.3494224649208943, + "tokens_seen": 2997747712 + }, + { + "epoch": 0.82, + "learning_rate": 9.259779338014041e-05, + "loss": 1.2095, + "theoretical_loss": 3.349416882033275, + "tokens_seen": 2997813248 + }, + { + "epoch": 0.82, + "learning_rate": 9.257773319959879e-05, + "loss": 1.2887, + "theoretical_loss": 3.3494112993018765, + "tokens_seen": 2997878784 + }, + { + "epoch": 0.82, + "learning_rate": 9.255767301905717e-05, + "loss": 1.2139, + "theoretical_loss": 3.3494057167266904, + "tokens_seen": 2997944320 + }, + { + "epoch": 0.82, + "learning_rate": 9.253761283851554e-05, + "loss": 1.2625, + "theoretical_loss": 3.3494001343077096, + "tokens_seen": 2998009856 + }, + { + "epoch": 0.82, + "learning_rate": 9.251755265797393e-05, + "loss": 1.2471, + "theoretical_loss": 3.3493945520449255, + "tokens_seen": 2998075392 + }, + { + "epoch": 0.82, + "learning_rate": 9.249749247743231e-05, + "loss": 1.102, + "theoretical_loss": 3.3493889699383312, + "tokens_seen": 2998140928 + }, + { + "epoch": 0.82, + "learning_rate": 9.247743229689068e-05, + "loss": 1.2317, + "theoretical_loss": 3.3493833879879182, + "tokens_seen": 2998206464 + }, + { + "epoch": 0.82, + "learning_rate": 9.245737211634906e-05, + "loss": 1.2532, + "theoretical_loss": 3.3493778061936794, + "tokens_seen": 2998272000 + }, + { + "epoch": 0.82, + "learning_rate": 9.243731193580742e-05, + "loss": 1.3422, + "theoretical_loss": 3.3493722245556063, + "tokens_seen": 2998337536 + }, + { + "epoch": 0.82, + "learning_rate": 9.24172517552658e-05, + "loss": 1.4117, + "theoretical_loss": 3.349366643073692, + "tokens_seen": 2998403072 + }, + { + "epoch": 0.82, + "learning_rate": 9.239719157472417e-05, + "loss": 1.2225, + "theoretical_loss": 3.3493610617479277, + "tokens_seen": 2998468608 + }, + { + "epoch": 0.82, + "learning_rate": 9.237713139418255e-05, + "loss": 1.265, + "theoretical_loss": 3.349355480578306, + "tokens_seen": 2998534144 + }, + { + "epoch": 0.82, + "learning_rate": 9.235707121364092e-05, + "loss": 1.2602, + "theoretical_loss": 3.3493498995648197, + "tokens_seen": 2998599680 + }, + { + "epoch": 0.82, + "learning_rate": 9.23370110330993e-05, + "loss": 1.3106, + "theoretical_loss": 3.3493443187074603, + "tokens_seen": 2998665216 + }, + { + "epoch": 0.82, + "learning_rate": 9.231695085255768e-05, + "loss": 1.2469, + "theoretical_loss": 3.34933873800622, + "tokens_seen": 2998730752 + }, + { + "epoch": 0.82, + "learning_rate": 9.229689067201605e-05, + "loss": 1.3093, + "theoretical_loss": 3.349333157461092, + "tokens_seen": 2998796288 + }, + { + "epoch": 0.82, + "learning_rate": 9.227683049147443e-05, + "loss": 1.2008, + "theoretical_loss": 3.349327577072067, + "tokens_seen": 2998861824 + }, + { + "epoch": 0.82, + "learning_rate": 9.22567703109328e-05, + "loss": 1.2446, + "theoretical_loss": 3.349321996839138, + "tokens_seen": 2998927360 + }, + { + "epoch": 0.82, + "learning_rate": 9.223671013039117e-05, + "loss": 1.1966, + "theoretical_loss": 3.349316416762298, + "tokens_seen": 2998992896 + }, + { + "epoch": 0.82, + "learning_rate": 9.221664994984954e-05, + "loss": 1.26, + "theoretical_loss": 3.3493108368415383, + "tokens_seen": 2999058432 + }, + { + "epoch": 0.82, + "learning_rate": 9.219658976930792e-05, + "loss": 1.2055, + "theoretical_loss": 3.3493052570768507, + "tokens_seen": 2999123968 + }, + { + "epoch": 0.82, + "learning_rate": 9.217652958876629e-05, + "loss": 1.3078, + "theoretical_loss": 3.3492996774682284, + "tokens_seen": 2999189504 + }, + { + "epoch": 0.82, + "learning_rate": 9.215646940822467e-05, + "loss": 1.2523, + "theoretical_loss": 3.3492940980156636, + "tokens_seen": 2999255040 + }, + { + "epoch": 0.82, + "learning_rate": 9.213640922768306e-05, + "loss": 1.2008, + "theoretical_loss": 3.3492885187191477, + "tokens_seen": 2999320576 + }, + { + "epoch": 0.82, + "learning_rate": 9.211634904714143e-05, + "loss": 1.2375, + "theoretical_loss": 3.3492829395786736, + "tokens_seen": 2999386112 + }, + { + "epoch": 0.82, + "learning_rate": 9.209628886659981e-05, + "loss": 1.2988, + "theoretical_loss": 3.349277360594233, + "tokens_seen": 2999451648 + }, + { + "epoch": 0.82, + "learning_rate": 9.207622868605819e-05, + "loss": 1.1091, + "theoretical_loss": 3.3492717817658186, + "tokens_seen": 2999517184 + }, + { + "epoch": 0.82, + "learning_rate": 9.205616850551656e-05, + "loss": 1.1626, + "theoretical_loss": 3.3492662030934226, + "tokens_seen": 2999582720 + }, + { + "epoch": 0.82, + "learning_rate": 9.203610832497492e-05, + "loss": 1.1813, + "theoretical_loss": 3.3492606245770373, + "tokens_seen": 2999648256 + }, + { + "epoch": 0.82, + "learning_rate": 9.20160481444333e-05, + "loss": 1.2513, + "theoretical_loss": 3.3492550462166544, + "tokens_seen": 2999713792 + }, + { + "epoch": 0.82, + "learning_rate": 9.199598796389168e-05, + "loss": 1.1508, + "theoretical_loss": 3.3492494680122666, + "tokens_seen": 2999779328 + }, + { + "epoch": 0.82, + "learning_rate": 9.197592778335005e-05, + "loss": 1.2984, + "theoretical_loss": 3.3492438899638657, + "tokens_seen": 2999844864 + }, + { + "epoch": 0.82, + "learning_rate": 9.195586760280843e-05, + "loss": 1.2597, + "theoretical_loss": 3.349238312071445, + "tokens_seen": 2999910400 + }, + { + "epoch": 0.82, + "learning_rate": 9.19358074222668e-05, + "loss": 1.2251, + "theoretical_loss": 3.3492327343349952, + "tokens_seen": 2999975936 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1090669, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.322437047958374, + "objective/train/theoretical_loss": 3.3492271567545093, + "objective/train/tokens_used": 1370500576, + "theoretical_loss": 3.3492271567545093, + "tokens_seen": 3000041472 + }, + { + "epoch": 0.82, + "learning_rate": 9.191574724172518e-05, + "loss": 1.402, + "theoretical_loss": 3.3492271567545093, + "tokens_seen": 3000041472 + }, + { + "epoch": 0.82, + "learning_rate": 9.189568706118355e-05, + "loss": 1.1713, + "theoretical_loss": 3.34922157932998, + "tokens_seen": 3000107008 + }, + { + "epoch": 0.82, + "learning_rate": 9.187562688064193e-05, + "loss": 1.1902, + "theoretical_loss": 3.3492160020613984, + "tokens_seen": 3000172544 + }, + { + "epoch": 0.82, + "learning_rate": 9.185556670010029e-05, + "loss": 1.2054, + "theoretical_loss": 3.349210424948758, + "tokens_seen": 3000238080 + }, + { + "epoch": 0.82, + "learning_rate": 9.183550651955867e-05, + "loss": 1.2394, + "theoretical_loss": 3.34920484799205, + "tokens_seen": 3000303616 + }, + { + "epoch": 0.82, + "learning_rate": 9.181544633901704e-05, + "loss": 1.2867, + "theoretical_loss": 3.3491992711912673, + "tokens_seen": 3000369152 + }, + { + "epoch": 0.82, + "learning_rate": 9.179538615847542e-05, + "loss": 1.2708, + "theoretical_loss": 3.3491936945464023, + "tokens_seen": 3000434688 + }, + { + "epoch": 0.82, + "learning_rate": 9.177532597793381e-05, + "loss": 1.3449, + "theoretical_loss": 3.349188118057446, + "tokens_seen": 3000500224 + }, + { + "epoch": 0.82, + "learning_rate": 9.175526579739218e-05, + "loss": 1.2743, + "theoretical_loss": 3.349182541724392, + "tokens_seen": 3000565760 + }, + { + "epoch": 0.82, + "learning_rate": 9.173520561685056e-05, + "loss": 1.2459, + "theoretical_loss": 3.3491769655472314, + "tokens_seen": 3000631296 + }, + { + "epoch": 0.82, + "learning_rate": 9.171514543630894e-05, + "loss": 1.223, + "theoretical_loss": 3.3491713895259574, + "tokens_seen": 3000696832 + }, + { + "epoch": 0.82, + "learning_rate": 9.169508525576731e-05, + "loss": 1.1506, + "theoretical_loss": 3.349165813660562, + "tokens_seen": 3000762368 + }, + { + "epoch": 0.82, + "learning_rate": 9.167502507522569e-05, + "loss": 1.1472, + "theoretical_loss": 3.3491602379510375, + "tokens_seen": 3000827904 + }, + { + "epoch": 0.82, + "learning_rate": 9.165496489468405e-05, + "loss": 1.2445, + "theoretical_loss": 3.3491546623973756, + "tokens_seen": 3000893440 + }, + { + "epoch": 0.82, + "learning_rate": 9.163490471414243e-05, + "loss": 1.3146, + "theoretical_loss": 3.349149086999569, + "tokens_seen": 3000958976 + }, + { + "epoch": 0.82, + "learning_rate": 9.16148445336008e-05, + "loss": 1.3349, + "theoretical_loss": 3.3491435117576094, + "tokens_seen": 3001024512 + }, + { + "epoch": 0.82, + "learning_rate": 9.159478435305918e-05, + "loss": 1.4089, + "theoretical_loss": 3.34913793667149, + "tokens_seen": 3001090048 + }, + { + "epoch": 0.82, + "learning_rate": 9.157472417251755e-05, + "loss": 1.1726, + "theoretical_loss": 3.3491323617412023, + "tokens_seen": 3001155584 + }, + { + "epoch": 0.82, + "learning_rate": 9.155466399197593e-05, + "loss": 1.2012, + "theoretical_loss": 3.349126786966739, + "tokens_seen": 3001221120 + }, + { + "epoch": 0.82, + "learning_rate": 9.15346038114343e-05, + "loss": 1.2898, + "theoretical_loss": 3.349121212348092, + "tokens_seen": 3001286656 + }, + { + "epoch": 0.82, + "learning_rate": 9.151454363089268e-05, + "loss": 1.3385, + "theoretical_loss": 3.3491156378852533, + "tokens_seen": 3001352192 + }, + { + "epoch": 0.82, + "learning_rate": 9.149448345035106e-05, + "loss": 1.0968, + "theoretical_loss": 3.3491100635782156, + "tokens_seen": 3001417728 + }, + { + "epoch": 0.82, + "learning_rate": 9.147442326980943e-05, + "loss": 1.1905, + "theoretical_loss": 3.349104489426971, + "tokens_seen": 3001483264 + }, + { + "epoch": 0.82, + "learning_rate": 9.14543630892678e-05, + "loss": 1.2295, + "theoretical_loss": 3.349098915431512, + "tokens_seen": 3001548800 + }, + { + "epoch": 0.82, + "learning_rate": 9.143430290872617e-05, + "loss": 1.1305, + "theoretical_loss": 3.3490933415918303, + "tokens_seen": 3001614336 + }, + { + "epoch": 0.82, + "learning_rate": 9.141424272818455e-05, + "loss": 1.1321, + "theoretical_loss": 3.349087767907919, + "tokens_seen": 3001679872 + }, + { + "epoch": 0.82, + "learning_rate": 9.139418254764294e-05, + "loss": 1.2404, + "theoretical_loss": 3.3490821943797693, + "tokens_seen": 3001745408 + }, + { + "epoch": 0.82, + "learning_rate": 9.137412236710131e-05, + "loss": 1.3083, + "theoretical_loss": 3.349076621007374, + "tokens_seen": 3001810944 + }, + { + "epoch": 0.82, + "learning_rate": 9.135406218655969e-05, + "loss": 1.2789, + "theoretical_loss": 3.3490710477907255, + "tokens_seen": 3001876480 + }, + { + "epoch": 0.82, + "learning_rate": 9.133400200601806e-05, + "loss": 1.3399, + "theoretical_loss": 3.3490654747298154, + "tokens_seen": 3001942016 + }, + { + "epoch": 0.82, + "learning_rate": 9.131394182547644e-05, + "loss": 1.2181, + "theoretical_loss": 3.3490599018246368, + "tokens_seen": 3002007552 + }, + { + "epoch": 0.82, + "learning_rate": 9.129388164493481e-05, + "loss": 1.3606, + "theoretical_loss": 3.3490543290751815, + "tokens_seen": 3002073088 + }, + { + "epoch": 0.82, + "learning_rate": 9.127382146439319e-05, + "loss": 1.2325, + "theoretical_loss": 3.3490487564814417, + "tokens_seen": 3002138624 + }, + { + "epoch": 0.82, + "learning_rate": 9.125376128385155e-05, + "loss": 1.3946, + "theoretical_loss": 3.34904318404341, + "tokens_seen": 3002204160 + }, + { + "epoch": 0.82, + "learning_rate": 9.123370110330993e-05, + "loss": 1.2212, + "theoretical_loss": 3.3490376117610783, + "tokens_seen": 3002269696 + }, + { + "epoch": 0.82, + "learning_rate": 9.12136409227683e-05, + "loss": 1.2657, + "theoretical_loss": 3.3490320396344386, + "tokens_seen": 3002335232 + }, + { + "epoch": 0.82, + "learning_rate": 9.119358074222668e-05, + "loss": 1.2478, + "theoretical_loss": 3.3490264676634833, + "tokens_seen": 3002400768 + }, + { + "epoch": 0.82, + "learning_rate": 9.117352056168506e-05, + "loss": 1.3307, + "theoretical_loss": 3.3490208958482057, + "tokens_seen": 3002466304 + }, + { + "epoch": 0.82, + "learning_rate": 9.115346038114343e-05, + "loss": 1.3479, + "theoretical_loss": 3.3490153241885965, + "tokens_seen": 3002531840 + }, + { + "epoch": 0.82, + "learning_rate": 9.113340020060181e-05, + "loss": 1.2969, + "theoretical_loss": 3.349009752684649, + "tokens_seen": 3002597376 + }, + { + "epoch": 0.82, + "learning_rate": 9.111334002006018e-05, + "loss": 1.2547, + "theoretical_loss": 3.3490041813363547, + "tokens_seen": 3002662912 + }, + { + "epoch": 0.82, + "learning_rate": 9.109327983951856e-05, + "loss": 1.3736, + "theoretical_loss": 3.3489986101437066, + "tokens_seen": 3002728448 + }, + { + "epoch": 0.82, + "learning_rate": 9.107321965897692e-05, + "loss": 1.2581, + "theoretical_loss": 3.3489930391066967, + "tokens_seen": 3002793984 + }, + { + "epoch": 0.82, + "learning_rate": 9.10531594784353e-05, + "loss": 1.3076, + "theoretical_loss": 3.348987468225317, + "tokens_seen": 3002859520 + }, + { + "epoch": 0.82, + "learning_rate": 9.103309929789367e-05, + "loss": 1.2217, + "theoretical_loss": 3.34898189749956, + "tokens_seen": 3002925056 + }, + { + "epoch": 0.82, + "learning_rate": 9.101303911735206e-05, + "loss": 1.1875, + "theoretical_loss": 3.3489763269294177, + "tokens_seen": 3002990592 + }, + { + "epoch": 0.82, + "learning_rate": 9.099297893681044e-05, + "loss": 1.2169, + "theoretical_loss": 3.3489707565148827, + "tokens_seen": 3003056128 + }, + { + "epoch": 0.82, + "learning_rate": 9.097291875626881e-05, + "loss": 1.3784, + "theoretical_loss": 3.348965186255947, + "tokens_seen": 3003121664 + }, + { + "epoch": 0.82, + "learning_rate": 9.095285857572719e-05, + "loss": 1.2711, + "theoretical_loss": 3.348959616152603, + "tokens_seen": 3003187200 + }, + { + "epoch": 0.82, + "learning_rate": 9.093279839518557e-05, + "loss": 1.2648, + "theoretical_loss": 3.348954046204843, + "tokens_seen": 3003252736 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1091879, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2252734899520874, + "objective/train/theoretical_loss": 3.3489484764126587, + "objective/train/tokens_used": 1373777376, + "theoretical_loss": 3.3489484764126587, + "tokens_seen": 3003318272 + }, + { + "epoch": 0.82, + "learning_rate": 9.091273821464394e-05, + "loss": 1.2283, + "theoretical_loss": 3.3489484764126587, + "tokens_seen": 3003318272 + }, + { + "epoch": 0.82, + "learning_rate": 9.089267803410232e-05, + "loss": 1.3223, + "theoretical_loss": 3.3489429067760437, + "tokens_seen": 3003383808 + }, + { + "epoch": 0.82, + "learning_rate": 9.087261785356068e-05, + "loss": 1.3736, + "theoretical_loss": 3.3489373372949887, + "tokens_seen": 3003449344 + }, + { + "epoch": 0.82, + "learning_rate": 9.085255767301905e-05, + "loss": 1.2659, + "theoretical_loss": 3.3489317679694866, + "tokens_seen": 3003514880 + }, + { + "epoch": 0.82, + "learning_rate": 9.083249749247743e-05, + "loss": 1.2242, + "theoretical_loss": 3.3489261987995302, + "tokens_seen": 3003580416 + }, + { + "epoch": 0.82, + "learning_rate": 9.08124373119358e-05, + "loss": 1.2974, + "theoretical_loss": 3.348920629785111, + "tokens_seen": 3003645952 + }, + { + "epoch": 0.82, + "learning_rate": 9.079237713139418e-05, + "loss": 1.2522, + "theoretical_loss": 3.3489150609262217, + "tokens_seen": 3003711488 + }, + { + "epoch": 0.82, + "learning_rate": 9.077231695085256e-05, + "loss": 1.2489, + "theoretical_loss": 3.348909492222854, + "tokens_seen": 3003777024 + }, + { + "epoch": 0.82, + "learning_rate": 9.075225677031093e-05, + "loss": 1.1544, + "theoretical_loss": 3.348903923675001, + "tokens_seen": 3003842560 + }, + { + "epoch": 0.82, + "learning_rate": 9.073219658976931e-05, + "loss": 1.2372, + "theoretical_loss": 3.348898355282654, + "tokens_seen": 3003908096 + }, + { + "epoch": 0.82, + "learning_rate": 9.071213640922769e-05, + "loss": 1.2946, + "theoretical_loss": 3.3488927870458065, + "tokens_seen": 3003973632 + }, + { + "epoch": 0.82, + "learning_rate": 9.069207622868606e-05, + "loss": 1.2697, + "theoretical_loss": 3.3488872189644496, + "tokens_seen": 3004039168 + }, + { + "epoch": 0.82, + "learning_rate": 9.067201604814442e-05, + "loss": 1.2513, + "theoretical_loss": 3.3488816510385764, + "tokens_seen": 3004104704 + }, + { + "epoch": 0.82, + "learning_rate": 9.065195586760281e-05, + "loss": 1.2259, + "theoretical_loss": 3.3488760832681783, + "tokens_seen": 3004170240 + }, + { + "epoch": 0.82, + "learning_rate": 9.063189568706119e-05, + "loss": 1.1897, + "theoretical_loss": 3.3488705156532483, + "tokens_seen": 3004235776 + }, + { + "epoch": 0.82, + "learning_rate": 9.061183550651956e-05, + "loss": 1.26, + "theoretical_loss": 3.3488649481937784, + "tokens_seen": 3004301312 + }, + { + "epoch": 0.82, + "learning_rate": 9.059177532597794e-05, + "loss": 1.1755, + "theoretical_loss": 3.348859380889761, + "tokens_seen": 3004366848 + }, + { + "epoch": 0.82, + "learning_rate": 9.057171514543632e-05, + "loss": 1.2211, + "theoretical_loss": 3.3488538137411883, + "tokens_seen": 3004432384 + }, + { + "epoch": 0.82, + "learning_rate": 9.055165496489469e-05, + "loss": 1.2378, + "theoretical_loss": 3.3488482467480525, + "tokens_seen": 3004497920 + }, + { + "epoch": 0.82, + "learning_rate": 9.053159478435307e-05, + "loss": 1.209, + "theoretical_loss": 3.3488426799103457, + "tokens_seen": 3004563456 + }, + { + "epoch": 0.82, + "learning_rate": 9.051153460381144e-05, + "loss": 1.261, + "theoretical_loss": 3.3488371132280603, + "tokens_seen": 3004628992 + }, + { + "epoch": 0.82, + "learning_rate": 9.04914744232698e-05, + "loss": 1.1495, + "theoretical_loss": 3.348831546701189, + "tokens_seen": 3004694528 + }, + { + "epoch": 0.82, + "learning_rate": 9.047141424272818e-05, + "loss": 1.3219, + "theoretical_loss": 3.3488259803297233, + "tokens_seen": 3004760064 + }, + { + "epoch": 0.82, + "learning_rate": 9.045135406218656e-05, + "loss": 1.1674, + "theoretical_loss": 3.348820414113656, + "tokens_seen": 3004825600 + }, + { + "epoch": 0.82, + "learning_rate": 9.043129388164493e-05, + "loss": 1.2354, + "theoretical_loss": 3.34881484805298, + "tokens_seen": 3004891136 + }, + { + "epoch": 0.82, + "learning_rate": 9.041123370110331e-05, + "loss": 1.1841, + "theoretical_loss": 3.348809282147686, + "tokens_seen": 3004956672 + }, + { + "epoch": 0.82, + "learning_rate": 9.039117352056168e-05, + "loss": 1.1809, + "theoretical_loss": 3.348803716397767, + "tokens_seen": 3005022208 + }, + { + "epoch": 0.82, + "learning_rate": 9.037111334002006e-05, + "loss": 1.272, + "theoretical_loss": 3.3487981508032156, + "tokens_seen": 3005087744 + }, + { + "epoch": 0.82, + "learning_rate": 9.035105315947844e-05, + "loss": 1.2214, + "theoretical_loss": 3.3487925853640244, + "tokens_seen": 3005153280 + }, + { + "epoch": 0.82, + "learning_rate": 9.033099297893681e-05, + "loss": 1.2514, + "theoretical_loss": 3.3487870200801844, + "tokens_seen": 3005218816 + }, + { + "epoch": 0.82, + "learning_rate": 9.031093279839519e-05, + "loss": 1.2047, + "theoretical_loss": 3.348781454951689, + "tokens_seen": 3005284352 + }, + { + "epoch": 0.82, + "learning_rate": 9.029087261785355e-05, + "loss": 1.1444, + "theoretical_loss": 3.34877588997853, + "tokens_seen": 3005349888 + }, + { + "epoch": 0.82, + "learning_rate": 9.027081243731194e-05, + "loss": 1.3247, + "theoretical_loss": 3.3487703251606993, + "tokens_seen": 3005415424 + }, + { + "epoch": 0.82, + "learning_rate": 9.025075225677032e-05, + "loss": 1.2842, + "theoretical_loss": 3.3487647604981903, + "tokens_seen": 3005480960 + }, + { + "epoch": 0.82, + "learning_rate": 9.023069207622869e-05, + "loss": 1.248, + "theoretical_loss": 3.3487591959909944, + "tokens_seen": 3005546496 + }, + { + "epoch": 0.82, + "learning_rate": 9.021063189568707e-05, + "loss": 1.1967, + "theoretical_loss": 3.348753631639104, + "tokens_seen": 3005612032 + }, + { + "epoch": 0.82, + "learning_rate": 9.019057171514544e-05, + "loss": 1.2675, + "theoretical_loss": 3.3487480674425116, + "tokens_seen": 3005677568 + }, + { + "epoch": 0.82, + "learning_rate": 9.017051153460382e-05, + "loss": 1.2585, + "theoretical_loss": 3.3487425034012093, + "tokens_seen": 3005743104 + }, + { + "epoch": 0.82, + "learning_rate": 9.01504513540622e-05, + "loss": 1.2739, + "theoretical_loss": 3.3487369395151894, + "tokens_seen": 3005808640 + }, + { + "epoch": 0.82, + "learning_rate": 9.013039117352057e-05, + "loss": 1.2026, + "theoretical_loss": 3.3487313757844444, + "tokens_seen": 3005874176 + }, + { + "epoch": 0.82, + "learning_rate": 9.011033099297895e-05, + "loss": 1.148, + "theoretical_loss": 3.348725812208966, + "tokens_seen": 3005939712 + }, + { + "epoch": 0.82, + "learning_rate": 9.009027081243731e-05, + "loss": 1.1544, + "theoretical_loss": 3.348720248788747, + "tokens_seen": 3006005248 + }, + { + "epoch": 0.82, + "learning_rate": 9.007021063189568e-05, + "loss": 1.1707, + "theoretical_loss": 3.3487146855237797, + "tokens_seen": 3006070784 + }, + { + "epoch": 0.82, + "learning_rate": 9.005015045135406e-05, + "loss": 1.1891, + "theoretical_loss": 3.348709122414056, + "tokens_seen": 3006136320 + }, + { + "epoch": 0.82, + "learning_rate": 9.003009027081244e-05, + "loss": 1.1441, + "theoretical_loss": 3.348703559459569, + "tokens_seen": 3006201856 + }, + { + "epoch": 0.82, + "learning_rate": 9.001003009027081e-05, + "loss": 1.203, + "theoretical_loss": 3.34869799666031, + "tokens_seen": 3006267392 + }, + { + "epoch": 0.82, + "learning_rate": 8.998996990972919e-05, + "loss": 1.2838, + "theoretical_loss": 3.3486924340162716, + "tokens_seen": 3006332928 + }, + { + "epoch": 0.82, + "learning_rate": 8.996990972918756e-05, + "loss": 1.3152, + "theoretical_loss": 3.3486868715274465, + "tokens_seen": 3006398464 + }, + { + "epoch": 0.82, + "learning_rate": 8.994984954864594e-05, + "loss": 1.1836, + "theoretical_loss": 3.3486813091938266, + "tokens_seen": 3006464000 + }, + { + "epoch": 0.82, + "learning_rate": 8.992978936810431e-05, + "loss": 1.2567, + "theoretical_loss": 3.3486757470154043, + "tokens_seen": 3006529536 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1093144, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2492966651916504, + "objective/train/theoretical_loss": 3.348670184992171, + "objective/train/tokens_used": 1377054176, + "theoretical_loss": 3.348670184992171, + "tokens_seen": 3006595072 + }, + { + "epoch": 0.82, + "learning_rate": 8.990972918756269e-05, + "loss": 1.2636, + "theoretical_loss": 3.348670184992171, + "tokens_seen": 3006595072 + }, + { + "epoch": 0.82, + "learning_rate": 8.988966900702107e-05, + "loss": 1.1011, + "theoretical_loss": 3.348664623124121, + "tokens_seen": 3006660608 + }, + { + "epoch": 0.82, + "learning_rate": 8.986960882647944e-05, + "loss": 1.3603, + "theoretical_loss": 3.3486590614112446, + "tokens_seen": 3006726144 + }, + { + "epoch": 0.82, + "learning_rate": 8.984954864593782e-05, + "loss": 1.2726, + "theoretical_loss": 3.3486534998535356, + "tokens_seen": 3006791680 + }, + { + "epoch": 0.82, + "learning_rate": 8.982948846539619e-05, + "loss": 1.2783, + "theoretical_loss": 3.348647938450985, + "tokens_seen": 3006857216 + }, + { + "epoch": 0.82, + "learning_rate": 8.980942828485457e-05, + "loss": 1.2424, + "theoretical_loss": 3.348642377203586, + "tokens_seen": 3006922752 + }, + { + "epoch": 0.82, + "learning_rate": 8.978936810431294e-05, + "loss": 1.1692, + "theoretical_loss": 3.3486368161113305, + "tokens_seen": 3006988288 + }, + { + "epoch": 0.82, + "learning_rate": 8.976930792377132e-05, + "loss": 1.0817, + "theoretical_loss": 3.348631255174211, + "tokens_seen": 3007053824 + }, + { + "epoch": 0.82, + "learning_rate": 8.97492477432297e-05, + "loss": 1.3148, + "theoretical_loss": 3.3486256943922195, + "tokens_seen": 3007119360 + }, + { + "epoch": 0.82, + "learning_rate": 8.972918756268807e-05, + "loss": 1.2218, + "theoretical_loss": 3.348620133765348, + "tokens_seen": 3007184896 + }, + { + "epoch": 0.82, + "learning_rate": 8.970912738214643e-05, + "loss": 1.3164, + "theoretical_loss": 3.34861457329359, + "tokens_seen": 3007250432 + }, + { + "epoch": 0.82, + "learning_rate": 8.968906720160481e-05, + "loss": 1.2174, + "theoretical_loss": 3.3486090129769366, + "tokens_seen": 3007315968 + }, + { + "epoch": 0.82, + "learning_rate": 8.966900702106319e-05, + "loss": 1.1073, + "theoretical_loss": 3.3486034528153805, + "tokens_seen": 3007381504 + }, + { + "epoch": 0.82, + "learning_rate": 8.964894684052156e-05, + "loss": 1.2947, + "theoretical_loss": 3.348597892808914, + "tokens_seen": 3007447040 + }, + { + "epoch": 0.82, + "learning_rate": 8.962888665997994e-05, + "loss": 1.2749, + "theoretical_loss": 3.34859233295753, + "tokens_seen": 3007512576 + }, + { + "epoch": 0.82, + "learning_rate": 8.960882647943831e-05, + "loss": 1.1767, + "theoretical_loss": 3.3485867732612196, + "tokens_seen": 3007578112 + }, + { + "epoch": 0.82, + "learning_rate": 8.958876629889669e-05, + "loss": 1.4104, + "theoretical_loss": 3.3485812137199757, + "tokens_seen": 3007643648 + }, + { + "epoch": 0.82, + "learning_rate": 8.956870611835506e-05, + "loss": 1.1561, + "theoretical_loss": 3.348575654333791, + "tokens_seen": 3007709184 + }, + { + "epoch": 0.82, + "learning_rate": 8.954864593781344e-05, + "loss": 1.2311, + "theoretical_loss": 3.348570095102657, + "tokens_seen": 3007774720 + }, + { + "epoch": 0.82, + "learning_rate": 8.952858575727182e-05, + "loss": 1.2704, + "theoretical_loss": 3.348564536026567, + "tokens_seen": 3007840256 + }, + { + "epoch": 0.82, + "learning_rate": 8.950852557673019e-05, + "loss": 1.2921, + "theoretical_loss": 3.3485589771055118, + "tokens_seen": 3007905792 + }, + { + "epoch": 0.82, + "learning_rate": 8.948846539618857e-05, + "loss": 1.2852, + "theoretical_loss": 3.348553418339485, + "tokens_seen": 3007971328 + }, + { + "epoch": 0.82, + "learning_rate": 8.946840521564694e-05, + "loss": 1.1555, + "theoretical_loss": 3.3485478597284786, + "tokens_seen": 3008036864 + }, + { + "epoch": 0.82, + "learning_rate": 8.944834503510532e-05, + "loss": 1.1866, + "theoretical_loss": 3.348542301272485, + "tokens_seen": 3008102400 + }, + { + "epoch": 0.82, + "learning_rate": 8.94282848545637e-05, + "loss": 1.2488, + "theoretical_loss": 3.348536742971496, + "tokens_seen": 3008167936 + }, + { + "epoch": 0.82, + "learning_rate": 8.940822467402207e-05, + "loss": 1.2058, + "theoretical_loss": 3.348531184825504, + "tokens_seen": 3008233472 + }, + { + "epoch": 0.82, + "learning_rate": 8.938816449348045e-05, + "loss": 1.2506, + "theoretical_loss": 3.348525626834502, + "tokens_seen": 3008299008 + }, + { + "epoch": 0.82, + "learning_rate": 8.936810431293882e-05, + "loss": 1.1255, + "theoretical_loss": 3.3485200689984813, + "tokens_seen": 3008364544 + }, + { + "epoch": 0.82, + "learning_rate": 8.93480441323972e-05, + "loss": 1.2304, + "theoretical_loss": 3.348514511317435, + "tokens_seen": 3008430080 + }, + { + "epoch": 0.82, + "learning_rate": 8.932798395185557e-05, + "loss": 1.1621, + "theoretical_loss": 3.348508953791355, + "tokens_seen": 3008495616 + }, + { + "epoch": 0.82, + "learning_rate": 8.930792377131394e-05, + "loss": 1.2395, + "theoretical_loss": 3.3485033964202335, + "tokens_seen": 3008561152 + }, + { + "epoch": 0.82, + "learning_rate": 8.928786359077231e-05, + "loss": 1.2761, + "theoretical_loss": 3.348497839204063, + "tokens_seen": 3008626688 + }, + { + "epoch": 0.82, + "learning_rate": 8.926780341023069e-05, + "loss": 1.2425, + "theoretical_loss": 3.3484922821428365, + "tokens_seen": 3008692224 + }, + { + "epoch": 0.82, + "learning_rate": 8.924774322968906e-05, + "loss": 1.2567, + "theoretical_loss": 3.348486725236545, + "tokens_seen": 3008757760 + }, + { + "epoch": 0.82, + "learning_rate": 8.922768304914744e-05, + "loss": 1.311, + "theoretical_loss": 3.3484811684851814, + "tokens_seen": 3008823296 + }, + { + "epoch": 0.82, + "learning_rate": 8.920762286860582e-05, + "loss": 1.1931, + "theoretical_loss": 3.3484756118887384, + "tokens_seen": 3008888832 + }, + { + "epoch": 0.82, + "learning_rate": 8.918756268806419e-05, + "loss": 1.3059, + "theoretical_loss": 3.3484700554472075, + "tokens_seen": 3008954368 + }, + { + "epoch": 0.82, + "learning_rate": 8.916750250752257e-05, + "loss": 1.1465, + "theoretical_loss": 3.348464499160582, + "tokens_seen": 3009019904 + }, + { + "epoch": 0.82, + "learning_rate": 8.914744232698096e-05, + "loss": 1.2023, + "theoretical_loss": 3.348458943028853, + "tokens_seen": 3009085440 + }, + { + "epoch": 0.82, + "learning_rate": 8.912738214643933e-05, + "loss": 1.349, + "theoretical_loss": 3.3484533870520137, + "tokens_seen": 3009150976 + }, + { + "epoch": 0.82, + "learning_rate": 8.91073219658977e-05, + "loss": 1.2251, + "theoretical_loss": 3.3484478312300565, + "tokens_seen": 3009216512 + }, + { + "epoch": 0.82, + "learning_rate": 8.908726178535607e-05, + "loss": 1.3079, + "theoretical_loss": 3.348442275562973, + "tokens_seen": 3009282048 + }, + { + "epoch": 0.82, + "learning_rate": 8.906720160481445e-05, + "loss": 1.1821, + "theoretical_loss": 3.3484367200507563, + "tokens_seen": 3009347584 + }, + { + "epoch": 0.82, + "learning_rate": 8.904714142427282e-05, + "loss": 1.2439, + "theoretical_loss": 3.3484311646933977, + "tokens_seen": 3009413120 + }, + { + "epoch": 0.82, + "learning_rate": 8.90270812437312e-05, + "loss": 1.2722, + "theoretical_loss": 3.3484256094908904, + "tokens_seen": 3009478656 + }, + { + "epoch": 0.82, + "learning_rate": 8.900702106318957e-05, + "loss": 1.222, + "theoretical_loss": 3.3484200544432263, + "tokens_seen": 3009544192 + }, + { + "epoch": 0.82, + "learning_rate": 8.898696088264795e-05, + "loss": 1.2082, + "theoretical_loss": 3.3484144995503984, + "tokens_seen": 3009609728 + }, + { + "epoch": 0.82, + "learning_rate": 8.896690070210633e-05, + "loss": 1.2133, + "theoretical_loss": 3.3484089448123977, + "tokens_seen": 3009675264 + }, + { + "epoch": 0.82, + "learning_rate": 8.89468405215647e-05, + "loss": 1.2566, + "theoretical_loss": 3.3484033902292176, + "tokens_seen": 3009740800 + }, + { + "epoch": 0.82, + "learning_rate": 8.892678034102306e-05, + "loss": 1.2148, + "theoretical_loss": 3.34839783580085, + "tokens_seen": 3009806336 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1094320, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2957937717437744, + "objective/train/theoretical_loss": 3.3483922815272873, + "objective/train/tokens_used": 1380330976, + "theoretical_loss": 3.3483922815272873, + "tokens_seen": 3009871872 + }, + { + "epoch": 0.82, + "learning_rate": 8.890672016048144e-05, + "loss": 1.3301, + "theoretical_loss": 3.3483922815272873, + "tokens_seen": 3009871872 + }, + { + "epoch": 0.82, + "learning_rate": 8.888665997993981e-05, + "loss": 1.218, + "theoretical_loss": 3.348386727408522, + "tokens_seen": 3009937408 + }, + { + "epoch": 0.82, + "learning_rate": 8.886659979939819e-05, + "loss": 1.2582, + "theoretical_loss": 3.3483811734445457, + "tokens_seen": 3010002944 + }, + { + "epoch": 0.82, + "learning_rate": 8.884653961885657e-05, + "loss": 1.1732, + "theoretical_loss": 3.348375619635352, + "tokens_seen": 3010068480 + }, + { + "epoch": 0.82, + "learning_rate": 8.882647943831494e-05, + "loss": 1.2053, + "theoretical_loss": 3.348370065980932, + "tokens_seen": 3010134016 + }, + { + "epoch": 0.82, + "learning_rate": 8.880641925777332e-05, + "loss": 1.2596, + "theoretical_loss": 3.3483645124812784, + "tokens_seen": 3010199552 + }, + { + "epoch": 0.82, + "learning_rate": 8.87863590772317e-05, + "loss": 1.1889, + "theoretical_loss": 3.348358959136384, + "tokens_seen": 3010265088 + }, + { + "epoch": 0.82, + "learning_rate": 8.876629889669008e-05, + "loss": 1.3915, + "theoretical_loss": 3.34835340594624, + "tokens_seen": 3010330624 + }, + { + "epoch": 0.82, + "learning_rate": 8.874623871614846e-05, + "loss": 1.171, + "theoretical_loss": 3.3483478529108397, + "tokens_seen": 3010396160 + }, + { + "epoch": 0.82, + "learning_rate": 8.872617853560682e-05, + "loss": 1.2031, + "theoretical_loss": 3.3483423000301755, + "tokens_seen": 3010461696 + }, + { + "epoch": 0.82, + "learning_rate": 8.87061183550652e-05, + "loss": 1.2315, + "theoretical_loss": 3.348336747304239, + "tokens_seen": 3010527232 + }, + { + "epoch": 0.82, + "learning_rate": 8.868605817452357e-05, + "loss": 1.2525, + "theoretical_loss": 3.348331194733023, + "tokens_seen": 3010592768 + }, + { + "epoch": 0.82, + "learning_rate": 8.866599799398195e-05, + "loss": 1.3551, + "theoretical_loss": 3.3483256423165195, + "tokens_seen": 3010658304 + }, + { + "epoch": 0.82, + "learning_rate": 8.864593781344032e-05, + "loss": 1.2163, + "theoretical_loss": 3.348320090054721, + "tokens_seen": 3010723840 + }, + { + "epoch": 0.82, + "learning_rate": 8.86258776328987e-05, + "loss": 1.2434, + "theoretical_loss": 3.3483145379476205, + "tokens_seen": 3010789376 + }, + { + "epoch": 0.82, + "learning_rate": 8.860581745235708e-05, + "loss": 1.2509, + "theoretical_loss": 3.3483089859952093, + "tokens_seen": 3010854912 + }, + { + "epoch": 0.82, + "learning_rate": 8.858575727181545e-05, + "loss": 1.2318, + "theoretical_loss": 3.3483034341974798, + "tokens_seen": 3010920448 + }, + { + "epoch": 0.82, + "learning_rate": 8.856569709127383e-05, + "loss": 1.3126, + "theoretical_loss": 3.3482978825544247, + "tokens_seen": 3010985984 + }, + { + "epoch": 0.82, + "learning_rate": 8.85456369107322e-05, + "loss": 1.2681, + "theoretical_loss": 3.3482923310660366, + "tokens_seen": 3011051520 + }, + { + "epoch": 0.82, + "learning_rate": 8.852557673019057e-05, + "loss": 1.209, + "theoretical_loss": 3.348286779732307, + "tokens_seen": 3011117056 + }, + { + "epoch": 0.82, + "learning_rate": 8.850551654964894e-05, + "loss": 1.1812, + "theoretical_loss": 3.3482812285532293, + "tokens_seen": 3011182592 + }, + { + "epoch": 0.82, + "learning_rate": 8.848545636910732e-05, + "loss": 1.2183, + "theoretical_loss": 3.348275677528795, + "tokens_seen": 3011248128 + }, + { + "epoch": 0.83, + "learning_rate": 8.846539618856569e-05, + "loss": 1.1619, + "theoretical_loss": 3.3482701266589965, + "tokens_seen": 3011313664 + }, + { + "epoch": 0.83, + "learning_rate": 8.844533600802407e-05, + "loss": 1.2842, + "theoretical_loss": 3.348264575943826, + "tokens_seen": 3011379200 + }, + { + "epoch": 0.83, + "learning_rate": 8.842527582748244e-05, + "loss": 1.2232, + "theoretical_loss": 3.3482590253832765, + "tokens_seen": 3011444736 + }, + { + "epoch": 0.83, + "learning_rate": 8.840521564694082e-05, + "loss": 1.242, + "theoretical_loss": 3.34825347497734, + "tokens_seen": 3011510272 + }, + { + "epoch": 0.83, + "learning_rate": 8.838515546639921e-05, + "loss": 1.3083, + "theoretical_loss": 3.3482479247260084, + "tokens_seen": 3011575808 + }, + { + "epoch": 0.83, + "learning_rate": 8.836509528585759e-05, + "loss": 1.2316, + "theoretical_loss": 3.348242374629274, + "tokens_seen": 3011641344 + }, + { + "epoch": 0.83, + "learning_rate": 8.834503510531596e-05, + "loss": 1.2961, + "theoretical_loss": 3.34823682468713, + "tokens_seen": 3011706880 + }, + { + "epoch": 0.83, + "learning_rate": 8.832497492477432e-05, + "loss": 1.3022, + "theoretical_loss": 3.3482312748995686, + "tokens_seen": 3011772416 + }, + { + "epoch": 0.83, + "learning_rate": 8.83049147442327e-05, + "loss": 1.1723, + "theoretical_loss": 3.3482257252665817, + "tokens_seen": 3011837952 + }, + { + "epoch": 0.83, + "learning_rate": 8.828485456369108e-05, + "loss": 1.2396, + "theoretical_loss": 3.3482201757881613, + "tokens_seen": 3011903488 + }, + { + "epoch": 0.83, + "learning_rate": 8.826479438314945e-05, + "loss": 1.2796, + "theoretical_loss": 3.3482146264643005, + "tokens_seen": 3011969024 + }, + { + "epoch": 0.83, + "learning_rate": 8.824473420260783e-05, + "loss": 1.2475, + "theoretical_loss": 3.3482090772949906, + "tokens_seen": 3012034560 + }, + { + "epoch": 0.83, + "learning_rate": 8.82246740220662e-05, + "loss": 1.1937, + "theoretical_loss": 3.348203528280225, + "tokens_seen": 3012100096 + }, + { + "epoch": 0.83, + "learning_rate": 8.820461384152458e-05, + "loss": 1.2166, + "theoretical_loss": 3.348197979419996, + "tokens_seen": 3012165632 + }, + { + "epoch": 0.83, + "learning_rate": 8.818455366098295e-05, + "loss": 1.1765, + "theoretical_loss": 3.3481924307142954, + "tokens_seen": 3012231168 + }, + { + "epoch": 0.83, + "learning_rate": 8.816449348044133e-05, + "loss": 1.3187, + "theoretical_loss": 3.3481868821631156, + "tokens_seen": 3012296704 + }, + { + "epoch": 0.83, + "learning_rate": 8.814443329989969e-05, + "loss": 1.1384, + "theoretical_loss": 3.348181333766449, + "tokens_seen": 3012362240 + }, + { + "epoch": 0.83, + "learning_rate": 8.812437311935807e-05, + "loss": 1.1879, + "theoretical_loss": 3.348175785524288, + "tokens_seen": 3012427776 + }, + { + "epoch": 0.83, + "learning_rate": 8.810431293881644e-05, + "loss": 1.179, + "theoretical_loss": 3.3481702374366247, + "tokens_seen": 3012493312 + }, + { + "epoch": 0.83, + "learning_rate": 8.808425275827482e-05, + "loss": 1.2718, + "theoretical_loss": 3.3481646895034523, + "tokens_seen": 3012558848 + }, + { + "epoch": 0.83, + "learning_rate": 8.80641925777332e-05, + "loss": 1.3446, + "theoretical_loss": 3.3481591417247616, + "tokens_seen": 3012624384 + }, + { + "epoch": 0.83, + "learning_rate": 8.804413239719157e-05, + "loss": 1.3166, + "theoretical_loss": 3.3481535941005465, + "tokens_seen": 3012689920 + }, + { + "epoch": 0.83, + "learning_rate": 8.802407221664996e-05, + "loss": 1.2108, + "theoretical_loss": 3.3481480466307985, + "tokens_seen": 3012755456 + }, + { + "epoch": 0.83, + "learning_rate": 8.800401203610834e-05, + "loss": 1.1505, + "theoretical_loss": 3.34814249931551, + "tokens_seen": 3012820992 + }, + { + "epoch": 0.83, + "learning_rate": 8.798395185556671e-05, + "loss": 1.3459, + "theoretical_loss": 3.3481369521546736, + "tokens_seen": 3012886528 + }, + { + "epoch": 0.83, + "learning_rate": 8.796389167502509e-05, + "loss": 1.2998, + "theoretical_loss": 3.3481314051482816, + "tokens_seen": 3012952064 + }, + { + "epoch": 0.83, + "learning_rate": 8.794383149448345e-05, + "loss": 1.3141, + "theoretical_loss": 3.3481258582963256, + "tokens_seen": 3013017600 + }, + { + "epoch": 0.83, + "learning_rate": 8.792377131394183e-05, + "loss": 1.188, + "theoretical_loss": 3.348120311598799, + "tokens_seen": 3013083136 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1095529, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2712202072143555, + "objective/train/theoretical_loss": 3.3481147650556937, + "objective/train/tokens_used": 1383607776, + "theoretical_loss": 3.3481147650556937, + "tokens_seen": 3013148672 + }, + { + "epoch": 0.83, + "learning_rate": 8.79037111334002e-05, + "loss": 1.2587, + "theoretical_loss": 3.3481147650556937, + "tokens_seen": 3013148672 + }, + { + "epoch": 0.83, + "learning_rate": 8.788365095285858e-05, + "loss": 1.2639, + "theoretical_loss": 3.348109218667002, + "tokens_seen": 3013214208 + }, + { + "epoch": 0.83, + "learning_rate": 8.786359077231695e-05, + "loss": 1.2714, + "theoretical_loss": 3.3481036724327167, + "tokens_seen": 3013279744 + }, + { + "epoch": 0.83, + "learning_rate": 8.784353059177533e-05, + "loss": 1.2016, + "theoretical_loss": 3.348098126352829, + "tokens_seen": 3013345280 + }, + { + "epoch": 0.83, + "learning_rate": 8.78234704112337e-05, + "loss": 1.2157, + "theoretical_loss": 3.3480925804273323, + "tokens_seen": 3013410816 + }, + { + "epoch": 0.83, + "learning_rate": 8.780341023069208e-05, + "loss": 1.275, + "theoretical_loss": 3.348087034656219, + "tokens_seen": 3013476352 + }, + { + "epoch": 0.83, + "learning_rate": 8.778335005015046e-05, + "loss": 1.1858, + "theoretical_loss": 3.3480814890394806, + "tokens_seen": 3013541888 + }, + { + "epoch": 0.83, + "learning_rate": 8.776328986960883e-05, + "loss": 1.2648, + "theoretical_loss": 3.34807594357711, + "tokens_seen": 3013607424 + }, + { + "epoch": 0.83, + "learning_rate": 8.77432296890672e-05, + "loss": 1.2293, + "theoretical_loss": 3.3480703982690994, + "tokens_seen": 3013672960 + }, + { + "epoch": 0.83, + "learning_rate": 8.772316950852557e-05, + "loss": 1.2157, + "theoretical_loss": 3.3480648531154413, + "tokens_seen": 3013738496 + }, + { + "epoch": 0.83, + "learning_rate": 8.770310932798395e-05, + "loss": 1.2538, + "theoretical_loss": 3.348059308116128, + "tokens_seen": 3013804032 + }, + { + "epoch": 0.83, + "learning_rate": 8.768304914744232e-05, + "loss": 1.2893, + "theoretical_loss": 3.348053763271152, + "tokens_seen": 3013869568 + }, + { + "epoch": 0.83, + "learning_rate": 8.76629889669007e-05, + "loss": 1.2555, + "theoretical_loss": 3.348048218580505, + "tokens_seen": 3013935104 + }, + { + "epoch": 0.83, + "learning_rate": 8.764292878635909e-05, + "loss": 1.3168, + "theoretical_loss": 3.34804267404418, + "tokens_seen": 3014000640 + }, + { + "epoch": 0.83, + "learning_rate": 8.762286860581746e-05, + "loss": 1.1971, + "theoretical_loss": 3.348037129662169, + "tokens_seen": 3014066176 + }, + { + "epoch": 0.83, + "learning_rate": 8.760280842527584e-05, + "loss": 1.3072, + "theoretical_loss": 3.348031585434465, + "tokens_seen": 3014131712 + }, + { + "epoch": 0.83, + "learning_rate": 8.758274824473421e-05, + "loss": 1.285, + "theoretical_loss": 3.3480260413610594, + "tokens_seen": 3014197248 + }, + { + "epoch": 0.83, + "learning_rate": 8.756268806419259e-05, + "loss": 1.2853, + "theoretical_loss": 3.348020497441945, + "tokens_seen": 3014262784 + }, + { + "epoch": 0.83, + "learning_rate": 8.754262788365095e-05, + "loss": 1.1665, + "theoretical_loss": 3.348014953677114, + "tokens_seen": 3014328320 + }, + { + "epoch": 0.83, + "learning_rate": 8.752256770310933e-05, + "loss": 1.3399, + "theoretical_loss": 3.3480094100665596, + "tokens_seen": 3014393856 + }, + { + "epoch": 0.83, + "learning_rate": 8.75025075225677e-05, + "loss": 1.2851, + "theoretical_loss": 3.3480038666102727, + "tokens_seen": 3014459392 + }, + { + "epoch": 0.83, + "learning_rate": 8.748244734202608e-05, + "loss": 1.2702, + "theoretical_loss": 3.347998323308247, + "tokens_seen": 3014524928 + }, + { + "epoch": 0.83, + "learning_rate": 8.746238716148446e-05, + "loss": 1.2397, + "theoretical_loss": 3.347992780160474, + "tokens_seen": 3014590464 + }, + { + "epoch": 0.83, + "learning_rate": 8.744232698094283e-05, + "loss": 1.2501, + "theoretical_loss": 3.3479872371669464, + "tokens_seen": 3014656000 + }, + { + "epoch": 0.83, + "learning_rate": 8.742226680040121e-05, + "loss": 1.2013, + "theoretical_loss": 3.3479816943276566, + "tokens_seen": 3014721536 + }, + { + "epoch": 0.83, + "learning_rate": 8.740220661985958e-05, + "loss": 1.3071, + "theoretical_loss": 3.3479761516425963, + "tokens_seen": 3014787072 + }, + { + "epoch": 0.83, + "learning_rate": 8.738214643931796e-05, + "loss": 1.2229, + "theoretical_loss": 3.347970609111759, + "tokens_seen": 3014852608 + }, + { + "epoch": 0.83, + "learning_rate": 8.736208625877632e-05, + "loss": 1.2146, + "theoretical_loss": 3.347965066735136, + "tokens_seen": 3014918144 + }, + { + "epoch": 0.83, + "learning_rate": 8.73420260782347e-05, + "loss": 1.2681, + "theoretical_loss": 3.3479595245127203, + "tokens_seen": 3014983680 + }, + { + "epoch": 0.83, + "learning_rate": 8.732196589769307e-05, + "loss": 1.3636, + "theoretical_loss": 3.347953982444504, + "tokens_seen": 3015049216 + }, + { + "epoch": 0.83, + "learning_rate": 8.730190571715145e-05, + "loss": 1.2775, + "theoretical_loss": 3.3479484405304794, + "tokens_seen": 3015114752 + }, + { + "epoch": 0.83, + "learning_rate": 8.728184553660982e-05, + "loss": 1.2842, + "theoretical_loss": 3.347942898770639, + "tokens_seen": 3015180288 + }, + { + "epoch": 0.83, + "learning_rate": 8.726178535606821e-05, + "loss": 1.2808, + "theoretical_loss": 3.3479373571649753, + "tokens_seen": 3015245824 + }, + { + "epoch": 0.83, + "learning_rate": 8.724172517552659e-05, + "loss": 1.229, + "theoretical_loss": 3.3479318157134808, + "tokens_seen": 3015311360 + }, + { + "epoch": 0.83, + "learning_rate": 8.722166499498496e-05, + "loss": 1.2065, + "theoretical_loss": 3.347926274416147, + "tokens_seen": 3015376896 + }, + { + "epoch": 0.83, + "learning_rate": 8.720160481444334e-05, + "loss": 1.3117, + "theoretical_loss": 3.347920733272967, + "tokens_seen": 3015442432 + }, + { + "epoch": 0.83, + "learning_rate": 8.718154463390172e-05, + "loss": 1.2117, + "theoretical_loss": 3.3479151922839328, + "tokens_seen": 3015507968 + }, + { + "epoch": 0.83, + "learning_rate": 8.716148445336008e-05, + "loss": 1.2977, + "theoretical_loss": 3.3479096514490374, + "tokens_seen": 3015573504 + }, + { + "epoch": 0.83, + "learning_rate": 8.714142427281845e-05, + "loss": 1.367, + "theoretical_loss": 3.347904110768272, + "tokens_seen": 3015639040 + }, + { + "epoch": 0.83, + "learning_rate": 8.712136409227683e-05, + "loss": 1.1617, + "theoretical_loss": 3.34789857024163, + "tokens_seen": 3015704576 + }, + { + "epoch": 0.83, + "learning_rate": 8.71013039117352e-05, + "loss": 1.156, + "theoretical_loss": 3.3478930298691036, + "tokens_seen": 3015770112 + }, + { + "epoch": 0.83, + "learning_rate": 8.708124373119358e-05, + "loss": 1.2682, + "theoretical_loss": 3.347887489650685, + "tokens_seen": 3015835648 + }, + { + "epoch": 0.83, + "learning_rate": 8.706118355065196e-05, + "loss": 1.1781, + "theoretical_loss": 3.3478819495863665, + "tokens_seen": 3015901184 + }, + { + "epoch": 0.83, + "learning_rate": 8.704112337011033e-05, + "loss": 1.1598, + "theoretical_loss": 3.3478764096761404, + "tokens_seen": 3015966720 + }, + { + "epoch": 0.83, + "learning_rate": 8.702106318956871e-05, + "loss": 1.2099, + "theoretical_loss": 3.3478708699199995, + "tokens_seen": 3016032256 + }, + { + "epoch": 0.83, + "learning_rate": 8.700100300902709e-05, + "loss": 1.1565, + "theoretical_loss": 3.3478653303179353, + "tokens_seen": 3016097792 + }, + { + "epoch": 0.83, + "learning_rate": 8.698094282848546e-05, + "loss": 1.2102, + "theoretical_loss": 3.3478597908699412, + "tokens_seen": 3016163328 + }, + { + "epoch": 0.83, + "learning_rate": 8.696088264794382e-05, + "loss": 1.227, + "theoretical_loss": 3.347854251576009, + "tokens_seen": 3016228864 + }, + { + "epoch": 0.83, + "learning_rate": 8.69408224674022e-05, + "loss": 1.187, + "theoretical_loss": 3.347848712436131, + "tokens_seen": 3016294400 + }, + { + "epoch": 0.83, + "learning_rate": 8.692076228686057e-05, + "loss": 1.2563, + "theoretical_loss": 3.3478431734502996, + "tokens_seen": 3016359936 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1096638, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3409316539764404, + "objective/train/theoretical_loss": 3.3478376346185077, + "objective/train/tokens_used": 1386884576, + "theoretical_loss": 3.3478376346185077, + "tokens_seen": 3016425472 + }, + { + "epoch": 0.83, + "learning_rate": 8.690070210631895e-05, + "loss": 1.3796, + "theoretical_loss": 3.3478376346185077, + "tokens_seen": 3016425472 + }, + { + "epoch": 0.83, + "learning_rate": 8.688064192577734e-05, + "loss": 1.1939, + "theoretical_loss": 3.347832095940747, + "tokens_seen": 3016491008 + }, + { + "epoch": 0.83, + "learning_rate": 8.686058174523572e-05, + "loss": 1.3035, + "theoretical_loss": 3.3478265574170103, + "tokens_seen": 3016556544 + }, + { + "epoch": 0.83, + "learning_rate": 8.684052156469409e-05, + "loss": 1.2408, + "theoretical_loss": 3.3478210190472897, + "tokens_seen": 3016622080 + }, + { + "epoch": 0.83, + "learning_rate": 8.682046138415247e-05, + "loss": 1.2729, + "theoretical_loss": 3.3478154808315774, + "tokens_seen": 3016687616 + }, + { + "epoch": 0.83, + "learning_rate": 8.680040120361084e-05, + "loss": 1.1604, + "theoretical_loss": 3.3478099427698664, + "tokens_seen": 3016753152 + }, + { + "epoch": 0.83, + "learning_rate": 8.67803410230692e-05, + "loss": 1.3099, + "theoretical_loss": 3.3478044048621487, + "tokens_seen": 3016818688 + }, + { + "epoch": 0.83, + "learning_rate": 8.676028084252758e-05, + "loss": 1.2885, + "theoretical_loss": 3.3477988671084167, + "tokens_seen": 3016884224 + }, + { + "epoch": 0.83, + "learning_rate": 8.674022066198596e-05, + "loss": 1.2425, + "theoretical_loss": 3.3477933295086624, + "tokens_seen": 3016949760 + }, + { + "epoch": 0.83, + "learning_rate": 8.672016048144433e-05, + "loss": 1.1738, + "theoretical_loss": 3.3477877920628787, + "tokens_seen": 3017015296 + }, + { + "epoch": 0.83, + "learning_rate": 8.670010030090271e-05, + "loss": 1.3217, + "theoretical_loss": 3.347782254771058, + "tokens_seen": 3017080832 + }, + { + "epoch": 0.83, + "learning_rate": 8.668004012036108e-05, + "loss": 1.2361, + "theoretical_loss": 3.3477767176331925, + "tokens_seen": 3017146368 + }, + { + "epoch": 0.83, + "learning_rate": 8.665997993981946e-05, + "loss": 1.2087, + "theoretical_loss": 3.3477711806492745, + "tokens_seen": 3017211904 + }, + { + "epoch": 0.83, + "learning_rate": 8.663991975927784e-05, + "loss": 1.2087, + "theoretical_loss": 3.3477656438192964, + "tokens_seen": 3017277440 + }, + { + "epoch": 0.83, + "learning_rate": 8.661985957873621e-05, + "loss": 1.2083, + "theoretical_loss": 3.347760107143251, + "tokens_seen": 3017342976 + }, + { + "epoch": 0.83, + "learning_rate": 8.659979939819459e-05, + "loss": 1.1481, + "theoretical_loss": 3.3477545706211296, + "tokens_seen": 3017408512 + }, + { + "epoch": 0.83, + "learning_rate": 8.657973921765295e-05, + "loss": 1.1471, + "theoretical_loss": 3.3477490342529257, + "tokens_seen": 3017474048 + }, + { + "epoch": 0.83, + "learning_rate": 8.655967903711133e-05, + "loss": 1.2685, + "theoretical_loss": 3.347743498038631, + "tokens_seen": 3017539584 + }, + { + "epoch": 0.83, + "learning_rate": 8.65396188565697e-05, + "loss": 1.2001, + "theoretical_loss": 3.3477379619782384, + "tokens_seen": 3017605120 + }, + { + "epoch": 0.83, + "learning_rate": 8.651955867602809e-05, + "loss": 1.2827, + "theoretical_loss": 3.34773242607174, + "tokens_seen": 3017670656 + }, + { + "epoch": 0.83, + "learning_rate": 8.649949849548647e-05, + "loss": 1.2835, + "theoretical_loss": 3.347726890319128, + "tokens_seen": 3017736192 + }, + { + "epoch": 0.83, + "learning_rate": 8.647943831494484e-05, + "loss": 1.359, + "theoretical_loss": 3.3477213547203952, + "tokens_seen": 3017801728 + }, + { + "epoch": 0.83, + "learning_rate": 8.645937813440322e-05, + "loss": 1.2911, + "theoretical_loss": 3.3477158192755336, + "tokens_seen": 3017867264 + }, + { + "epoch": 0.83, + "learning_rate": 8.64393179538616e-05, + "loss": 1.1404, + "theoretical_loss": 3.3477102839845356, + "tokens_seen": 3017932800 + }, + { + "epoch": 0.83, + "learning_rate": 8.641925777331997e-05, + "loss": 1.1919, + "theoretical_loss": 3.3477047488473937, + "tokens_seen": 3017998336 + }, + { + "epoch": 0.83, + "learning_rate": 8.639919759277835e-05, + "loss": 1.2773, + "theoretical_loss": 3.3476992138641006, + "tokens_seen": 3018063872 + }, + { + "epoch": 0.83, + "learning_rate": 8.637913741223671e-05, + "loss": 1.2141, + "theoretical_loss": 3.347693679034648, + "tokens_seen": 3018129408 + }, + { + "epoch": 0.83, + "learning_rate": 8.635907723169508e-05, + "loss": 1.225, + "theoretical_loss": 3.347688144359029, + "tokens_seen": 3018194944 + }, + { + "epoch": 0.83, + "learning_rate": 8.633901705115346e-05, + "loss": 1.2655, + "theoretical_loss": 3.3476826098372356, + "tokens_seen": 3018260480 + }, + { + "epoch": 0.83, + "learning_rate": 8.631895687061183e-05, + "loss": 1.2521, + "theoretical_loss": 3.34767707546926, + "tokens_seen": 3018326016 + }, + { + "epoch": 0.83, + "learning_rate": 8.629889669007021e-05, + "loss": 1.2985, + "theoretical_loss": 3.347671541255095, + "tokens_seen": 3018391552 + }, + { + "epoch": 0.83, + "learning_rate": 8.627883650952859e-05, + "loss": 1.2537, + "theoretical_loss": 3.3476660071947326, + "tokens_seen": 3018457088 + }, + { + "epoch": 0.83, + "learning_rate": 8.625877632898696e-05, + "loss": 1.269, + "theoretical_loss": 3.347660473288166, + "tokens_seen": 3018522624 + }, + { + "epoch": 0.83, + "learning_rate": 8.623871614844534e-05, + "loss": 1.2807, + "theoretical_loss": 3.3476549395353863, + "tokens_seen": 3018588160 + }, + { + "epoch": 0.83, + "learning_rate": 8.621865596790371e-05, + "loss": 1.2069, + "theoretical_loss": 3.347649405936387, + "tokens_seen": 3018653696 + }, + { + "epoch": 0.83, + "learning_rate": 8.619859578736209e-05, + "loss": 1.2406, + "theoretical_loss": 3.34764387249116, + "tokens_seen": 3018719232 + }, + { + "epoch": 0.83, + "learning_rate": 8.617853560682045e-05, + "loss": 1.132, + "theoretical_loss": 3.347638339199697, + "tokens_seen": 3018784768 + }, + { + "epoch": 0.83, + "learning_rate": 8.615847542627883e-05, + "loss": 1.2233, + "theoretical_loss": 3.347632806061992, + "tokens_seen": 3018850304 + }, + { + "epoch": 0.83, + "learning_rate": 8.613841524573722e-05, + "loss": 1.2389, + "theoretical_loss": 3.3476272730780363, + "tokens_seen": 3018915840 + }, + { + "epoch": 0.83, + "learning_rate": 8.611835506519559e-05, + "loss": 1.3935, + "theoretical_loss": 3.3476217402478223, + "tokens_seen": 3018981376 + }, + { + "epoch": 0.83, + "learning_rate": 8.609829488465397e-05, + "loss": 1.2559, + "theoretical_loss": 3.3476162075713427, + "tokens_seen": 3019046912 + }, + { + "epoch": 0.83, + "learning_rate": 8.607823470411234e-05, + "loss": 1.1895, + "theoretical_loss": 3.34761067504859, + "tokens_seen": 3019112448 + }, + { + "epoch": 0.83, + "learning_rate": 8.605817452357072e-05, + "loss": 1.1821, + "theoretical_loss": 3.3476051426795563, + "tokens_seen": 3019177984 + }, + { + "epoch": 0.83, + "learning_rate": 8.60381143430291e-05, + "loss": 1.266, + "theoretical_loss": 3.347599610464234, + "tokens_seen": 3019243520 + }, + { + "epoch": 0.83, + "learning_rate": 8.601805416248747e-05, + "loss": 1.3396, + "theoretical_loss": 3.3475940784026155, + "tokens_seen": 3019309056 + }, + { + "epoch": 0.83, + "learning_rate": 8.599799398194583e-05, + "loss": 1.2026, + "theoretical_loss": 3.347588546494693, + "tokens_seen": 3019374592 + }, + { + "epoch": 0.83, + "learning_rate": 8.597793380140421e-05, + "loss": 1.2665, + "theoretical_loss": 3.3475830147404597, + "tokens_seen": 3019440128 + }, + { + "epoch": 0.83, + "learning_rate": 8.595787362086259e-05, + "loss": 1.2466, + "theoretical_loss": 3.3475774831399074, + "tokens_seen": 3019505664 + }, + { + "epoch": 0.83, + "learning_rate": 8.593781344032096e-05, + "loss": 1.3391, + "theoretical_loss": 3.3475719516930287, + "tokens_seen": 3019571200 + }, + { + "epoch": 0.83, + "learning_rate": 8.591775325977934e-05, + "loss": 1.2207, + "theoretical_loss": 3.347566420399815, + "tokens_seen": 3019636736 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1097674, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2506718635559082, + "objective/train/theoretical_loss": 3.34756088926026, + "objective/train/tokens_used": 1390161376, + "theoretical_loss": 3.34756088926026, + "tokens_seen": 3019702272 + }, + { + "epoch": 0.83, + "learning_rate": 8.589769307923771e-05, + "loss": 1.245, + "theoretical_loss": 3.34756088926026, + "tokens_seen": 3019702272 + }, + { + "epoch": 0.83, + "learning_rate": 8.587763289869609e-05, + "loss": 1.3835, + "theoretical_loss": 3.347555358274356, + "tokens_seen": 3019767808 + }, + { + "epoch": 0.83, + "learning_rate": 8.585757271815446e-05, + "loss": 1.2042, + "theoretical_loss": 3.3475498274420947, + "tokens_seen": 3019833344 + }, + { + "epoch": 0.83, + "learning_rate": 8.583751253761284e-05, + "loss": 1.2671, + "theoretical_loss": 3.347544296763469, + "tokens_seen": 3019898880 + }, + { + "epoch": 0.83, + "learning_rate": 8.581745235707122e-05, + "loss": 1.2828, + "theoretical_loss": 3.347538766238471, + "tokens_seen": 3019964416 + }, + { + "epoch": 0.83, + "learning_rate": 8.579739217652958e-05, + "loss": 1.1757, + "theoretical_loss": 3.3475332358670933, + "tokens_seen": 3020029952 + }, + { + "epoch": 0.83, + "learning_rate": 8.577733199598795e-05, + "loss": 1.254, + "theoretical_loss": 3.347527705649328, + "tokens_seen": 3020095488 + }, + { + "epoch": 0.83, + "learning_rate": 8.575727181544634e-05, + "loss": 1.2534, + "theoretical_loss": 3.347522175585168, + "tokens_seen": 3020161024 + }, + { + "epoch": 0.83, + "learning_rate": 8.573721163490472e-05, + "loss": 1.2634, + "theoretical_loss": 3.347516645674605, + "tokens_seen": 3020226560 + }, + { + "epoch": 0.83, + "learning_rate": 8.57171514543631e-05, + "loss": 1.3176, + "theoretical_loss": 3.3475111159176323, + "tokens_seen": 3020292096 + }, + { + "epoch": 0.83, + "learning_rate": 8.569709127382147e-05, + "loss": 1.2092, + "theoretical_loss": 3.3475055863142416, + "tokens_seen": 3020357632 + }, + { + "epoch": 0.83, + "learning_rate": 8.567703109327985e-05, + "loss": 1.2491, + "theoretical_loss": 3.3475000568644258, + "tokens_seen": 3020423168 + }, + { + "epoch": 0.83, + "learning_rate": 8.565697091273822e-05, + "loss": 1.2049, + "theoretical_loss": 3.3474945275681764, + "tokens_seen": 3020488704 + }, + { + "epoch": 0.83, + "learning_rate": 8.56369107321966e-05, + "loss": 1.2714, + "theoretical_loss": 3.347488998425487, + "tokens_seen": 3020554240 + }, + { + "epoch": 0.83, + "learning_rate": 8.561685055165497e-05, + "loss": 1.2141, + "theoretical_loss": 3.3474834694363493, + "tokens_seen": 3020619776 + }, + { + "epoch": 0.83, + "learning_rate": 8.559679037111334e-05, + "loss": 1.1889, + "theoretical_loss": 3.347477940600756, + "tokens_seen": 3020685312 + }, + { + "epoch": 0.83, + "learning_rate": 8.557673019057171e-05, + "loss": 1.2976, + "theoretical_loss": 3.3474724119186994, + "tokens_seen": 3020750848 + }, + { + "epoch": 0.83, + "learning_rate": 8.555667001003009e-05, + "loss": 1.1912, + "theoretical_loss": 3.3474668833901715, + "tokens_seen": 3020816384 + }, + { + "epoch": 0.83, + "learning_rate": 8.553660982948846e-05, + "loss": 1.2756, + "theoretical_loss": 3.3474613550151653, + "tokens_seen": 3020881920 + }, + { + "epoch": 0.83, + "learning_rate": 8.551654964894684e-05, + "loss": 1.2968, + "theoretical_loss": 3.3474558267936727, + "tokens_seen": 3020947456 + }, + { + "epoch": 0.83, + "learning_rate": 8.549648946840522e-05, + "loss": 1.1994, + "theoretical_loss": 3.3474502987256867, + "tokens_seen": 3021012992 + }, + { + "epoch": 0.83, + "learning_rate": 8.547642928786359e-05, + "loss": 1.2415, + "theoretical_loss": 3.347444770811199, + "tokens_seen": 3021078528 + }, + { + "epoch": 0.83, + "learning_rate": 8.545636910732197e-05, + "loss": 1.3217, + "theoretical_loss": 3.3474392430502027, + "tokens_seen": 3021144064 + }, + { + "epoch": 0.83, + "learning_rate": 8.543630892678034e-05, + "loss": 1.2839, + "theoretical_loss": 3.34743371544269, + "tokens_seen": 3021209600 + }, + { + "epoch": 0.83, + "learning_rate": 8.54162487462387e-05, + "loss": 1.2385, + "theoretical_loss": 3.347428187988653, + "tokens_seen": 3021275136 + }, + { + "epoch": 0.83, + "learning_rate": 8.53961885656971e-05, + "loss": 1.3743, + "theoretical_loss": 3.3474226606880846, + "tokens_seen": 3021340672 + }, + { + "epoch": 0.83, + "learning_rate": 8.537612838515547e-05, + "loss": 1.2457, + "theoretical_loss": 3.3474171335409766, + "tokens_seen": 3021406208 + }, + { + "epoch": 0.83, + "learning_rate": 8.535606820461385e-05, + "loss": 1.2583, + "theoretical_loss": 3.347411606547322, + "tokens_seen": 3021471744 + }, + { + "epoch": 0.83, + "learning_rate": 8.533600802407222e-05, + "loss": 1.1858, + "theoretical_loss": 3.3474060797071123, + "tokens_seen": 3021537280 + }, + { + "epoch": 0.83, + "learning_rate": 8.53159478435306e-05, + "loss": 1.2406, + "theoretical_loss": 3.347400553020341, + "tokens_seen": 3021602816 + }, + { + "epoch": 0.83, + "learning_rate": 8.529588766298897e-05, + "loss": 1.247, + "theoretical_loss": 3.3473950264870003, + "tokens_seen": 3021668352 + }, + { + "epoch": 0.83, + "learning_rate": 8.527582748244735e-05, + "loss": 1.244, + "theoretical_loss": 3.347389500107082, + "tokens_seen": 3021733888 + }, + { + "epoch": 0.83, + "learning_rate": 8.525576730190572e-05, + "loss": 1.3037, + "theoretical_loss": 3.3473839738805795, + "tokens_seen": 3021799424 + }, + { + "epoch": 0.83, + "learning_rate": 8.52357071213641e-05, + "loss": 1.2018, + "theoretical_loss": 3.3473784478074844, + "tokens_seen": 3021864960 + }, + { + "epoch": 0.83, + "learning_rate": 8.521564694082246e-05, + "loss": 1.3602, + "theoretical_loss": 3.347372921887789, + "tokens_seen": 3021930496 + }, + { + "epoch": 0.83, + "learning_rate": 8.519558676028084e-05, + "loss": 1.2057, + "theoretical_loss": 3.347367396121486, + "tokens_seen": 3021996032 + }, + { + "epoch": 0.83, + "learning_rate": 8.517552657973921e-05, + "loss": 1.1948, + "theoretical_loss": 3.347361870508568, + "tokens_seen": 3022061568 + }, + { + "epoch": 0.83, + "learning_rate": 8.515546639919759e-05, + "loss": 1.1635, + "theoretical_loss": 3.3473563450490276, + "tokens_seen": 3022127104 + }, + { + "epoch": 0.83, + "learning_rate": 8.513540621865597e-05, + "loss": 1.3319, + "theoretical_loss": 3.3473508197428563, + "tokens_seen": 3022192640 + }, + { + "epoch": 0.83, + "learning_rate": 8.511534603811434e-05, + "loss": 1.244, + "theoretical_loss": 3.3473452945900477, + "tokens_seen": 3022258176 + }, + { + "epoch": 0.83, + "learning_rate": 8.509528585757272e-05, + "loss": 1.2806, + "theoretical_loss": 3.3473397695905933, + "tokens_seen": 3022323712 + }, + { + "epoch": 0.83, + "learning_rate": 8.50752256770311e-05, + "loss": 1.3208, + "theoretical_loss": 3.347334244744486, + "tokens_seen": 3022389248 + }, + { + "epoch": 0.83, + "learning_rate": 8.505516549648947e-05, + "loss": 1.1542, + "theoretical_loss": 3.347328720051718, + "tokens_seen": 3022454784 + }, + { + "epoch": 0.83, + "learning_rate": 8.503510531594785e-05, + "loss": 1.2812, + "theoretical_loss": 3.3473231955122817, + "tokens_seen": 3022520320 + }, + { + "epoch": 0.83, + "learning_rate": 8.501504513540622e-05, + "loss": 1.1626, + "theoretical_loss": 3.3473176711261696, + "tokens_seen": 3022585856 + }, + { + "epoch": 0.83, + "learning_rate": 8.49949849548646e-05, + "loss": 1.2693, + "theoretical_loss": 3.3473121468933744, + "tokens_seen": 3022651392 + }, + { + "epoch": 0.83, + "learning_rate": 8.497492477432297e-05, + "loss": 1.1774, + "theoretical_loss": 3.347306622813888, + "tokens_seen": 3022716928 + }, + { + "epoch": 0.83, + "learning_rate": 8.495486459378135e-05, + "loss": 1.2772, + "theoretical_loss": 3.347301098887703, + "tokens_seen": 3022782464 + }, + { + "epoch": 0.83, + "learning_rate": 8.493480441323972e-05, + "loss": 1.1391, + "theoretical_loss": 3.347295575114812, + "tokens_seen": 3022848000 + }, + { + "epoch": 0.83, + "learning_rate": 8.49147442326981e-05, + "loss": 1.2397, + "theoretical_loss": 3.3472900514952073, + "tokens_seen": 3022913536 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1098835, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1438367366790771, + "objective/train/theoretical_loss": 3.347284528028881, + "objective/train/tokens_used": 1393438176, + "theoretical_loss": 3.347284528028881, + "tokens_seen": 3022979072 + }, + { + "epoch": 0.83, + "learning_rate": 8.489468405215648e-05, + "loss": 1.2045, + "theoretical_loss": 3.347284528028881, + "tokens_seen": 3022979072 + }, + { + "epoch": 0.83, + "learning_rate": 8.487462387161485e-05, + "loss": 1.1539, + "theoretical_loss": 3.3472790047158267, + "tokens_seen": 3023044608 + }, + { + "epoch": 0.83, + "learning_rate": 8.485456369107323e-05, + "loss": 1.2967, + "theoretical_loss": 3.3472734815560354, + "tokens_seen": 3023110144 + }, + { + "epoch": 0.83, + "learning_rate": 8.48345035105316e-05, + "loss": 1.2734, + "theoretical_loss": 3.3472679585495, + "tokens_seen": 3023175680 + }, + { + "epoch": 0.83, + "learning_rate": 8.481444332998997e-05, + "loss": 1.2409, + "theoretical_loss": 3.3472624356962135, + "tokens_seen": 3023241216 + }, + { + "epoch": 0.83, + "learning_rate": 8.479438314944834e-05, + "loss": 1.207, + "theoretical_loss": 3.3472569129961673, + "tokens_seen": 3023306752 + }, + { + "epoch": 0.83, + "learning_rate": 8.477432296890672e-05, + "loss": 1.3755, + "theoretical_loss": 3.347251390449355, + "tokens_seen": 3023372288 + }, + { + "epoch": 0.83, + "learning_rate": 8.475426278836509e-05, + "loss": 1.1983, + "theoretical_loss": 3.347245868055768, + "tokens_seen": 3023437824 + }, + { + "epoch": 0.83, + "learning_rate": 8.473420260782347e-05, + "loss": 1.1823, + "theoretical_loss": 3.347240345815399, + "tokens_seen": 3023503360 + }, + { + "epoch": 0.83, + "learning_rate": 8.471414242728184e-05, + "loss": 1.2533, + "theoretical_loss": 3.347234823728241, + "tokens_seen": 3023568896 + }, + { + "epoch": 0.83, + "learning_rate": 8.469408224674022e-05, + "loss": 1.2937, + "theoretical_loss": 3.347229301794286, + "tokens_seen": 3023634432 + }, + { + "epoch": 0.83, + "learning_rate": 8.46740220661986e-05, + "loss": 1.1643, + "theoretical_loss": 3.3472237800135263, + "tokens_seen": 3023699968 + }, + { + "epoch": 0.83, + "learning_rate": 8.465396188565697e-05, + "loss": 1.3158, + "theoretical_loss": 3.347218258385954, + "tokens_seen": 3023765504 + }, + { + "epoch": 0.83, + "learning_rate": 8.463390170511536e-05, + "loss": 1.2567, + "theoretical_loss": 3.3472127369115627, + "tokens_seen": 3023831040 + }, + { + "epoch": 0.83, + "learning_rate": 8.461384152457372e-05, + "loss": 1.1237, + "theoretical_loss": 3.347207215590344, + "tokens_seen": 3023896576 + }, + { + "epoch": 0.83, + "learning_rate": 8.45937813440321e-05, + "loss": 1.1584, + "theoretical_loss": 3.34720169442229, + "tokens_seen": 3023962112 + }, + { + "epoch": 0.83, + "learning_rate": 8.457372116349047e-05, + "loss": 1.2243, + "theoretical_loss": 3.347196173407394, + "tokens_seen": 3024027648 + }, + { + "epoch": 0.83, + "learning_rate": 8.455366098294885e-05, + "loss": 1.2768, + "theoretical_loss": 3.347190652545648, + "tokens_seen": 3024093184 + }, + { + "epoch": 0.83, + "learning_rate": 8.453360080240723e-05, + "loss": 1.2058, + "theoretical_loss": 3.347185131837044, + "tokens_seen": 3024158720 + }, + { + "epoch": 0.83, + "learning_rate": 8.45135406218656e-05, + "loss": 1.1982, + "theoretical_loss": 3.3471796112815753, + "tokens_seen": 3024224256 + }, + { + "epoch": 0.83, + "learning_rate": 8.449348044132398e-05, + "loss": 1.2272, + "theoretical_loss": 3.347174090879234, + "tokens_seen": 3024289792 + }, + { + "epoch": 0.83, + "learning_rate": 8.447342026078235e-05, + "loss": 1.1686, + "theoretical_loss": 3.347168570630012, + "tokens_seen": 3024355328 + }, + { + "epoch": 0.83, + "learning_rate": 8.445336008024073e-05, + "loss": 1.3128, + "theoretical_loss": 3.3471630505339025, + "tokens_seen": 3024420864 + }, + { + "epoch": 0.83, + "learning_rate": 8.443329989969909e-05, + "loss": 1.3129, + "theoretical_loss": 3.3471575305908976, + "tokens_seen": 3024486400 + }, + { + "epoch": 0.83, + "learning_rate": 8.441323971915747e-05, + "loss": 1.2675, + "theoretical_loss": 3.3471520108009893, + "tokens_seen": 3024551936 + }, + { + "epoch": 0.83, + "learning_rate": 8.439317953861584e-05, + "loss": 1.366, + "theoretical_loss": 3.347146491164171, + "tokens_seen": 3024617472 + }, + { + "epoch": 0.83, + "learning_rate": 8.437311935807422e-05, + "loss": 1.2788, + "theoretical_loss": 3.3471409716804343, + "tokens_seen": 3024683008 + }, + { + "epoch": 0.83, + "learning_rate": 8.43530591775326e-05, + "loss": 1.1619, + "theoretical_loss": 3.3471354523497725, + "tokens_seen": 3024748544 + }, + { + "epoch": 0.83, + "learning_rate": 8.433299899699097e-05, + "loss": 1.1911, + "theoretical_loss": 3.347129933172177, + "tokens_seen": 3024814080 + }, + { + "epoch": 0.83, + "learning_rate": 8.431293881644935e-05, + "loss": 1.3469, + "theoretical_loss": 3.347124414147641, + "tokens_seen": 3024879616 + }, + { + "epoch": 0.83, + "learning_rate": 8.429287863590772e-05, + "loss": 1.2297, + "theoretical_loss": 3.347118895276157, + "tokens_seen": 3024945152 + }, + { + "epoch": 0.83, + "learning_rate": 8.42728184553661e-05, + "loss": 1.2985, + "theoretical_loss": 3.3471133765577163, + "tokens_seen": 3025010688 + }, + { + "epoch": 0.83, + "learning_rate": 8.425275827482449e-05, + "loss": 1.261, + "theoretical_loss": 3.3471078579923126, + "tokens_seen": 3025076224 + }, + { + "epoch": 0.83, + "learning_rate": 8.423269809428285e-05, + "loss": 1.2284, + "theoretical_loss": 3.347102339579938, + "tokens_seen": 3025141760 + }, + { + "epoch": 0.83, + "learning_rate": 8.421263791374123e-05, + "loss": 1.2061, + "theoretical_loss": 3.3470968213205845, + "tokens_seen": 3025207296 + }, + { + "epoch": 0.83, + "learning_rate": 8.41925777331996e-05, + "loss": 1.2685, + "theoretical_loss": 3.3470913032142455, + "tokens_seen": 3025272832 + }, + { + "epoch": 0.83, + "learning_rate": 8.417251755265798e-05, + "loss": 1.2281, + "theoretical_loss": 3.347085785260912, + "tokens_seen": 3025338368 + }, + { + "epoch": 0.83, + "learning_rate": 8.415245737211635e-05, + "loss": 1.2292, + "theoretical_loss": 3.347080267460578, + "tokens_seen": 3025403904 + }, + { + "epoch": 0.83, + "learning_rate": 8.413239719157473e-05, + "loss": 1.2333, + "theoretical_loss": 3.347074749813235, + "tokens_seen": 3025469440 + }, + { + "epoch": 0.83, + "learning_rate": 8.41123370110331e-05, + "loss": 1.4134, + "theoretical_loss": 3.3470692323188755, + "tokens_seen": 3025534976 + }, + { + "epoch": 0.83, + "learning_rate": 8.409227683049148e-05, + "loss": 1.313, + "theoretical_loss": 3.347063714977492, + "tokens_seen": 3025600512 + }, + { + "epoch": 0.83, + "learning_rate": 8.407221664994986e-05, + "loss": 1.1043, + "theoretical_loss": 3.3470581977890776, + "tokens_seen": 3025666048 + }, + { + "epoch": 0.83, + "learning_rate": 8.405215646940823e-05, + "loss": 1.2497, + "theoretical_loss": 3.347052680753624, + "tokens_seen": 3025731584 + }, + { + "epoch": 0.83, + "learning_rate": 8.40320962888666e-05, + "loss": 1.3059, + "theoretical_loss": 3.3470471638711237, + "tokens_seen": 3025797120 + }, + { + "epoch": 0.83, + "learning_rate": 8.401203610832497e-05, + "loss": 1.2897, + "theoretical_loss": 3.347041647141569, + "tokens_seen": 3025862656 + }, + { + "epoch": 0.83, + "learning_rate": 8.399197592778335e-05, + "loss": 1.1873, + "theoretical_loss": 3.3470361305649527, + "tokens_seen": 3025928192 + }, + { + "epoch": 0.83, + "learning_rate": 8.397191574724172e-05, + "loss": 1.2478, + "theoretical_loss": 3.347030614141268, + "tokens_seen": 3025993728 + }, + { + "epoch": 0.83, + "learning_rate": 8.39518555667001e-05, + "loss": 1.2939, + "theoretical_loss": 3.347025097870506, + "tokens_seen": 3026059264 + }, + { + "epoch": 0.83, + "learning_rate": 8.393179538615847e-05, + "loss": 1.2998, + "theoretical_loss": 3.3470195817526593, + "tokens_seen": 3026124800 + }, + { + "epoch": 0.83, + "learning_rate": 8.391173520561685e-05, + "loss": 1.2668, + "theoretical_loss": 3.347014065787721, + "tokens_seen": 3026190336 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1099986, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3417669534683228, + "objective/train/theoretical_loss": 3.3470085499756834, + "objective/train/tokens_used": 1396714976, + "theoretical_loss": 3.3470085499756834, + "tokens_seen": 3026255872 + }, + { + "epoch": 0.83, + "learning_rate": 8.389167502507524e-05, + "loss": 1.2867, + "theoretical_loss": 3.3470085499756834, + "tokens_seen": 3026255872 + }, + { + "epoch": 0.83, + "learning_rate": 8.387161484453361e-05, + "loss": 1.4052, + "theoretical_loss": 3.347003034316539, + "tokens_seen": 3026321408 + }, + { + "epoch": 0.83, + "learning_rate": 8.385155466399198e-05, + "loss": 1.2444, + "theoretical_loss": 3.3469975188102796, + "tokens_seen": 3026386944 + }, + { + "epoch": 0.83, + "learning_rate": 8.383149448345035e-05, + "loss": 1.3747, + "theoretical_loss": 3.3469920034568985, + "tokens_seen": 3026452480 + }, + { + "epoch": 0.83, + "learning_rate": 8.381143430290873e-05, + "loss": 1.2653, + "theoretical_loss": 3.3469864882563876, + "tokens_seen": 3026518016 + }, + { + "epoch": 0.83, + "learning_rate": 8.37913741223671e-05, + "loss": 1.3196, + "theoretical_loss": 3.3469809732087397, + "tokens_seen": 3026583552 + }, + { + "epoch": 0.83, + "learning_rate": 8.377131394182548e-05, + "loss": 1.302, + "theoretical_loss": 3.346975458313947, + "tokens_seen": 3026649088 + }, + { + "epoch": 0.83, + "learning_rate": 8.375125376128386e-05, + "loss": 1.214, + "theoretical_loss": 3.346969943572002, + "tokens_seen": 3026714624 + }, + { + "epoch": 0.83, + "learning_rate": 8.373119358074223e-05, + "loss": 1.238, + "theoretical_loss": 3.346964428982897, + "tokens_seen": 3026780160 + }, + { + "epoch": 0.83, + "learning_rate": 8.37111334002006e-05, + "loss": 1.2741, + "theoretical_loss": 3.346958914546625, + "tokens_seen": 3026845696 + }, + { + "epoch": 0.83, + "learning_rate": 8.369107321965898e-05, + "loss": 1.2887, + "theoretical_loss": 3.3469534002631782, + "tokens_seen": 3026911232 + }, + { + "epoch": 0.83, + "learning_rate": 8.367101303911736e-05, + "loss": 1.2009, + "theoretical_loss": 3.346947886132549, + "tokens_seen": 3026976768 + }, + { + "epoch": 0.83, + "learning_rate": 8.365095285857572e-05, + "loss": 1.2155, + "theoretical_loss": 3.3469423721547296, + "tokens_seen": 3027042304 + }, + { + "epoch": 0.83, + "learning_rate": 8.36308926780341e-05, + "loss": 1.2345, + "theoretical_loss": 3.3469368583297125, + "tokens_seen": 3027107840 + }, + { + "epoch": 0.83, + "learning_rate": 8.361083249749247e-05, + "loss": 1.3069, + "theoretical_loss": 3.3469313446574906, + "tokens_seen": 3027173376 + }, + { + "epoch": 0.83, + "learning_rate": 8.359077231695085e-05, + "loss": 1.2563, + "theoretical_loss": 3.346925831138056, + "tokens_seen": 3027238912 + }, + { + "epoch": 0.83, + "learning_rate": 8.357071213640922e-05, + "loss": 1.3279, + "theoretical_loss": 3.346920317771401, + "tokens_seen": 3027304448 + }, + { + "epoch": 0.83, + "learning_rate": 8.35506519558676e-05, + "loss": 1.2119, + "theoretical_loss": 3.3469148045575188, + "tokens_seen": 3027369984 + }, + { + "epoch": 0.83, + "learning_rate": 8.353059177532598e-05, + "loss": 1.3831, + "theoretical_loss": 3.3469092914964014, + "tokens_seen": 3027435520 + }, + { + "epoch": 0.83, + "learning_rate": 8.351053159478436e-05, + "loss": 1.2556, + "theoretical_loss": 3.346903778588041, + "tokens_seen": 3027501056 + }, + { + "epoch": 0.83, + "learning_rate": 8.349047141424274e-05, + "loss": 1.2751, + "theoretical_loss": 3.34689826583243, + "tokens_seen": 3027566592 + }, + { + "epoch": 0.83, + "learning_rate": 8.347041123370112e-05, + "loss": 1.2597, + "theoretical_loss": 3.346892753229562, + "tokens_seen": 3027632128 + }, + { + "epoch": 0.83, + "learning_rate": 8.345035105315948e-05, + "loss": 1.2845, + "theoretical_loss": 3.3468872407794277, + "tokens_seen": 3027697664 + }, + { + "epoch": 0.84, + "learning_rate": 8.343029087261785e-05, + "loss": 1.2944, + "theoretical_loss": 3.346881728482021, + "tokens_seen": 3027763200 + }, + { + "epoch": 0.84, + "learning_rate": 8.341023069207623e-05, + "loss": 1.228, + "theoretical_loss": 3.346876216337334, + "tokens_seen": 3027828736 + }, + { + "epoch": 0.84, + "learning_rate": 8.33901705115346e-05, + "loss": 1.3207, + "theoretical_loss": 3.346870704345358, + "tokens_seen": 3027894272 + }, + { + "epoch": 0.84, + "learning_rate": 8.337011033099298e-05, + "loss": 1.2631, + "theoretical_loss": 3.3468651925060877, + "tokens_seen": 3027959808 + }, + { + "epoch": 0.84, + "learning_rate": 8.335005015045136e-05, + "loss": 1.1998, + "theoretical_loss": 3.346859680819514, + "tokens_seen": 3028025344 + }, + { + "epoch": 0.84, + "learning_rate": 8.332998996990973e-05, + "loss": 1.2605, + "theoretical_loss": 3.3468541692856295, + "tokens_seen": 3028090880 + }, + { + "epoch": 0.84, + "learning_rate": 8.330992978936811e-05, + "loss": 1.1669, + "theoretical_loss": 3.3468486579044265, + "tokens_seen": 3028156416 + }, + { + "epoch": 0.84, + "learning_rate": 8.328986960882648e-05, + "loss": 1.2171, + "theoretical_loss": 3.3468431466758983, + "tokens_seen": 3028221952 + }, + { + "epoch": 0.84, + "learning_rate": 8.326980942828486e-05, + "loss": 1.2282, + "theoretical_loss": 3.346837635600037, + "tokens_seen": 3028287488 + }, + { + "epoch": 0.84, + "learning_rate": 8.324974924774322e-05, + "loss": 1.138, + "theoretical_loss": 3.3468321246768347, + "tokens_seen": 3028353024 + }, + { + "epoch": 0.84, + "learning_rate": 8.32296890672016e-05, + "loss": 1.2675, + "theoretical_loss": 3.346826613906284, + "tokens_seen": 3028418560 + }, + { + "epoch": 0.84, + "learning_rate": 8.320962888665997e-05, + "loss": 1.2174, + "theoretical_loss": 3.346821103288378, + "tokens_seen": 3028484096 + }, + { + "epoch": 0.84, + "learning_rate": 8.318956870611835e-05, + "loss": 1.1805, + "theoretical_loss": 3.3468155928231083, + "tokens_seen": 3028549632 + }, + { + "epoch": 0.84, + "learning_rate": 8.316950852557673e-05, + "loss": 1.1702, + "theoretical_loss": 3.346810082510468, + "tokens_seen": 3028615168 + }, + { + "epoch": 0.84, + "learning_rate": 8.31494483450351e-05, + "loss": 1.2969, + "theoretical_loss": 3.346804572350449, + "tokens_seen": 3028680704 + }, + { + "epoch": 0.84, + "learning_rate": 8.312938816449349e-05, + "loss": 1.2471, + "theoretical_loss": 3.3467990623430444, + "tokens_seen": 3028746240 + }, + { + "epoch": 0.84, + "learning_rate": 8.310932798395187e-05, + "loss": 1.3173, + "theoretical_loss": 3.346793552488246, + "tokens_seen": 3028811776 + }, + { + "epoch": 0.84, + "learning_rate": 8.308926780341024e-05, + "loss": 1.3538, + "theoretical_loss": 3.346788042786047, + "tokens_seen": 3028877312 + }, + { + "epoch": 0.84, + "learning_rate": 8.30692076228686e-05, + "loss": 1.2902, + "theoretical_loss": 3.346782533236439, + "tokens_seen": 3028942848 + }, + { + "epoch": 0.84, + "learning_rate": 8.304914744232698e-05, + "loss": 1.3039, + "theoretical_loss": 3.3467770238394157, + "tokens_seen": 3029008384 + }, + { + "epoch": 0.84, + "learning_rate": 8.302908726178536e-05, + "loss": 1.312, + "theoretical_loss": 3.3467715145949684, + "tokens_seen": 3029073920 + }, + { + "epoch": 0.84, + "learning_rate": 8.300902708124373e-05, + "loss": 1.2209, + "theoretical_loss": 3.34676600550309, + "tokens_seen": 3029139456 + }, + { + "epoch": 0.84, + "learning_rate": 8.298896690070211e-05, + "loss": 1.259, + "theoretical_loss": 3.346760496563773, + "tokens_seen": 3029204992 + }, + { + "epoch": 0.84, + "learning_rate": 8.296890672016048e-05, + "loss": 1.2818, + "theoretical_loss": 3.3467549877770097, + "tokens_seen": 3029270528 + }, + { + "epoch": 0.84, + "learning_rate": 8.294884653961886e-05, + "loss": 1.2281, + "theoretical_loss": 3.346749479142793, + "tokens_seen": 3029336064 + }, + { + "epoch": 0.84, + "learning_rate": 8.292878635907724e-05, + "loss": 1.2505, + "theoretical_loss": 3.346743970661115, + "tokens_seen": 3029401600 + }, + { + "epoch": 0.84, + "learning_rate": 8.290872617853561e-05, + "loss": 1.2435, + "theoretical_loss": 3.3467384623319685, + "tokens_seen": 3029467136 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1101281, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2901899814605713, + "objective/train/theoretical_loss": 3.3467329541553457, + "objective/train/tokens_used": 1399991776, + "theoretical_loss": 3.3467329541553457, + "tokens_seen": 3029532672 + }, + { + "epoch": 0.84, + "learning_rate": 8.288866599799399e-05, + "loss": 1.3329, + "theoretical_loss": 3.3467329541553457, + "tokens_seen": 3029532672 + }, + { + "epoch": 0.84, + "learning_rate": 8.286860581745235e-05, + "loss": 1.2278, + "theoretical_loss": 3.346727446131239, + "tokens_seen": 3029598208 + }, + { + "epoch": 0.84, + "learning_rate": 8.284854563691073e-05, + "loss": 1.2291, + "theoretical_loss": 3.346721938259641, + "tokens_seen": 3029663744 + }, + { + "epoch": 0.84, + "learning_rate": 8.28284854563691e-05, + "loss": 1.2509, + "theoretical_loss": 3.3467164305405444, + "tokens_seen": 3029729280 + }, + { + "epoch": 0.84, + "learning_rate": 8.280842527582748e-05, + "loss": 1.1136, + "theoretical_loss": 3.346710922973941, + "tokens_seen": 3029794816 + }, + { + "epoch": 0.84, + "learning_rate": 8.278836509528585e-05, + "loss": 1.3434, + "theoretical_loss": 3.3467054155598244, + "tokens_seen": 3029860352 + }, + { + "epoch": 0.84, + "learning_rate": 8.276830491474424e-05, + "loss": 1.2186, + "theoretical_loss": 3.346699908298186, + "tokens_seen": 3029925888 + }, + { + "epoch": 0.84, + "learning_rate": 8.274824473420262e-05, + "loss": 1.272, + "theoretical_loss": 3.346694401189019, + "tokens_seen": 3029991424 + }, + { + "epoch": 0.84, + "learning_rate": 8.2728184553661e-05, + "loss": 1.2801, + "theoretical_loss": 3.346688894232315, + "tokens_seen": 3030056960 + }, + { + "epoch": 0.84, + "learning_rate": 8.270812437311937e-05, + "loss": 1.1944, + "theoretical_loss": 3.346683387428068, + "tokens_seen": 3030122496 + }, + { + "epoch": 0.84, + "learning_rate": 8.268806419257775e-05, + "loss": 1.245, + "theoretical_loss": 3.3466778807762694, + "tokens_seen": 3030188032 + }, + { + "epoch": 0.84, + "learning_rate": 8.266800401203611e-05, + "loss": 1.277, + "theoretical_loss": 3.346672374276911, + "tokens_seen": 3030253568 + }, + { + "epoch": 0.84, + "learning_rate": 8.264794383149448e-05, + "loss": 1.2613, + "theoretical_loss": 3.346666867929987, + "tokens_seen": 3030319104 + }, + { + "epoch": 0.84, + "learning_rate": 8.262788365095286e-05, + "loss": 1.3421, + "theoretical_loss": 3.346661361735489, + "tokens_seen": 3030384640 + }, + { + "epoch": 0.84, + "learning_rate": 8.260782347041123e-05, + "loss": 1.2026, + "theoretical_loss": 3.3466558556934087, + "tokens_seen": 3030450176 + }, + { + "epoch": 0.84, + "learning_rate": 8.258776328986961e-05, + "loss": 1.1613, + "theoretical_loss": 3.34665034980374, + "tokens_seen": 3030515712 + }, + { + "epoch": 0.84, + "learning_rate": 8.256770310932799e-05, + "loss": 1.2367, + "theoretical_loss": 3.346644844066475, + "tokens_seen": 3030581248 + }, + { + "epoch": 0.84, + "learning_rate": 8.254764292878636e-05, + "loss": 1.3307, + "theoretical_loss": 3.346639338481605, + "tokens_seen": 3030646784 + }, + { + "epoch": 0.84, + "learning_rate": 8.252758274824474e-05, + "loss": 1.2711, + "theoretical_loss": 3.3466338330491245, + "tokens_seen": 3030712320 + }, + { + "epoch": 0.84, + "learning_rate": 8.250752256770311e-05, + "loss": 1.3061, + "theoretical_loss": 3.3466283277690243, + "tokens_seen": 3030777856 + }, + { + "epoch": 0.84, + "learning_rate": 8.248746238716148e-05, + "loss": 1.2815, + "theoretical_loss": 3.3466228226412977, + "tokens_seen": 3030843392 + }, + { + "epoch": 0.84, + "learning_rate": 8.246740220661985e-05, + "loss": 1.3228, + "theoretical_loss": 3.3466173176659373, + "tokens_seen": 3030908928 + }, + { + "epoch": 0.84, + "learning_rate": 8.244734202607823e-05, + "loss": 1.2693, + "theoretical_loss": 3.346611812842935, + "tokens_seen": 3030974464 + }, + { + "epoch": 0.84, + "learning_rate": 8.24272818455366e-05, + "loss": 1.2147, + "theoretical_loss": 3.3466063081722837, + "tokens_seen": 3031040000 + }, + { + "epoch": 0.84, + "learning_rate": 8.240722166499498e-05, + "loss": 1.2751, + "theoretical_loss": 3.3466008036539754, + "tokens_seen": 3031105536 + }, + { + "epoch": 0.84, + "learning_rate": 8.238716148445337e-05, + "loss": 1.2167, + "theoretical_loss": 3.3465952992880035, + "tokens_seen": 3031171072 + }, + { + "epoch": 0.84, + "learning_rate": 8.236710130391174e-05, + "loss": 1.2379, + "theoretical_loss": 3.3465897950743595, + "tokens_seen": 3031236608 + }, + { + "epoch": 0.84, + "learning_rate": 8.234704112337012e-05, + "loss": 1.287, + "theoretical_loss": 3.3465842910130363, + "tokens_seen": 3031302144 + }, + { + "epoch": 0.84, + "learning_rate": 8.23269809428285e-05, + "loss": 1.2713, + "theoretical_loss": 3.346578787104027, + "tokens_seen": 3031367680 + }, + { + "epoch": 0.84, + "learning_rate": 8.230692076228687e-05, + "loss": 1.4265, + "theoretical_loss": 3.3465732833473227, + "tokens_seen": 3031433216 + }, + { + "epoch": 0.84, + "learning_rate": 8.228686058174523e-05, + "loss": 1.2834, + "theoretical_loss": 3.3465677797429176, + "tokens_seen": 3031498752 + }, + { + "epoch": 0.84, + "learning_rate": 8.226680040120361e-05, + "loss": 1.2391, + "theoretical_loss": 3.346562276290803, + "tokens_seen": 3031564288 + }, + { + "epoch": 0.84, + "learning_rate": 8.224674022066199e-05, + "loss": 1.2795, + "theoretical_loss": 3.3465567729909713, + "tokens_seen": 3031629824 + }, + { + "epoch": 0.84, + "learning_rate": 8.222668004012036e-05, + "loss": 1.2308, + "theoretical_loss": 3.346551269843416, + "tokens_seen": 3031695360 + }, + { + "epoch": 0.84, + "learning_rate": 8.220661985957874e-05, + "loss": 1.3756, + "theoretical_loss": 3.346545766848129, + "tokens_seen": 3031760896 + }, + { + "epoch": 0.84, + "learning_rate": 8.218655967903711e-05, + "loss": 1.3254, + "theoretical_loss": 3.3465402640051023, + "tokens_seen": 3031826432 + }, + { + "epoch": 0.84, + "learning_rate": 8.216649949849549e-05, + "loss": 1.1679, + "theoretical_loss": 3.346534761314329, + "tokens_seen": 3031891968 + }, + { + "epoch": 0.84, + "learning_rate": 8.214643931795386e-05, + "loss": 1.3625, + "theoretical_loss": 3.3465292587758015, + "tokens_seen": 3031957504 + }, + { + "epoch": 0.84, + "learning_rate": 8.212637913741224e-05, + "loss": 1.3352, + "theoretical_loss": 3.3465237563895123, + "tokens_seen": 3032023040 + }, + { + "epoch": 0.84, + "learning_rate": 8.210631895687062e-05, + "loss": 1.3267, + "theoretical_loss": 3.346518254155454, + "tokens_seen": 3032088576 + }, + { + "epoch": 0.84, + "learning_rate": 8.208625877632898e-05, + "loss": 1.3399, + "theoretical_loss": 3.346512752073619, + "tokens_seen": 3032154112 + }, + { + "epoch": 0.84, + "learning_rate": 8.206619859578735e-05, + "loss": 1.1764, + "theoretical_loss": 3.3465072501439996, + "tokens_seen": 3032219648 + }, + { + "epoch": 0.84, + "learning_rate": 8.204613841524573e-05, + "loss": 1.2189, + "theoretical_loss": 3.3465017483665886, + "tokens_seen": 3032285184 + }, + { + "epoch": 0.84, + "learning_rate": 8.20260782347041e-05, + "loss": 1.3206, + "theoretical_loss": 3.3464962467413786, + "tokens_seen": 3032350720 + }, + { + "epoch": 0.84, + "learning_rate": 8.20060180541625e-05, + "loss": 1.3172, + "theoretical_loss": 3.3464907452683614, + "tokens_seen": 3032416256 + }, + { + "epoch": 0.84, + "learning_rate": 8.198595787362087e-05, + "loss": 1.2168, + "theoretical_loss": 3.34648524394753, + "tokens_seen": 3032481792 + }, + { + "epoch": 0.84, + "learning_rate": 8.196589769307925e-05, + "loss": 1.2237, + "theoretical_loss": 3.3464797427788775, + "tokens_seen": 3032547328 + }, + { + "epoch": 0.84, + "learning_rate": 8.194583751253762e-05, + "loss": 1.2582, + "theoretical_loss": 3.346474241762395, + "tokens_seen": 3032612864 + }, + { + "epoch": 0.84, + "learning_rate": 8.1925777331996e-05, + "loss": 1.345, + "theoretical_loss": 3.3464687408980764, + "tokens_seen": 3032678400 + }, + { + "epoch": 0.84, + "learning_rate": 8.190571715145437e-05, + "loss": 1.2798, + "theoretical_loss": 3.3464632401859133, + "tokens_seen": 3032743936 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1102459, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2690143585205078, + "objective/train/theoretical_loss": 3.3464577396258983, + "objective/train/tokens_used": 1403268576, + "theoretical_loss": 3.3464577396258983, + "tokens_seen": 3032809472 + }, + { + "epoch": 0.84, + "learning_rate": 8.188565697091274e-05, + "loss": 1.2467, + "theoretical_loss": 3.3464577396258983, + "tokens_seen": 3032809472 + }, + { + "epoch": 0.84, + "learning_rate": 8.186559679037111e-05, + "loss": 1.1818, + "theoretical_loss": 3.3464522392180247, + "tokens_seen": 3032875008 + }, + { + "epoch": 0.84, + "learning_rate": 8.184553660982949e-05, + "loss": 1.2801, + "theoretical_loss": 3.346446738962284, + "tokens_seen": 3032940544 + }, + { + "epoch": 0.84, + "learning_rate": 8.182547642928786e-05, + "loss": 1.3231, + "theoretical_loss": 3.3464412388586693, + "tokens_seen": 3033006080 + }, + { + "epoch": 0.84, + "learning_rate": 8.180541624874624e-05, + "loss": 1.1731, + "theoretical_loss": 3.3464357389071724, + "tokens_seen": 3033071616 + }, + { + "epoch": 0.84, + "learning_rate": 8.178535606820462e-05, + "loss": 1.2901, + "theoretical_loss": 3.3464302391077867, + "tokens_seen": 3033137152 + }, + { + "epoch": 0.84, + "learning_rate": 8.176529588766299e-05, + "loss": 1.1553, + "theoretical_loss": 3.3464247394605042, + "tokens_seen": 3033202688 + }, + { + "epoch": 0.84, + "learning_rate": 8.174523570712137e-05, + "loss": 1.2106, + "theoretical_loss": 3.3464192399653174, + "tokens_seen": 3033268224 + }, + { + "epoch": 0.84, + "learning_rate": 8.172517552657974e-05, + "loss": 1.217, + "theoretical_loss": 3.346413740622219, + "tokens_seen": 3033333760 + }, + { + "epoch": 0.84, + "learning_rate": 8.17051153460381e-05, + "loss": 1.2563, + "theoretical_loss": 3.346408241431202, + "tokens_seen": 3033399296 + }, + { + "epoch": 0.84, + "learning_rate": 8.168505516549648e-05, + "loss": 1.2503, + "theoretical_loss": 3.3464027423922578, + "tokens_seen": 3033464832 + }, + { + "epoch": 0.84, + "learning_rate": 8.166499498495486e-05, + "loss": 1.2143, + "theoretical_loss": 3.3463972435053795, + "tokens_seen": 3033530368 + }, + { + "epoch": 0.84, + "learning_rate": 8.164493480441323e-05, + "loss": 1.2482, + "theoretical_loss": 3.3463917447705596, + "tokens_seen": 3033595904 + }, + { + "epoch": 0.84, + "learning_rate": 8.162487462387162e-05, + "loss": 1.3331, + "theoretical_loss": 3.346386246187791, + "tokens_seen": 3033661440 + }, + { + "epoch": 0.84, + "learning_rate": 8.160481444333e-05, + "loss": 1.2306, + "theoretical_loss": 3.346380747757065, + "tokens_seen": 3033726976 + }, + { + "epoch": 0.84, + "learning_rate": 8.158475426278837e-05, + "loss": 1.3014, + "theoretical_loss": 3.346375249478376, + "tokens_seen": 3033792512 + }, + { + "epoch": 0.84, + "learning_rate": 8.156469408224675e-05, + "loss": 1.2728, + "theoretical_loss": 3.3463697513517143, + "tokens_seen": 3033858048 + }, + { + "epoch": 0.84, + "learning_rate": 8.154463390170512e-05, + "loss": 1.2889, + "theoretical_loss": 3.346364253377074, + "tokens_seen": 3033923584 + }, + { + "epoch": 0.84, + "learning_rate": 8.15245737211635e-05, + "loss": 1.2392, + "theoretical_loss": 3.346358755554447, + "tokens_seen": 3033989120 + }, + { + "epoch": 0.84, + "learning_rate": 8.150451354062186e-05, + "loss": 1.2126, + "theoretical_loss": 3.346353257883826, + "tokens_seen": 3034054656 + }, + { + "epoch": 0.84, + "learning_rate": 8.148445336008024e-05, + "loss": 1.4059, + "theoretical_loss": 3.3463477603652034, + "tokens_seen": 3034120192 + }, + { + "epoch": 0.84, + "learning_rate": 8.146439317953861e-05, + "loss": 1.3293, + "theoretical_loss": 3.346342262998572, + "tokens_seen": 3034185728 + }, + { + "epoch": 0.84, + "learning_rate": 8.144433299899699e-05, + "loss": 1.2444, + "theoretical_loss": 3.3463367657839242, + "tokens_seen": 3034251264 + }, + { + "epoch": 0.84, + "learning_rate": 8.142427281845537e-05, + "loss": 1.2232, + "theoretical_loss": 3.3463312687212525, + "tokens_seen": 3034316800 + }, + { + "epoch": 0.84, + "learning_rate": 8.140421263791374e-05, + "loss": 1.2317, + "theoretical_loss": 3.3463257718105486, + "tokens_seen": 3034382336 + }, + { + "epoch": 0.84, + "learning_rate": 8.138415245737212e-05, + "loss": 1.3124, + "theoretical_loss": 3.3463202750518066, + "tokens_seen": 3034447872 + }, + { + "epoch": 0.84, + "learning_rate": 8.13640922768305e-05, + "loss": 1.3169, + "theoretical_loss": 3.3463147784450173, + "tokens_seen": 3034513408 + }, + { + "epoch": 0.84, + "learning_rate": 8.134403209628887e-05, + "loss": 1.2331, + "theoretical_loss": 3.346309281990175, + "tokens_seen": 3034578944 + }, + { + "epoch": 0.84, + "learning_rate": 8.132397191574724e-05, + "loss": 1.1598, + "theoretical_loss": 3.3463037856872706, + "tokens_seen": 3034644480 + }, + { + "epoch": 0.84, + "learning_rate": 8.130391173520561e-05, + "loss": 1.3199, + "theoretical_loss": 3.3462982895362976, + "tokens_seen": 3034710016 + }, + { + "epoch": 0.84, + "learning_rate": 8.128385155466398e-05, + "loss": 1.331, + "theoretical_loss": 3.3462927935372484, + "tokens_seen": 3034775552 + }, + { + "epoch": 0.84, + "learning_rate": 8.126379137412237e-05, + "loss": 1.1731, + "theoretical_loss": 3.346287297690115, + "tokens_seen": 3034841088 + }, + { + "epoch": 0.84, + "learning_rate": 8.124373119358075e-05, + "loss": 1.2192, + "theoretical_loss": 3.3462818019948903, + "tokens_seen": 3034906624 + }, + { + "epoch": 0.84, + "learning_rate": 8.122367101303912e-05, + "loss": 1.2987, + "theoretical_loss": 3.3462763064515673, + "tokens_seen": 3034972160 + }, + { + "epoch": 0.84, + "learning_rate": 8.12036108324975e-05, + "loss": 1.2632, + "theoretical_loss": 3.3462708110601374, + "tokens_seen": 3035037696 + }, + { + "epoch": 0.84, + "learning_rate": 8.118355065195588e-05, + "loss": 1.247, + "theoretical_loss": 3.3462653158205944, + "tokens_seen": 3035103232 + }, + { + "epoch": 0.84, + "learning_rate": 8.116349047141425e-05, + "loss": 1.2606, + "theoretical_loss": 3.3462598207329295, + "tokens_seen": 3035168768 + }, + { + "epoch": 0.84, + "learning_rate": 8.114343029087263e-05, + "loss": 1.368, + "theoretical_loss": 3.346254325797136, + "tokens_seen": 3035234304 + }, + { + "epoch": 0.84, + "learning_rate": 8.1123370110331e-05, + "loss": 1.2334, + "theoretical_loss": 3.346248831013207, + "tokens_seen": 3035299840 + }, + { + "epoch": 0.84, + "learning_rate": 8.110330992978937e-05, + "loss": 1.2119, + "theoretical_loss": 3.3462433363811335, + "tokens_seen": 3035365376 + }, + { + "epoch": 0.84, + "learning_rate": 8.108324974924774e-05, + "loss": 1.3174, + "theoretical_loss": 3.3462378419009093, + "tokens_seen": 3035430912 + }, + { + "epoch": 0.84, + "learning_rate": 8.106318956870612e-05, + "loss": 1.3344, + "theoretical_loss": 3.346232347572526, + "tokens_seen": 3035496448 + }, + { + "epoch": 0.84, + "learning_rate": 8.104312938816449e-05, + "loss": 1.2341, + "theoretical_loss": 3.346226853395977, + "tokens_seen": 3035561984 + }, + { + "epoch": 0.84, + "learning_rate": 8.102306920762287e-05, + "loss": 1.2935, + "theoretical_loss": 3.3462213593712544, + "tokens_seen": 3035627520 + }, + { + "epoch": 0.84, + "learning_rate": 8.100300902708124e-05, + "loss": 1.2427, + "theoretical_loss": 3.346215865498351, + "tokens_seen": 3035693056 + }, + { + "epoch": 0.84, + "learning_rate": 8.098294884653962e-05, + "loss": 1.3289, + "theoretical_loss": 3.346210371777259, + "tokens_seen": 3035758592 + }, + { + "epoch": 0.84, + "learning_rate": 8.0962888665998e-05, + "loss": 1.2463, + "theoretical_loss": 3.346204878207971, + "tokens_seen": 3035824128 + }, + { + "epoch": 0.84, + "learning_rate": 8.094282848545637e-05, + "loss": 1.3251, + "theoretical_loss": 3.3461993847904794, + "tokens_seen": 3035889664 + }, + { + "epoch": 0.84, + "learning_rate": 8.092276830491473e-05, + "loss": 1.3799, + "theoretical_loss": 3.346193891524777, + "tokens_seen": 3035955200 + }, + { + "epoch": 0.84, + "learning_rate": 8.090270812437311e-05, + "loss": 1.272, + "theoretical_loss": 3.346188398410856, + "tokens_seen": 3036020736 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1103322, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2968523502349854, + "objective/train/theoretical_loss": 3.346182905448709, + "objective/train/tokens_used": 1406545376, + "theoretical_loss": 3.346182905448709, + "tokens_seen": 3036086272 + }, + { + "epoch": 0.84, + "learning_rate": 8.08826479438315e-05, + "loss": 1.3275, + "theoretical_loss": 3.346182905448709, + "tokens_seen": 3036086272 + }, + { + "epoch": 0.84, + "learning_rate": 8.086258776328987e-05, + "loss": 1.352, + "theoretical_loss": 3.3461774126383292, + "tokens_seen": 3036151808 + }, + { + "epoch": 0.84, + "learning_rate": 8.084252758274825e-05, + "loss": 1.3025, + "theoretical_loss": 3.3461719199797084, + "tokens_seen": 3036217344 + }, + { + "epoch": 0.84, + "learning_rate": 8.082246740220663e-05, + "loss": 1.2462, + "theoretical_loss": 3.346166427472839, + "tokens_seen": 3036282880 + }, + { + "epoch": 0.84, + "learning_rate": 8.0802407221665e-05, + "loss": 1.1911, + "theoretical_loss": 3.346160935117714, + "tokens_seen": 3036348416 + }, + { + "epoch": 0.84, + "learning_rate": 8.078234704112338e-05, + "loss": 1.247, + "theoretical_loss": 3.346155442914326, + "tokens_seen": 3036413952 + }, + { + "epoch": 0.84, + "learning_rate": 8.076228686058175e-05, + "loss": 1.3837, + "theoretical_loss": 3.3461499508626673, + "tokens_seen": 3036479488 + }, + { + "epoch": 0.84, + "learning_rate": 8.074222668004013e-05, + "loss": 1.3276, + "theoretical_loss": 3.3461444589627307, + "tokens_seen": 3036545024 + }, + { + "epoch": 0.84, + "learning_rate": 8.072216649949849e-05, + "loss": 1.2771, + "theoretical_loss": 3.3461389672145083, + "tokens_seen": 3036610560 + }, + { + "epoch": 0.84, + "learning_rate": 8.070210631895687e-05, + "loss": 1.3017, + "theoretical_loss": 3.3461334756179926, + "tokens_seen": 3036676096 + }, + { + "epoch": 0.84, + "learning_rate": 8.068204613841524e-05, + "loss": 1.3708, + "theoretical_loss": 3.3461279841731764, + "tokens_seen": 3036741632 + }, + { + "epoch": 0.84, + "learning_rate": 8.066198595787362e-05, + "loss": 1.2164, + "theoretical_loss": 3.346122492880052, + "tokens_seen": 3036807168 + }, + { + "epoch": 0.84, + "learning_rate": 8.0641925777332e-05, + "loss": 1.3123, + "theoretical_loss": 3.346117001738613, + "tokens_seen": 3036872704 + }, + { + "epoch": 0.84, + "learning_rate": 8.062186559679037e-05, + "loss": 1.204, + "theoretical_loss": 3.3461115107488504, + "tokens_seen": 3036938240 + }, + { + "epoch": 0.84, + "learning_rate": 8.060180541624875e-05, + "loss": 1.1967, + "theoretical_loss": 3.3461060199107573, + "tokens_seen": 3037003776 + }, + { + "epoch": 0.84, + "learning_rate": 8.058174523570712e-05, + "loss": 1.2383, + "theoretical_loss": 3.3461005292243264, + "tokens_seen": 3037069312 + }, + { + "epoch": 0.84, + "learning_rate": 8.05616850551655e-05, + "loss": 1.3572, + "theoretical_loss": 3.3460950386895507, + "tokens_seen": 3037134848 + }, + { + "epoch": 0.84, + "learning_rate": 8.054162487462387e-05, + "loss": 1.3348, + "theoretical_loss": 3.3460895483064217, + "tokens_seen": 3037200384 + }, + { + "epoch": 0.84, + "learning_rate": 8.052156469408224e-05, + "loss": 1.2998, + "theoretical_loss": 3.3460840580749327, + "tokens_seen": 3037265920 + }, + { + "epoch": 0.84, + "learning_rate": 8.050150451354063e-05, + "loss": 1.2046, + "theoretical_loss": 3.346078567995076, + "tokens_seen": 3037331456 + }, + { + "epoch": 0.84, + "learning_rate": 8.0481444332999e-05, + "loss": 1.3022, + "theoretical_loss": 3.346073078066844, + "tokens_seen": 3037396992 + }, + { + "epoch": 0.84, + "learning_rate": 8.046138415245738e-05, + "loss": 1.2237, + "theoretical_loss": 3.3460675882902295, + "tokens_seen": 3037462528 + }, + { + "epoch": 0.84, + "learning_rate": 8.044132397191575e-05, + "loss": 1.2374, + "theoretical_loss": 3.346062098665225, + "tokens_seen": 3037528064 + }, + { + "epoch": 0.84, + "learning_rate": 8.042126379137413e-05, + "loss": 1.264, + "theoretical_loss": 3.346056609191823, + "tokens_seen": 3037593600 + }, + { + "epoch": 0.84, + "learning_rate": 8.04012036108325e-05, + "loss": 1.3168, + "theoretical_loss": 3.3460511198700162, + "tokens_seen": 3037659136 + }, + { + "epoch": 0.84, + "learning_rate": 8.038114343029088e-05, + "loss": 1.3568, + "theoretical_loss": 3.3460456306997965, + "tokens_seen": 3037724672 + }, + { + "epoch": 0.84, + "learning_rate": 8.036108324974926e-05, + "loss": 1.2589, + "theoretical_loss": 3.346040141681157, + "tokens_seen": 3037790208 + }, + { + "epoch": 0.84, + "learning_rate": 8.034102306920763e-05, + "loss": 1.2909, + "theoretical_loss": 3.3460346528140903, + "tokens_seen": 3037855744 + }, + { + "epoch": 0.84, + "learning_rate": 8.0320962888666e-05, + "loss": 1.2367, + "theoretical_loss": 3.3460291640985886, + "tokens_seen": 3037921280 + }, + { + "epoch": 0.84, + "learning_rate": 8.030090270812437e-05, + "loss": 1.2584, + "theoretical_loss": 3.346023675534645, + "tokens_seen": 3037986816 + }, + { + "epoch": 0.84, + "learning_rate": 8.028084252758275e-05, + "loss": 1.2641, + "theoretical_loss": 3.3460181871222514, + "tokens_seen": 3038052352 + }, + { + "epoch": 0.84, + "learning_rate": 8.026078234704112e-05, + "loss": 1.217, + "theoretical_loss": 3.3460126988614007, + "tokens_seen": 3038117888 + }, + { + "epoch": 0.84, + "learning_rate": 8.02407221664995e-05, + "loss": 1.3233, + "theoretical_loss": 3.3460072107520853, + "tokens_seen": 3038183424 + }, + { + "epoch": 0.84, + "learning_rate": 8.022066198595787e-05, + "loss": 1.1944, + "theoretical_loss": 3.346001722794298, + "tokens_seen": 3038248960 + }, + { + "epoch": 0.84, + "learning_rate": 8.020060180541625e-05, + "loss": 1.3055, + "theoretical_loss": 3.345996234988031, + "tokens_seen": 3038314496 + }, + { + "epoch": 0.84, + "learning_rate": 8.018054162487462e-05, + "loss": 1.2639, + "theoretical_loss": 3.345990747333277, + "tokens_seen": 3038380032 + }, + { + "epoch": 0.84, + "learning_rate": 8.0160481444333e-05, + "loss": 1.1755, + "theoretical_loss": 3.3459852598300284, + "tokens_seen": 3038445568 + }, + { + "epoch": 0.84, + "learning_rate": 8.014042126379138e-05, + "loss": 1.1279, + "theoretical_loss": 3.3459797724782785, + "tokens_seen": 3038511104 + }, + { + "epoch": 0.84, + "learning_rate": 8.012036108324975e-05, + "loss": 1.2043, + "theoretical_loss": 3.3459742852780185, + "tokens_seen": 3038576640 + }, + { + "epoch": 0.84, + "learning_rate": 8.010030090270813e-05, + "loss": 1.3112, + "theoretical_loss": 3.3459687982292423, + "tokens_seen": 3038642176 + }, + { + "epoch": 0.84, + "learning_rate": 8.00802407221665e-05, + "loss": 1.1954, + "theoretical_loss": 3.345963311331942, + "tokens_seen": 3038707712 + }, + { + "epoch": 0.84, + "learning_rate": 8.006018054162488e-05, + "loss": 1.1452, + "theoretical_loss": 3.3459578245861095, + "tokens_seen": 3038773248 + }, + { + "epoch": 0.84, + "learning_rate": 8.004012036108325e-05, + "loss": 1.3296, + "theoretical_loss": 3.345952337991738, + "tokens_seen": 3038838784 + }, + { + "epoch": 0.84, + "learning_rate": 8.002006018054163e-05, + "loss": 1.2302, + "theoretical_loss": 3.34594685154882, + "tokens_seen": 3038904320 + }, + { + "epoch": 0.84, + "learning_rate": 8e-05, + "loss": 1.2714, + "theoretical_loss": 3.345941365257348, + "tokens_seen": 3038969856 + }, + { + "epoch": 0.84, + "learning_rate": 7.997993981945838e-05, + "loss": 1.1288, + "theoretical_loss": 3.3459358791173144, + "tokens_seen": 3039035392 + }, + { + "epoch": 0.84, + "learning_rate": 7.995987963891676e-05, + "loss": 1.2883, + "theoretical_loss": 3.345930393128712, + "tokens_seen": 3039100928 + }, + { + "epoch": 0.84, + "learning_rate": 7.993981945837512e-05, + "loss": 1.3435, + "theoretical_loss": 3.3459249072915336, + "tokens_seen": 3039166464 + }, + { + "epoch": 0.84, + "learning_rate": 7.99197592778335e-05, + "loss": 1.2817, + "theoretical_loss": 3.345919421605771, + "tokens_seen": 3039232000 + }, + { + "epoch": 0.84, + "learning_rate": 7.989969909729187e-05, + "loss": 1.2168, + "theoretical_loss": 3.345913936071417, + "tokens_seen": 3039297536 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1104383, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1367526054382324, + "objective/train/theoretical_loss": 3.3459084506884644, + "objective/train/tokens_used": 1409822176, + "theoretical_loss": 3.3459084506884644, + "tokens_seen": 3039363072 + }, + { + "epoch": 0.84, + "learning_rate": 7.987963891675025e-05, + "loss": 1.2062, + "theoretical_loss": 3.3459084506884644, + "tokens_seen": 3039363072 + }, + { + "epoch": 0.84, + "learning_rate": 7.985957873620862e-05, + "loss": 1.2118, + "theoretical_loss": 3.345902965456906, + "tokens_seen": 3039428608 + }, + { + "epoch": 0.84, + "learning_rate": 7.9839518555667e-05, + "loss": 1.2667, + "theoretical_loss": 3.345897480376734, + "tokens_seen": 3039494144 + }, + { + "epoch": 0.84, + "learning_rate": 7.981945837512538e-05, + "loss": 1.3485, + "theoretical_loss": 3.3458919954479405, + "tokens_seen": 3039559680 + }, + { + "epoch": 0.84, + "learning_rate": 7.979939819458375e-05, + "loss": 1.2898, + "theoretical_loss": 3.345886510670519, + "tokens_seen": 3039625216 + }, + { + "epoch": 0.84, + "learning_rate": 7.977933801404213e-05, + "loss": 1.2881, + "theoretical_loss": 3.345881026044461, + "tokens_seen": 3039690752 + }, + { + "epoch": 0.84, + "learning_rate": 7.975927783350052e-05, + "loss": 1.2018, + "theoretical_loss": 3.34587554156976, + "tokens_seen": 3039756288 + }, + { + "epoch": 0.84, + "learning_rate": 7.973921765295888e-05, + "loss": 1.299, + "theoretical_loss": 3.3458700572464086, + "tokens_seen": 3039821824 + }, + { + "epoch": 0.84, + "learning_rate": 7.971915747241725e-05, + "loss": 1.2923, + "theoretical_loss": 3.3458645730743988, + "tokens_seen": 3039887360 + }, + { + "epoch": 0.84, + "learning_rate": 7.969909729187563e-05, + "loss": 1.1834, + "theoretical_loss": 3.345859089053723, + "tokens_seen": 3039952896 + }, + { + "epoch": 0.84, + "learning_rate": 7.9679037111334e-05, + "loss": 1.2605, + "theoretical_loss": 3.345853605184374, + "tokens_seen": 3040018432 + }, + { + "epoch": 0.84, + "learning_rate": 7.965897693079238e-05, + "loss": 1.1839, + "theoretical_loss": 3.345848121466345, + "tokens_seen": 3040083968 + }, + { + "epoch": 0.84, + "learning_rate": 7.963891675025076e-05, + "loss": 1.2249, + "theoretical_loss": 3.345842637899628, + "tokens_seen": 3040149504 + }, + { + "epoch": 0.84, + "learning_rate": 7.961885656970913e-05, + "loss": 1.2604, + "theoretical_loss": 3.345837154484215, + "tokens_seen": 3040215040 + }, + { + "epoch": 0.84, + "learning_rate": 7.959879638916751e-05, + "loss": 1.2952, + "theoretical_loss": 3.3458316712201, + "tokens_seen": 3040280576 + }, + { + "epoch": 0.84, + "learning_rate": 7.957873620862588e-05, + "loss": 1.2312, + "theoretical_loss": 3.345826188107274, + "tokens_seen": 3040346112 + }, + { + "epoch": 0.84, + "learning_rate": 7.955867602808425e-05, + "loss": 1.1766, + "theoretical_loss": 3.3458207051457305, + "tokens_seen": 3040411648 + }, + { + "epoch": 0.84, + "learning_rate": 7.953861584754262e-05, + "loss": 1.207, + "theoretical_loss": 3.345815222335462, + "tokens_seen": 3040477184 + }, + { + "epoch": 0.84, + "learning_rate": 7.9518555667001e-05, + "loss": 1.2766, + "theoretical_loss": 3.3458097396764606, + "tokens_seen": 3040542720 + }, + { + "epoch": 0.84, + "learning_rate": 7.949849548645937e-05, + "loss": 1.2518, + "theoretical_loss": 3.3458042571687194, + "tokens_seen": 3040608256 + }, + { + "epoch": 0.84, + "learning_rate": 7.947843530591775e-05, + "loss": 1.2565, + "theoretical_loss": 3.3457987748122306, + "tokens_seen": 3040673792 + }, + { + "epoch": 0.84, + "learning_rate": 7.945837512537613e-05, + "loss": 1.2685, + "theoretical_loss": 3.345793292606987, + "tokens_seen": 3040739328 + }, + { + "epoch": 0.84, + "learning_rate": 7.94383149448345e-05, + "loss": 1.3532, + "theoretical_loss": 3.3457878105529812, + "tokens_seen": 3040804864 + }, + { + "epoch": 0.84, + "learning_rate": 7.941825476429288e-05, + "loss": 1.2469, + "theoretical_loss": 3.3457823286502055, + "tokens_seen": 3040870400 + }, + { + "epoch": 0.84, + "learning_rate": 7.939819458375125e-05, + "loss": 1.2776, + "theoretical_loss": 3.345776846898653, + "tokens_seen": 3040935936 + }, + { + "epoch": 0.84, + "learning_rate": 7.937813440320964e-05, + "loss": 1.1849, + "theoretical_loss": 3.345771365298315, + "tokens_seen": 3041001472 + }, + { + "epoch": 0.84, + "learning_rate": 7.9358074222668e-05, + "loss": 1.2494, + "theoretical_loss": 3.345765883849186, + "tokens_seen": 3041067008 + }, + { + "epoch": 0.84, + "learning_rate": 7.933801404212638e-05, + "loss": 1.2891, + "theoretical_loss": 3.345760402551257, + "tokens_seen": 3041132544 + }, + { + "epoch": 0.84, + "learning_rate": 7.931795386158476e-05, + "loss": 1.3831, + "theoretical_loss": 3.345754921404521, + "tokens_seen": 3041198080 + }, + { + "epoch": 0.84, + "learning_rate": 7.929789368104313e-05, + "loss": 1.2123, + "theoretical_loss": 3.345749440408971, + "tokens_seen": 3041263616 + }, + { + "epoch": 0.84, + "learning_rate": 7.927783350050151e-05, + "loss": 1.3638, + "theoretical_loss": 3.345743959564599, + "tokens_seen": 3041329152 + }, + { + "epoch": 0.84, + "learning_rate": 7.925777331995988e-05, + "loss": 1.2335, + "theoretical_loss": 3.345738478871398, + "tokens_seen": 3041394688 + }, + { + "epoch": 0.84, + "learning_rate": 7.923771313941826e-05, + "loss": 1.2094, + "theoretical_loss": 3.34573299832936, + "tokens_seen": 3041460224 + }, + { + "epoch": 0.84, + "learning_rate": 7.921765295887664e-05, + "loss": 1.2788, + "theoretical_loss": 3.3457275179384784, + "tokens_seen": 3041525760 + }, + { + "epoch": 0.84, + "learning_rate": 7.919759277833501e-05, + "loss": 1.2579, + "theoretical_loss": 3.345722037698745, + "tokens_seen": 3041591296 + }, + { + "epoch": 0.84, + "learning_rate": 7.917753259779339e-05, + "loss": 1.2977, + "theoretical_loss": 3.345716557610153, + "tokens_seen": 3041656832 + }, + { + "epoch": 0.84, + "learning_rate": 7.915747241725175e-05, + "loss": 1.2508, + "theoretical_loss": 3.3457110776726946, + "tokens_seen": 3041722368 + }, + { + "epoch": 0.84, + "learning_rate": 7.913741223671012e-05, + "loss": 1.3873, + "theoretical_loss": 3.3457055978863623, + "tokens_seen": 3041787904 + }, + { + "epoch": 0.84, + "learning_rate": 7.91173520561685e-05, + "loss": 1.3208, + "theoretical_loss": 3.345700118251149, + "tokens_seen": 3041853440 + }, + { + "epoch": 0.84, + "learning_rate": 7.909729187562688e-05, + "loss": 1.17, + "theoretical_loss": 3.345694638767047, + "tokens_seen": 3041918976 + }, + { + "epoch": 0.84, + "learning_rate": 7.907723169508525e-05, + "loss": 1.2843, + "theoretical_loss": 3.345689159434049, + "tokens_seen": 3041984512 + }, + { + "epoch": 0.84, + "learning_rate": 7.905717151454363e-05, + "loss": 1.2524, + "theoretical_loss": 3.345683680252148, + "tokens_seen": 3042050048 + }, + { + "epoch": 0.84, + "learning_rate": 7.9037111334002e-05, + "loss": 1.1907, + "theoretical_loss": 3.345678201221336, + "tokens_seen": 3042115584 + }, + { + "epoch": 0.84, + "learning_rate": 7.901705115346038e-05, + "loss": 1.4159, + "theoretical_loss": 3.345672722341605, + "tokens_seen": 3042181120 + }, + { + "epoch": 0.84, + "learning_rate": 7.899699097291877e-05, + "loss": 1.4238, + "theoretical_loss": 3.345667243612949, + "tokens_seen": 3042246656 + }, + { + "epoch": 0.84, + "learning_rate": 7.897693079237714e-05, + "loss": 1.2007, + "theoretical_loss": 3.3456617650353597, + "tokens_seen": 3042312192 + }, + { + "epoch": 0.84, + "learning_rate": 7.895687061183551e-05, + "loss": 1.291, + "theoretical_loss": 3.34565628660883, + "tokens_seen": 3042377728 + }, + { + "epoch": 0.84, + "learning_rate": 7.893681043129388e-05, + "loss": 1.2181, + "theoretical_loss": 3.3456508083333523, + "tokens_seen": 3042443264 + }, + { + "epoch": 0.84, + "learning_rate": 7.891675025075226e-05, + "loss": 1.3695, + "theoretical_loss": 3.345645330208919, + "tokens_seen": 3042508800 + }, + { + "epoch": 0.84, + "learning_rate": 7.889669007021063e-05, + "loss": 1.295, + "theoretical_loss": 3.3456398522355233, + "tokens_seen": 3042574336 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1105520, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.30586838722229, + "objective/train/theoretical_loss": 3.345634374413157, + "objective/train/tokens_used": 1413098976, + "theoretical_loss": 3.345634374413157, + "tokens_seen": 3042639872 + }, + { + "epoch": 0.84, + "learning_rate": 7.887662988966901e-05, + "loss": 1.3113, + "theoretical_loss": 3.345634374413157, + "tokens_seen": 3042639872 + }, + { + "epoch": 0.84, + "learning_rate": 7.885656970912739e-05, + "loss": 1.0604, + "theoretical_loss": 3.3456288967418133, + "tokens_seen": 3042705408 + }, + { + "epoch": 0.84, + "learning_rate": 7.883650952858576e-05, + "loss": 1.2855, + "theoretical_loss": 3.3456234192214844, + "tokens_seen": 3042770944 + }, + { + "epoch": 0.84, + "learning_rate": 7.881644934804414e-05, + "loss": 1.4043, + "theoretical_loss": 3.345617941852163, + "tokens_seen": 3042836480 + }, + { + "epoch": 0.84, + "learning_rate": 7.879638916750251e-05, + "loss": 1.3062, + "theoretical_loss": 3.345612464633842, + "tokens_seen": 3042902016 + }, + { + "epoch": 0.84, + "learning_rate": 7.877632898696088e-05, + "loss": 1.2596, + "theoretical_loss": 3.345606987566514, + "tokens_seen": 3042967552 + }, + { + "epoch": 0.84, + "learning_rate": 7.875626880641925e-05, + "loss": 1.2816, + "theoretical_loss": 3.345601510650171, + "tokens_seen": 3043033088 + }, + { + "epoch": 0.84, + "learning_rate": 7.873620862587763e-05, + "loss": 1.3625, + "theoretical_loss": 3.3455960338848056, + "tokens_seen": 3043098624 + }, + { + "epoch": 0.84, + "learning_rate": 7.8716148445336e-05, + "loss": 1.3028, + "theoretical_loss": 3.3455905572704108, + "tokens_seen": 3043164160 + }, + { + "epoch": 0.84, + "learning_rate": 7.869608826479438e-05, + "loss": 1.1424, + "theoretical_loss": 3.345585080806979, + "tokens_seen": 3043229696 + }, + { + "epoch": 0.84, + "learning_rate": 7.867602808425275e-05, + "loss": 1.1993, + "theoretical_loss": 3.345579604494503, + "tokens_seen": 3043295232 + }, + { + "epoch": 0.84, + "learning_rate": 7.865596790371113e-05, + "loss": 1.2826, + "theoretical_loss": 3.3455741283329754, + "tokens_seen": 3043360768 + }, + { + "epoch": 0.84, + "learning_rate": 7.863590772316952e-05, + "loss": 1.2021, + "theoretical_loss": 3.345568652322388, + "tokens_seen": 3043426304 + }, + { + "epoch": 0.84, + "learning_rate": 7.86158475426279e-05, + "loss": 1.4122, + "theoretical_loss": 3.345563176462735, + "tokens_seen": 3043491840 + }, + { + "epoch": 0.84, + "learning_rate": 7.859578736208627e-05, + "loss": 1.2292, + "theoretical_loss": 3.345557700754007, + "tokens_seen": 3043557376 + }, + { + "epoch": 0.84, + "learning_rate": 7.857572718154463e-05, + "loss": 1.2743, + "theoretical_loss": 3.345552225196198, + "tokens_seen": 3043622912 + }, + { + "epoch": 0.84, + "learning_rate": 7.855566700100301e-05, + "loss": 1.2662, + "theoretical_loss": 3.3455467497893006, + "tokens_seen": 3043688448 + }, + { + "epoch": 0.84, + "learning_rate": 7.853560682046139e-05, + "loss": 1.2496, + "theoretical_loss": 3.3455412745333066, + "tokens_seen": 3043753984 + }, + { + "epoch": 0.84, + "learning_rate": 7.851554663991976e-05, + "loss": 1.2065, + "theoretical_loss": 3.345535799428209, + "tokens_seen": 3043819520 + }, + { + "epoch": 0.84, + "learning_rate": 7.849548645937814e-05, + "loss": 1.2576, + "theoretical_loss": 3.3455303244740007, + "tokens_seen": 3043885056 + }, + { + "epoch": 0.84, + "learning_rate": 7.847542627883651e-05, + "loss": 1.3144, + "theoretical_loss": 3.3455248496706735, + "tokens_seen": 3043950592 + }, + { + "epoch": 0.84, + "learning_rate": 7.845536609829489e-05, + "loss": 1.245, + "theoretical_loss": 3.3455193750182204, + "tokens_seen": 3044016128 + }, + { + "epoch": 0.84, + "learning_rate": 7.843530591775326e-05, + "loss": 1.3172, + "theoretical_loss": 3.345513900516634, + "tokens_seen": 3044081664 + }, + { + "epoch": 0.84, + "learning_rate": 7.841524573721164e-05, + "loss": 1.2299, + "theoretical_loss": 3.3455084261659076, + "tokens_seen": 3044147200 + }, + { + "epoch": 0.84, + "learning_rate": 7.839518555667002e-05, + "loss": 1.2205, + "theoretical_loss": 3.345502951966033, + "tokens_seen": 3044212736 + }, + { + "epoch": 0.85, + "learning_rate": 7.837512537612838e-05, + "loss": 1.2837, + "theoretical_loss": 3.3454974779170024, + "tokens_seen": 3044278272 + }, + { + "epoch": 0.85, + "learning_rate": 7.835506519558675e-05, + "loss": 1.2658, + "theoretical_loss": 3.345492004018809, + "tokens_seen": 3044343808 + }, + { + "epoch": 0.85, + "learning_rate": 7.833500501504513e-05, + "loss": 1.3151, + "theoretical_loss": 3.3454865302714456, + "tokens_seen": 3044409344 + }, + { + "epoch": 0.85, + "learning_rate": 7.83149448345035e-05, + "loss": 1.2643, + "theoretical_loss": 3.3454810566749043, + "tokens_seen": 3044474880 + }, + { + "epoch": 0.85, + "learning_rate": 7.829488465396188e-05, + "loss": 1.3304, + "theoretical_loss": 3.3454755832291783, + "tokens_seen": 3044540416 + }, + { + "epoch": 0.85, + "learning_rate": 7.827482447342026e-05, + "loss": 1.2834, + "theoretical_loss": 3.3454701099342596, + "tokens_seen": 3044605952 + }, + { + "epoch": 0.85, + "learning_rate": 7.825476429287865e-05, + "loss": 1.1928, + "theoretical_loss": 3.345464636790141, + "tokens_seen": 3044671488 + }, + { + "epoch": 0.85, + "learning_rate": 7.823470411233702e-05, + "loss": 1.2542, + "theoretical_loss": 3.3454591637968147, + "tokens_seen": 3044737024 + }, + { + "epoch": 0.85, + "learning_rate": 7.82146439317954e-05, + "loss": 1.2294, + "theoretical_loss": 3.3454536909542743, + "tokens_seen": 3044802560 + }, + { + "epoch": 0.85, + "learning_rate": 7.819458375125377e-05, + "loss": 1.3037, + "theoretical_loss": 3.3454482182625114, + "tokens_seen": 3044868096 + }, + { + "epoch": 0.85, + "learning_rate": 7.817452357071214e-05, + "loss": 1.2858, + "theoretical_loss": 3.3454427457215195, + "tokens_seen": 3044933632 + }, + { + "epoch": 0.85, + "learning_rate": 7.815446339017051e-05, + "loss": 1.296, + "theoretical_loss": 3.3454372733312905, + "tokens_seen": 3044999168 + }, + { + "epoch": 0.85, + "learning_rate": 7.813440320962889e-05, + "loss": 1.3191, + "theoretical_loss": 3.345431801091817, + "tokens_seen": 3045064704 + }, + { + "epoch": 0.85, + "learning_rate": 7.811434302908726e-05, + "loss": 1.3258, + "theoretical_loss": 3.345426329003092, + "tokens_seen": 3045130240 + }, + { + "epoch": 0.85, + "learning_rate": 7.809428284854564e-05, + "loss": 1.2766, + "theoretical_loss": 3.3454208570651076, + "tokens_seen": 3045195776 + }, + { + "epoch": 0.85, + "learning_rate": 7.807422266800401e-05, + "loss": 1.2456, + "theoretical_loss": 3.345415385277857, + "tokens_seen": 3045261312 + }, + { + "epoch": 0.85, + "learning_rate": 7.805416248746239e-05, + "loss": 1.1379, + "theoretical_loss": 3.345409913641333, + "tokens_seen": 3045326848 + }, + { + "epoch": 0.85, + "learning_rate": 7.803410230692077e-05, + "loss": 1.2734, + "theoretical_loss": 3.345404442155527, + "tokens_seen": 3045392384 + }, + { + "epoch": 0.85, + "learning_rate": 7.801404212637914e-05, + "loss": 1.1134, + "theoretical_loss": 3.345398970820433, + "tokens_seen": 3045457920 + }, + { + "epoch": 0.85, + "learning_rate": 7.79939819458375e-05, + "loss": 1.1874, + "theoretical_loss": 3.3453934996360424, + "tokens_seen": 3045523456 + }, + { + "epoch": 0.85, + "learning_rate": 7.797392176529588e-05, + "loss": 1.3489, + "theoretical_loss": 3.3453880286023487, + "tokens_seen": 3045588992 + }, + { + "epoch": 0.85, + "learning_rate": 7.795386158475426e-05, + "loss": 1.2671, + "theoretical_loss": 3.345382557719344, + "tokens_seen": 3045654528 + }, + { + "epoch": 0.85, + "learning_rate": 7.793380140421263e-05, + "loss": 1.2418, + "theoretical_loss": 3.345377086987021, + "tokens_seen": 3045720064 + }, + { + "epoch": 0.85, + "learning_rate": 7.791374122367101e-05, + "loss": 1.2697, + "theoretical_loss": 3.345371616405372, + "tokens_seen": 3045785600 + }, + { + "epoch": 0.85, + "learning_rate": 7.789368104312938e-05, + "loss": 1.3273, + "theoretical_loss": 3.345366145974391, + "tokens_seen": 3045851136 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1106703, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1013998985290527, + "objective/train/theoretical_loss": 3.345360675694069, + "objective/train/tokens_used": 1416375776, + "theoretical_loss": 3.345360675694069, + "tokens_seen": 3045916672 + }, + { + "epoch": 0.85, + "learning_rate": 7.787362086258777e-05, + "loss": 1.2358, + "theoretical_loss": 3.345360675694069, + "tokens_seen": 3045916672 + }, + { + "epoch": 0.85, + "learning_rate": 7.785356068204615e-05, + "loss": 1.2663, + "theoretical_loss": 3.345355205564399, + "tokens_seen": 3045982208 + }, + { + "epoch": 0.85, + "learning_rate": 7.783350050150452e-05, + "loss": 1.2875, + "theoretical_loss": 3.3453497355853745, + "tokens_seen": 3046047744 + }, + { + "epoch": 0.85, + "learning_rate": 7.78134403209629e-05, + "loss": 1.272, + "theoretical_loss": 3.345344265756987, + "tokens_seen": 3046113280 + }, + { + "epoch": 0.85, + "learning_rate": 7.779338014042126e-05, + "loss": 1.2552, + "theoretical_loss": 3.345338796079229, + "tokens_seen": 3046178816 + }, + { + "epoch": 0.85, + "learning_rate": 7.777331995987964e-05, + "loss": 1.3269, + "theoretical_loss": 3.3453333265520944, + "tokens_seen": 3046244352 + }, + { + "epoch": 0.85, + "learning_rate": 7.775325977933801e-05, + "loss": 1.1895, + "theoretical_loss": 3.345327857175575, + "tokens_seen": 3046309888 + }, + { + "epoch": 0.85, + "learning_rate": 7.773319959879639e-05, + "loss": 1.281, + "theoretical_loss": 3.345322387949663, + "tokens_seen": 3046375424 + }, + { + "epoch": 0.85, + "learning_rate": 7.771313941825477e-05, + "loss": 1.2038, + "theoretical_loss": 3.345316918874352, + "tokens_seen": 3046440960 + }, + { + "epoch": 0.85, + "learning_rate": 7.769307923771314e-05, + "loss": 1.2617, + "theoretical_loss": 3.3453114499496337, + "tokens_seen": 3046506496 + }, + { + "epoch": 0.85, + "learning_rate": 7.767301905717152e-05, + "loss": 1.264, + "theoretical_loss": 3.3453059811755015, + "tokens_seen": 3046572032 + }, + { + "epoch": 0.85, + "learning_rate": 7.765295887662989e-05, + "loss": 1.2784, + "theoretical_loss": 3.345300512551947, + "tokens_seen": 3046637568 + }, + { + "epoch": 0.85, + "learning_rate": 7.763289869608827e-05, + "loss": 1.2607, + "theoretical_loss": 3.3452950440789637, + "tokens_seen": 3046703104 + }, + { + "epoch": 0.85, + "learning_rate": 7.761283851554664e-05, + "loss": 1.2717, + "theoretical_loss": 3.3452895757565444, + "tokens_seen": 3046768640 + }, + { + "epoch": 0.85, + "learning_rate": 7.759277833500501e-05, + "loss": 1.2472, + "theoretical_loss": 3.345284107584681, + "tokens_seen": 3046834176 + }, + { + "epoch": 0.85, + "learning_rate": 7.757271815446338e-05, + "loss": 1.3483, + "theoretical_loss": 3.3452786395633662, + "tokens_seen": 3046899712 + }, + { + "epoch": 0.85, + "learning_rate": 7.755265797392176e-05, + "loss": 1.2527, + "theoretical_loss": 3.3452731716925928, + "tokens_seen": 3046965248 + }, + { + "epoch": 0.85, + "learning_rate": 7.753259779338013e-05, + "loss": 1.2011, + "theoretical_loss": 3.345267703972354, + "tokens_seen": 3047030784 + }, + { + "epoch": 0.85, + "learning_rate": 7.751253761283852e-05, + "loss": 1.2876, + "theoretical_loss": 3.345262236402641, + "tokens_seen": 3047096320 + }, + { + "epoch": 0.85, + "learning_rate": 7.74924774322969e-05, + "loss": 1.3082, + "theoretical_loss": 3.3452567689834476, + "tokens_seen": 3047161856 + }, + { + "epoch": 0.85, + "learning_rate": 7.747241725175528e-05, + "loss": 1.2859, + "theoretical_loss": 3.3452513017147663, + "tokens_seen": 3047227392 + }, + { + "epoch": 0.85, + "learning_rate": 7.745235707121365e-05, + "loss": 1.3824, + "theoretical_loss": 3.3452458345965894, + "tokens_seen": 3047292928 + }, + { + "epoch": 0.85, + "learning_rate": 7.743229689067203e-05, + "loss": 1.2301, + "theoretical_loss": 3.3452403676289095, + "tokens_seen": 3047358464 + }, + { + "epoch": 0.85, + "learning_rate": 7.74122367101304e-05, + "loss": 1.2148, + "theoretical_loss": 3.3452349008117195, + "tokens_seen": 3047424000 + }, + { + "epoch": 0.85, + "learning_rate": 7.739217652958876e-05, + "loss": 1.302, + "theoretical_loss": 3.3452294341450117, + "tokens_seen": 3047489536 + }, + { + "epoch": 0.85, + "learning_rate": 7.737211634904714e-05, + "loss": 1.2552, + "theoretical_loss": 3.3452239676287787, + "tokens_seen": 3047555072 + }, + { + "epoch": 0.85, + "learning_rate": 7.735205616850552e-05, + "loss": 1.2735, + "theoretical_loss": 3.345218501263014, + "tokens_seen": 3047620608 + }, + { + "epoch": 0.85, + "learning_rate": 7.733199598796389e-05, + "loss": 1.3161, + "theoretical_loss": 3.345213035047709, + "tokens_seen": 3047686144 + }, + { + "epoch": 0.85, + "learning_rate": 7.731193580742227e-05, + "loss": 1.262, + "theoretical_loss": 3.3452075689828566, + "tokens_seen": 3047751680 + }, + { + "epoch": 0.85, + "learning_rate": 7.729187562688064e-05, + "loss": 1.3165, + "theoretical_loss": 3.3452021030684502, + "tokens_seen": 3047817216 + }, + { + "epoch": 0.85, + "learning_rate": 7.727181544633902e-05, + "loss": 1.3408, + "theoretical_loss": 3.3451966373044817, + "tokens_seen": 3047882752 + }, + { + "epoch": 0.85, + "learning_rate": 7.72517552657974e-05, + "loss": 1.2513, + "theoretical_loss": 3.345191171690944, + "tokens_seen": 3047948288 + }, + { + "epoch": 0.85, + "learning_rate": 7.723169508525577e-05, + "loss": 1.3317, + "theoretical_loss": 3.345185706227829, + "tokens_seen": 3048013824 + }, + { + "epoch": 0.85, + "learning_rate": 7.721163490471413e-05, + "loss": 1.2668, + "theoretical_loss": 3.3451802409151306, + "tokens_seen": 3048079360 + }, + { + "epoch": 0.85, + "learning_rate": 7.719157472417251e-05, + "loss": 1.3075, + "theoretical_loss": 3.3451747757528407, + "tokens_seen": 3048144896 + }, + { + "epoch": 0.85, + "learning_rate": 7.717151454363088e-05, + "loss": 1.2835, + "theoretical_loss": 3.3451693107409524, + "tokens_seen": 3048210432 + }, + { + "epoch": 0.85, + "learning_rate": 7.715145436308926e-05, + "loss": 1.2745, + "theoretical_loss": 3.3451638458794575, + "tokens_seen": 3048275968 + }, + { + "epoch": 0.85, + "learning_rate": 7.713139418254765e-05, + "loss": 1.2469, + "theoretical_loss": 3.345158381168349, + "tokens_seen": 3048341504 + }, + { + "epoch": 0.85, + "learning_rate": 7.711133400200603e-05, + "loss": 1.2074, + "theoretical_loss": 3.3451529166076197, + "tokens_seen": 3048407040 + }, + { + "epoch": 0.85, + "learning_rate": 7.70912738214644e-05, + "loss": 1.3387, + "theoretical_loss": 3.3451474521972626, + "tokens_seen": 3048472576 + }, + { + "epoch": 0.85, + "learning_rate": 7.707121364092278e-05, + "loss": 1.1404, + "theoretical_loss": 3.3451419879372692, + "tokens_seen": 3048538112 + }, + { + "epoch": 0.85, + "learning_rate": 7.705115346038115e-05, + "loss": 1.1787, + "theoretical_loss": 3.3451365238276334, + "tokens_seen": 3048603648 + }, + { + "epoch": 0.85, + "learning_rate": 7.703109327983953e-05, + "loss": 1.3549, + "theoretical_loss": 3.345131059868347, + "tokens_seen": 3048669184 + }, + { + "epoch": 0.85, + "learning_rate": 7.701103309929789e-05, + "loss": 1.2844, + "theoretical_loss": 3.3451255960594026, + "tokens_seen": 3048734720 + }, + { + "epoch": 0.85, + "learning_rate": 7.699097291875627e-05, + "loss": 1.2752, + "theoretical_loss": 3.345120132400793, + "tokens_seen": 3048800256 + }, + { + "epoch": 0.85, + "learning_rate": 7.697091273821464e-05, + "loss": 1.3509, + "theoretical_loss": 3.3451146688925117, + "tokens_seen": 3048865792 + }, + { + "epoch": 0.85, + "learning_rate": 7.695085255767302e-05, + "loss": 1.2119, + "theoretical_loss": 3.3451092055345497, + "tokens_seen": 3048931328 + }, + { + "epoch": 0.85, + "learning_rate": 7.69307923771314e-05, + "loss": 1.3092, + "theoretical_loss": 3.345103742326901, + "tokens_seen": 3048996864 + }, + { + "epoch": 0.85, + "learning_rate": 7.691073219658977e-05, + "loss": 1.1888, + "theoretical_loss": 3.345098279269558, + "tokens_seen": 3049062400 + }, + { + "epoch": 0.85, + "learning_rate": 7.689067201604815e-05, + "loss": 1.239, + "theoretical_loss": 3.345092816362512, + "tokens_seen": 3049127936 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1107994, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2179710865020752, + "objective/train/theoretical_loss": 3.3450873536057575, + "objective/train/tokens_used": 1419652576, + "theoretical_loss": 3.3450873536057575, + "tokens_seen": 3049193472 + }, + { + "epoch": 0.85, + "learning_rate": 7.687061183550652e-05, + "loss": 1.1453, + "theoretical_loss": 3.3450873536057575, + "tokens_seen": 3049193472 + }, + { + "epoch": 0.85, + "learning_rate": 7.68505516549649e-05, + "loss": 1.2262, + "theoretical_loss": 3.3450818909992863, + "tokens_seen": 3049259008 + }, + { + "epoch": 0.85, + "learning_rate": 7.683049147442327e-05, + "loss": 1.3077, + "theoretical_loss": 3.345076428543091, + "tokens_seen": 3049324544 + }, + { + "epoch": 0.85, + "learning_rate": 7.681043129388164e-05, + "loss": 1.2779, + "theoretical_loss": 3.3450709662371643, + "tokens_seen": 3049390080 + }, + { + "epoch": 0.85, + "learning_rate": 7.679037111334001e-05, + "loss": 1.2862, + "theoretical_loss": 3.3450655040814987, + "tokens_seen": 3049455616 + }, + { + "epoch": 0.85, + "learning_rate": 7.677031093279839e-05, + "loss": 1.3399, + "theoretical_loss": 3.3450600420760868, + "tokens_seen": 3049521152 + }, + { + "epoch": 0.85, + "learning_rate": 7.675025075225678e-05, + "loss": 1.2185, + "theoretical_loss": 3.3450545802209217, + "tokens_seen": 3049586688 + }, + { + "epoch": 0.85, + "learning_rate": 7.673019057171515e-05, + "loss": 1.2631, + "theoretical_loss": 3.3450491185159956, + "tokens_seen": 3049652224 + }, + { + "epoch": 0.85, + "learning_rate": 7.671013039117353e-05, + "loss": 1.2031, + "theoretical_loss": 3.345043656961302, + "tokens_seen": 3049717760 + }, + { + "epoch": 0.85, + "learning_rate": 7.66900702106319e-05, + "loss": 1.2488, + "theoretical_loss": 3.345038195556832, + "tokens_seen": 3049783296 + }, + { + "epoch": 0.85, + "learning_rate": 7.667001003009028e-05, + "loss": 1.261, + "theoretical_loss": 3.3450327343025794, + "tokens_seen": 3049848832 + }, + { + "epoch": 0.85, + "learning_rate": 7.664994984954866e-05, + "loss": 1.2556, + "theoretical_loss": 3.345027273198536, + "tokens_seen": 3049914368 + }, + { + "epoch": 0.85, + "learning_rate": 7.662988966900703e-05, + "loss": 1.1875, + "theoretical_loss": 3.3450218122446955, + "tokens_seen": 3049979904 + }, + { + "epoch": 0.85, + "learning_rate": 7.66098294884654e-05, + "loss": 1.2153, + "theoretical_loss": 3.34501635144105, + "tokens_seen": 3050045440 + }, + { + "epoch": 0.85, + "learning_rate": 7.658976930792377e-05, + "loss": 1.3107, + "theoretical_loss": 3.345010890787592, + "tokens_seen": 3050110976 + }, + { + "epoch": 0.85, + "learning_rate": 7.656970912738215e-05, + "loss": 1.1388, + "theoretical_loss": 3.3450054302843144, + "tokens_seen": 3050176512 + }, + { + "epoch": 0.85, + "learning_rate": 7.654964894684052e-05, + "loss": 1.3107, + "theoretical_loss": 3.3449999699312096, + "tokens_seen": 3050242048 + }, + { + "epoch": 0.85, + "learning_rate": 7.65295887662989e-05, + "loss": 1.2439, + "theoretical_loss": 3.34499450972827, + "tokens_seen": 3050307584 + }, + { + "epoch": 0.85, + "learning_rate": 7.650952858575727e-05, + "loss": 1.328, + "theoretical_loss": 3.3449890496754895, + "tokens_seen": 3050373120 + }, + { + "epoch": 0.85, + "learning_rate": 7.648946840521565e-05, + "loss": 1.2692, + "theoretical_loss": 3.344983589772859, + "tokens_seen": 3050438656 + }, + { + "epoch": 0.85, + "learning_rate": 7.646940822467402e-05, + "loss": 1.2059, + "theoretical_loss": 3.3449781300203725, + "tokens_seen": 3050504192 + }, + { + "epoch": 0.85, + "learning_rate": 7.64493480441324e-05, + "loss": 1.3123, + "theoretical_loss": 3.344972670418022, + "tokens_seen": 3050569728 + }, + { + "epoch": 0.85, + "learning_rate": 7.642928786359076e-05, + "loss": 1.2797, + "theoretical_loss": 3.3449672109658, + "tokens_seen": 3050635264 + }, + { + "epoch": 0.85, + "learning_rate": 7.640922768304914e-05, + "loss": 1.258, + "theoretical_loss": 3.3449617516637, + "tokens_seen": 3050700800 + }, + { + "epoch": 0.85, + "learning_rate": 7.638916750250751e-05, + "loss": 1.269, + "theoretical_loss": 3.3449562925117133, + "tokens_seen": 3050766336 + }, + { + "epoch": 0.85, + "learning_rate": 7.63691073219659e-05, + "loss": 1.3114, + "theoretical_loss": 3.344950833509834, + "tokens_seen": 3050831872 + }, + { + "epoch": 0.85, + "learning_rate": 7.634904714142428e-05, + "loss": 1.243, + "theoretical_loss": 3.344945374658054, + "tokens_seen": 3050897408 + }, + { + "epoch": 0.85, + "learning_rate": 7.632898696088265e-05, + "loss": 1.1236, + "theoretical_loss": 3.344939915956366, + "tokens_seen": 3050962944 + }, + { + "epoch": 0.85, + "learning_rate": 7.630892678034103e-05, + "loss": 1.2043, + "theoretical_loss": 3.3449344574047624, + "tokens_seen": 3051028480 + }, + { + "epoch": 0.85, + "learning_rate": 7.62888665997994e-05, + "loss": 1.2946, + "theoretical_loss": 3.3449289990032365, + "tokens_seen": 3051094016 + }, + { + "epoch": 0.85, + "learning_rate": 7.626880641925778e-05, + "loss": 1.2968, + "theoretical_loss": 3.3449235407517803, + "tokens_seen": 3051159552 + }, + { + "epoch": 0.85, + "learning_rate": 7.624874623871616e-05, + "loss": 1.2791, + "theoretical_loss": 3.344918082650387, + "tokens_seen": 3051225088 + }, + { + "epoch": 0.85, + "learning_rate": 7.622868605817452e-05, + "loss": 1.3451, + "theoretical_loss": 3.344912624699049, + "tokens_seen": 3051290624 + }, + { + "epoch": 0.85, + "learning_rate": 7.62086258776329e-05, + "loss": 1.1661, + "theoretical_loss": 3.3449071668977584, + "tokens_seen": 3051356160 + }, + { + "epoch": 0.85, + "learning_rate": 7.618856569709127e-05, + "loss": 1.288, + "theoretical_loss": 3.3449017092465088, + "tokens_seen": 3051421696 + }, + { + "epoch": 0.85, + "learning_rate": 7.616850551654965e-05, + "loss": 1.1701, + "theoretical_loss": 3.3448962517452925, + "tokens_seen": 3051487232 + }, + { + "epoch": 0.85, + "learning_rate": 7.614844533600802e-05, + "loss": 1.1225, + "theoretical_loss": 3.344890794394102, + "tokens_seen": 3051552768 + }, + { + "epoch": 0.85, + "learning_rate": 7.61283851554664e-05, + "loss": 1.2746, + "theoretical_loss": 3.3448853371929297, + "tokens_seen": 3051618304 + }, + { + "epoch": 0.85, + "learning_rate": 7.610832497492477e-05, + "loss": 1.2091, + "theoretical_loss": 3.344879880141769, + "tokens_seen": 3051683840 + }, + { + "epoch": 0.85, + "learning_rate": 7.608826479438315e-05, + "loss": 1.2292, + "theoretical_loss": 3.3448744232406122, + "tokens_seen": 3051749376 + }, + { + "epoch": 0.85, + "learning_rate": 7.606820461384153e-05, + "loss": 1.2552, + "theoretical_loss": 3.3448689664894515, + "tokens_seen": 3051814912 + }, + { + "epoch": 0.85, + "learning_rate": 7.60481444332999e-05, + "loss": 1.3826, + "theoretical_loss": 3.34486350988828, + "tokens_seen": 3051880448 + }, + { + "epoch": 0.85, + "learning_rate": 7.602808425275826e-05, + "loss": 1.1521, + "theoretical_loss": 3.3448580534370906, + "tokens_seen": 3051945984 + }, + { + "epoch": 0.85, + "learning_rate": 7.600802407221665e-05, + "loss": 1.3393, + "theoretical_loss": 3.344852597135876, + "tokens_seen": 3052011520 + }, + { + "epoch": 0.85, + "learning_rate": 7.598796389167503e-05, + "loss": 1.2678, + "theoretical_loss": 3.344847140984628, + "tokens_seen": 3052077056 + }, + { + "epoch": 0.85, + "learning_rate": 7.59679037111334e-05, + "loss": 1.2322, + "theoretical_loss": 3.34484168498334, + "tokens_seen": 3052142592 + }, + { + "epoch": 0.85, + "learning_rate": 7.594784353059178e-05, + "loss": 1.3441, + "theoretical_loss": 3.3448362291320044, + "tokens_seen": 3052208128 + }, + { + "epoch": 0.85, + "learning_rate": 7.592778335005016e-05, + "loss": 1.2018, + "theoretical_loss": 3.344830773430614, + "tokens_seen": 3052273664 + }, + { + "epoch": 0.85, + "learning_rate": 7.590772316950853e-05, + "loss": 1.2171, + "theoretical_loss": 3.3448253178791614, + "tokens_seen": 3052339200 + }, + { + "epoch": 0.85, + "learning_rate": 7.588766298896691e-05, + "loss": 1.2493, + "theoretical_loss": 3.3448198624776393, + "tokens_seen": 3052404736 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1109048, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2406476736068726, + "objective/train/theoretical_loss": 3.34481440722604, + "objective/train/tokens_used": 1422929376, + "theoretical_loss": 3.34481440722604, + "tokens_seen": 3052470272 + }, + { + "epoch": 0.85, + "learning_rate": 7.586760280842528e-05, + "loss": 1.2638, + "theoretical_loss": 3.34481440722604, + "tokens_seen": 3052470272 + }, + { + "epoch": 0.85, + "learning_rate": 7.584754262788365e-05, + "loss": 1.1846, + "theoretical_loss": 3.3448089521243567, + "tokens_seen": 3052535808 + }, + { + "epoch": 0.85, + "learning_rate": 7.582748244734202e-05, + "loss": 1.3334, + "theoretical_loss": 3.3448034971725815, + "tokens_seen": 3052601344 + }, + { + "epoch": 0.85, + "learning_rate": 7.58074222668004e-05, + "loss": 1.3485, + "theoretical_loss": 3.3447980423707078, + "tokens_seen": 3052666880 + }, + { + "epoch": 0.85, + "learning_rate": 7.578736208625877e-05, + "loss": 1.2987, + "theoretical_loss": 3.3447925877187275, + "tokens_seen": 3052732416 + }, + { + "epoch": 0.85, + "learning_rate": 7.576730190571715e-05, + "loss": 1.2404, + "theoretical_loss": 3.3447871332166335, + "tokens_seen": 3052797952 + }, + { + "epoch": 0.85, + "learning_rate": 7.574724172517553e-05, + "loss": 1.2917, + "theoretical_loss": 3.344781678864419, + "tokens_seen": 3052863488 + }, + { + "epoch": 0.85, + "learning_rate": 7.57271815446339e-05, + "loss": 1.135, + "theoretical_loss": 3.344776224662076, + "tokens_seen": 3052929024 + }, + { + "epoch": 0.85, + "learning_rate": 7.570712136409228e-05, + "loss": 1.1667, + "theoretical_loss": 3.3447707706095975, + "tokens_seen": 3052994560 + }, + { + "epoch": 0.85, + "learning_rate": 7.568706118355065e-05, + "loss": 1.2028, + "theoretical_loss": 3.344765316706976, + "tokens_seen": 3053060096 + }, + { + "epoch": 0.85, + "learning_rate": 7.566700100300903e-05, + "loss": 1.2377, + "theoretical_loss": 3.344759862954204, + "tokens_seen": 3053125632 + }, + { + "epoch": 0.85, + "learning_rate": 7.564694082246739e-05, + "loss": 1.3642, + "theoretical_loss": 3.3447544093512747, + "tokens_seen": 3053191168 + }, + { + "epoch": 0.85, + "learning_rate": 7.562688064192578e-05, + "loss": 1.212, + "theoretical_loss": 3.34474895589818, + "tokens_seen": 3053256704 + }, + { + "epoch": 0.85, + "learning_rate": 7.560682046138416e-05, + "loss": 1.1834, + "theoretical_loss": 3.344743502594914, + "tokens_seen": 3053322240 + }, + { + "epoch": 0.85, + "learning_rate": 7.558676028084253e-05, + "loss": 1.3133, + "theoretical_loss": 3.3447380494414674, + "tokens_seen": 3053387776 + }, + { + "epoch": 0.85, + "learning_rate": 7.556670010030091e-05, + "loss": 1.2072, + "theoretical_loss": 3.3447325964378347, + "tokens_seen": 3053453312 + }, + { + "epoch": 0.85, + "learning_rate": 7.554663991975928e-05, + "loss": 1.3418, + "theoretical_loss": 3.344727143584007, + "tokens_seen": 3053518848 + }, + { + "epoch": 0.85, + "learning_rate": 7.552657973921766e-05, + "loss": 1.3296, + "theoretical_loss": 3.344721690879978, + "tokens_seen": 3053584384 + }, + { + "epoch": 0.85, + "learning_rate": 7.550651955867604e-05, + "loss": 1.2802, + "theoretical_loss": 3.3447162383257405, + "tokens_seen": 3053649920 + }, + { + "epoch": 0.85, + "learning_rate": 7.548645937813441e-05, + "loss": 1.2681, + "theoretical_loss": 3.3447107859212863, + "tokens_seen": 3053715456 + }, + { + "epoch": 0.85, + "learning_rate": 7.546639919759279e-05, + "loss": 1.1581, + "theoretical_loss": 3.3447053336666084, + "tokens_seen": 3053780992 + }, + { + "epoch": 0.85, + "learning_rate": 7.544633901705115e-05, + "loss": 1.2152, + "theoretical_loss": 3.3446998815616995, + "tokens_seen": 3053846528 + }, + { + "epoch": 0.85, + "learning_rate": 7.542627883650952e-05, + "loss": 1.3208, + "theoretical_loss": 3.3446944296065526, + "tokens_seen": 3053912064 + }, + { + "epoch": 0.85, + "learning_rate": 7.54062186559679e-05, + "loss": 1.2924, + "theoretical_loss": 3.34468897780116, + "tokens_seen": 3053977600 + }, + { + "epoch": 0.85, + "learning_rate": 7.538615847542628e-05, + "loss": 1.2169, + "theoretical_loss": 3.344683526145515, + "tokens_seen": 3054043136 + }, + { + "epoch": 0.85, + "learning_rate": 7.536609829488465e-05, + "loss": 1.1749, + "theoretical_loss": 3.344678074639609, + "tokens_seen": 3054108672 + }, + { + "epoch": 0.85, + "learning_rate": 7.534603811434303e-05, + "loss": 1.2251, + "theoretical_loss": 3.344672623283436, + "tokens_seen": 3054174208 + }, + { + "epoch": 0.85, + "learning_rate": 7.53259779338014e-05, + "loss": 1.1302, + "theoretical_loss": 3.344667172076988, + "tokens_seen": 3054239744 + }, + { + "epoch": 0.85, + "learning_rate": 7.530591775325978e-05, + "loss": 1.2795, + "theoretical_loss": 3.344661721020258, + "tokens_seen": 3054305280 + }, + { + "epoch": 0.85, + "learning_rate": 7.528585757271816e-05, + "loss": 1.3009, + "theoretical_loss": 3.344656270113238, + "tokens_seen": 3054370816 + }, + { + "epoch": 0.85, + "learning_rate": 7.526579739217652e-05, + "loss": 1.2062, + "theoretical_loss": 3.3446508193559215, + "tokens_seen": 3054436352 + }, + { + "epoch": 0.85, + "learning_rate": 7.524573721163491e-05, + "loss": 1.1535, + "theoretical_loss": 3.3446453687483007, + "tokens_seen": 3054501888 + }, + { + "epoch": 0.85, + "learning_rate": 7.522567703109328e-05, + "loss": 1.2599, + "theoretical_loss": 3.3446399182903686, + "tokens_seen": 3054567424 + }, + { + "epoch": 0.85, + "learning_rate": 7.520561685055166e-05, + "loss": 1.2164, + "theoretical_loss": 3.344634467982117, + "tokens_seen": 3054632960 + }, + { + "epoch": 0.85, + "learning_rate": 7.518555667001003e-05, + "loss": 1.2359, + "theoretical_loss": 3.3446290178235403, + "tokens_seen": 3054698496 + }, + { + "epoch": 0.85, + "learning_rate": 7.516549648946841e-05, + "loss": 1.2397, + "theoretical_loss": 3.3446235678146294, + "tokens_seen": 3054764032 + }, + { + "epoch": 0.85, + "learning_rate": 7.514543630892679e-05, + "loss": 1.1268, + "theoretical_loss": 3.344618117955378, + "tokens_seen": 3054829568 + }, + { + "epoch": 0.85, + "learning_rate": 7.512537612838516e-05, + "loss": 1.182, + "theoretical_loss": 3.3446126682457784, + "tokens_seen": 3054895104 + }, + { + "epoch": 0.85, + "learning_rate": 7.510531594784354e-05, + "loss": 1.1957, + "theoretical_loss": 3.3446072186858236, + "tokens_seen": 3054960640 + }, + { + "epoch": 0.85, + "learning_rate": 7.508525576730191e-05, + "loss": 1.1926, + "theoretical_loss": 3.344601769275506, + "tokens_seen": 3055026176 + }, + { + "epoch": 0.85, + "learning_rate": 7.506519558676028e-05, + "loss": 1.2495, + "theoretical_loss": 3.3445963200148183, + "tokens_seen": 3055091712 + }, + { + "epoch": 0.85, + "learning_rate": 7.504513540621865e-05, + "loss": 1.1646, + "theoretical_loss": 3.344590870903753, + "tokens_seen": 3055157248 + }, + { + "epoch": 0.85, + "learning_rate": 7.502507522567703e-05, + "loss": 1.2367, + "theoretical_loss": 3.3445854219423032, + "tokens_seen": 3055222784 + }, + { + "epoch": 0.85, + "learning_rate": 7.50050150451354e-05, + "loss": 1.1632, + "theoretical_loss": 3.3445799731304615, + "tokens_seen": 3055288320 + }, + { + "epoch": 0.85, + "learning_rate": 7.498495486459378e-05, + "loss": 1.3074, + "theoretical_loss": 3.34457452446822, + "tokens_seen": 3055353856 + }, + { + "epoch": 0.85, + "learning_rate": 7.496489468405215e-05, + "loss": 1.3392, + "theoretical_loss": 3.3445690759555724, + "tokens_seen": 3055419392 + }, + { + "epoch": 0.85, + "learning_rate": 7.494483450351053e-05, + "loss": 1.2089, + "theoretical_loss": 3.344563627592511, + "tokens_seen": 3055484928 + }, + { + "epoch": 0.85, + "learning_rate": 7.49247743229689e-05, + "loss": 1.3018, + "theoretical_loss": 3.3445581793790278, + "tokens_seen": 3055550464 + }, + { + "epoch": 0.85, + "learning_rate": 7.490471414242728e-05, + "loss": 1.2053, + "theoretical_loss": 3.344552731315116, + "tokens_seen": 3055616000 + }, + { + "epoch": 0.85, + "learning_rate": 7.488465396188567e-05, + "loss": 1.317, + "theoretical_loss": 3.3445472834007686, + "tokens_seen": 3055681536 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1110166, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.205101490020752, + "objective/train/theoretical_loss": 3.3445418356359777, + "objective/train/tokens_used": 1426206176, + "theoretical_loss": 3.3445418356359777, + "tokens_seen": 3055747072 + }, + { + "epoch": 0.85, + "learning_rate": 7.486459378134403e-05, + "loss": 1.2743, + "theoretical_loss": 3.3445418356359777, + "tokens_seen": 3055747072 + }, + { + "epoch": 0.85, + "learning_rate": 7.484453360080241e-05, + "loss": 1.0473, + "theoretical_loss": 3.3445363880207366, + "tokens_seen": 3055812608 + }, + { + "epoch": 0.85, + "learning_rate": 7.482447342026079e-05, + "loss": 1.2773, + "theoretical_loss": 3.344530940555037, + "tokens_seen": 3055878144 + }, + { + "epoch": 0.85, + "learning_rate": 7.480441323971916e-05, + "loss": 1.2894, + "theoretical_loss": 3.3445254932388733, + "tokens_seen": 3055943680 + }, + { + "epoch": 0.85, + "learning_rate": 7.478435305917754e-05, + "loss": 1.1828, + "theoretical_loss": 3.3445200460722364, + "tokens_seen": 3056009216 + }, + { + "epoch": 0.85, + "learning_rate": 7.476429287863591e-05, + "loss": 1.2666, + "theoretical_loss": 3.34451459905512, + "tokens_seen": 3056074752 + }, + { + "epoch": 0.85, + "learning_rate": 7.474423269809429e-05, + "loss": 1.2722, + "theoretical_loss": 3.3445091521875163, + "tokens_seen": 3056140288 + }, + { + "epoch": 0.85, + "learning_rate": 7.472417251755266e-05, + "loss": 1.2677, + "theoretical_loss": 3.3445037054694184, + "tokens_seen": 3056205824 + }, + { + "epoch": 0.85, + "learning_rate": 7.470411233701104e-05, + "loss": 1.2063, + "theoretical_loss": 3.3444982589008188, + "tokens_seen": 3056271360 + }, + { + "epoch": 0.85, + "learning_rate": 7.468405215646942e-05, + "loss": 1.3205, + "theoretical_loss": 3.34449281248171, + "tokens_seen": 3056336896 + }, + { + "epoch": 0.85, + "learning_rate": 7.466399197592778e-05, + "loss": 1.2702, + "theoretical_loss": 3.344487366212085, + "tokens_seen": 3056402432 + }, + { + "epoch": 0.85, + "learning_rate": 7.464393179538615e-05, + "loss": 1.1827, + "theoretical_loss": 3.3444819200919365, + "tokens_seen": 3056467968 + }, + { + "epoch": 0.85, + "learning_rate": 7.462387161484453e-05, + "loss": 1.2308, + "theoretical_loss": 3.3444764741212567, + "tokens_seen": 3056533504 + }, + { + "epoch": 0.85, + "learning_rate": 7.46038114343029e-05, + "loss": 1.139, + "theoretical_loss": 3.3444710283000387, + "tokens_seen": 3056599040 + }, + { + "epoch": 0.85, + "learning_rate": 7.458375125376128e-05, + "loss": 1.2342, + "theoretical_loss": 3.3444655826282754, + "tokens_seen": 3056664576 + }, + { + "epoch": 0.85, + "learning_rate": 7.456369107321966e-05, + "loss": 1.2063, + "theoretical_loss": 3.3444601371059592, + "tokens_seen": 3056730112 + }, + { + "epoch": 0.85, + "learning_rate": 7.454363089267803e-05, + "loss": 1.2658, + "theoretical_loss": 3.344454691733083, + "tokens_seen": 3056795648 + }, + { + "epoch": 0.85, + "learning_rate": 7.452357071213641e-05, + "loss": 1.2649, + "theoretical_loss": 3.3444492465096394, + "tokens_seen": 3056861184 + }, + { + "epoch": 0.85, + "learning_rate": 7.45035105315948e-05, + "loss": 1.2681, + "theoretical_loss": 3.3444438014356206, + "tokens_seen": 3056926720 + }, + { + "epoch": 0.85, + "learning_rate": 7.448345035105317e-05, + "loss": 1.199, + "theoretical_loss": 3.3444383565110196, + "tokens_seen": 3056992256 + }, + { + "epoch": 0.85, + "learning_rate": 7.446339017051154e-05, + "loss": 1.2944, + "theoretical_loss": 3.3444329117358294, + "tokens_seen": 3057057792 + }, + { + "epoch": 0.85, + "learning_rate": 7.444332998996991e-05, + "loss": 1.277, + "theoretical_loss": 3.3444274671100427, + "tokens_seen": 3057123328 + }, + { + "epoch": 0.85, + "learning_rate": 7.442326980942829e-05, + "loss": 1.1964, + "theoretical_loss": 3.344422022633652, + "tokens_seen": 3057188864 + }, + { + "epoch": 0.85, + "learning_rate": 7.440320962888666e-05, + "loss": 1.2089, + "theoretical_loss": 3.34441657830665, + "tokens_seen": 3057254400 + }, + { + "epoch": 0.85, + "learning_rate": 7.438314944834504e-05, + "loss": 1.1274, + "theoretical_loss": 3.3444111341290297, + "tokens_seen": 3057319936 + }, + { + "epoch": 0.85, + "learning_rate": 7.436308926780341e-05, + "loss": 1.2457, + "theoretical_loss": 3.344405690100783, + "tokens_seen": 3057385472 + }, + { + "epoch": 0.85, + "learning_rate": 7.434302908726179e-05, + "loss": 1.1875, + "theoretical_loss": 3.344400246221903, + "tokens_seen": 3057451008 + }, + { + "epoch": 0.85, + "learning_rate": 7.432296890672017e-05, + "loss": 1.2643, + "theoretical_loss": 3.344394802492383, + "tokens_seen": 3057516544 + }, + { + "epoch": 0.85, + "learning_rate": 7.430290872617854e-05, + "loss": 1.3029, + "theoretical_loss": 3.344389358912215, + "tokens_seen": 3057582080 + }, + { + "epoch": 0.85, + "learning_rate": 7.42828485456369e-05, + "loss": 1.2308, + "theoretical_loss": 3.344383915481392, + "tokens_seen": 3057647616 + }, + { + "epoch": 0.85, + "learning_rate": 7.426278836509528e-05, + "loss": 1.0972, + "theoretical_loss": 3.3443784721999066, + "tokens_seen": 3057713152 + }, + { + "epoch": 0.85, + "learning_rate": 7.424272818455366e-05, + "loss": 1.1398, + "theoretical_loss": 3.3443730290677514, + "tokens_seen": 3057778688 + }, + { + "epoch": 0.85, + "learning_rate": 7.422266800401203e-05, + "loss": 1.2086, + "theoretical_loss": 3.3443675860849194, + "tokens_seen": 3057844224 + }, + { + "epoch": 0.85, + "learning_rate": 7.420260782347041e-05, + "loss": 1.2069, + "theoretical_loss": 3.344362143251403, + "tokens_seen": 3057909760 + }, + { + "epoch": 0.85, + "learning_rate": 7.418254764292878e-05, + "loss": 1.2721, + "theoretical_loss": 3.344356700567195, + "tokens_seen": 3057975296 + }, + { + "epoch": 0.85, + "learning_rate": 7.416248746238716e-05, + "loss": 1.1739, + "theoretical_loss": 3.3443512580322885, + "tokens_seen": 3058040832 + }, + { + "epoch": 0.85, + "learning_rate": 7.414242728184553e-05, + "loss": 1.2772, + "theoretical_loss": 3.344345815646675, + "tokens_seen": 3058106368 + }, + { + "epoch": 0.85, + "learning_rate": 7.412236710130392e-05, + "loss": 1.1791, + "theoretical_loss": 3.3443403734103487, + "tokens_seen": 3058171904 + }, + { + "epoch": 0.85, + "learning_rate": 7.41023069207623e-05, + "loss": 1.1926, + "theoretical_loss": 3.344334931323302, + "tokens_seen": 3058237440 + }, + { + "epoch": 0.85, + "learning_rate": 7.408224674022066e-05, + "loss": 1.1356, + "theoretical_loss": 3.3443294893855264, + "tokens_seen": 3058302976 + }, + { + "epoch": 0.85, + "learning_rate": 7.406218655967904e-05, + "loss": 1.2867, + "theoretical_loss": 3.344324047597016, + "tokens_seen": 3058368512 + }, + { + "epoch": 0.85, + "learning_rate": 7.404212637913741e-05, + "loss": 1.2529, + "theoretical_loss": 3.344318605957763, + "tokens_seen": 3058434048 + }, + { + "epoch": 0.85, + "learning_rate": 7.402206619859579e-05, + "loss": 1.3529, + "theoretical_loss": 3.34431316446776, + "tokens_seen": 3058499584 + }, + { + "epoch": 0.85, + "learning_rate": 7.400200601805417e-05, + "loss": 1.346, + "theoretical_loss": 3.3443077231269993, + "tokens_seen": 3058565120 + }, + { + "epoch": 0.85, + "learning_rate": 7.398194583751254e-05, + "loss": 1.3231, + "theoretical_loss": 3.3443022819354744, + "tokens_seen": 3058630656 + }, + { + "epoch": 0.85, + "learning_rate": 7.396188565697092e-05, + "loss": 1.0801, + "theoretical_loss": 3.344296840893178, + "tokens_seen": 3058696192 + }, + { + "epoch": 0.85, + "learning_rate": 7.394182547642929e-05, + "loss": 1.119, + "theoretical_loss": 3.3442914000001025, + "tokens_seen": 3058761728 + }, + { + "epoch": 0.85, + "learning_rate": 7.392176529588767e-05, + "loss": 1.2655, + "theoretical_loss": 3.34428595925624, + "tokens_seen": 3058827264 + }, + { + "epoch": 0.85, + "learning_rate": 7.390170511534604e-05, + "loss": 1.2394, + "theoretical_loss": 3.3442805186615843, + "tokens_seen": 3058892800 + }, + { + "epoch": 0.85, + "learning_rate": 7.38816449348044e-05, + "loss": 1.1967, + "theoretical_loss": 3.344275078216128, + "tokens_seen": 3058958336 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1111330, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.197962999343872, + "objective/train/theoretical_loss": 3.344269637919863, + "objective/train/tokens_used": 1429482976, + "theoretical_loss": 3.344269637919863, + "tokens_seen": 3059023872 + }, + { + "epoch": 0.85, + "learning_rate": 7.386158475426278e-05, + "loss": 1.1617, + "theoretical_loss": 3.344269637919863, + "tokens_seen": 3059023872 + }, + { + "epoch": 0.85, + "learning_rate": 7.384152457372116e-05, + "loss": 1.2778, + "theoretical_loss": 3.3442641977727825, + "tokens_seen": 3059089408 + }, + { + "epoch": 0.85, + "learning_rate": 7.382146439317953e-05, + "loss": 1.277, + "theoretical_loss": 3.3442587577748792, + "tokens_seen": 3059154944 + }, + { + "epoch": 0.85, + "learning_rate": 7.380140421263791e-05, + "loss": 1.3115, + "theoretical_loss": 3.344253317926146, + "tokens_seen": 3059220480 + }, + { + "epoch": 0.85, + "learning_rate": 7.378134403209629e-05, + "loss": 1.1773, + "theoretical_loss": 3.344247878226575, + "tokens_seen": 3059286016 + }, + { + "epoch": 0.85, + "learning_rate": 7.376128385155466e-05, + "loss": 1.2668, + "theoretical_loss": 3.34424243867616, + "tokens_seen": 3059351552 + }, + { + "epoch": 0.85, + "learning_rate": 7.374122367101305e-05, + "loss": 1.2853, + "theoretical_loss": 3.3442369992748926, + "tokens_seen": 3059417088 + }, + { + "epoch": 0.85, + "learning_rate": 7.372116349047143e-05, + "loss": 1.2472, + "theoretical_loss": 3.3442315600227657, + "tokens_seen": 3059482624 + }, + { + "epoch": 0.85, + "learning_rate": 7.37011033099298e-05, + "loss": 1.1235, + "theoretical_loss": 3.344226120919773, + "tokens_seen": 3059548160 + }, + { + "epoch": 0.85, + "learning_rate": 7.368104312938816e-05, + "loss": 1.1502, + "theoretical_loss": 3.344220681965906, + "tokens_seen": 3059613696 + }, + { + "epoch": 0.85, + "learning_rate": 7.366098294884654e-05, + "loss": 1.2667, + "theoretical_loss": 3.344215243161158, + "tokens_seen": 3059679232 + }, + { + "epoch": 0.85, + "learning_rate": 7.364092276830492e-05, + "loss": 1.2644, + "theoretical_loss": 3.3442098045055215, + "tokens_seen": 3059744768 + }, + { + "epoch": 0.85, + "learning_rate": 7.362086258776329e-05, + "loss": 1.1519, + "theoretical_loss": 3.3442043659989893, + "tokens_seen": 3059810304 + }, + { + "epoch": 0.85, + "learning_rate": 7.360080240722167e-05, + "loss": 1.2404, + "theoretical_loss": 3.3441989276415542, + "tokens_seen": 3059875840 + }, + { + "epoch": 0.85, + "learning_rate": 7.358074222668004e-05, + "loss": 1.1967, + "theoretical_loss": 3.3441934894332093, + "tokens_seen": 3059941376 + }, + { + "epoch": 0.85, + "learning_rate": 7.356068204613842e-05, + "loss": 1.3028, + "theoretical_loss": 3.3441880513739464, + "tokens_seen": 3060006912 + }, + { + "epoch": 0.85, + "learning_rate": 7.35406218655968e-05, + "loss": 1.1793, + "theoretical_loss": 3.344182613463759, + "tokens_seen": 3060072448 + }, + { + "epoch": 0.85, + "learning_rate": 7.352056168505517e-05, + "loss": 1.3346, + "theoretical_loss": 3.3441771757026397, + "tokens_seen": 3060137984 + }, + { + "epoch": 0.85, + "learning_rate": 7.350050150451353e-05, + "loss": 1.2836, + "theoretical_loss": 3.344171738090581, + "tokens_seen": 3060203520 + }, + { + "epoch": 0.85, + "learning_rate": 7.348044132397191e-05, + "loss": 1.1588, + "theoretical_loss": 3.3441663006275753, + "tokens_seen": 3060269056 + }, + { + "epoch": 0.85, + "learning_rate": 7.346038114343028e-05, + "loss": 1.2061, + "theoretical_loss": 3.344160863313616, + "tokens_seen": 3060334592 + }, + { + "epoch": 0.85, + "learning_rate": 7.344032096288866e-05, + "loss": 1.2305, + "theoretical_loss": 3.3441554261486957, + "tokens_seen": 3060400128 + }, + { + "epoch": 0.85, + "learning_rate": 7.342026078234704e-05, + "loss": 1.2737, + "theoretical_loss": 3.3441499891328066, + "tokens_seen": 3060465664 + }, + { + "epoch": 0.85, + "learning_rate": 7.340020060180541e-05, + "loss": 1.2271, + "theoretical_loss": 3.344144552265942, + "tokens_seen": 3060531200 + }, + { + "epoch": 0.85, + "learning_rate": 7.33801404212638e-05, + "loss": 1.2824, + "theoretical_loss": 3.3441391155480944, + "tokens_seen": 3060596736 + }, + { + "epoch": 0.85, + "learning_rate": 7.336008024072218e-05, + "loss": 1.1939, + "theoretical_loss": 3.344133678979256, + "tokens_seen": 3060662272 + }, + { + "epoch": 0.85, + "learning_rate": 7.334002006018055e-05, + "loss": 1.2042, + "theoretical_loss": 3.344128242559421, + "tokens_seen": 3060727808 + }, + { + "epoch": 0.86, + "learning_rate": 7.331995987963893e-05, + "loss": 1.3652, + "theoretical_loss": 3.344122806288581, + "tokens_seen": 3060793344 + }, + { + "epoch": 0.86, + "learning_rate": 7.329989969909729e-05, + "loss": 1.2253, + "theoretical_loss": 3.344117370166728, + "tokens_seen": 3060858880 + }, + { + "epoch": 0.86, + "learning_rate": 7.327983951855567e-05, + "loss": 1.1215, + "theoretical_loss": 3.3441119341938568, + "tokens_seen": 3060924416 + }, + { + "epoch": 0.86, + "learning_rate": 7.325977933801404e-05, + "loss": 1.2646, + "theoretical_loss": 3.344106498369958, + "tokens_seen": 3060989952 + }, + { + "epoch": 0.86, + "learning_rate": 7.323971915747242e-05, + "loss": 1.2584, + "theoretical_loss": 3.344101062695026, + "tokens_seen": 3061055488 + }, + { + "epoch": 0.86, + "learning_rate": 7.32196589769308e-05, + "loss": 1.1661, + "theoretical_loss": 3.3440956271690525, + "tokens_seen": 3061121024 + }, + { + "epoch": 0.86, + "learning_rate": 7.319959879638917e-05, + "loss": 1.277, + "theoretical_loss": 3.3440901917920307, + "tokens_seen": 3061186560 + }, + { + "epoch": 0.86, + "learning_rate": 7.317953861584755e-05, + "loss": 1.1809, + "theoretical_loss": 3.344084756563953, + "tokens_seen": 3061252096 + }, + { + "epoch": 0.86, + "learning_rate": 7.315947843530592e-05, + "loss": 1.1162, + "theoretical_loss": 3.3440793214848124, + "tokens_seen": 3061317632 + }, + { + "epoch": 0.86, + "learning_rate": 7.31394182547643e-05, + "loss": 1.115, + "theoretical_loss": 3.3440738865546016, + "tokens_seen": 3061383168 + }, + { + "epoch": 0.86, + "learning_rate": 7.311935807422267e-05, + "loss": 1.3299, + "theoretical_loss": 3.344068451773313, + "tokens_seen": 3061448704 + }, + { + "epoch": 0.86, + "learning_rate": 7.309929789368104e-05, + "loss": 1.2333, + "theoretical_loss": 3.34406301714094, + "tokens_seen": 3061514240 + }, + { + "epoch": 0.86, + "learning_rate": 7.307923771313941e-05, + "loss": 1.2458, + "theoretical_loss": 3.3440575826574745, + "tokens_seen": 3061579776 + }, + { + "epoch": 0.86, + "learning_rate": 7.305917753259779e-05, + "loss": 1.2766, + "theoretical_loss": 3.34405214832291, + "tokens_seen": 3061645312 + }, + { + "epoch": 0.86, + "learning_rate": 7.303911735205616e-05, + "loss": 1.1243, + "theoretical_loss": 3.3440467141372388, + "tokens_seen": 3061710848 + }, + { + "epoch": 0.86, + "learning_rate": 7.301905717151454e-05, + "loss": 1.3593, + "theoretical_loss": 3.3440412801004538, + "tokens_seen": 3061776384 + }, + { + "epoch": 0.86, + "learning_rate": 7.299899699097293e-05, + "loss": 1.2892, + "theoretical_loss": 3.3440358462125475, + "tokens_seen": 3061841920 + }, + { + "epoch": 0.86, + "learning_rate": 7.29789368104313e-05, + "loss": 1.1931, + "theoretical_loss": 3.344030412473513, + "tokens_seen": 3061907456 + }, + { + "epoch": 0.86, + "learning_rate": 7.295887662988968e-05, + "loss": 1.3246, + "theoretical_loss": 3.3440249788833425, + "tokens_seen": 3061972992 + }, + { + "epoch": 0.86, + "learning_rate": 7.293881644934806e-05, + "loss": 1.3444, + "theoretical_loss": 3.3440195454420296, + "tokens_seen": 3062038528 + }, + { + "epoch": 0.86, + "learning_rate": 7.291875626880642e-05, + "loss": 1.2291, + "theoretical_loss": 3.344014112149566, + "tokens_seen": 3062104064 + }, + { + "epoch": 0.86, + "learning_rate": 7.28986960882648e-05, + "loss": 1.264, + "theoretical_loss": 3.3440086790059453, + "tokens_seen": 3062169600 + }, + { + "epoch": 0.86, + "learning_rate": 7.287863590772317e-05, + "loss": 1.2129, + "theoretical_loss": 3.3440032460111597, + "tokens_seen": 3062235136 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1112415, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.172864556312561, + "objective/train/theoretical_loss": 3.343997813165202, + "objective/train/tokens_used": 1432759776, + "theoretical_loss": 3.343997813165202, + "tokens_seen": 3062300672 + }, + { + "epoch": 0.86, + "learning_rate": 7.285857572718154e-05, + "loss": 1.1978, + "theoretical_loss": 3.343997813165202, + "tokens_seen": 3062300672 + }, + { + "epoch": 0.86, + "learning_rate": 7.283851554663992e-05, + "loss": 1.2873, + "theoretical_loss": 3.3439923804680656, + "tokens_seen": 3062366208 + }, + { + "epoch": 0.86, + "learning_rate": 7.28184553660983e-05, + "loss": 1.335, + "theoretical_loss": 3.343986947919742, + "tokens_seen": 3062431744 + }, + { + "epoch": 0.86, + "learning_rate": 7.279839518555667e-05, + "loss": 1.2176, + "theoretical_loss": 3.343981515520225, + "tokens_seen": 3062497280 + }, + { + "epoch": 0.86, + "learning_rate": 7.277833500501505e-05, + "loss": 1.343, + "theoretical_loss": 3.343976083269507, + "tokens_seen": 3062562816 + }, + { + "epoch": 0.86, + "learning_rate": 7.275827482447342e-05, + "loss": 1.356, + "theoretical_loss": 3.343970651167581, + "tokens_seen": 3062628352 + }, + { + "epoch": 0.86, + "learning_rate": 7.27382146439318e-05, + "loss": 1.3856, + "theoretical_loss": 3.343965219214439, + "tokens_seen": 3062693888 + }, + { + "epoch": 0.86, + "learning_rate": 7.271815446339016e-05, + "loss": 1.246, + "theoretical_loss": 3.343959787410074, + "tokens_seen": 3062759424 + }, + { + "epoch": 0.86, + "learning_rate": 7.269809428284854e-05, + "loss": 1.2012, + "theoretical_loss": 3.3439543557544793, + "tokens_seen": 3062824960 + }, + { + "epoch": 0.86, + "learning_rate": 7.267803410230691e-05, + "loss": 1.2288, + "theoretical_loss": 3.3439489242476474, + "tokens_seen": 3062890496 + }, + { + "epoch": 0.86, + "learning_rate": 7.265797392176529e-05, + "loss": 1.282, + "theoretical_loss": 3.343943492889571, + "tokens_seen": 3062956032 + }, + { + "epoch": 0.86, + "learning_rate": 7.263791374122367e-05, + "loss": 1.2314, + "theoretical_loss": 3.3439380616802423, + "tokens_seen": 3063021568 + }, + { + "epoch": 0.86, + "learning_rate": 7.261785356068205e-05, + "loss": 1.1952, + "theoretical_loss": 3.343932630619655, + "tokens_seen": 3063087104 + }, + { + "epoch": 0.86, + "learning_rate": 7.259779338014043e-05, + "loss": 1.2614, + "theoretical_loss": 3.343927199707801, + "tokens_seen": 3063152640 + }, + { + "epoch": 0.86, + "learning_rate": 7.25777331995988e-05, + "loss": 1.2516, + "theoretical_loss": 3.343921768944673, + "tokens_seen": 3063218176 + }, + { + "epoch": 0.86, + "learning_rate": 7.255767301905718e-05, + "loss": 1.3118, + "theoretical_loss": 3.343916338330265, + "tokens_seen": 3063283712 + }, + { + "epoch": 0.86, + "learning_rate": 7.253761283851556e-05, + "loss": 1.2396, + "theoretical_loss": 3.3439109078645686, + "tokens_seen": 3063349248 + }, + { + "epoch": 0.86, + "learning_rate": 7.251755265797392e-05, + "loss": 1.4123, + "theoretical_loss": 3.3439054775475765, + "tokens_seen": 3063414784 + }, + { + "epoch": 0.86, + "learning_rate": 7.24974924774323e-05, + "loss": 1.1974, + "theoretical_loss": 3.3439000473792824, + "tokens_seen": 3063480320 + }, + { + "epoch": 0.86, + "learning_rate": 7.247743229689067e-05, + "loss": 1.2987, + "theoretical_loss": 3.3438946173596777, + "tokens_seen": 3063545856 + }, + { + "epoch": 0.86, + "learning_rate": 7.245737211634905e-05, + "loss": 1.2119, + "theoretical_loss": 3.3438891874887564, + "tokens_seen": 3063611392 + }, + { + "epoch": 0.86, + "learning_rate": 7.243731193580742e-05, + "loss": 1.3068, + "theoretical_loss": 3.3438837577665104, + "tokens_seen": 3063676928 + }, + { + "epoch": 0.86, + "learning_rate": 7.24172517552658e-05, + "loss": 1.2663, + "theoretical_loss": 3.3438783281929334, + "tokens_seen": 3063742464 + }, + { + "epoch": 0.86, + "learning_rate": 7.239719157472417e-05, + "loss": 1.2644, + "theoretical_loss": 3.3438728987680166, + "tokens_seen": 3063808000 + }, + { + "epoch": 0.86, + "learning_rate": 7.237713139418255e-05, + "loss": 1.2199, + "theoretical_loss": 3.3438674694917543, + "tokens_seen": 3063873536 + }, + { + "epoch": 0.86, + "learning_rate": 7.235707121364093e-05, + "loss": 1.2371, + "theoretical_loss": 3.343862040364139, + "tokens_seen": 3063939072 + }, + { + "epoch": 0.86, + "learning_rate": 7.23370110330993e-05, + "loss": 1.2474, + "theoretical_loss": 3.343856611385162, + "tokens_seen": 3064004608 + }, + { + "epoch": 0.86, + "learning_rate": 7.231695085255766e-05, + "loss": 1.3425, + "theoretical_loss": 3.343851182554818, + "tokens_seen": 3064070144 + }, + { + "epoch": 0.86, + "learning_rate": 7.229689067201604e-05, + "loss": 1.309, + "theoretical_loss": 3.3438457538730986, + "tokens_seen": 3064135680 + }, + { + "epoch": 0.86, + "learning_rate": 7.227683049147442e-05, + "loss": 1.2838, + "theoretical_loss": 3.343840325339997, + "tokens_seen": 3064201216 + }, + { + "epoch": 0.86, + "learning_rate": 7.22567703109328e-05, + "loss": 1.2505, + "theoretical_loss": 3.3438348969555056, + "tokens_seen": 3064266752 + }, + { + "epoch": 0.86, + "learning_rate": 7.223671013039118e-05, + "loss": 1.1515, + "theoretical_loss": 3.3438294687196173, + "tokens_seen": 3064332288 + }, + { + "epoch": 0.86, + "learning_rate": 7.221664994984956e-05, + "loss": 1.2391, + "theoretical_loss": 3.3438240406323247, + "tokens_seen": 3064397824 + }, + { + "epoch": 0.86, + "learning_rate": 7.219658976930793e-05, + "loss": 1.1839, + "theoretical_loss": 3.343818612693621, + "tokens_seen": 3064463360 + }, + { + "epoch": 0.86, + "learning_rate": 7.217652958876631e-05, + "loss": 1.1778, + "theoretical_loss": 3.3438131849034987, + "tokens_seen": 3064528896 + }, + { + "epoch": 0.86, + "learning_rate": 7.215646940822468e-05, + "loss": 1.2616, + "theoretical_loss": 3.343807757261951, + "tokens_seen": 3064594432 + }, + { + "epoch": 0.86, + "learning_rate": 7.213640922768305e-05, + "loss": 1.2842, + "theoretical_loss": 3.3438023297689696, + "tokens_seen": 3064659968 + }, + { + "epoch": 0.86, + "learning_rate": 7.211634904714142e-05, + "loss": 1.1668, + "theoretical_loss": 3.3437969024245477, + "tokens_seen": 3064725504 + }, + { + "epoch": 0.86, + "learning_rate": 7.20962888665998e-05, + "loss": 1.252, + "theoretical_loss": 3.343791475228679, + "tokens_seen": 3064791040 + }, + { + "epoch": 0.86, + "learning_rate": 7.207622868605817e-05, + "loss": 1.184, + "theoretical_loss": 3.3437860481813546, + "tokens_seen": 3064856576 + }, + { + "epoch": 0.86, + "learning_rate": 7.205616850551655e-05, + "loss": 1.2375, + "theoretical_loss": 3.3437806212825687, + "tokens_seen": 3064922112 + }, + { + "epoch": 0.86, + "learning_rate": 7.203610832497493e-05, + "loss": 1.2314, + "theoretical_loss": 3.3437751945323138, + "tokens_seen": 3064987648 + }, + { + "epoch": 0.86, + "learning_rate": 7.20160481444333e-05, + "loss": 1.3149, + "theoretical_loss": 3.343769767930582, + "tokens_seen": 3065053184 + }, + { + "epoch": 0.86, + "learning_rate": 7.199598796389168e-05, + "loss": 1.256, + "theoretical_loss": 3.343764341477366, + "tokens_seen": 3065118720 + }, + { + "epoch": 0.86, + "learning_rate": 7.197592778335005e-05, + "loss": 1.2844, + "theoretical_loss": 3.3437589151726597, + "tokens_seen": 3065184256 + }, + { + "epoch": 0.86, + "learning_rate": 7.195586760280843e-05, + "loss": 1.2267, + "theoretical_loss": 3.3437534890164544, + "tokens_seen": 3065249792 + }, + { + "epoch": 0.86, + "learning_rate": 7.193580742226679e-05, + "loss": 1.1909, + "theoretical_loss": 3.343748063008744, + "tokens_seen": 3065315328 + }, + { + "epoch": 0.86, + "learning_rate": 7.191574724172517e-05, + "loss": 1.1851, + "theoretical_loss": 3.3437426371495214, + "tokens_seen": 3065380864 + }, + { + "epoch": 0.86, + "learning_rate": 7.189568706118354e-05, + "loss": 1.2988, + "theoretical_loss": 3.343737211438778, + "tokens_seen": 3065446400 + }, + { + "epoch": 0.86, + "learning_rate": 7.187562688064193e-05, + "loss": 1.2925, + "theoretical_loss": 3.343731785876508, + "tokens_seen": 3065511936 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1113675, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.228407859802246, + "objective/train/theoretical_loss": 3.343726360462703, + "objective/train/tokens_used": 1436036576, + "theoretical_loss": 3.343726360462703, + "tokens_seen": 3065577472 + }, + { + "epoch": 0.86, + "learning_rate": 7.185556670010031e-05, + "loss": 1.2218, + "theoretical_loss": 3.343726360462703, + "tokens_seen": 3065577472 + }, + { + "epoch": 0.86, + "learning_rate": 7.183550651955868e-05, + "loss": 1.1769, + "theoretical_loss": 3.3437209351973567, + "tokens_seen": 3065643008 + }, + { + "epoch": 0.86, + "learning_rate": 7.181544633901706e-05, + "loss": 1.3058, + "theoretical_loss": 3.3437155100804614, + "tokens_seen": 3065708544 + }, + { + "epoch": 0.86, + "learning_rate": 7.179538615847543e-05, + "loss": 1.3262, + "theoretical_loss": 3.3437100851120096, + "tokens_seen": 3065774080 + }, + { + "epoch": 0.86, + "learning_rate": 7.177532597793381e-05, + "loss": 1.1986, + "theoretical_loss": 3.343704660291995, + "tokens_seen": 3065839616 + }, + { + "epoch": 0.86, + "learning_rate": 7.175526579739219e-05, + "loss": 1.2743, + "theoretical_loss": 3.3436992356204094, + "tokens_seen": 3065905152 + }, + { + "epoch": 0.86, + "learning_rate": 7.173520561685055e-05, + "loss": 1.248, + "theoretical_loss": 3.3436938110972463, + "tokens_seen": 3065970688 + }, + { + "epoch": 0.86, + "learning_rate": 7.171514543630892e-05, + "loss": 1.3278, + "theoretical_loss": 3.343688386722498, + "tokens_seen": 3066036224 + }, + { + "epoch": 0.86, + "learning_rate": 7.16950852557673e-05, + "loss": 1.3025, + "theoretical_loss": 3.343682962496157, + "tokens_seen": 3066101760 + }, + { + "epoch": 0.86, + "learning_rate": 7.167502507522568e-05, + "loss": 1.1593, + "theoretical_loss": 3.3436775384182162, + "tokens_seen": 3066167296 + }, + { + "epoch": 0.86, + "learning_rate": 7.165496489468405e-05, + "loss": 1.2305, + "theoretical_loss": 3.3436721144886694, + "tokens_seen": 3066232832 + }, + { + "epoch": 0.86, + "learning_rate": 7.163490471414243e-05, + "loss": 1.2691, + "theoretical_loss": 3.3436666907075083, + "tokens_seen": 3066298368 + }, + { + "epoch": 0.86, + "learning_rate": 7.16148445336008e-05, + "loss": 1.2365, + "theoretical_loss": 3.343661267074726, + "tokens_seen": 3066363904 + }, + { + "epoch": 0.86, + "learning_rate": 7.159478435305918e-05, + "loss": 1.2524, + "theoretical_loss": 3.343655843590315, + "tokens_seen": 3066429440 + }, + { + "epoch": 0.86, + "learning_rate": 7.157472417251756e-05, + "loss": 1.1999, + "theoretical_loss": 3.3436504202542685, + "tokens_seen": 3066494976 + }, + { + "epoch": 0.86, + "learning_rate": 7.155466399197592e-05, + "loss": 1.1164, + "theoretical_loss": 3.343644997066579, + "tokens_seen": 3066560512 + }, + { + "epoch": 0.86, + "learning_rate": 7.153460381143429e-05, + "loss": 1.2325, + "theoretical_loss": 3.343639574027239, + "tokens_seen": 3066626048 + }, + { + "epoch": 0.86, + "learning_rate": 7.151454363089267e-05, + "loss": 1.2105, + "theoretical_loss": 3.343634151136242, + "tokens_seen": 3066691584 + }, + { + "epoch": 0.86, + "learning_rate": 7.149448345035106e-05, + "loss": 1.2958, + "theoretical_loss": 3.3436287283935804, + "tokens_seen": 3066757120 + }, + { + "epoch": 0.86, + "learning_rate": 7.147442326980943e-05, + "loss": 1.2587, + "theoretical_loss": 3.343623305799247, + "tokens_seen": 3066822656 + }, + { + "epoch": 0.86, + "learning_rate": 7.145436308926781e-05, + "loss": 1.2295, + "theoretical_loss": 3.343617883353234, + "tokens_seen": 3066888192 + }, + { + "epoch": 0.86, + "learning_rate": 7.143430290872619e-05, + "loss": 1.1695, + "theoretical_loss": 3.3436124610555353, + "tokens_seen": 3066953728 + }, + { + "epoch": 0.86, + "learning_rate": 7.141424272818456e-05, + "loss": 1.3045, + "theoretical_loss": 3.3436070389061427, + "tokens_seen": 3067019264 + }, + { + "epoch": 0.86, + "learning_rate": 7.139418254764294e-05, + "loss": 1.3137, + "theoretical_loss": 3.3436016169050493, + "tokens_seen": 3067084800 + }, + { + "epoch": 0.86, + "learning_rate": 7.137412236710131e-05, + "loss": 1.2387, + "theoretical_loss": 3.343596195052248, + "tokens_seen": 3067150336 + }, + { + "epoch": 0.86, + "learning_rate": 7.135406218655968e-05, + "loss": 1.2782, + "theoretical_loss": 3.3435907733477315, + "tokens_seen": 3067215872 + }, + { + "epoch": 0.86, + "learning_rate": 7.133400200601805e-05, + "loss": 1.2218, + "theoretical_loss": 3.343585351791493, + "tokens_seen": 3067281408 + }, + { + "epoch": 0.86, + "learning_rate": 7.131394182547643e-05, + "loss": 1.2396, + "theoretical_loss": 3.343579930383524, + "tokens_seen": 3067346944 + }, + { + "epoch": 0.86, + "learning_rate": 7.12938816449348e-05, + "loss": 1.2849, + "theoretical_loss": 3.343574509123819, + "tokens_seen": 3067412480 + }, + { + "epoch": 0.86, + "learning_rate": 7.127382146439318e-05, + "loss": 1.221, + "theoretical_loss": 3.3435690880123694, + "tokens_seen": 3067478016 + }, + { + "epoch": 0.86, + "learning_rate": 7.125376128385155e-05, + "loss": 1.2896, + "theoretical_loss": 3.3435636670491684, + "tokens_seen": 3067543552 + }, + { + "epoch": 0.86, + "learning_rate": 7.123370110330993e-05, + "loss": 1.1655, + "theoretical_loss": 3.3435582462342093, + "tokens_seen": 3067609088 + }, + { + "epoch": 0.86, + "learning_rate": 7.12136409227683e-05, + "loss": 1.3117, + "theoretical_loss": 3.3435528255674845, + "tokens_seen": 3067674624 + }, + { + "epoch": 0.86, + "learning_rate": 7.119358074222668e-05, + "loss": 1.173, + "theoretical_loss": 3.343547405048986, + "tokens_seen": 3067740160 + }, + { + "epoch": 0.86, + "learning_rate": 7.117352056168506e-05, + "loss": 1.1512, + "theoretical_loss": 3.3435419846787076, + "tokens_seen": 3067805696 + }, + { + "epoch": 0.86, + "learning_rate": 7.115346038114342e-05, + "loss": 1.1919, + "theoretical_loss": 3.343536564456642, + "tokens_seen": 3067871232 + }, + { + "epoch": 0.86, + "learning_rate": 7.113340020060181e-05, + "loss": 1.198, + "theoretical_loss": 3.3435311443827818, + "tokens_seen": 3067936768 + }, + { + "epoch": 0.86, + "learning_rate": 7.111334002006018e-05, + "loss": 1.2253, + "theoretical_loss": 3.3435257244571197, + "tokens_seen": 3068002304 + }, + { + "epoch": 0.86, + "learning_rate": 7.109327983951856e-05, + "loss": 1.2969, + "theoretical_loss": 3.3435203046796484, + "tokens_seen": 3068067840 + }, + { + "epoch": 0.86, + "learning_rate": 7.107321965897694e-05, + "loss": 1.2193, + "theoretical_loss": 3.343514885050361, + "tokens_seen": 3068133376 + }, + { + "epoch": 0.86, + "learning_rate": 7.105315947843531e-05, + "loss": 1.2216, + "theoretical_loss": 3.34350946556925, + "tokens_seen": 3068198912 + }, + { + "epoch": 0.86, + "learning_rate": 7.103309929789369e-05, + "loss": 1.2122, + "theoretical_loss": 3.343504046236308, + "tokens_seen": 3068264448 + }, + { + "epoch": 0.86, + "learning_rate": 7.101303911735206e-05, + "loss": 1.2154, + "theoretical_loss": 3.343498627051528, + "tokens_seen": 3068329984 + }, + { + "epoch": 0.86, + "learning_rate": 7.099297893681044e-05, + "loss": 1.3754, + "theoretical_loss": 3.343493208014903, + "tokens_seen": 3068395520 + }, + { + "epoch": 0.86, + "learning_rate": 7.097291875626882e-05, + "loss": 1.28, + "theoretical_loss": 3.343487789126426, + "tokens_seen": 3068461056 + }, + { + "epoch": 0.86, + "learning_rate": 7.095285857572718e-05, + "loss": 1.3348, + "theoretical_loss": 3.3434823703860888, + "tokens_seen": 3068526592 + }, + { + "epoch": 0.86, + "learning_rate": 7.093279839518555e-05, + "loss": 1.1923, + "theoretical_loss": 3.3434769517938854, + "tokens_seen": 3068592128 + }, + { + "epoch": 0.86, + "learning_rate": 7.091273821464393e-05, + "loss": 1.3415, + "theoretical_loss": 3.343471533349808, + "tokens_seen": 3068657664 + }, + { + "epoch": 0.86, + "learning_rate": 7.08926780341023e-05, + "loss": 1.1897, + "theoretical_loss": 3.343466115053849, + "tokens_seen": 3068723200 + }, + { + "epoch": 0.86, + "learning_rate": 7.087261785356068e-05, + "loss": 1.2685, + "theoretical_loss": 3.3434606969060017, + "tokens_seen": 3068788736 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1114913, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1459938287734985, + "objective/train/theoretical_loss": 3.3434552789062586, + "objective/train/tokens_used": 1439313376, + "theoretical_loss": 3.3434552789062586, + "tokens_seen": 3068854272 + }, + { + "epoch": 0.86, + "learning_rate": 7.085255767301906e-05, + "loss": 1.1436, + "theoretical_loss": 3.3434552789062586, + "tokens_seen": 3068854272 + }, + { + "epoch": 0.86, + "learning_rate": 7.083249749247743e-05, + "loss": 1.2042, + "theoretical_loss": 3.3434498610546126, + "tokens_seen": 3068919808 + }, + { + "epoch": 0.86, + "learning_rate": 7.081243731193581e-05, + "loss": 1.1617, + "theoretical_loss": 3.343444443351057, + "tokens_seen": 3068985344 + }, + { + "epoch": 0.86, + "learning_rate": 7.079237713139418e-05, + "loss": 1.2073, + "theoretical_loss": 3.343439025795584, + "tokens_seen": 3069050880 + }, + { + "epoch": 0.86, + "learning_rate": 7.077231695085255e-05, + "loss": 1.3145, + "theoretical_loss": 3.343433608388186, + "tokens_seen": 3069116416 + }, + { + "epoch": 0.86, + "learning_rate": 7.075225677031094e-05, + "loss": 1.2751, + "theoretical_loss": 3.343428191128857, + "tokens_seen": 3069181952 + }, + { + "epoch": 0.86, + "learning_rate": 7.073219658976931e-05, + "loss": 1.2873, + "theoretical_loss": 3.3434227740175886, + "tokens_seen": 3069247488 + }, + { + "epoch": 0.86, + "learning_rate": 7.071213640922769e-05, + "loss": 1.1155, + "theoretical_loss": 3.343417357054374, + "tokens_seen": 3069313024 + }, + { + "epoch": 0.86, + "learning_rate": 7.069207622868606e-05, + "loss": 1.3589, + "theoretical_loss": 3.3434119402392066, + "tokens_seen": 3069378560 + }, + { + "epoch": 0.86, + "learning_rate": 7.067201604814444e-05, + "loss": 1.2517, + "theoretical_loss": 3.343406523572078, + "tokens_seen": 3069444096 + }, + { + "epoch": 0.86, + "learning_rate": 7.065195586760281e-05, + "loss": 1.2636, + "theoretical_loss": 3.3434011070529825, + "tokens_seen": 3069509632 + }, + { + "epoch": 0.86, + "learning_rate": 7.063189568706119e-05, + "loss": 1.1962, + "theoretical_loss": 3.3433956906819113, + "tokens_seen": 3069575168 + }, + { + "epoch": 0.86, + "learning_rate": 7.061183550651957e-05, + "loss": 1.2496, + "theoretical_loss": 3.3433902744588586, + "tokens_seen": 3069640704 + }, + { + "epoch": 0.86, + "learning_rate": 7.059177532597794e-05, + "loss": 1.2855, + "theoretical_loss": 3.343384858383816, + "tokens_seen": 3069706240 + }, + { + "epoch": 0.86, + "learning_rate": 7.05717151454363e-05, + "loss": 1.3152, + "theoretical_loss": 3.343379442456777, + "tokens_seen": 3069771776 + }, + { + "epoch": 0.86, + "learning_rate": 7.055165496489468e-05, + "loss": 1.2697, + "theoretical_loss": 3.3433740266777345, + "tokens_seen": 3069837312 + }, + { + "epoch": 0.86, + "learning_rate": 7.053159478435306e-05, + "loss": 1.3434, + "theoretical_loss": 3.3433686110466807, + "tokens_seen": 3069902848 + }, + { + "epoch": 0.86, + "learning_rate": 7.051153460381143e-05, + "loss": 1.293, + "theoretical_loss": 3.343363195563609, + "tokens_seen": 3069968384 + }, + { + "epoch": 0.86, + "learning_rate": 7.049147442326981e-05, + "loss": 1.3148, + "theoretical_loss": 3.3433577802285117, + "tokens_seen": 3070033920 + }, + { + "epoch": 0.86, + "learning_rate": 7.047141424272818e-05, + "loss": 1.3099, + "theoretical_loss": 3.343352365041382, + "tokens_seen": 3070099456 + }, + { + "epoch": 0.86, + "learning_rate": 7.045135406218656e-05, + "loss": 1.244, + "theoretical_loss": 3.3433469500022124, + "tokens_seen": 3070164992 + }, + { + "epoch": 0.86, + "learning_rate": 7.043129388164493e-05, + "loss": 1.3104, + "theoretical_loss": 3.343341535110996, + "tokens_seen": 3070230528 + }, + { + "epoch": 0.86, + "learning_rate": 7.041123370110331e-05, + "loss": 1.2672, + "theoretical_loss": 3.343336120367725, + "tokens_seen": 3070296064 + }, + { + "epoch": 0.86, + "learning_rate": 7.039117352056169e-05, + "loss": 1.2072, + "theoretical_loss": 3.3433307057723933, + "tokens_seen": 3070361600 + }, + { + "epoch": 0.86, + "learning_rate": 7.037111334002006e-05, + "loss": 1.2719, + "theoretical_loss": 3.3433252913249927, + "tokens_seen": 3070427136 + }, + { + "epoch": 0.86, + "learning_rate": 7.035105315947844e-05, + "loss": 1.1981, + "theoretical_loss": 3.343319877025516, + "tokens_seen": 3070492672 + }, + { + "epoch": 0.86, + "learning_rate": 7.033099297893681e-05, + "loss": 1.3416, + "theoretical_loss": 3.3433144628739564, + "tokens_seen": 3070558208 + }, + { + "epoch": 0.86, + "learning_rate": 7.031093279839519e-05, + "loss": 1.2398, + "theoretical_loss": 3.343309048870307, + "tokens_seen": 3070623744 + }, + { + "epoch": 0.86, + "learning_rate": 7.029087261785357e-05, + "loss": 1.27, + "theoretical_loss": 3.34330363501456, + "tokens_seen": 3070689280 + }, + { + "epoch": 0.86, + "learning_rate": 7.027081243731194e-05, + "loss": 1.2646, + "theoretical_loss": 3.343298221306709, + "tokens_seen": 3070754816 + }, + { + "epoch": 0.86, + "learning_rate": 7.025075225677032e-05, + "loss": 1.2659, + "theoretical_loss": 3.3432928077467454, + "tokens_seen": 3070820352 + }, + { + "epoch": 0.86, + "learning_rate": 7.023069207622869e-05, + "loss": 1.3048, + "theoretical_loss": 3.3432873943346633, + "tokens_seen": 3070885888 + }, + { + "epoch": 0.86, + "learning_rate": 7.021063189568707e-05, + "loss": 1.2921, + "theoretical_loss": 3.343281981070455, + "tokens_seen": 3070951424 + }, + { + "epoch": 0.86, + "learning_rate": 7.019057171514544e-05, + "loss": 1.2769, + "theoretical_loss": 3.3432765679541134, + "tokens_seen": 3071016960 + }, + { + "epoch": 0.86, + "learning_rate": 7.01705115346038e-05, + "loss": 1.2028, + "theoretical_loss": 3.343271154985631, + "tokens_seen": 3071082496 + }, + { + "epoch": 0.86, + "learning_rate": 7.015045135406218e-05, + "loss": 1.2634, + "theoretical_loss": 3.343265742165001, + "tokens_seen": 3071148032 + }, + { + "epoch": 0.86, + "learning_rate": 7.013039117352056e-05, + "loss": 1.2181, + "theoretical_loss": 3.343260329492216, + "tokens_seen": 3071213568 + }, + { + "epoch": 0.86, + "learning_rate": 7.011033099297893e-05, + "loss": 1.2297, + "theoretical_loss": 3.343254916967269, + "tokens_seen": 3071279104 + }, + { + "epoch": 0.86, + "learning_rate": 7.009027081243731e-05, + "loss": 1.2876, + "theoretical_loss": 3.343249504590153, + "tokens_seen": 3071344640 + }, + { + "epoch": 0.86, + "learning_rate": 7.007021063189569e-05, + "loss": 1.2028, + "theoretical_loss": 3.34324409236086, + "tokens_seen": 3071410176 + }, + { + "epoch": 0.86, + "learning_rate": 7.005015045135406e-05, + "loss": 1.2566, + "theoretical_loss": 3.3432386802793834, + "tokens_seen": 3071475712 + }, + { + "epoch": 0.86, + "learning_rate": 7.003009027081244e-05, + "loss": 1.3447, + "theoretical_loss": 3.343233268345716, + "tokens_seen": 3071541248 + }, + { + "epoch": 0.86, + "learning_rate": 7.001003009027081e-05, + "loss": 1.1817, + "theoretical_loss": 3.3432278565598508, + "tokens_seen": 3071606784 + }, + { + "epoch": 0.86, + "learning_rate": 6.998996990972919e-05, + "loss": 1.3316, + "theoretical_loss": 3.34322244492178, + "tokens_seen": 3071672320 + }, + { + "epoch": 0.86, + "learning_rate": 6.996990972918756e-05, + "loss": 1.3311, + "theoretical_loss": 3.343217033431497, + "tokens_seen": 3071737856 + }, + { + "epoch": 0.86, + "learning_rate": 6.994984954864594e-05, + "loss": 1.2085, + "theoretical_loss": 3.343211622088994, + "tokens_seen": 3071803392 + }, + { + "epoch": 0.86, + "learning_rate": 6.992978936810432e-05, + "loss": 1.3125, + "theoretical_loss": 3.343206210894264, + "tokens_seen": 3071868928 + }, + { + "epoch": 0.86, + "learning_rate": 6.990972918756269e-05, + "loss": 1.3013, + "theoretical_loss": 3.3432007998473003, + "tokens_seen": 3071934464 + }, + { + "epoch": 0.86, + "learning_rate": 6.988966900702107e-05, + "loss": 1.319, + "theoretical_loss": 3.3431953889480956, + "tokens_seen": 3072000000 + }, + { + "epoch": 0.86, + "learning_rate": 6.986960882647944e-05, + "loss": 1.2937, + "theoretical_loss": 3.343189978196642, + "tokens_seen": 3072065536 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1116127, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2608907222747803, + "objective/train/theoretical_loss": 3.3431845675929335, + "objective/train/tokens_used": 1442590176, + "theoretical_loss": 3.3431845675929335, + "tokens_seen": 3072131072 + }, + { + "epoch": 0.86, + "learning_rate": 6.984954864593782e-05, + "loss": 1.205, + "theoretical_loss": 3.3431845675929335, + "tokens_seen": 3072131072 + }, + { + "epoch": 0.86, + "learning_rate": 6.98294884653962e-05, + "loss": 1.2421, + "theoretical_loss": 3.3431791571369613, + "tokens_seen": 3072196608 + }, + { + "epoch": 0.86, + "learning_rate": 6.980942828485457e-05, + "loss": 1.2326, + "theoretical_loss": 3.34317374682872, + "tokens_seen": 3072262144 + }, + { + "epoch": 0.86, + "learning_rate": 6.978936810431293e-05, + "loss": 1.3149, + "theoretical_loss": 3.3431683366682012, + "tokens_seen": 3072327680 + }, + { + "epoch": 0.86, + "learning_rate": 6.976930792377131e-05, + "loss": 1.24, + "theoretical_loss": 3.343162926655398, + "tokens_seen": 3072393216 + }, + { + "epoch": 0.86, + "learning_rate": 6.974924774322968e-05, + "loss": 1.2194, + "theoretical_loss": 3.3431575167903036, + "tokens_seen": 3072458752 + }, + { + "epoch": 0.86, + "learning_rate": 6.972918756268806e-05, + "loss": 1.2076, + "theoretical_loss": 3.3431521070729104, + "tokens_seen": 3072524288 + }, + { + "epoch": 0.86, + "learning_rate": 6.970912738214644e-05, + "loss": 1.2361, + "theoretical_loss": 3.343146697503211, + "tokens_seen": 3072589824 + }, + { + "epoch": 0.86, + "learning_rate": 6.968906720160481e-05, + "loss": 1.2779, + "theoretical_loss": 3.343141288081199, + "tokens_seen": 3072655360 + }, + { + "epoch": 0.86, + "learning_rate": 6.966900702106319e-05, + "loss": 1.2237, + "theoretical_loss": 3.3431358788068666, + "tokens_seen": 3072720896 + }, + { + "epoch": 0.86, + "learning_rate": 6.964894684052156e-05, + "loss": 1.3128, + "theoretical_loss": 3.3431304696802067, + "tokens_seen": 3072786432 + }, + { + "epoch": 0.86, + "learning_rate": 6.962888665997995e-05, + "loss": 1.309, + "theoretical_loss": 3.343125060701212, + "tokens_seen": 3072851968 + }, + { + "epoch": 0.86, + "learning_rate": 6.960882647943833e-05, + "loss": 1.2526, + "theoretical_loss": 3.3431196518698756, + "tokens_seen": 3072917504 + }, + { + "epoch": 0.86, + "learning_rate": 6.958876629889669e-05, + "loss": 1.2836, + "theoretical_loss": 3.3431142431861907, + "tokens_seen": 3072983040 + }, + { + "epoch": 0.86, + "learning_rate": 6.956870611835507e-05, + "loss": 1.1674, + "theoretical_loss": 3.3431088346501494, + "tokens_seen": 3073048576 + }, + { + "epoch": 0.86, + "learning_rate": 6.954864593781344e-05, + "loss": 1.2051, + "theoretical_loss": 3.3431034262617443, + "tokens_seen": 3073114112 + }, + { + "epoch": 0.86, + "learning_rate": 6.952858575727182e-05, + "loss": 1.1876, + "theoretical_loss": 3.3430980180209695, + "tokens_seen": 3073179648 + }, + { + "epoch": 0.86, + "learning_rate": 6.95085255767302e-05, + "loss": 1.2835, + "theoretical_loss": 3.3430926099278166, + "tokens_seen": 3073245184 + }, + { + "epoch": 0.86, + "learning_rate": 6.948846539618857e-05, + "loss": 1.2837, + "theoretical_loss": 3.343087201982279, + "tokens_seen": 3073310720 + }, + { + "epoch": 0.86, + "learning_rate": 6.946840521564695e-05, + "loss": 1.1802, + "theoretical_loss": 3.343081794184349, + "tokens_seen": 3073376256 + }, + { + "epoch": 0.86, + "learning_rate": 6.944834503510532e-05, + "loss": 1.1966, + "theoretical_loss": 3.3430763865340203, + "tokens_seen": 3073441792 + }, + { + "epoch": 0.86, + "learning_rate": 6.94282848545637e-05, + "loss": 1.3509, + "theoretical_loss": 3.343070979031285, + "tokens_seen": 3073507328 + }, + { + "epoch": 0.86, + "learning_rate": 6.940822467402207e-05, + "loss": 1.2425, + "theoretical_loss": 3.343065571676136, + "tokens_seen": 3073572864 + }, + { + "epoch": 0.86, + "learning_rate": 6.938816449348044e-05, + "loss": 1.2831, + "theoretical_loss": 3.3430601644685662, + "tokens_seen": 3073638400 + }, + { + "epoch": 0.86, + "learning_rate": 6.936810431293881e-05, + "loss": 1.3233, + "theoretical_loss": 3.343054757408569, + "tokens_seen": 3073703936 + }, + { + "epoch": 0.86, + "learning_rate": 6.934804413239719e-05, + "loss": 1.1616, + "theoretical_loss": 3.3430493504961363, + "tokens_seen": 3073769472 + }, + { + "epoch": 0.86, + "learning_rate": 6.932798395185556e-05, + "loss": 1.1886, + "theoretical_loss": 3.343043943731262, + "tokens_seen": 3073835008 + }, + { + "epoch": 0.86, + "learning_rate": 6.930792377131394e-05, + "loss": 1.256, + "theoretical_loss": 3.3430385371139373, + "tokens_seen": 3073900544 + }, + { + "epoch": 0.86, + "learning_rate": 6.928786359077231e-05, + "loss": 1.2462, + "theoretical_loss": 3.3430331306441565, + "tokens_seen": 3073966080 + }, + { + "epoch": 0.86, + "learning_rate": 6.926780341023069e-05, + "loss": 1.2439, + "theoretical_loss": 3.343027724321912, + "tokens_seen": 3074031616 + }, + { + "epoch": 0.86, + "learning_rate": 6.924774322968908e-05, + "loss": 1.2824, + "theoretical_loss": 3.343022318147196, + "tokens_seen": 3074097152 + }, + { + "epoch": 0.86, + "learning_rate": 6.922768304914746e-05, + "loss": 1.2256, + "theoretical_loss": 3.3430169121200026, + "tokens_seen": 3074162688 + }, + { + "epoch": 0.86, + "learning_rate": 6.920762286860582e-05, + "loss": 1.2876, + "theoretical_loss": 3.3430115062403236, + "tokens_seen": 3074228224 + }, + { + "epoch": 0.86, + "learning_rate": 6.918756268806419e-05, + "loss": 1.2345, + "theoretical_loss": 3.3430061005081524, + "tokens_seen": 3074293760 + }, + { + "epoch": 0.86, + "learning_rate": 6.916750250752257e-05, + "loss": 1.1778, + "theoretical_loss": 3.3430006949234814, + "tokens_seen": 3074359296 + }, + { + "epoch": 0.86, + "learning_rate": 6.914744232698094e-05, + "loss": 1.2957, + "theoretical_loss": 3.3429952894863035, + "tokens_seen": 3074424832 + }, + { + "epoch": 0.86, + "learning_rate": 6.912738214643932e-05, + "loss": 1.2909, + "theoretical_loss": 3.3429898841966117, + "tokens_seen": 3074490368 + }, + { + "epoch": 0.86, + "learning_rate": 6.91073219658977e-05, + "loss": 1.1998, + "theoretical_loss": 3.3429844790543988, + "tokens_seen": 3074555904 + }, + { + "epoch": 0.86, + "learning_rate": 6.908726178535607e-05, + "loss": 1.2823, + "theoretical_loss": 3.3429790740596577, + "tokens_seen": 3074621440 + }, + { + "epoch": 0.86, + "learning_rate": 6.906720160481445e-05, + "loss": 1.312, + "theoretical_loss": 3.342973669212381, + "tokens_seen": 3074686976 + }, + { + "epoch": 0.86, + "learning_rate": 6.904714142427282e-05, + "loss": 1.1901, + "theoretical_loss": 3.3429682645125616, + "tokens_seen": 3074752512 + }, + { + "epoch": 0.86, + "learning_rate": 6.90270812437312e-05, + "loss": 1.2371, + "theoretical_loss": 3.3429628599601924, + "tokens_seen": 3074818048 + }, + { + "epoch": 0.86, + "learning_rate": 6.900702106318956e-05, + "loss": 1.0938, + "theoretical_loss": 3.3429574555552666, + "tokens_seen": 3074883584 + }, + { + "epoch": 0.86, + "learning_rate": 6.898696088264794e-05, + "loss": 1.2901, + "theoretical_loss": 3.342952051297776, + "tokens_seen": 3074949120 + }, + { + "epoch": 0.86, + "learning_rate": 6.896690070210631e-05, + "loss": 1.1924, + "theoretical_loss": 3.342946647187715, + "tokens_seen": 3075014656 + }, + { + "epoch": 0.86, + "learning_rate": 6.894684052156469e-05, + "loss": 1.3023, + "theoretical_loss": 3.342941243225075, + "tokens_seen": 3075080192 + }, + { + "epoch": 0.86, + "learning_rate": 6.892678034102306e-05, + "loss": 1.2962, + "theoretical_loss": 3.3429358394098494, + "tokens_seen": 3075145728 + }, + { + "epoch": 0.86, + "learning_rate": 6.890672016048144e-05, + "loss": 1.3097, + "theoretical_loss": 3.3429304357420313, + "tokens_seen": 3075211264 + }, + { + "epoch": 0.86, + "learning_rate": 6.888665997993982e-05, + "loss": 1.236, + "theoretical_loss": 3.342925032221613, + "tokens_seen": 3075276800 + }, + { + "epoch": 0.86, + "learning_rate": 6.88665997993982e-05, + "loss": 1.2196, + "theoretical_loss": 3.3429196288485876, + "tokens_seen": 3075342336 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1117177, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3147614002227783, + "objective/train/theoretical_loss": 3.3429142256229483, + "objective/train/tokens_used": 1445866976, + "theoretical_loss": 3.3429142256229483, + "tokens_seen": 3075407872 + }, + { + "epoch": 0.86, + "learning_rate": 6.884653961885658e-05, + "loss": 1.277, + "theoretical_loss": 3.3429142256229483, + "tokens_seen": 3075407872 + }, + { + "epoch": 0.86, + "learning_rate": 6.882647943831496e-05, + "loss": 1.2309, + "theoretical_loss": 3.342908822544687, + "tokens_seen": 3075473408 + }, + { + "epoch": 0.86, + "learning_rate": 6.880641925777332e-05, + "loss": 1.2897, + "theoretical_loss": 3.3429034196137977, + "tokens_seen": 3075538944 + }, + { + "epoch": 0.86, + "learning_rate": 6.87863590772317e-05, + "loss": 1.3641, + "theoretical_loss": 3.342898016830272, + "tokens_seen": 3075604480 + }, + { + "epoch": 0.86, + "learning_rate": 6.876629889669007e-05, + "loss": 1.2228, + "theoretical_loss": 3.342892614194104, + "tokens_seen": 3075670016 + }, + { + "epoch": 0.86, + "learning_rate": 6.874623871614845e-05, + "loss": 1.2597, + "theoretical_loss": 3.3428872117052855, + "tokens_seen": 3075735552 + }, + { + "epoch": 0.86, + "learning_rate": 6.872617853560682e-05, + "loss": 1.1883, + "theoretical_loss": 3.34288180936381, + "tokens_seen": 3075801088 + }, + { + "epoch": 0.86, + "learning_rate": 6.87061183550652e-05, + "loss": 1.1291, + "theoretical_loss": 3.3428764071696704, + "tokens_seen": 3075866624 + }, + { + "epoch": 0.86, + "learning_rate": 6.868605817452357e-05, + "loss": 1.215, + "theoretical_loss": 3.3428710051228587, + "tokens_seen": 3075932160 + }, + { + "epoch": 0.86, + "learning_rate": 6.866599799398195e-05, + "loss": 1.1792, + "theoretical_loss": 3.342865603223369, + "tokens_seen": 3075997696 + }, + { + "epoch": 0.86, + "learning_rate": 6.864593781344033e-05, + "loss": 1.2926, + "theoretical_loss": 3.3428602014711926, + "tokens_seen": 3076063232 + }, + { + "epoch": 0.86, + "learning_rate": 6.862587763289869e-05, + "loss": 1.2479, + "theoretical_loss": 3.342854799866324, + "tokens_seen": 3076128768 + }, + { + "epoch": 0.86, + "learning_rate": 6.860581745235706e-05, + "loss": 1.2506, + "theoretical_loss": 3.3428493984087546, + "tokens_seen": 3076194304 + }, + { + "epoch": 0.86, + "learning_rate": 6.858575727181544e-05, + "loss": 1.2604, + "theoretical_loss": 3.3428439970984782, + "tokens_seen": 3076259840 + }, + { + "epoch": 0.86, + "learning_rate": 6.856569709127382e-05, + "loss": 1.3367, + "theoretical_loss": 3.342838595935487, + "tokens_seen": 3076325376 + }, + { + "epoch": 0.86, + "learning_rate": 6.854563691073219e-05, + "loss": 1.2501, + "theoretical_loss": 3.3428331949197747, + "tokens_seen": 3076390912 + }, + { + "epoch": 0.86, + "learning_rate": 6.852557673019057e-05, + "loss": 1.1985, + "theoretical_loss": 3.3428277940513333, + "tokens_seen": 3076456448 + }, + { + "epoch": 0.86, + "learning_rate": 6.850551654964896e-05, + "loss": 1.297, + "theoretical_loss": 3.342822393330156, + "tokens_seen": 3076521984 + }, + { + "epoch": 0.86, + "learning_rate": 6.848545636910733e-05, + "loss": 1.2036, + "theoretical_loss": 3.342816992756236, + "tokens_seen": 3076587520 + }, + { + "epoch": 0.86, + "learning_rate": 6.846539618856571e-05, + "loss": 1.1435, + "theoretical_loss": 3.342811592329565, + "tokens_seen": 3076653056 + }, + { + "epoch": 0.86, + "learning_rate": 6.844533600802408e-05, + "loss": 1.1949, + "theoretical_loss": 3.3428061920501375, + "tokens_seen": 3076718592 + }, + { + "epoch": 0.86, + "learning_rate": 6.842527582748245e-05, + "loss": 1.2049, + "theoretical_loss": 3.342800791917945, + "tokens_seen": 3076784128 + }, + { + "epoch": 0.86, + "learning_rate": 6.840521564694082e-05, + "loss": 1.1968, + "theoretical_loss": 3.342795391932981, + "tokens_seen": 3076849664 + }, + { + "epoch": 0.86, + "learning_rate": 6.83851554663992e-05, + "loss": 1.2568, + "theoretical_loss": 3.3427899920952377, + "tokens_seen": 3076915200 + }, + { + "epoch": 0.86, + "learning_rate": 6.836509528585757e-05, + "loss": 1.2301, + "theoretical_loss": 3.342784592404709, + "tokens_seen": 3076980736 + }, + { + "epoch": 0.86, + "learning_rate": 6.834503510531595e-05, + "loss": 1.2297, + "theoretical_loss": 3.3427791928613866, + "tokens_seen": 3077046272 + }, + { + "epoch": 0.86, + "learning_rate": 6.832497492477433e-05, + "loss": 1.2204, + "theoretical_loss": 3.3427737934652644, + "tokens_seen": 3077111808 + }, + { + "epoch": 0.86, + "learning_rate": 6.83049147442327e-05, + "loss": 1.2332, + "theoretical_loss": 3.3427683942163346, + "tokens_seen": 3077177344 + }, + { + "epoch": 0.86, + "learning_rate": 6.828485456369108e-05, + "loss": 1.3122, + "theoretical_loss": 3.3427629951145903, + "tokens_seen": 3077242880 + }, + { + "epoch": 0.87, + "learning_rate": 6.826479438314945e-05, + "loss": 1.2319, + "theoretical_loss": 3.342757596160024, + "tokens_seen": 3077308416 + }, + { + "epoch": 0.87, + "learning_rate": 6.824473420260783e-05, + "loss": 1.2666, + "theoretical_loss": 3.3427521973526293, + "tokens_seen": 3077373952 + }, + { + "epoch": 0.87, + "learning_rate": 6.822467402206619e-05, + "loss": 1.2914, + "theoretical_loss": 3.3427467986923984, + "tokens_seen": 3077439488 + }, + { + "epoch": 0.87, + "learning_rate": 6.820461384152457e-05, + "loss": 1.2557, + "theoretical_loss": 3.342741400179324, + "tokens_seen": 3077505024 + }, + { + "epoch": 0.87, + "learning_rate": 6.818455366098294e-05, + "loss": 1.3353, + "theoretical_loss": 3.3427360018133996, + "tokens_seen": 3077570560 + }, + { + "epoch": 0.87, + "learning_rate": 6.816449348044132e-05, + "loss": 1.164, + "theoretical_loss": 3.342730603594618, + "tokens_seen": 3077636096 + }, + { + "epoch": 0.87, + "learning_rate": 6.81444332998997e-05, + "loss": 1.2923, + "theoretical_loss": 3.3427252055229713, + "tokens_seen": 3077701632 + }, + { + "epoch": 0.87, + "learning_rate": 6.812437311935808e-05, + "loss": 1.2777, + "theoretical_loss": 3.342719807598453, + "tokens_seen": 3077767168 + }, + { + "epoch": 0.87, + "learning_rate": 6.810431293881646e-05, + "loss": 1.2387, + "theoretical_loss": 3.342714409821056, + "tokens_seen": 3077832704 + }, + { + "epoch": 0.87, + "learning_rate": 6.808425275827483e-05, + "loss": 1.2553, + "theoretical_loss": 3.342709012190773, + "tokens_seen": 3077898240 + }, + { + "epoch": 0.87, + "learning_rate": 6.806419257773321e-05, + "loss": 1.2568, + "theoretical_loss": 3.3427036147075966, + "tokens_seen": 3077963776 + }, + { + "epoch": 0.87, + "learning_rate": 6.804413239719159e-05, + "loss": 1.3009, + "theoretical_loss": 3.3426982173715203, + "tokens_seen": 3078029312 + }, + { + "epoch": 0.87, + "learning_rate": 6.802407221664995e-05, + "loss": 1.2258, + "theoretical_loss": 3.342692820182536, + "tokens_seen": 3078094848 + }, + { + "epoch": 0.87, + "learning_rate": 6.800401203610832e-05, + "loss": 1.2202, + "theoretical_loss": 3.3426874231406374, + "tokens_seen": 3078160384 + }, + { + "epoch": 0.87, + "learning_rate": 6.79839518555667e-05, + "loss": 1.2813, + "theoretical_loss": 3.342682026245817, + "tokens_seen": 3078225920 + }, + { + "epoch": 0.87, + "learning_rate": 6.796389167502508e-05, + "loss": 1.2353, + "theoretical_loss": 3.342676629498068, + "tokens_seen": 3078291456 + }, + { + "epoch": 0.87, + "learning_rate": 6.794383149448345e-05, + "loss": 1.2144, + "theoretical_loss": 3.3426712328973824, + "tokens_seen": 3078356992 + }, + { + "epoch": 0.87, + "learning_rate": 6.792377131394183e-05, + "loss": 1.1927, + "theoretical_loss": 3.3426658364437545, + "tokens_seen": 3078422528 + }, + { + "epoch": 0.87, + "learning_rate": 6.79037111334002e-05, + "loss": 1.2933, + "theoretical_loss": 3.3426604401371756, + "tokens_seen": 3078488064 + }, + { + "epoch": 0.87, + "learning_rate": 6.788365095285858e-05, + "loss": 1.2723, + "theoretical_loss": 3.34265504397764, + "tokens_seen": 3078553600 + }, + { + "epoch": 0.87, + "learning_rate": 6.786359077231695e-05, + "loss": 1.272, + "theoretical_loss": 3.342649647965139, + "tokens_seen": 3078619136 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1118389, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3177624940872192, + "objective/train/theoretical_loss": 3.342644252099667, + "objective/train/tokens_used": 1449143776, + "theoretical_loss": 3.342644252099667, + "tokens_seen": 3078684672 + }, + { + "epoch": 0.87, + "learning_rate": 6.784353059177532e-05, + "loss": 1.2908, + "theoretical_loss": 3.342644252099667, + "tokens_seen": 3078684672 + }, + { + "epoch": 0.87, + "learning_rate": 6.782347041123369e-05, + "loss": 1.2306, + "theoretical_loss": 3.3426388563812157, + "tokens_seen": 3078750208 + }, + { + "epoch": 0.87, + "learning_rate": 6.780341023069207e-05, + "loss": 1.2217, + "theoretical_loss": 3.3426334608097785, + "tokens_seen": 3078815744 + }, + { + "epoch": 0.87, + "learning_rate": 6.778335005015044e-05, + "loss": 1.2074, + "theoretical_loss": 3.342628065385348, + "tokens_seen": 3078881280 + }, + { + "epoch": 0.87, + "learning_rate": 6.776328986960882e-05, + "loss": 1.198, + "theoretical_loss": 3.342622670107918, + "tokens_seen": 3078946816 + }, + { + "epoch": 0.87, + "learning_rate": 6.774322968906721e-05, + "loss": 1.1445, + "theoretical_loss": 3.34261727497748, + "tokens_seen": 3079012352 + }, + { + "epoch": 0.87, + "learning_rate": 6.772316950852559e-05, + "loss": 1.1809, + "theoretical_loss": 3.342611879994028, + "tokens_seen": 3079077888 + }, + { + "epoch": 0.87, + "learning_rate": 6.770310932798396e-05, + "loss": 1.2591, + "theoretical_loss": 3.342606485157554, + "tokens_seen": 3079143424 + }, + { + "epoch": 0.87, + "learning_rate": 6.768304914744234e-05, + "loss": 1.266, + "theoretical_loss": 3.3426010904680514, + "tokens_seen": 3079208960 + }, + { + "epoch": 0.87, + "learning_rate": 6.766298896690071e-05, + "loss": 1.2726, + "theoretical_loss": 3.342595695925513, + "tokens_seen": 3079274496 + }, + { + "epoch": 0.87, + "learning_rate": 6.764292878635908e-05, + "loss": 1.2961, + "theoretical_loss": 3.342590301529931, + "tokens_seen": 3079340032 + }, + { + "epoch": 0.87, + "learning_rate": 6.762286860581745e-05, + "loss": 1.227, + "theoretical_loss": 3.3425849072812994, + "tokens_seen": 3079405568 + }, + { + "epoch": 0.87, + "learning_rate": 6.760280842527583e-05, + "loss": 1.2108, + "theoretical_loss": 3.3425795131796106, + "tokens_seen": 3079471104 + }, + { + "epoch": 0.87, + "learning_rate": 6.75827482447342e-05, + "loss": 1.2739, + "theoretical_loss": 3.342574119224857, + "tokens_seen": 3079536640 + }, + { + "epoch": 0.87, + "learning_rate": 6.756268806419258e-05, + "loss": 1.2701, + "theoretical_loss": 3.342568725417032, + "tokens_seen": 3079602176 + }, + { + "epoch": 0.87, + "learning_rate": 6.754262788365095e-05, + "loss": 1.2868, + "theoretical_loss": 3.3425633317561285, + "tokens_seen": 3079667712 + }, + { + "epoch": 0.87, + "learning_rate": 6.752256770310933e-05, + "loss": 1.2752, + "theoretical_loss": 3.3425579382421393, + "tokens_seen": 3079733248 + }, + { + "epoch": 0.87, + "learning_rate": 6.75025075225677e-05, + "loss": 1.2418, + "theoretical_loss": 3.3425525448750566, + "tokens_seen": 3079798784 + }, + { + "epoch": 0.87, + "learning_rate": 6.748244734202608e-05, + "loss": 1.2966, + "theoretical_loss": 3.342547151654874, + "tokens_seen": 3079864320 + }, + { + "epoch": 0.87, + "learning_rate": 6.746238716148446e-05, + "loss": 1.267, + "theoretical_loss": 3.342541758581585, + "tokens_seen": 3079929856 + }, + { + "epoch": 0.87, + "learning_rate": 6.744232698094282e-05, + "loss": 1.2535, + "theoretical_loss": 3.3425363656551808, + "tokens_seen": 3079995392 + }, + { + "epoch": 0.87, + "learning_rate": 6.74222668004012e-05, + "loss": 1.2703, + "theoretical_loss": 3.342530972875655, + "tokens_seen": 3080060928 + }, + { + "epoch": 0.87, + "learning_rate": 6.740220661985957e-05, + "loss": 1.2227, + "theoretical_loss": 3.3425255802430014, + "tokens_seen": 3080126464 + }, + { + "epoch": 0.87, + "learning_rate": 6.738214643931795e-05, + "loss": 1.2854, + "theoretical_loss": 3.342520187757212, + "tokens_seen": 3080192000 + }, + { + "epoch": 0.87, + "learning_rate": 6.736208625877634e-05, + "loss": 1.3179, + "theoretical_loss": 3.3425147954182792, + "tokens_seen": 3080257536 + }, + { + "epoch": 0.87, + "learning_rate": 6.734202607823471e-05, + "loss": 1.1473, + "theoretical_loss": 3.342509403226197, + "tokens_seen": 3080323072 + }, + { + "epoch": 0.87, + "learning_rate": 6.732196589769309e-05, + "loss": 1.2411, + "theoretical_loss": 3.3425040111809574, + "tokens_seen": 3080388608 + }, + { + "epoch": 0.87, + "learning_rate": 6.730190571715146e-05, + "loss": 1.2664, + "theoretical_loss": 3.342498619282554, + "tokens_seen": 3080454144 + }, + { + "epoch": 0.87, + "learning_rate": 6.728184553660984e-05, + "loss": 1.2762, + "theoretical_loss": 3.342493227530979, + "tokens_seen": 3080519680 + }, + { + "epoch": 0.87, + "learning_rate": 6.726178535606822e-05, + "loss": 1.2477, + "theoretical_loss": 3.3424878359262253, + "tokens_seen": 3080585216 + }, + { + "epoch": 0.87, + "learning_rate": 6.724172517552658e-05, + "loss": 1.3185, + "theoretical_loss": 3.3424824444682866, + "tokens_seen": 3080650752 + }, + { + "epoch": 0.87, + "learning_rate": 6.722166499498495e-05, + "loss": 1.1908, + "theoretical_loss": 3.342477053157155, + "tokens_seen": 3080716288 + }, + { + "epoch": 0.87, + "learning_rate": 6.720160481444333e-05, + "loss": 1.2373, + "theoretical_loss": 3.342471661992824, + "tokens_seen": 3080781824 + }, + { + "epoch": 0.87, + "learning_rate": 6.71815446339017e-05, + "loss": 1.2854, + "theoretical_loss": 3.3424662709752857, + "tokens_seen": 3080847360 + }, + { + "epoch": 0.87, + "learning_rate": 6.716148445336008e-05, + "loss": 1.2182, + "theoretical_loss": 3.3424608801045332, + "tokens_seen": 3080912896 + }, + { + "epoch": 0.87, + "learning_rate": 6.714142427281846e-05, + "loss": 1.2567, + "theoretical_loss": 3.34245548938056, + "tokens_seen": 3080978432 + }, + { + "epoch": 0.87, + "learning_rate": 6.712136409227683e-05, + "loss": 1.26, + "theoretical_loss": 3.342450098803358, + "tokens_seen": 3081043968 + }, + { + "epoch": 0.87, + "learning_rate": 6.710130391173521e-05, + "loss": 1.2625, + "theoretical_loss": 3.3424447083729207, + "tokens_seen": 3081109504 + }, + { + "epoch": 0.87, + "learning_rate": 6.708124373119358e-05, + "loss": 1.3398, + "theoretical_loss": 3.3424393180892413, + "tokens_seen": 3081175040 + }, + { + "epoch": 0.87, + "learning_rate": 6.706118355065195e-05, + "loss": 1.2339, + "theoretical_loss": 3.342433927952312, + "tokens_seen": 3081240576 + }, + { + "epoch": 0.87, + "learning_rate": 6.704112337011032e-05, + "loss": 1.265, + "theoretical_loss": 3.342428537962126, + "tokens_seen": 3081306112 + }, + { + "epoch": 0.87, + "learning_rate": 6.70210631895687e-05, + "loss": 1.2587, + "theoretical_loss": 3.3424231481186766, + "tokens_seen": 3081371648 + }, + { + "epoch": 0.87, + "learning_rate": 6.700100300902709e-05, + "loss": 1.339, + "theoretical_loss": 3.3424177584219557, + "tokens_seen": 3081437184 + }, + { + "epoch": 0.87, + "learning_rate": 6.698094282848546e-05, + "loss": 1.2229, + "theoretical_loss": 3.3424123688719565, + "tokens_seen": 3081502720 + }, + { + "epoch": 0.87, + "learning_rate": 6.696088264794384e-05, + "loss": 1.288, + "theoretical_loss": 3.342406979468673, + "tokens_seen": 3081568256 + }, + { + "epoch": 0.87, + "learning_rate": 6.694082246740221e-05, + "loss": 1.2738, + "theoretical_loss": 3.3424015902120963, + "tokens_seen": 3081633792 + }, + { + "epoch": 0.87, + "learning_rate": 6.692076228686059e-05, + "loss": 1.3918, + "theoretical_loss": 3.3423962011022206, + "tokens_seen": 3081699328 + }, + { + "epoch": 0.87, + "learning_rate": 6.690070210631897e-05, + "loss": 1.2622, + "theoretical_loss": 3.342390812139038, + "tokens_seen": 3081764864 + }, + { + "epoch": 0.87, + "learning_rate": 6.688064192577734e-05, + "loss": 1.1636, + "theoretical_loss": 3.342385423322542, + "tokens_seen": 3081830400 + }, + { + "epoch": 0.87, + "learning_rate": 6.68605817452357e-05, + "loss": 1.2475, + "theoretical_loss": 3.3423800346527255, + "tokens_seen": 3081895936 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1119488, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2091344594955444, + "objective/train/theoretical_loss": 3.342374646129581, + "objective/train/tokens_used": 1452420576, + "theoretical_loss": 3.342374646129581, + "tokens_seen": 3081961472 + }, + { + "epoch": 0.87, + "learning_rate": 6.684052156469408e-05, + "loss": 1.1885, + "theoretical_loss": 3.342374646129581, + "tokens_seen": 3081961472 + }, + { + "epoch": 0.87, + "learning_rate": 6.682046138415246e-05, + "loss": 1.1314, + "theoretical_loss": 3.3423692577531012, + "tokens_seen": 3082027008 + }, + { + "epoch": 0.87, + "learning_rate": 6.680040120361083e-05, + "loss": 1.1791, + "theoretical_loss": 3.34236386952328, + "tokens_seen": 3082092544 + }, + { + "epoch": 0.87, + "learning_rate": 6.678034102306921e-05, + "loss": 1.2749, + "theoretical_loss": 3.3423584814401086, + "tokens_seen": 3082158080 + }, + { + "epoch": 0.87, + "learning_rate": 6.676028084252758e-05, + "loss": 1.1877, + "theoretical_loss": 3.3423530935035815, + "tokens_seen": 3082223616 + }, + { + "epoch": 0.87, + "learning_rate": 6.674022066198596e-05, + "loss": 1.1862, + "theoretical_loss": 3.342347705713691, + "tokens_seen": 3082289152 + }, + { + "epoch": 0.87, + "learning_rate": 6.672016048144433e-05, + "loss": 1.2283, + "theoretical_loss": 3.34234231807043, + "tokens_seen": 3082354688 + }, + { + "epoch": 0.87, + "learning_rate": 6.670010030090271e-05, + "loss": 1.2037, + "theoretical_loss": 3.342336930573791, + "tokens_seen": 3082420224 + }, + { + "epoch": 0.87, + "learning_rate": 6.668004012036109e-05, + "loss": 1.2889, + "theoretical_loss": 3.3423315432237675, + "tokens_seen": 3082485760 + }, + { + "epoch": 0.87, + "learning_rate": 6.665997993981945e-05, + "loss": 1.3585, + "theoretical_loss": 3.342326156020352, + "tokens_seen": 3082551296 + }, + { + "epoch": 0.87, + "learning_rate": 6.663991975927782e-05, + "loss": 1.2095, + "theoretical_loss": 3.342320768963538, + "tokens_seen": 3082616832 + }, + { + "epoch": 0.87, + "learning_rate": 6.661985957873621e-05, + "loss": 1.2005, + "theoretical_loss": 3.342315382053317, + "tokens_seen": 3082682368 + }, + { + "epoch": 0.87, + "learning_rate": 6.659979939819459e-05, + "loss": 1.2736, + "theoretical_loss": 3.3423099952896838, + "tokens_seen": 3082747904 + }, + { + "epoch": 0.87, + "learning_rate": 6.657973921765296e-05, + "loss": 1.2442, + "theoretical_loss": 3.34230460867263, + "tokens_seen": 3082813440 + }, + { + "epoch": 0.87, + "learning_rate": 6.655967903711134e-05, + "loss": 1.2503, + "theoretical_loss": 3.342299222202149, + "tokens_seen": 3082878976 + }, + { + "epoch": 0.87, + "learning_rate": 6.653961885656972e-05, + "loss": 1.1524, + "theoretical_loss": 3.342293835878233, + "tokens_seen": 3082944512 + }, + { + "epoch": 0.87, + "learning_rate": 6.651955867602809e-05, + "loss": 1.2455, + "theoretical_loss": 3.3422884497008756, + "tokens_seen": 3083010048 + }, + { + "epoch": 0.87, + "learning_rate": 6.649949849548647e-05, + "loss": 1.2621, + "theoretical_loss": 3.3422830636700698, + "tokens_seen": 3083075584 + }, + { + "epoch": 0.87, + "learning_rate": 6.647943831494484e-05, + "loss": 1.2678, + "theoretical_loss": 3.342277677785808, + "tokens_seen": 3083141120 + }, + { + "epoch": 0.87, + "learning_rate": 6.64593781344032e-05, + "loss": 1.2377, + "theoretical_loss": 3.3422722920480834, + "tokens_seen": 3083206656 + }, + { + "epoch": 0.87, + "learning_rate": 6.643931795386158e-05, + "loss": 1.2339, + "theoretical_loss": 3.3422669064568886, + "tokens_seen": 3083272192 + }, + { + "epoch": 0.87, + "learning_rate": 6.641925777331996e-05, + "loss": 1.2712, + "theoretical_loss": 3.342261521012217, + "tokens_seen": 3083337728 + }, + { + "epoch": 0.87, + "learning_rate": 6.639919759277833e-05, + "loss": 1.2293, + "theoretical_loss": 3.3422561357140608, + "tokens_seen": 3083403264 + }, + { + "epoch": 0.87, + "learning_rate": 6.637913741223671e-05, + "loss": 1.2505, + "theoretical_loss": 3.3422507505624135, + "tokens_seen": 3083468800 + }, + { + "epoch": 0.87, + "learning_rate": 6.635907723169509e-05, + "loss": 1.2392, + "theoretical_loss": 3.342245365557268, + "tokens_seen": 3083534336 + }, + { + "epoch": 0.87, + "learning_rate": 6.633901705115346e-05, + "loss": 1.2362, + "theoretical_loss": 3.3422399806986167, + "tokens_seen": 3083599872 + }, + { + "epoch": 0.87, + "learning_rate": 6.631895687061184e-05, + "loss": 1.3112, + "theoretical_loss": 3.342234595986453, + "tokens_seen": 3083665408 + }, + { + "epoch": 0.87, + "learning_rate": 6.629889669007021e-05, + "loss": 1.262, + "theoretical_loss": 3.3422292114207695, + "tokens_seen": 3083730944 + }, + { + "epoch": 0.87, + "learning_rate": 6.627883650952857e-05, + "loss": 1.1807, + "theoretical_loss": 3.3422238270015594, + "tokens_seen": 3083796480 + }, + { + "epoch": 0.87, + "learning_rate": 6.625877632898695e-05, + "loss": 1.1751, + "theoretical_loss": 3.3422184427288153, + "tokens_seen": 3083862016 + }, + { + "epoch": 0.87, + "learning_rate": 6.623871614844534e-05, + "loss": 1.2099, + "theoretical_loss": 3.34221305860253, + "tokens_seen": 3083927552 + }, + { + "epoch": 0.87, + "learning_rate": 6.621865596790372e-05, + "loss": 1.3728, + "theoretical_loss": 3.3422076746226965, + "tokens_seen": 3083993088 + }, + { + "epoch": 0.87, + "learning_rate": 6.619859578736209e-05, + "loss": 1.1834, + "theoretical_loss": 3.3422022907893085, + "tokens_seen": 3084058624 + }, + { + "epoch": 0.87, + "learning_rate": 6.617853560682047e-05, + "loss": 1.1474, + "theoretical_loss": 3.3421969071023576, + "tokens_seen": 3084124160 + }, + { + "epoch": 0.87, + "learning_rate": 6.615847542627884e-05, + "loss": 1.2994, + "theoretical_loss": 3.3421915235618376, + "tokens_seen": 3084189696 + }, + { + "epoch": 0.87, + "learning_rate": 6.613841524573722e-05, + "loss": 1.2128, + "theoretical_loss": 3.3421861401677413, + "tokens_seen": 3084255232 + }, + { + "epoch": 0.87, + "learning_rate": 6.61183550651956e-05, + "loss": 1.3078, + "theoretical_loss": 3.3421807569200612, + "tokens_seen": 3084320768 + }, + { + "epoch": 0.87, + "learning_rate": 6.609829488465397e-05, + "loss": 1.283, + "theoretical_loss": 3.3421753738187903, + "tokens_seen": 3084386304 + }, + { + "epoch": 0.87, + "learning_rate": 6.607823470411233e-05, + "loss": 1.2978, + "theoretical_loss": 3.342169990863922, + "tokens_seen": 3084451840 + }, + { + "epoch": 0.87, + "learning_rate": 6.605817452357071e-05, + "loss": 1.1953, + "theoretical_loss": 3.342164608055449, + "tokens_seen": 3084517376 + }, + { + "epoch": 0.87, + "learning_rate": 6.603811434302908e-05, + "loss": 1.1716, + "theoretical_loss": 3.3421592253933636, + "tokens_seen": 3084582912 + }, + { + "epoch": 0.87, + "learning_rate": 6.601805416248746e-05, + "loss": 1.2957, + "theoretical_loss": 3.3421538428776594, + "tokens_seen": 3084648448 + }, + { + "epoch": 0.87, + "learning_rate": 6.599799398194584e-05, + "loss": 1.2203, + "theoretical_loss": 3.342148460508329, + "tokens_seen": 3084713984 + }, + { + "epoch": 0.87, + "learning_rate": 6.597793380140421e-05, + "loss": 1.2808, + "theoretical_loss": 3.342143078285366, + "tokens_seen": 3084779520 + }, + { + "epoch": 0.87, + "learning_rate": 6.595787362086259e-05, + "loss": 1.3213, + "theoretical_loss": 3.342137696208762, + "tokens_seen": 3084845056 + }, + { + "epoch": 0.87, + "learning_rate": 6.593781344032096e-05, + "loss": 1.2702, + "theoretical_loss": 3.3421323142785107, + "tokens_seen": 3084910592 + }, + { + "epoch": 0.87, + "learning_rate": 6.591775325977934e-05, + "loss": 1.2366, + "theoretical_loss": 3.342126932494605, + "tokens_seen": 3084976128 + }, + { + "epoch": 0.87, + "learning_rate": 6.589769307923771e-05, + "loss": 1.2645, + "theoretical_loss": 3.342121550857038, + "tokens_seen": 3085041664 + }, + { + "epoch": 0.87, + "learning_rate": 6.587763289869609e-05, + "loss": 1.278, + "theoretical_loss": 3.342116169365802, + "tokens_seen": 3085107200 + }, + { + "epoch": 0.87, + "learning_rate": 6.585757271815447e-05, + "loss": 1.1933, + "theoretical_loss": 3.3421107880208902, + "tokens_seen": 3085172736 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1120785, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3070566654205322, + "objective/train/theoretical_loss": 3.342105406822296, + "objective/train/tokens_used": 1455697376, + "theoretical_loss": 3.342105406822296, + "tokens_seen": 3085238272 + }, + { + "epoch": 0.87, + "learning_rate": 6.583751253761284e-05, + "loss": 1.3057, + "theoretical_loss": 3.342105406822296, + "tokens_seen": 3085238272 + }, + { + "epoch": 0.87, + "learning_rate": 6.581745235707122e-05, + "loss": 1.2691, + "theoretical_loss": 3.3421000257700118, + "tokens_seen": 3085303808 + }, + { + "epoch": 0.87, + "learning_rate": 6.57973921765296e-05, + "loss": 1.2937, + "theoretical_loss": 3.3420946448640305, + "tokens_seen": 3085369344 + }, + { + "epoch": 0.87, + "learning_rate": 6.577733199598797e-05, + "loss": 1.1958, + "theoretical_loss": 3.342089264104345, + "tokens_seen": 3085434880 + }, + { + "epoch": 0.87, + "learning_rate": 6.575727181544635e-05, + "loss": 1.2442, + "theoretical_loss": 3.3420838834909485, + "tokens_seen": 3085500416 + }, + { + "epoch": 0.87, + "learning_rate": 6.573721163490472e-05, + "loss": 1.4305, + "theoretical_loss": 3.342078503023834, + "tokens_seen": 3085565952 + }, + { + "epoch": 0.87, + "learning_rate": 6.57171514543631e-05, + "loss": 1.1885, + "theoretical_loss": 3.342073122702994, + "tokens_seen": 3085631488 + }, + { + "epoch": 0.87, + "learning_rate": 6.569709127382147e-05, + "loss": 1.2693, + "theoretical_loss": 3.3420677425284215, + "tokens_seen": 3085697024 + }, + { + "epoch": 0.87, + "learning_rate": 6.567703109327984e-05, + "loss": 1.2065, + "theoretical_loss": 3.3420623625001094, + "tokens_seen": 3085762560 + }, + { + "epoch": 0.87, + "learning_rate": 6.565697091273821e-05, + "loss": 1.3117, + "theoretical_loss": 3.342056982618051, + "tokens_seen": 3085828096 + }, + { + "epoch": 0.87, + "learning_rate": 6.563691073219659e-05, + "loss": 1.2711, + "theoretical_loss": 3.342051602882239, + "tokens_seen": 3085893632 + }, + { + "epoch": 0.87, + "learning_rate": 6.561685055165496e-05, + "loss": 1.2963, + "theoretical_loss": 3.342046223292666, + "tokens_seen": 3085959168 + }, + { + "epoch": 0.87, + "learning_rate": 6.559679037111334e-05, + "loss": 1.1932, + "theoretical_loss": 3.3420408438493254, + "tokens_seen": 3086024704 + }, + { + "epoch": 0.87, + "learning_rate": 6.557673019057171e-05, + "loss": 1.176, + "theoretical_loss": 3.3420354645522097, + "tokens_seen": 3086090240 + }, + { + "epoch": 0.87, + "learning_rate": 6.555667001003009e-05, + "loss": 1.2568, + "theoretical_loss": 3.342030085401312, + "tokens_seen": 3086155776 + }, + { + "epoch": 0.87, + "learning_rate": 6.553660982948847e-05, + "loss": 1.218, + "theoretical_loss": 3.3420247063966255, + "tokens_seen": 3086221312 + }, + { + "epoch": 0.87, + "learning_rate": 6.551654964894684e-05, + "loss": 1.2161, + "theoretical_loss": 3.3420193275381425, + "tokens_seen": 3086286848 + }, + { + "epoch": 0.87, + "learning_rate": 6.549648946840522e-05, + "loss": 1.2674, + "theoretical_loss": 3.3420139488258567, + "tokens_seen": 3086352384 + }, + { + "epoch": 0.87, + "learning_rate": 6.547642928786359e-05, + "loss": 1.2717, + "theoretical_loss": 3.34200857025976, + "tokens_seen": 3086417920 + }, + { + "epoch": 0.87, + "learning_rate": 6.545636910732197e-05, + "loss": 1.2085, + "theoretical_loss": 3.3420031918398467, + "tokens_seen": 3086483456 + }, + { + "epoch": 0.87, + "learning_rate": 6.543630892678034e-05, + "loss": 1.1755, + "theoretical_loss": 3.3419978135661084, + "tokens_seen": 3086548992 + }, + { + "epoch": 0.87, + "learning_rate": 6.541624874623872e-05, + "loss": 1.2001, + "theoretical_loss": 3.341992435438539, + "tokens_seen": 3086614528 + }, + { + "epoch": 0.87, + "learning_rate": 6.53961885656971e-05, + "loss": 1.1976, + "theoretical_loss": 3.3419870574571306, + "tokens_seen": 3086680064 + }, + { + "epoch": 0.87, + "learning_rate": 6.537612838515547e-05, + "loss": 1.2641, + "theoretical_loss": 3.3419816796218766, + "tokens_seen": 3086745600 + }, + { + "epoch": 0.87, + "learning_rate": 6.535606820461385e-05, + "loss": 1.3117, + "theoretical_loss": 3.34197630193277, + "tokens_seen": 3086811136 + }, + { + "epoch": 0.87, + "learning_rate": 6.533600802407222e-05, + "loss": 1.1794, + "theoretical_loss": 3.3419709243898037, + "tokens_seen": 3086876672 + }, + { + "epoch": 0.87, + "learning_rate": 6.53159478435306e-05, + "loss": 1.2965, + "theoretical_loss": 3.34196554699297, + "tokens_seen": 3086942208 + }, + { + "epoch": 0.87, + "learning_rate": 6.529588766298896e-05, + "loss": 1.1408, + "theoretical_loss": 3.341960169742263, + "tokens_seen": 3087007744 + }, + { + "epoch": 0.87, + "learning_rate": 6.527582748244734e-05, + "loss": 1.193, + "theoretical_loss": 3.341954792637675, + "tokens_seen": 3087073280 + }, + { + "epoch": 0.87, + "learning_rate": 6.525576730190571e-05, + "loss": 1.2064, + "theoretical_loss": 3.3419494156791982, + "tokens_seen": 3087138816 + }, + { + "epoch": 0.87, + "learning_rate": 6.523570712136409e-05, + "loss": 1.0993, + "theoretical_loss": 3.3419440388668264, + "tokens_seen": 3087204352 + }, + { + "epoch": 0.87, + "learning_rate": 6.521564694082246e-05, + "loss": 1.3056, + "theoretical_loss": 3.3419386622005525, + "tokens_seen": 3087269888 + }, + { + "epoch": 0.87, + "learning_rate": 6.519558676028084e-05, + "loss": 1.2574, + "theoretical_loss": 3.341933285680369, + "tokens_seen": 3087335424 + }, + { + "epoch": 0.87, + "learning_rate": 6.517552657973922e-05, + "loss": 1.2744, + "theoretical_loss": 3.3419279093062695, + "tokens_seen": 3087400960 + }, + { + "epoch": 0.87, + "learning_rate": 6.515546639919759e-05, + "loss": 1.2585, + "theoretical_loss": 3.3419225330782467, + "tokens_seen": 3087466496 + }, + { + "epoch": 0.87, + "learning_rate": 6.513540621865597e-05, + "loss": 1.2379, + "theoretical_loss": 3.341917156996293, + "tokens_seen": 3087532032 + }, + { + "epoch": 0.87, + "learning_rate": 6.511534603811436e-05, + "loss": 1.1762, + "theoretical_loss": 3.3419117810604018, + "tokens_seen": 3087597568 + }, + { + "epoch": 0.87, + "learning_rate": 6.509528585757272e-05, + "loss": 1.2455, + "theoretical_loss": 3.3419064052705654, + "tokens_seen": 3087663104 + }, + { + "epoch": 0.87, + "learning_rate": 6.50752256770311e-05, + "loss": 1.2362, + "theoretical_loss": 3.341901029626778, + "tokens_seen": 3087728640 + }, + { + "epoch": 0.87, + "learning_rate": 6.505516549648947e-05, + "loss": 1.2575, + "theoretical_loss": 3.3418956541290314, + "tokens_seen": 3087794176 + }, + { + "epoch": 0.87, + "learning_rate": 6.503510531594785e-05, + "loss": 1.2836, + "theoretical_loss": 3.341890278777319, + "tokens_seen": 3087859712 + }, + { + "epoch": 0.87, + "learning_rate": 6.501504513540622e-05, + "loss": 1.1844, + "theoretical_loss": 3.341884903571634, + "tokens_seen": 3087925248 + }, + { + "epoch": 0.87, + "learning_rate": 6.49949849548646e-05, + "loss": 1.1943, + "theoretical_loss": 3.3418795285119685, + "tokens_seen": 3087990784 + }, + { + "epoch": 0.87, + "learning_rate": 6.497492477432297e-05, + "loss": 1.22, + "theoretical_loss": 3.341874153598316, + "tokens_seen": 3088056320 + }, + { + "epoch": 0.87, + "learning_rate": 6.495486459378135e-05, + "loss": 1.2247, + "theoretical_loss": 3.341868778830669, + "tokens_seen": 3088121856 + }, + { + "epoch": 0.87, + "learning_rate": 6.493480441323973e-05, + "loss": 1.1761, + "theoretical_loss": 3.3418634042090214, + "tokens_seen": 3088187392 + }, + { + "epoch": 0.87, + "learning_rate": 6.491474423269809e-05, + "loss": 1.2604, + "theoretical_loss": 3.3418580297333653, + "tokens_seen": 3088252928 + }, + { + "epoch": 0.87, + "learning_rate": 6.489468405215646e-05, + "loss": 1.2859, + "theoretical_loss": 3.3418526554036942, + "tokens_seen": 3088318464 + }, + { + "epoch": 0.87, + "learning_rate": 6.487462387161484e-05, + "loss": 1.2304, + "theoretical_loss": 3.34184728122, + "tokens_seen": 3088384000 + }, + { + "epoch": 0.87, + "learning_rate": 6.485456369107322e-05, + "loss": 1.1519, + "theoretical_loss": 3.3418419071822774, + "tokens_seen": 3088449536 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1122042, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2953546047210693, + "objective/train/theoretical_loss": 3.3418365332905173, + "objective/train/tokens_used": 1458974176, + "theoretical_loss": 3.3418365332905173, + "tokens_seen": 3088515072 + }, + { + "epoch": 0.87, + "learning_rate": 6.483450351053159e-05, + "loss": 1.2519, + "theoretical_loss": 3.3418365332905173, + "tokens_seen": 3088515072 + }, + { + "epoch": 0.87, + "learning_rate": 6.481444332998997e-05, + "loss": 1.1937, + "theoretical_loss": 3.3418311595447143, + "tokens_seen": 3088580608 + }, + { + "epoch": 0.87, + "learning_rate": 6.479438314944834e-05, + "loss": 1.2097, + "theoretical_loss": 3.34182578594486, + "tokens_seen": 3088646144 + }, + { + "epoch": 0.87, + "learning_rate": 6.477432296890672e-05, + "loss": 1.3098, + "theoretical_loss": 3.3418204124909483, + "tokens_seen": 3088711680 + }, + { + "epoch": 0.87, + "learning_rate": 6.47542627883651e-05, + "loss": 1.2686, + "theoretical_loss": 3.341815039182972, + "tokens_seen": 3088777216 + }, + { + "epoch": 0.87, + "learning_rate": 6.473420260782348e-05, + "loss": 1.2339, + "theoretical_loss": 3.3418096660209238, + "tokens_seen": 3088842752 + }, + { + "epoch": 0.87, + "learning_rate": 6.471414242728185e-05, + "loss": 1.3011, + "theoretical_loss": 3.3418042930047966, + "tokens_seen": 3088908288 + }, + { + "epoch": 0.87, + "learning_rate": 6.469408224674022e-05, + "loss": 1.2508, + "theoretical_loss": 3.3417989201345835, + "tokens_seen": 3088973824 + }, + { + "epoch": 0.87, + "learning_rate": 6.46740220661986e-05, + "loss": 1.2932, + "theoretical_loss": 3.341793547410277, + "tokens_seen": 3089039360 + }, + { + "epoch": 0.87, + "learning_rate": 6.465396188565697e-05, + "loss": 1.2007, + "theoretical_loss": 3.341788174831871, + "tokens_seen": 3089104896 + }, + { + "epoch": 0.87, + "learning_rate": 6.463390170511535e-05, + "loss": 1.1841, + "theoretical_loss": 3.341782802399358, + "tokens_seen": 3089170432 + }, + { + "epoch": 0.87, + "learning_rate": 6.461384152457372e-05, + "loss": 1.2498, + "theoretical_loss": 3.3417774301127303, + "tokens_seen": 3089235968 + }, + { + "epoch": 0.87, + "learning_rate": 6.45937813440321e-05, + "loss": 1.2002, + "theoretical_loss": 3.3417720579719816, + "tokens_seen": 3089301504 + }, + { + "epoch": 0.87, + "learning_rate": 6.457372116349048e-05, + "loss": 1.2387, + "theoretical_loss": 3.3417666859771047, + "tokens_seen": 3089367040 + }, + { + "epoch": 0.87, + "learning_rate": 6.455366098294885e-05, + "loss": 1.2, + "theoretical_loss": 3.3417613141280924, + "tokens_seen": 3089432576 + }, + { + "epoch": 0.87, + "learning_rate": 6.453360080240723e-05, + "loss": 1.2969, + "theoretical_loss": 3.3417559424249377, + "tokens_seen": 3089498112 + }, + { + "epoch": 0.87, + "learning_rate": 6.451354062186559e-05, + "loss": 1.2499, + "theoretical_loss": 3.3417505708676334, + "tokens_seen": 3089563648 + }, + { + "epoch": 0.87, + "learning_rate": 6.449348044132397e-05, + "loss": 1.3111, + "theoretical_loss": 3.341745199456173, + "tokens_seen": 3089629184 + }, + { + "epoch": 0.87, + "learning_rate": 6.447342026078234e-05, + "loss": 1.2121, + "theoretical_loss": 3.3417398281905486, + "tokens_seen": 3089694720 + }, + { + "epoch": 0.87, + "learning_rate": 6.445336008024072e-05, + "loss": 1.2425, + "theoretical_loss": 3.341734457070754, + "tokens_seen": 3089760256 + }, + { + "epoch": 0.87, + "learning_rate": 6.44332998996991e-05, + "loss": 1.2979, + "theoretical_loss": 3.341729086096781, + "tokens_seen": 3089825792 + }, + { + "epoch": 0.87, + "learning_rate": 6.441323971915747e-05, + "loss": 1.3125, + "theoretical_loss": 3.3417237152686243, + "tokens_seen": 3089891328 + }, + { + "epoch": 0.87, + "learning_rate": 6.439317953861585e-05, + "loss": 1.3464, + "theoretical_loss": 3.3417183445862753, + "tokens_seen": 3089956864 + }, + { + "epoch": 0.87, + "learning_rate": 6.437311935807423e-05, + "loss": 1.185, + "theoretical_loss": 3.3417129740497273, + "tokens_seen": 3090022400 + }, + { + "epoch": 0.87, + "learning_rate": 6.435305917753261e-05, + "loss": 1.2053, + "theoretical_loss": 3.341707603658974, + "tokens_seen": 3090087936 + }, + { + "epoch": 0.87, + "learning_rate": 6.433299899699099e-05, + "loss": 1.2993, + "theoretical_loss": 3.3417022334140074, + "tokens_seen": 3090153472 + }, + { + "epoch": 0.87, + "learning_rate": 6.431293881644935e-05, + "loss": 1.2495, + "theoretical_loss": 3.341696863314821, + "tokens_seen": 3090219008 + }, + { + "epoch": 0.87, + "learning_rate": 6.429287863590772e-05, + "loss": 1.1847, + "theoretical_loss": 3.3416914933614073, + "tokens_seen": 3090284544 + }, + { + "epoch": 0.87, + "learning_rate": 6.42728184553661e-05, + "loss": 1.1721, + "theoretical_loss": 3.34168612355376, + "tokens_seen": 3090350080 + }, + { + "epoch": 0.87, + "learning_rate": 6.425275827482448e-05, + "loss": 1.131, + "theoretical_loss": 3.341680753891871, + "tokens_seen": 3090415616 + }, + { + "epoch": 0.87, + "learning_rate": 6.423269809428285e-05, + "loss": 1.2716, + "theoretical_loss": 3.3416753843757343, + "tokens_seen": 3090481152 + }, + { + "epoch": 0.87, + "learning_rate": 6.421263791374123e-05, + "loss": 1.1549, + "theoretical_loss": 3.3416700150053424, + "tokens_seen": 3090546688 + }, + { + "epoch": 0.87, + "learning_rate": 6.41925777331996e-05, + "loss": 1.216, + "theoretical_loss": 3.341664645780688, + "tokens_seen": 3090612224 + }, + { + "epoch": 0.87, + "learning_rate": 6.417251755265798e-05, + "loss": 1.214, + "theoretical_loss": 3.3416592767017645, + "tokens_seen": 3090677760 + }, + { + "epoch": 0.87, + "learning_rate": 6.415245737211635e-05, + "loss": 1.267, + "theoretical_loss": 3.3416539077685643, + "tokens_seen": 3090743296 + }, + { + "epoch": 0.87, + "learning_rate": 6.413239719157472e-05, + "loss": 1.2497, + "theoretical_loss": 3.3416485389810813, + "tokens_seen": 3090808832 + }, + { + "epoch": 0.87, + "learning_rate": 6.411233701103309e-05, + "loss": 1.2378, + "theoretical_loss": 3.3416431703393075, + "tokens_seen": 3090874368 + }, + { + "epoch": 0.87, + "learning_rate": 6.409227683049147e-05, + "loss": 1.1487, + "theoretical_loss": 3.341637801843236, + "tokens_seen": 3090939904 + }, + { + "epoch": 0.87, + "learning_rate": 6.407221664994984e-05, + "loss": 1.1276, + "theoretical_loss": 3.3416324334928604, + "tokens_seen": 3091005440 + }, + { + "epoch": 0.87, + "learning_rate": 6.405215646940822e-05, + "loss": 1.1784, + "theoretical_loss": 3.341627065288173, + "tokens_seen": 3091070976 + }, + { + "epoch": 0.87, + "learning_rate": 6.40320962888666e-05, + "loss": 1.2342, + "theoretical_loss": 3.3416216972291672, + "tokens_seen": 3091136512 + }, + { + "epoch": 0.87, + "learning_rate": 6.401203610832497e-05, + "loss": 1.2473, + "theoretical_loss": 3.3416163293158356, + "tokens_seen": 3091202048 + }, + { + "epoch": 0.87, + "learning_rate": 6.399197592778336e-05, + "loss": 1.2983, + "theoretical_loss": 3.3416109615481715, + "tokens_seen": 3091267584 + }, + { + "epoch": 0.87, + "learning_rate": 6.397191574724174e-05, + "loss": 1.2388, + "theoretical_loss": 3.3416055939261673, + "tokens_seen": 3091333120 + }, + { + "epoch": 0.87, + "learning_rate": 6.395185556670011e-05, + "loss": 1.1963, + "theoretical_loss": 3.341600226449817, + "tokens_seen": 3091398656 + }, + { + "epoch": 0.87, + "learning_rate": 6.393179538615847e-05, + "loss": 1.1633, + "theoretical_loss": 3.341594859119112, + "tokens_seen": 3091464192 + }, + { + "epoch": 0.87, + "learning_rate": 6.391173520561685e-05, + "loss": 1.2009, + "theoretical_loss": 3.3415894919340468, + "tokens_seen": 3091529728 + }, + { + "epoch": 0.87, + "learning_rate": 6.389167502507523e-05, + "loss": 1.3144, + "theoretical_loss": 3.3415841248946134, + "tokens_seen": 3091595264 + }, + { + "epoch": 0.87, + "learning_rate": 6.38716148445336e-05, + "loss": 1.2147, + "theoretical_loss": 3.3415787580008054, + "tokens_seen": 3091660800 + }, + { + "epoch": 0.87, + "learning_rate": 6.385155466399198e-05, + "loss": 1.245, + "theoretical_loss": 3.341573391252615, + "tokens_seen": 3091726336 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1123259, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2203683853149414, + "objective/train/theoretical_loss": 3.341568024650036, + "objective/train/tokens_used": 1462250976, + "theoretical_loss": 3.341568024650036, + "tokens_seen": 3091791872 + }, + { + "epoch": 0.87, + "learning_rate": 6.383149448345035e-05, + "loss": 1.2435, + "theoretical_loss": 3.341568024650036, + "tokens_seen": 3091791872 + }, + { + "epoch": 0.87, + "learning_rate": 6.381143430290873e-05, + "loss": 1.1939, + "theoretical_loss": 3.3415626581930606, + "tokens_seen": 3091857408 + }, + { + "epoch": 0.87, + "learning_rate": 6.37913741223671e-05, + "loss": 1.2835, + "theoretical_loss": 3.3415572918816823, + "tokens_seen": 3091922944 + }, + { + "epoch": 0.87, + "learning_rate": 6.377131394182548e-05, + "loss": 1.4183, + "theoretical_loss": 3.341551925715894, + "tokens_seen": 3091988480 + }, + { + "epoch": 0.87, + "learning_rate": 6.375125376128386e-05, + "loss": 1.264, + "theoretical_loss": 3.3415465596956886, + "tokens_seen": 3092054016 + }, + { + "epoch": 0.87, + "learning_rate": 6.373119358074222e-05, + "loss": 1.3195, + "theoretical_loss": 3.3415411938210586, + "tokens_seen": 3092119552 + }, + { + "epoch": 0.87, + "learning_rate": 6.37111334002006e-05, + "loss": 1.3256, + "theoretical_loss": 3.341535828091998, + "tokens_seen": 3092185088 + }, + { + "epoch": 0.87, + "learning_rate": 6.369107321965897e-05, + "loss": 1.203, + "theoretical_loss": 3.3415304625084987, + "tokens_seen": 3092250624 + }, + { + "epoch": 0.87, + "learning_rate": 6.367101303911735e-05, + "loss": 1.2702, + "theoretical_loss": 3.341525097070554, + "tokens_seen": 3092316160 + }, + { + "epoch": 0.87, + "learning_rate": 6.365095285857572e-05, + "loss": 1.3098, + "theoretical_loss": 3.3415197317781575, + "tokens_seen": 3092381696 + }, + { + "epoch": 0.87, + "learning_rate": 6.36308926780341e-05, + "loss": 1.2132, + "theoretical_loss": 3.3415143666313014, + "tokens_seen": 3092447232 + }, + { + "epoch": 0.87, + "learning_rate": 6.361083249749249e-05, + "loss": 1.3137, + "theoretical_loss": 3.3415090016299787, + "tokens_seen": 3092512768 + }, + { + "epoch": 0.87, + "learning_rate": 6.359077231695086e-05, + "loss": 1.2093, + "theoretical_loss": 3.341503636774183, + "tokens_seen": 3092578304 + }, + { + "epoch": 0.87, + "learning_rate": 6.357071213640924e-05, + "loss": 1.2278, + "theoretical_loss": 3.3414982720639066, + "tokens_seen": 3092643840 + }, + { + "epoch": 0.87, + "learning_rate": 6.355065195586761e-05, + "loss": 1.2795, + "theoretical_loss": 3.3414929074991426, + "tokens_seen": 3092709376 + }, + { + "epoch": 0.87, + "learning_rate": 6.353059177532598e-05, + "loss": 1.2789, + "theoretical_loss": 3.341487543079884, + "tokens_seen": 3092774912 + }, + { + "epoch": 0.87, + "learning_rate": 6.351053159478435e-05, + "loss": 1.3066, + "theoretical_loss": 3.341482178806124, + "tokens_seen": 3092840448 + }, + { + "epoch": 0.87, + "learning_rate": 6.349047141424273e-05, + "loss": 1.2518, + "theoretical_loss": 3.341476814677856, + "tokens_seen": 3092905984 + }, + { + "epoch": 0.87, + "learning_rate": 6.34704112337011e-05, + "loss": 1.2864, + "theoretical_loss": 3.341471450695072, + "tokens_seen": 3092971520 + }, + { + "epoch": 0.87, + "learning_rate": 6.345035105315948e-05, + "loss": 1.1896, + "theoretical_loss": 3.341466086857765, + "tokens_seen": 3093037056 + }, + { + "epoch": 0.87, + "learning_rate": 6.343029087261786e-05, + "loss": 1.2855, + "theoretical_loss": 3.3414607231659286, + "tokens_seen": 3093102592 + }, + { + "epoch": 0.87, + "learning_rate": 6.341023069207623e-05, + "loss": 1.2324, + "theoretical_loss": 3.3414553596195558, + "tokens_seen": 3093168128 + }, + { + "epoch": 0.87, + "learning_rate": 6.339017051153461e-05, + "loss": 1.1845, + "theoretical_loss": 3.341449996218639, + "tokens_seen": 3093233664 + }, + { + "epoch": 0.87, + "learning_rate": 6.337011033099298e-05, + "loss": 1.2195, + "theoretical_loss": 3.3414446329631717, + "tokens_seen": 3093299200 + }, + { + "epoch": 0.87, + "learning_rate": 6.335005015045135e-05, + "loss": 1.258, + "theoretical_loss": 3.3414392698531463, + "tokens_seen": 3093364736 + }, + { + "epoch": 0.87, + "learning_rate": 6.332998996990972e-05, + "loss": 1.4057, + "theoretical_loss": 3.3414339068885566, + "tokens_seen": 3093430272 + }, + { + "epoch": 0.87, + "learning_rate": 6.33099297893681e-05, + "loss": 1.18, + "theoretical_loss": 3.3414285440693945, + "tokens_seen": 3093495808 + }, + { + "epoch": 0.87, + "learning_rate": 6.328986960882647e-05, + "loss": 1.3387, + "theoretical_loss": 3.341423181395654, + "tokens_seen": 3093561344 + }, + { + "epoch": 0.87, + "learning_rate": 6.326980942828485e-05, + "loss": 1.3045, + "theoretical_loss": 3.3414178188673276, + "tokens_seen": 3093626880 + }, + { + "epoch": 0.87, + "learning_rate": 6.324974924774324e-05, + "loss": 1.2816, + "theoretical_loss": 3.3414124564844085, + "tokens_seen": 3093692416 + }, + { + "epoch": 0.88, + "learning_rate": 6.322968906720161e-05, + "loss": 1.2431, + "theoretical_loss": 3.341407094246889, + "tokens_seen": 3093757952 + }, + { + "epoch": 0.88, + "learning_rate": 6.320962888665999e-05, + "loss": 1.3941, + "theoretical_loss": 3.341401732154763, + "tokens_seen": 3093823488 + }, + { + "epoch": 0.88, + "learning_rate": 6.318956870611837e-05, + "loss": 1.2937, + "theoretical_loss": 3.341396370208023, + "tokens_seen": 3093889024 + }, + { + "epoch": 0.88, + "learning_rate": 6.316950852557674e-05, + "loss": 1.2336, + "theoretical_loss": 3.341391008406662, + "tokens_seen": 3093954560 + }, + { + "epoch": 0.88, + "learning_rate": 6.31494483450351e-05, + "loss": 1.3068, + "theoretical_loss": 3.3413856467506724, + "tokens_seen": 3094020096 + }, + { + "epoch": 0.88, + "learning_rate": 6.312938816449348e-05, + "loss": 1.361, + "theoretical_loss": 3.3413802852400485, + "tokens_seen": 3094085632 + }, + { + "epoch": 0.88, + "learning_rate": 6.310932798395186e-05, + "loss": 1.3246, + "theoretical_loss": 3.3413749238747825, + "tokens_seen": 3094151168 + }, + { + "epoch": 0.88, + "learning_rate": 6.308926780341023e-05, + "loss": 1.2301, + "theoretical_loss": 3.3413695626548674, + "tokens_seen": 3094216704 + }, + { + "epoch": 0.88, + "learning_rate": 6.306920762286861e-05, + "loss": 1.2851, + "theoretical_loss": 3.341364201580296, + "tokens_seen": 3094282240 + }, + { + "epoch": 0.88, + "learning_rate": 6.304914744232698e-05, + "loss": 1.1271, + "theoretical_loss": 3.341358840651062, + "tokens_seen": 3094347776 + }, + { + "epoch": 0.88, + "learning_rate": 6.302908726178536e-05, + "loss": 1.2249, + "theoretical_loss": 3.3413534798671574, + "tokens_seen": 3094413312 + }, + { + "epoch": 0.88, + "learning_rate": 6.300902708124373e-05, + "loss": 1.1055, + "theoretical_loss": 3.341348119228576, + "tokens_seen": 3094478848 + }, + { + "epoch": 0.88, + "learning_rate": 6.298896690070211e-05, + "loss": 1.2265, + "theoretical_loss": 3.34134275873531, + "tokens_seen": 3094544384 + }, + { + "epoch": 0.88, + "learning_rate": 6.296890672016049e-05, + "loss": 1.2735, + "theoretical_loss": 3.3413373983873536, + "tokens_seen": 3094609920 + }, + { + "epoch": 0.88, + "learning_rate": 6.294884653961885e-05, + "loss": 1.2743, + "theoretical_loss": 3.3413320381846985, + "tokens_seen": 3094675456 + }, + { + "epoch": 0.88, + "learning_rate": 6.292878635907722e-05, + "loss": 1.1069, + "theoretical_loss": 3.3413266781273383, + "tokens_seen": 3094740992 + }, + { + "epoch": 0.88, + "learning_rate": 6.29087261785356e-05, + "loss": 1.3151, + "theoretical_loss": 3.341321318215266, + "tokens_seen": 3094806528 + }, + { + "epoch": 0.88, + "learning_rate": 6.288866599799398e-05, + "loss": 1.173, + "theoretical_loss": 3.341315958448474, + "tokens_seen": 3094872064 + }, + { + "epoch": 0.88, + "learning_rate": 6.286860581745236e-05, + "loss": 1.2616, + "theoretical_loss": 3.3413105988269565, + "tokens_seen": 3094937600 + }, + { + "epoch": 0.88, + "learning_rate": 6.284854563691074e-05, + "loss": 1.1863, + "theoretical_loss": 3.3413052393507057, + "tokens_seen": 3095003136 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1124452, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.351484775543213, + "objective/train/theoretical_loss": 3.3412998800197142, + "objective/train/tokens_used": 1465527776, + "theoretical_loss": 3.3412998800197142, + "tokens_seen": 3095068672 + }, + { + "epoch": 0.88, + "learning_rate": 6.282848545636912e-05, + "loss": 1.3052, + "theoretical_loss": 3.3412998800197142, + "tokens_seen": 3095068672 + }, + { + "epoch": 0.88, + "learning_rate": 6.280842527582749e-05, + "loss": 1.2323, + "theoretical_loss": 3.3412945208339755, + "tokens_seen": 3095134208 + }, + { + "epoch": 0.88, + "learning_rate": 6.278836509528587e-05, + "loss": 1.3331, + "theoretical_loss": 3.341289161793483, + "tokens_seen": 3095199744 + }, + { + "epoch": 0.88, + "learning_rate": 6.276830491474424e-05, + "loss": 1.1998, + "theoretical_loss": 3.3412838028982286, + "tokens_seen": 3095265280 + }, + { + "epoch": 0.88, + "learning_rate": 6.27482447342026e-05, + "loss": 1.2801, + "theoretical_loss": 3.341278444148206, + "tokens_seen": 3095330816 + }, + { + "epoch": 0.88, + "learning_rate": 6.272818455366098e-05, + "loss": 1.3901, + "theoretical_loss": 3.341273085543408, + "tokens_seen": 3095396352 + }, + { + "epoch": 0.88, + "learning_rate": 6.270812437311936e-05, + "loss": 1.3306, + "theoretical_loss": 3.341267727083828, + "tokens_seen": 3095461888 + }, + { + "epoch": 0.88, + "learning_rate": 6.268806419257773e-05, + "loss": 1.2727, + "theoretical_loss": 3.341262368769459, + "tokens_seen": 3095527424 + }, + { + "epoch": 0.88, + "learning_rate": 6.266800401203611e-05, + "loss": 1.1279, + "theoretical_loss": 3.341257010600293, + "tokens_seen": 3095592960 + }, + { + "epoch": 0.88, + "learning_rate": 6.264794383149448e-05, + "loss": 1.1551, + "theoretical_loss": 3.341251652576324, + "tokens_seen": 3095658496 + }, + { + "epoch": 0.88, + "learning_rate": 6.262788365095286e-05, + "loss": 1.3007, + "theoretical_loss": 3.3412462946975445, + "tokens_seen": 3095724032 + }, + { + "epoch": 0.88, + "learning_rate": 6.260782347041124e-05, + "loss": 1.2125, + "theoretical_loss": 3.3412409369639473, + "tokens_seen": 3095789568 + }, + { + "epoch": 0.88, + "learning_rate": 6.258776328986961e-05, + "loss": 1.1926, + "theoretical_loss": 3.341235579375526, + "tokens_seen": 3095855104 + }, + { + "epoch": 0.88, + "learning_rate": 6.256770310932797e-05, + "loss": 1.3037, + "theoretical_loss": 3.3412302219322734, + "tokens_seen": 3095920640 + }, + { + "epoch": 0.88, + "learning_rate": 6.254764292878635e-05, + "loss": 1.2668, + "theoretical_loss": 3.3412248646341824, + "tokens_seen": 3095986176 + }, + { + "epoch": 0.88, + "learning_rate": 6.252758274824473e-05, + "loss": 1.336, + "theoretical_loss": 3.341219507481246, + "tokens_seen": 3096051712 + }, + { + "epoch": 0.88, + "learning_rate": 6.25075225677031e-05, + "loss": 1.3127, + "theoretical_loss": 3.341214150473457, + "tokens_seen": 3096117248 + }, + { + "epoch": 0.88, + "learning_rate": 6.248746238716149e-05, + "loss": 1.2936, + "theoretical_loss": 3.341208793610809, + "tokens_seen": 3096182784 + }, + { + "epoch": 0.88, + "learning_rate": 6.246740220661985e-05, + "loss": 1.2775, + "theoretical_loss": 3.341203436893294, + "tokens_seen": 3096248320 + }, + { + "epoch": 0.88, + "learning_rate": 6.244734202607823e-05, + "loss": 1.1846, + "theoretical_loss": 3.3411980803209063, + "tokens_seen": 3096313856 + }, + { + "epoch": 0.88, + "learning_rate": 6.24272818455366e-05, + "loss": 1.2304, + "theoretical_loss": 3.341192723893638, + "tokens_seen": 3096379392 + }, + { + "epoch": 0.88, + "learning_rate": 6.240722166499498e-05, + "loss": 1.2387, + "theoretical_loss": 3.341187367611482, + "tokens_seen": 3096444928 + }, + { + "epoch": 0.88, + "learning_rate": 6.238716148445337e-05, + "loss": 1.1735, + "theoretical_loss": 3.3411820114744315, + "tokens_seen": 3096510464 + }, + { + "epoch": 0.88, + "learning_rate": 6.236710130391173e-05, + "loss": 1.168, + "theoretical_loss": 3.3411766554824798, + "tokens_seen": 3096576000 + }, + { + "epoch": 0.88, + "learning_rate": 6.234704112337011e-05, + "loss": 1.3184, + "theoretical_loss": 3.3411712996356195, + "tokens_seen": 3096641536 + }, + { + "epoch": 0.88, + "learning_rate": 6.232698094282848e-05, + "loss": 1.2845, + "theoretical_loss": 3.341165943933844, + "tokens_seen": 3096707072 + }, + { + "epoch": 0.88, + "learning_rate": 6.230692076228686e-05, + "loss": 1.2032, + "theoretical_loss": 3.341160588377146, + "tokens_seen": 3096772608 + }, + { + "epoch": 0.88, + "learning_rate": 6.228686058174524e-05, + "loss": 1.2316, + "theoretical_loss": 3.3411552329655185, + "tokens_seen": 3096838144 + }, + { + "epoch": 0.88, + "learning_rate": 6.226680040120361e-05, + "loss": 1.1589, + "theoretical_loss": 3.341149877698955, + "tokens_seen": 3096903680 + }, + { + "epoch": 0.88, + "learning_rate": 6.224674022066199e-05, + "loss": 1.156, + "theoretical_loss": 3.3411445225774474, + "tokens_seen": 3096969216 + }, + { + "epoch": 0.88, + "learning_rate": 6.222668004012036e-05, + "loss": 1.2716, + "theoretical_loss": 3.34113916760099, + "tokens_seen": 3097034752 + }, + { + "epoch": 0.88, + "learning_rate": 6.220661985957874e-05, + "loss": 1.3819, + "theoretical_loss": 3.3411338127695744, + "tokens_seen": 3097100288 + }, + { + "epoch": 0.88, + "learning_rate": 6.218655967903711e-05, + "loss": 1.1877, + "theoretical_loss": 3.341128458083195, + "tokens_seen": 3097165824 + }, + { + "epoch": 0.88, + "learning_rate": 6.216649949849549e-05, + "loss": 1.2291, + "theoretical_loss": 3.341123103541844, + "tokens_seen": 3097231360 + }, + { + "epoch": 0.88, + "learning_rate": 6.214643931795387e-05, + "loss": 1.2554, + "theoretical_loss": 3.3411177491455146, + "tokens_seen": 3097296896 + }, + { + "epoch": 0.88, + "learning_rate": 6.212637913741224e-05, + "loss": 1.2682, + "theoretical_loss": 3.3411123948942, + "tokens_seen": 3097362432 + }, + { + "epoch": 0.88, + "learning_rate": 6.210631895687062e-05, + "loss": 1.2829, + "theoretical_loss": 3.3411070407878927, + "tokens_seen": 3097427968 + }, + { + "epoch": 0.88, + "learning_rate": 6.2086258776329e-05, + "loss": 1.1434, + "theoretical_loss": 3.341101686826586, + "tokens_seen": 3097493504 + }, + { + "epoch": 0.88, + "learning_rate": 6.206619859578736e-05, + "loss": 1.1952, + "theoretical_loss": 3.341096333010273, + "tokens_seen": 3097559040 + }, + { + "epoch": 0.88, + "learning_rate": 6.204613841524573e-05, + "loss": 1.2751, + "theoretical_loss": 3.3410909793389463, + "tokens_seen": 3097624576 + }, + { + "epoch": 0.88, + "learning_rate": 6.202607823470412e-05, + "loss": 1.2913, + "theoretical_loss": 3.3410856258125996, + "tokens_seen": 3097690112 + }, + { + "epoch": 0.88, + "learning_rate": 6.20060180541625e-05, + "loss": 1.2717, + "theoretical_loss": 3.3410802724312254, + "tokens_seen": 3097755648 + }, + { + "epoch": 0.88, + "learning_rate": 6.198595787362086e-05, + "loss": 1.288, + "theoretical_loss": 3.341074919194817, + "tokens_seen": 3097821184 + }, + { + "epoch": 0.88, + "learning_rate": 6.196589769307923e-05, + "loss": 1.3195, + "theoretical_loss": 3.341069566103367, + "tokens_seen": 3097886720 + }, + { + "epoch": 0.88, + "learning_rate": 6.194583751253761e-05, + "loss": 1.2051, + "theoretical_loss": 3.341064213156869, + "tokens_seen": 3097952256 + }, + { + "epoch": 0.88, + "learning_rate": 6.192577733199599e-05, + "loss": 1.3285, + "theoretical_loss": 3.3410588603553153, + "tokens_seen": 3098017792 + }, + { + "epoch": 0.88, + "learning_rate": 6.190571715145436e-05, + "loss": 1.2204, + "theoretical_loss": 3.3410535076986996, + "tokens_seen": 3098083328 + }, + { + "epoch": 0.88, + "learning_rate": 6.188565697091274e-05, + "loss": 1.3401, + "theoretical_loss": 3.341048155187014, + "tokens_seen": 3098148864 + }, + { + "epoch": 0.88, + "learning_rate": 6.186559679037111e-05, + "loss": 1.304, + "theoretical_loss": 3.3410428028202523, + "tokens_seen": 3098214400 + }, + { + "epoch": 0.88, + "learning_rate": 6.184553660982949e-05, + "loss": 1.1779, + "theoretical_loss": 3.3410374505984075, + "tokens_seen": 3098279936 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1125757, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2427351474761963, + "objective/train/theoretical_loss": 3.3410320985214725, + "objective/train/tokens_used": 1468804576, + "theoretical_loss": 3.3410320985214725, + "tokens_seen": 3098345472 + }, + { + "epoch": 0.88, + "learning_rate": 6.182547642928787e-05, + "loss": 1.3082, + "theoretical_loss": 3.3410320985214725, + "tokens_seen": 3098345472 + }, + { + "epoch": 0.88, + "learning_rate": 6.180541624874624e-05, + "loss": 1.3113, + "theoretical_loss": 3.34102674658944, + "tokens_seen": 3098411008 + }, + { + "epoch": 0.88, + "learning_rate": 6.178535606820462e-05, + "loss": 1.1997, + "theoretical_loss": 3.3410213948023033, + "tokens_seen": 3098476544 + }, + { + "epoch": 0.88, + "learning_rate": 6.176529588766299e-05, + "loss": 1.1998, + "theoretical_loss": 3.3410160431600553, + "tokens_seen": 3098542080 + }, + { + "epoch": 0.88, + "learning_rate": 6.174523570712137e-05, + "loss": 1.2511, + "theoretical_loss": 3.3410106916626887, + "tokens_seen": 3098607616 + }, + { + "epoch": 0.88, + "learning_rate": 6.172517552657974e-05, + "loss": 1.2111, + "theoretical_loss": 3.3410053403101974, + "tokens_seen": 3098673152 + }, + { + "epoch": 0.88, + "learning_rate": 6.170511534603812e-05, + "loss": 1.318, + "theoretical_loss": 3.3409999891025737, + "tokens_seen": 3098738688 + }, + { + "epoch": 0.88, + "learning_rate": 6.168505516549648e-05, + "loss": 1.3577, + "theoretical_loss": 3.3409946380398106, + "tokens_seen": 3098804224 + }, + { + "epoch": 0.88, + "learning_rate": 6.166499498495486e-05, + "loss": 1.221, + "theoretical_loss": 3.340989287121902, + "tokens_seen": 3098869760 + }, + { + "epoch": 0.88, + "learning_rate": 6.164493480441325e-05, + "loss": 1.1947, + "theoretical_loss": 3.3409839363488394, + "tokens_seen": 3098935296 + }, + { + "epoch": 0.88, + "learning_rate": 6.162487462387162e-05, + "loss": 1.2318, + "theoretical_loss": 3.340978585720617, + "tokens_seen": 3099000832 + }, + { + "epoch": 0.88, + "learning_rate": 6.160481444333e-05, + "loss": 1.2255, + "theoretical_loss": 3.3409732352372274, + "tokens_seen": 3099066368 + }, + { + "epoch": 0.88, + "learning_rate": 6.158475426278836e-05, + "loss": 1.3245, + "theoretical_loss": 3.3409678848986637, + "tokens_seen": 3099131904 + }, + { + "epoch": 0.88, + "learning_rate": 6.156469408224674e-05, + "loss": 1.4005, + "theoretical_loss": 3.340962534704919, + "tokens_seen": 3099197440 + }, + { + "epoch": 0.88, + "learning_rate": 6.154463390170511e-05, + "loss": 1.2871, + "theoretical_loss": 3.340957184655986, + "tokens_seen": 3099262976 + }, + { + "epoch": 0.88, + "learning_rate": 6.152457372116349e-05, + "loss": 1.2999, + "theoretical_loss": 3.3409518347518583, + "tokens_seen": 3099328512 + }, + { + "epoch": 0.88, + "learning_rate": 6.150451354062186e-05, + "loss": 1.1839, + "theoretical_loss": 3.340946484992528, + "tokens_seen": 3099394048 + }, + { + "epoch": 0.88, + "learning_rate": 6.148445336008024e-05, + "loss": 1.2593, + "theoretical_loss": 3.340941135377989, + "tokens_seen": 3099459584 + }, + { + "epoch": 0.88, + "learning_rate": 6.146439317953862e-05, + "loss": 1.3403, + "theoretical_loss": 3.340935785908234, + "tokens_seen": 3099525120 + }, + { + "epoch": 0.88, + "learning_rate": 6.144433299899699e-05, + "loss": 1.2685, + "theoretical_loss": 3.340930436583256, + "tokens_seen": 3099590656 + }, + { + "epoch": 0.88, + "learning_rate": 6.142427281845537e-05, + "loss": 1.3122, + "theoretical_loss": 3.340925087403048, + "tokens_seen": 3099656192 + }, + { + "epoch": 0.88, + "learning_rate": 6.140421263791374e-05, + "loss": 1.1589, + "theoretical_loss": 3.340919738367603, + "tokens_seen": 3099721728 + }, + { + "epoch": 0.88, + "learning_rate": 6.138415245737212e-05, + "loss": 1.3281, + "theoretical_loss": 3.3409143894769144, + "tokens_seen": 3099787264 + }, + { + "epoch": 0.88, + "learning_rate": 6.13640922768305e-05, + "loss": 1.3311, + "theoretical_loss": 3.3409090407309745, + "tokens_seen": 3099852800 + }, + { + "epoch": 0.88, + "learning_rate": 6.134403209628887e-05, + "loss": 1.2322, + "theoretical_loss": 3.3409036921297766, + "tokens_seen": 3099918336 + }, + { + "epoch": 0.88, + "learning_rate": 6.132397191574725e-05, + "loss": 1.2501, + "theoretical_loss": 3.3408983436733144, + "tokens_seen": 3099983872 + }, + { + "epoch": 0.88, + "learning_rate": 6.130391173520561e-05, + "loss": 1.3634, + "theoretical_loss": 3.34089299536158, + "tokens_seen": 3100049408 + }, + { + "epoch": 0.88, + "learning_rate": 6.128385155466398e-05, + "loss": 1.2189, + "theoretical_loss": 3.3408876471945668, + "tokens_seen": 3100114944 + }, + { + "epoch": 0.88, + "learning_rate": 6.126379137412237e-05, + "loss": 1.2496, + "theoretical_loss": 3.340882299172268, + "tokens_seen": 3100180480 + }, + { + "epoch": 0.88, + "learning_rate": 6.124373119358075e-05, + "loss": 1.2441, + "theoretical_loss": 3.340876951294676, + "tokens_seen": 3100246016 + }, + { + "epoch": 0.88, + "learning_rate": 6.122367101303913e-05, + "loss": 1.2599, + "theoretical_loss": 3.3408716035617845, + "tokens_seen": 3100311552 + }, + { + "epoch": 0.88, + "learning_rate": 6.120361083249749e-05, + "loss": 1.1541, + "theoretical_loss": 3.3408662559735864, + "tokens_seen": 3100377088 + }, + { + "epoch": 0.88, + "learning_rate": 6.118355065195586e-05, + "loss": 1.3169, + "theoretical_loss": 3.3408609085300744, + "tokens_seen": 3100442624 + }, + { + "epoch": 0.88, + "learning_rate": 6.116349047141424e-05, + "loss": 1.1904, + "theoretical_loss": 3.3408555612312423, + "tokens_seen": 3100508160 + }, + { + "epoch": 0.88, + "learning_rate": 6.114343029087262e-05, + "loss": 1.3191, + "theoretical_loss": 3.340850214077082, + "tokens_seen": 3100573696 + }, + { + "epoch": 0.88, + "learning_rate": 6.112337011033099e-05, + "loss": 1.3495, + "theoretical_loss": 3.340844867067587, + "tokens_seen": 3100639232 + }, + { + "epoch": 0.88, + "learning_rate": 6.110330992978937e-05, + "loss": 1.2118, + "theoretical_loss": 3.340839520202751, + "tokens_seen": 3100704768 + }, + { + "epoch": 0.88, + "learning_rate": 6.108324974924774e-05, + "loss": 1.2634, + "theoretical_loss": 3.340834173482566, + "tokens_seen": 3100770304 + }, + { + "epoch": 0.88, + "learning_rate": 6.106318956870612e-05, + "loss": 1.3137, + "theoretical_loss": 3.3408288269070257, + "tokens_seen": 3100835840 + }, + { + "epoch": 0.88, + "learning_rate": 6.10431293881645e-05, + "loss": 1.2219, + "theoretical_loss": 3.340823480476123, + "tokens_seen": 3100901376 + }, + { + "epoch": 0.88, + "learning_rate": 6.102306920762287e-05, + "loss": 1.1932, + "theoretical_loss": 3.340818134189851, + "tokens_seen": 3100966912 + }, + { + "epoch": 0.88, + "learning_rate": 6.1003009027081246e-05, + "loss": 1.2275, + "theoretical_loss": 3.340812788048202, + "tokens_seen": 3101032448 + }, + { + "epoch": 0.88, + "learning_rate": 6.098294884653962e-05, + "loss": 1.2312, + "theoretical_loss": 3.3408074420511698, + "tokens_seen": 3101097984 + }, + { + "epoch": 0.88, + "learning_rate": 6.096288866599799e-05, + "loss": 1.2798, + "theoretical_loss": 3.3408020961987472, + "tokens_seen": 3101163520 + }, + { + "epoch": 0.88, + "learning_rate": 6.0942828485456366e-05, + "loss": 1.192, + "theoretical_loss": 3.3407967504909277, + "tokens_seen": 3101229056 + }, + { + "epoch": 0.88, + "learning_rate": 6.092276830491474e-05, + "loss": 1.1062, + "theoretical_loss": 3.3407914049277037, + "tokens_seen": 3101294592 + }, + { + "epoch": 0.88, + "learning_rate": 6.0902708124373125e-05, + "loss": 1.2974, + "theoretical_loss": 3.3407860595090684, + "tokens_seen": 3101360128 + }, + { + "epoch": 0.88, + "learning_rate": 6.08826479438315e-05, + "loss": 1.1511, + "theoretical_loss": 3.3407807142350148, + "tokens_seen": 3101425664 + }, + { + "epoch": 0.88, + "learning_rate": 6.086258776328987e-05, + "loss": 1.2025, + "theoretical_loss": 3.3407753691055357, + "tokens_seen": 3101491200 + }, + { + "epoch": 0.88, + "learning_rate": 6.0842527582748245e-05, + "loss": 1.1946, + "theoretical_loss": 3.340770024120625, + "tokens_seen": 3101556736 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1126928, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1931419372558594, + "objective/train/theoretical_loss": 3.3407646792802748, + "objective/train/tokens_used": 1472081376, + "theoretical_loss": 3.3407646792802748, + "tokens_seen": 3101622272 + }, + { + "epoch": 0.88, + "learning_rate": 6.082246740220662e-05, + "loss": 1.2137, + "theoretical_loss": 3.3407646792802748, + "tokens_seen": 3101622272 + }, + { + "epoch": 0.88, + "learning_rate": 6.0802407221665e-05, + "loss": 1.1793, + "theoretical_loss": 3.3407593345844786, + "tokens_seen": 3101687808 + }, + { + "epoch": 0.88, + "learning_rate": 6.078234704112337e-05, + "loss": 1.3168, + "theoretical_loss": 3.3407539900332295, + "tokens_seen": 3101753344 + }, + { + "epoch": 0.88, + "learning_rate": 6.076228686058174e-05, + "loss": 1.1753, + "theoretical_loss": 3.3407486456265203, + "tokens_seen": 3101818880 + }, + { + "epoch": 0.88, + "learning_rate": 6.074222668004012e-05, + "loss": 1.2804, + "theoretical_loss": 3.3407433013643444, + "tokens_seen": 3101884416 + }, + { + "epoch": 0.88, + "learning_rate": 6.072216649949849e-05, + "loss": 1.3099, + "theoretical_loss": 3.340737957246694, + "tokens_seen": 3101949952 + }, + { + "epoch": 0.88, + "learning_rate": 6.0702106318956876e-05, + "loss": 1.2686, + "theoretical_loss": 3.3407326132735635, + "tokens_seen": 3102015488 + }, + { + "epoch": 0.88, + "learning_rate": 6.068204613841525e-05, + "loss": 1.2872, + "theoretical_loss": 3.3407272694449444, + "tokens_seen": 3102081024 + }, + { + "epoch": 0.88, + "learning_rate": 6.066198595787362e-05, + "loss": 1.1462, + "theoretical_loss": 3.340721925760831, + "tokens_seen": 3102146560 + }, + { + "epoch": 0.88, + "learning_rate": 6.0641925777331996e-05, + "loss": 1.1678, + "theoretical_loss": 3.3407165822212153, + "tokens_seen": 3102212096 + }, + { + "epoch": 0.88, + "learning_rate": 6.062186559679037e-05, + "loss": 1.2045, + "theoretical_loss": 3.340711238826091, + "tokens_seen": 3102277632 + }, + { + "epoch": 0.88, + "learning_rate": 6.060180541624875e-05, + "loss": 1.278, + "theoretical_loss": 3.340705895575451, + "tokens_seen": 3102343168 + }, + { + "epoch": 0.88, + "learning_rate": 6.0581745235707124e-05, + "loss": 1.295, + "theoretical_loss": 3.3407005524692885, + "tokens_seen": 3102408704 + }, + { + "epoch": 0.88, + "learning_rate": 6.056168505516549e-05, + "loss": 1.2546, + "theoretical_loss": 3.3406952095075964, + "tokens_seen": 3102474240 + }, + { + "epoch": 0.88, + "learning_rate": 6.054162487462387e-05, + "loss": 1.2778, + "theoretical_loss": 3.3406898666903677, + "tokens_seen": 3102539776 + }, + { + "epoch": 0.88, + "learning_rate": 6.052156469408225e-05, + "loss": 1.318, + "theoretical_loss": 3.3406845240175955, + "tokens_seen": 3102605312 + }, + { + "epoch": 0.88, + "learning_rate": 6.050150451354063e-05, + "loss": 1.2752, + "theoretical_loss": 3.3406791814892727, + "tokens_seen": 3102670848 + }, + { + "epoch": 0.88, + "learning_rate": 6.0481444332999e-05, + "loss": 1.1638, + "theoretical_loss": 3.3406738391053925, + "tokens_seen": 3102736384 + }, + { + "epoch": 0.88, + "learning_rate": 6.046138415245737e-05, + "loss": 1.2792, + "theoretical_loss": 3.340668496865948, + "tokens_seen": 3102801920 + }, + { + "epoch": 0.88, + "learning_rate": 6.044132397191575e-05, + "loss": 1.1666, + "theoretical_loss": 3.340663154770932, + "tokens_seen": 3102867456 + }, + { + "epoch": 0.88, + "learning_rate": 6.0421263791374123e-05, + "loss": 1.1958, + "theoretical_loss": 3.3406578128203375, + "tokens_seen": 3102932992 + }, + { + "epoch": 0.88, + "learning_rate": 6.04012036108325e-05, + "loss": 1.196, + "theoretical_loss": 3.340652471014158, + "tokens_seen": 3102998528 + }, + { + "epoch": 0.88, + "learning_rate": 6.0381143430290875e-05, + "loss": 1.1192, + "theoretical_loss": 3.3406471293523863, + "tokens_seen": 3103064064 + }, + { + "epoch": 0.88, + "learning_rate": 6.0361083249749244e-05, + "loss": 1.2681, + "theoretical_loss": 3.340641787835015, + "tokens_seen": 3103129600 + }, + { + "epoch": 0.88, + "learning_rate": 6.0341023069207627e-05, + "loss": 1.3304, + "theoretical_loss": 3.340636446462038, + "tokens_seen": 3103195136 + }, + { + "epoch": 0.88, + "learning_rate": 6.0320962888666e-05, + "loss": 1.2637, + "theoretical_loss": 3.340631105233448, + "tokens_seen": 3103260672 + }, + { + "epoch": 0.88, + "learning_rate": 6.030090270812438e-05, + "loss": 1.2522, + "theoretical_loss": 3.340625764149238, + "tokens_seen": 3103326208 + }, + { + "epoch": 0.88, + "learning_rate": 6.0280842527582754e-05, + "loss": 1.233, + "theoretical_loss": 3.340620423209401, + "tokens_seen": 3103391744 + }, + { + "epoch": 0.88, + "learning_rate": 6.026078234704112e-05, + "loss": 1.2161, + "theoretical_loss": 3.3406150824139296, + "tokens_seen": 3103457280 + }, + { + "epoch": 0.88, + "learning_rate": 6.02407221664995e-05, + "loss": 1.2005, + "theoretical_loss": 3.3406097417628176, + "tokens_seen": 3103522816 + }, + { + "epoch": 0.88, + "learning_rate": 6.0220661985957875e-05, + "loss": 1.2195, + "theoretical_loss": 3.3406044012560576, + "tokens_seen": 3103588352 + }, + { + "epoch": 0.88, + "learning_rate": 6.020060180541625e-05, + "loss": 1.2648, + "theoretical_loss": 3.340599060893643, + "tokens_seen": 3103653888 + }, + { + "epoch": 0.88, + "learning_rate": 6.018054162487462e-05, + "loss": 1.2282, + "theoretical_loss": 3.3405937206755665, + "tokens_seen": 3103719424 + }, + { + "epoch": 0.88, + "learning_rate": 6.0160481444332995e-05, + "loss": 1.1573, + "theoretical_loss": 3.3405883806018215, + "tokens_seen": 3103784960 + }, + { + "epoch": 0.88, + "learning_rate": 6.014042126379138e-05, + "loss": 1.178, + "theoretical_loss": 3.340583040672401, + "tokens_seen": 3103850496 + }, + { + "epoch": 0.88, + "learning_rate": 6.0120361083249754e-05, + "loss": 1.3732, + "theoretical_loss": 3.3405777008872977, + "tokens_seen": 3103916032 + }, + { + "epoch": 0.88, + "learning_rate": 6.010030090270813e-05, + "loss": 1.3299, + "theoretical_loss": 3.340572361246505, + "tokens_seen": 3103981568 + }, + { + "epoch": 0.88, + "learning_rate": 6.00802407221665e-05, + "loss": 1.2345, + "theoretical_loss": 3.3405670217500156, + "tokens_seen": 3104047104 + }, + { + "epoch": 0.88, + "learning_rate": 6.0060180541624874e-05, + "loss": 1.1837, + "theoretical_loss": 3.340561682397823, + "tokens_seen": 3104112640 + }, + { + "epoch": 0.88, + "learning_rate": 6.004012036108325e-05, + "loss": 1.2317, + "theoretical_loss": 3.34055634318992, + "tokens_seen": 3104178176 + }, + { + "epoch": 0.88, + "learning_rate": 6.0020060180541626e-05, + "loss": 1.2507, + "theoretical_loss": 3.3405510041263, + "tokens_seen": 3104243712 + }, + { + "epoch": 0.88, + "learning_rate": 6e-05, + "loss": 1.2443, + "theoretical_loss": 3.340545665206955, + "tokens_seen": 3104309248 + }, + { + "epoch": 0.88, + "learning_rate": 5.997993981945837e-05, + "loss": 1.2084, + "theoretical_loss": 3.340540326431879, + "tokens_seen": 3104374784 + }, + { + "epoch": 0.88, + "learning_rate": 5.995987963891675e-05, + "loss": 1.2653, + "theoretical_loss": 3.3405349878010653, + "tokens_seen": 3104440320 + }, + { + "epoch": 0.88, + "learning_rate": 5.993981945837513e-05, + "loss": 1.2299, + "theoretical_loss": 3.3405296493145062, + "tokens_seen": 3104505856 + }, + { + "epoch": 0.88, + "learning_rate": 5.9919759277833505e-05, + "loss": 1.2184, + "theoretical_loss": 3.340524310972195, + "tokens_seen": 3104571392 + }, + { + "epoch": 0.88, + "learning_rate": 5.989969909729188e-05, + "loss": 1.1504, + "theoretical_loss": 3.340518972774125, + "tokens_seen": 3104636928 + }, + { + "epoch": 0.88, + "learning_rate": 5.987963891675025e-05, + "loss": 1.2239, + "theoretical_loss": 3.340513634720289, + "tokens_seen": 3104702464 + }, + { + "epoch": 0.88, + "learning_rate": 5.9859578736208625e-05, + "loss": 1.1887, + "theoretical_loss": 3.3405082968106803, + "tokens_seen": 3104768000 + }, + { + "epoch": 0.88, + "learning_rate": 5.9839518555667e-05, + "loss": 1.4095, + "theoretical_loss": 3.3405029590452915, + "tokens_seen": 3104833536 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1128234, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2622343301773071, + "objective/train/theoretical_loss": 3.340497621424116, + "objective/train/tokens_used": 1475358176, + "theoretical_loss": 3.340497621424116, + "tokens_seen": 3104899072 + }, + { + "epoch": 0.88, + "learning_rate": 5.981945837512538e-05, + "loss": 1.2339, + "theoretical_loss": 3.340497621424116, + "tokens_seen": 3104899072 + }, + { + "epoch": 0.88, + "learning_rate": 5.979939819458375e-05, + "loss": 1.2092, + "theoretical_loss": 3.340492283947147, + "tokens_seen": 3104964608 + }, + { + "epoch": 0.88, + "learning_rate": 5.977933801404212e-05, + "loss": 1.1484, + "theoretical_loss": 3.3404869466143774, + "tokens_seen": 3105030144 + }, + { + "epoch": 0.88, + "learning_rate": 5.9759277833500504e-05, + "loss": 1.2252, + "theoretical_loss": 3.3404816094258, + "tokens_seen": 3105095680 + }, + { + "epoch": 0.88, + "learning_rate": 5.973921765295888e-05, + "loss": 1.2858, + "theoretical_loss": 3.3404762723814083, + "tokens_seen": 3105161216 + }, + { + "epoch": 0.88, + "learning_rate": 5.9719157472417256e-05, + "loss": 1.2704, + "theoretical_loss": 3.3404709354811954, + "tokens_seen": 3105226752 + }, + { + "epoch": 0.88, + "learning_rate": 5.969909729187563e-05, + "loss": 1.3244, + "theoretical_loss": 3.340465598725154, + "tokens_seen": 3105292288 + }, + { + "epoch": 0.88, + "learning_rate": 5.9679037111334e-05, + "loss": 1.3497, + "theoretical_loss": 3.340460262113277, + "tokens_seen": 3105357824 + }, + { + "epoch": 0.88, + "learning_rate": 5.9658976930792376e-05, + "loss": 1.2545, + "theoretical_loss": 3.340454925645558, + "tokens_seen": 3105423360 + }, + { + "epoch": 0.88, + "learning_rate": 5.963891675025075e-05, + "loss": 1.3052, + "theoretical_loss": 3.3404495893219894, + "tokens_seen": 3105488896 + }, + { + "epoch": 0.88, + "learning_rate": 5.961885656970913e-05, + "loss": 1.2094, + "theoretical_loss": 3.3404442531425653, + "tokens_seen": 3105554432 + }, + { + "epoch": 0.88, + "learning_rate": 5.9598796389167504e-05, + "loss": 1.2439, + "theoretical_loss": 3.3404389171072775, + "tokens_seen": 3105619968 + }, + { + "epoch": 0.88, + "learning_rate": 5.957873620862588e-05, + "loss": 1.1886, + "theoretical_loss": 3.34043358121612, + "tokens_seen": 3105685504 + }, + { + "epoch": 0.88, + "learning_rate": 5.9558676028084255e-05, + "loss": 1.3585, + "theoretical_loss": 3.3404282454690857, + "tokens_seen": 3105751040 + }, + { + "epoch": 0.88, + "learning_rate": 5.953861584754263e-05, + "loss": 1.3469, + "theoretical_loss": 3.3404229098661675, + "tokens_seen": 3105816576 + }, + { + "epoch": 0.88, + "learning_rate": 5.951855566700101e-05, + "loss": 1.2002, + "theoretical_loss": 3.3404175744073585, + "tokens_seen": 3105882112 + }, + { + "epoch": 0.88, + "learning_rate": 5.949849548645938e-05, + "loss": 1.1512, + "theoretical_loss": 3.3404122390926516, + "tokens_seen": 3105947648 + }, + { + "epoch": 0.88, + "learning_rate": 5.947843530591775e-05, + "loss": 1.3575, + "theoretical_loss": 3.3404069039220405, + "tokens_seen": 3106013184 + }, + { + "epoch": 0.88, + "learning_rate": 5.945837512537613e-05, + "loss": 1.1912, + "theoretical_loss": 3.3404015688955173, + "tokens_seen": 3106078720 + }, + { + "epoch": 0.88, + "learning_rate": 5.9438314944834503e-05, + "loss": 1.2294, + "theoretical_loss": 3.3403962340130757, + "tokens_seen": 3106144256 + }, + { + "epoch": 0.88, + "learning_rate": 5.941825476429288e-05, + "loss": 1.254, + "theoretical_loss": 3.340390899274709, + "tokens_seen": 3106209792 + }, + { + "epoch": 0.88, + "learning_rate": 5.9398194583751255e-05, + "loss": 1.2046, + "theoretical_loss": 3.3403855646804095, + "tokens_seen": 3106275328 + }, + { + "epoch": 0.88, + "learning_rate": 5.937813440320963e-05, + "loss": 1.2866, + "theoretical_loss": 3.340380230230171, + "tokens_seen": 3106340864 + }, + { + "epoch": 0.88, + "learning_rate": 5.9358074222668007e-05, + "loss": 1.1637, + "theoretical_loss": 3.340374895923986, + "tokens_seen": 3106406400 + }, + { + "epoch": 0.88, + "learning_rate": 5.933801404212638e-05, + "loss": 1.264, + "theoretical_loss": 3.340369561761848, + "tokens_seen": 3106471936 + }, + { + "epoch": 0.88, + "learning_rate": 5.931795386158476e-05, + "loss": 1.2654, + "theoretical_loss": 3.3403642277437497, + "tokens_seen": 3106537472 + }, + { + "epoch": 0.88, + "learning_rate": 5.929789368104313e-05, + "loss": 1.2091, + "theoretical_loss": 3.3403588938696847, + "tokens_seen": 3106603008 + }, + { + "epoch": 0.88, + "learning_rate": 5.92778335005015e-05, + "loss": 1.3145, + "theoretical_loss": 3.3403535601396452, + "tokens_seen": 3106668544 + }, + { + "epoch": 0.88, + "learning_rate": 5.925777331995988e-05, + "loss": 1.1576, + "theoretical_loss": 3.3403482265536253, + "tokens_seen": 3106734080 + }, + { + "epoch": 0.88, + "learning_rate": 5.9237713139418255e-05, + "loss": 1.2603, + "theoretical_loss": 3.3403428931116177, + "tokens_seen": 3106799616 + }, + { + "epoch": 0.88, + "learning_rate": 5.921765295887663e-05, + "loss": 1.289, + "theoretical_loss": 3.340337559813615, + "tokens_seen": 3106865152 + }, + { + "epoch": 0.88, + "learning_rate": 5.9197592778335006e-05, + "loss": 1.3318, + "theoretical_loss": 3.3403322266596107, + "tokens_seen": 3106930688 + }, + { + "epoch": 0.88, + "learning_rate": 5.917753259779338e-05, + "loss": 1.2457, + "theoretical_loss": 3.340326893649598, + "tokens_seen": 3106996224 + }, + { + "epoch": 0.88, + "learning_rate": 5.915747241725176e-05, + "loss": 1.1894, + "theoretical_loss": 3.3403215607835697, + "tokens_seen": 3107061760 + }, + { + "epoch": 0.88, + "learning_rate": 5.9137412236710134e-05, + "loss": 1.1648, + "theoretical_loss": 3.3403162280615186, + "tokens_seen": 3107127296 + }, + { + "epoch": 0.88, + "learning_rate": 5.911735205616851e-05, + "loss": 1.2762, + "theoretical_loss": 3.3403108954834386, + "tokens_seen": 3107192832 + }, + { + "epoch": 0.88, + "learning_rate": 5.909729187562688e-05, + "loss": 1.2267, + "theoretical_loss": 3.340305563049322, + "tokens_seen": 3107258368 + }, + { + "epoch": 0.88, + "learning_rate": 5.9077231695085254e-05, + "loss": 1.3377, + "theoretical_loss": 3.3403002307591625, + "tokens_seen": 3107323904 + }, + { + "epoch": 0.88, + "learning_rate": 5.905717151454363e-05, + "loss": 1.1734, + "theoretical_loss": 3.340294898612953, + "tokens_seen": 3107389440 + }, + { + "epoch": 0.88, + "learning_rate": 5.9037111334002006e-05, + "loss": 1.2483, + "theoretical_loss": 3.340289566610686, + "tokens_seen": 3107454976 + }, + { + "epoch": 0.88, + "learning_rate": 5.901705115346039e-05, + "loss": 1.1824, + "theoretical_loss": 3.3402842347523554, + "tokens_seen": 3107520512 + }, + { + "epoch": 0.88, + "learning_rate": 5.899699097291876e-05, + "loss": 1.3196, + "theoretical_loss": 3.3402789030379534, + "tokens_seen": 3107586048 + }, + { + "epoch": 0.88, + "learning_rate": 5.897693079237713e-05, + "loss": 1.2971, + "theoretical_loss": 3.340273571467474, + "tokens_seen": 3107651584 + }, + { + "epoch": 0.88, + "learning_rate": 5.895687061183551e-05, + "loss": 1.3128, + "theoretical_loss": 3.34026824004091, + "tokens_seen": 3107717120 + }, + { + "epoch": 0.88, + "learning_rate": 5.8936810431293885e-05, + "loss": 1.2149, + "theoretical_loss": 3.3402629087582545, + "tokens_seen": 3107782656 + }, + { + "epoch": 0.88, + "learning_rate": 5.891675025075226e-05, + "loss": 1.2992, + "theoretical_loss": 3.3402575776195, + "tokens_seen": 3107848192 + }, + { + "epoch": 0.88, + "learning_rate": 5.889669007021063e-05, + "loss": 1.35, + "theoretical_loss": 3.34025224662464, + "tokens_seen": 3107913728 + }, + { + "epoch": 0.88, + "learning_rate": 5.8876629889669005e-05, + "loss": 1.2694, + "theoretical_loss": 3.3402469157736676, + "tokens_seen": 3107979264 + }, + { + "epoch": 0.88, + "learning_rate": 5.885656970912738e-05, + "loss": 1.2837, + "theoretical_loss": 3.340241585066576, + "tokens_seen": 3108044800 + }, + { + "epoch": 0.88, + "learning_rate": 5.8836509528585764e-05, + "loss": 1.2031, + "theoretical_loss": 3.340236254503358, + "tokens_seen": 3108110336 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1129377, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.267437219619751, + "objective/train/theoretical_loss": 3.340230924084007, + "objective/train/tokens_used": 1478634976, + "theoretical_loss": 3.340230924084007, + "tokens_seen": 3108175872 + }, + { + "epoch": 0.88, + "learning_rate": 5.881644934804414e-05, + "loss": 1.2679, + "theoretical_loss": 3.340230924084007, + "tokens_seen": 3108175872 + }, + { + "epoch": 0.88, + "learning_rate": 5.879638916750251e-05, + "loss": 1.2417, + "theoretical_loss": 3.340225593808516, + "tokens_seen": 3108241408 + }, + { + "epoch": 0.88, + "learning_rate": 5.8776328986960884e-05, + "loss": 1.1716, + "theoretical_loss": 3.3402202636768776, + "tokens_seen": 3108306944 + }, + { + "epoch": 0.88, + "learning_rate": 5.875626880641926e-05, + "loss": 1.1749, + "theoretical_loss": 3.3402149336890856, + "tokens_seen": 3108372480 + }, + { + "epoch": 0.88, + "learning_rate": 5.8736208625877636e-05, + "loss": 1.2726, + "theoretical_loss": 3.3402096038451328, + "tokens_seen": 3108438016 + }, + { + "epoch": 0.88, + "learning_rate": 5.8716148445336005e-05, + "loss": 1.2639, + "theoretical_loss": 3.3402042741450124, + "tokens_seen": 3108503552 + }, + { + "epoch": 0.88, + "learning_rate": 5.869608826479438e-05, + "loss": 1.2567, + "theoretical_loss": 3.3401989445887175, + "tokens_seen": 3108569088 + }, + { + "epoch": 0.88, + "learning_rate": 5.8676028084252756e-05, + "loss": 1.2731, + "theoretical_loss": 3.3401936151762404, + "tokens_seen": 3108634624 + }, + { + "epoch": 0.88, + "learning_rate": 5.865596790371113e-05, + "loss": 1.2396, + "theoretical_loss": 3.3401882859075753, + "tokens_seen": 3108700160 + }, + { + "epoch": 0.88, + "learning_rate": 5.8635907723169515e-05, + "loss": 1.2252, + "theoretical_loss": 3.340182956782715, + "tokens_seen": 3108765696 + }, + { + "epoch": 0.88, + "learning_rate": 5.8615847542627884e-05, + "loss": 1.3117, + "theoretical_loss": 3.3401776278016517, + "tokens_seen": 3108831232 + }, + { + "epoch": 0.88, + "learning_rate": 5.859578736208626e-05, + "loss": 1.2874, + "theoretical_loss": 3.3401722989643794, + "tokens_seen": 3108896768 + }, + { + "epoch": 0.88, + "learning_rate": 5.8575727181544635e-05, + "loss": 1.285, + "theoretical_loss": 3.340166970270891, + "tokens_seen": 3108962304 + }, + { + "epoch": 0.88, + "learning_rate": 5.855566700100301e-05, + "loss": 1.2626, + "theoretical_loss": 3.34016164172118, + "tokens_seen": 3109027840 + }, + { + "epoch": 0.88, + "learning_rate": 5.853560682046139e-05, + "loss": 1.1552, + "theoretical_loss": 3.340156313315239, + "tokens_seen": 3109093376 + }, + { + "epoch": 0.88, + "learning_rate": 5.8515546639919756e-05, + "loss": 1.2575, + "theoretical_loss": 3.340150985053061, + "tokens_seen": 3109158912 + }, + { + "epoch": 0.88, + "learning_rate": 5.849548645937813e-05, + "loss": 1.2307, + "theoretical_loss": 3.340145656934639, + "tokens_seen": 3109224448 + }, + { + "epoch": 0.88, + "learning_rate": 5.847542627883651e-05, + "loss": 1.3699, + "theoretical_loss": 3.3401403289599663, + "tokens_seen": 3109289984 + }, + { + "epoch": 0.88, + "learning_rate": 5.845536609829489e-05, + "loss": 1.3362, + "theoretical_loss": 3.3401350011290365, + "tokens_seen": 3109355520 + }, + { + "epoch": 0.88, + "learning_rate": 5.8435305917753266e-05, + "loss": 1.2085, + "theoretical_loss": 3.340129673441842, + "tokens_seen": 3109421056 + }, + { + "epoch": 0.88, + "learning_rate": 5.8415245737211635e-05, + "loss": 1.3115, + "theoretical_loss": 3.340124345898376, + "tokens_seen": 3109486592 + }, + { + "epoch": 0.88, + "learning_rate": 5.839518555667001e-05, + "loss": 1.2783, + "theoretical_loss": 3.3401190184986316, + "tokens_seen": 3109552128 + }, + { + "epoch": 0.88, + "learning_rate": 5.8375125376128387e-05, + "loss": 1.2641, + "theoretical_loss": 3.340113691242602, + "tokens_seen": 3109617664 + }, + { + "epoch": 0.88, + "learning_rate": 5.835506519558676e-05, + "loss": 1.3237, + "theoretical_loss": 3.3401083641302804, + "tokens_seen": 3109683200 + }, + { + "epoch": 0.88, + "learning_rate": 5.833500501504514e-05, + "loss": 1.2045, + "theoretical_loss": 3.34010303716166, + "tokens_seen": 3109748736 + }, + { + "epoch": 0.88, + "learning_rate": 5.831494483450351e-05, + "loss": 1.2759, + "theoretical_loss": 3.3400977103367335, + "tokens_seen": 3109814272 + }, + { + "epoch": 0.88, + "learning_rate": 5.829488465396188e-05, + "loss": 1.3155, + "theoretical_loss": 3.3400923836554943, + "tokens_seen": 3109879808 + }, + { + "epoch": 0.88, + "learning_rate": 5.8274824473420266e-05, + "loss": 1.2201, + "theoretical_loss": 3.340087057117935, + "tokens_seen": 3109945344 + }, + { + "epoch": 0.88, + "learning_rate": 5.825476429287864e-05, + "loss": 1.3192, + "theoretical_loss": 3.3400817307240493, + "tokens_seen": 3110010880 + }, + { + "epoch": 0.88, + "learning_rate": 5.823470411233702e-05, + "loss": 1.2344, + "theoretical_loss": 3.3400764044738303, + "tokens_seen": 3110076416 + }, + { + "epoch": 0.88, + "learning_rate": 5.8214643931795386e-05, + "loss": 1.1968, + "theoretical_loss": 3.3400710783672705, + "tokens_seen": 3110141952 + }, + { + "epoch": 0.88, + "learning_rate": 5.819458375125376e-05, + "loss": 1.2749, + "theoretical_loss": 3.3400657524043638, + "tokens_seen": 3110207488 + }, + { + "epoch": 0.89, + "learning_rate": 5.817452357071214e-05, + "loss": 1.2831, + "theoretical_loss": 3.3400604265851026, + "tokens_seen": 3110273024 + }, + { + "epoch": 0.89, + "learning_rate": 5.8154463390170514e-05, + "loss": 1.2416, + "theoretical_loss": 3.34005510090948, + "tokens_seen": 3110338560 + }, + { + "epoch": 0.89, + "learning_rate": 5.813440320962889e-05, + "loss": 1.1812, + "theoretical_loss": 3.34004977537749, + "tokens_seen": 3110404096 + }, + { + "epoch": 0.89, + "learning_rate": 5.811434302908726e-05, + "loss": 1.3911, + "theoretical_loss": 3.3400444499891244, + "tokens_seen": 3110469632 + }, + { + "epoch": 0.89, + "learning_rate": 5.8094282848545634e-05, + "loss": 1.3125, + "theoretical_loss": 3.3400391247443775, + "tokens_seen": 3110535168 + }, + { + "epoch": 0.89, + "learning_rate": 5.807422266800402e-05, + "loss": 1.1755, + "theoretical_loss": 3.3400337996432414, + "tokens_seen": 3110600704 + }, + { + "epoch": 0.89, + "learning_rate": 5.805416248746239e-05, + "loss": 1.2602, + "theoretical_loss": 3.3400284746857096, + "tokens_seen": 3110666240 + }, + { + "epoch": 0.89, + "learning_rate": 5.803410230692077e-05, + "loss": 1.3322, + "theoretical_loss": 3.340023149871776, + "tokens_seen": 3110731776 + }, + { + "epoch": 0.89, + "learning_rate": 5.801404212637914e-05, + "loss": 1.198, + "theoretical_loss": 3.340017825201432, + "tokens_seen": 3110797312 + }, + { + "epoch": 0.89, + "learning_rate": 5.799398194583751e-05, + "loss": 1.2571, + "theoretical_loss": 3.340012500674672, + "tokens_seen": 3110862848 + }, + { + "epoch": 0.89, + "learning_rate": 5.797392176529589e-05, + "loss": 1.1909, + "theoretical_loss": 3.3400071762914894, + "tokens_seen": 3110928384 + }, + { + "epoch": 0.89, + "learning_rate": 5.7953861584754265e-05, + "loss": 1.3194, + "theoretical_loss": 3.340001852051876, + "tokens_seen": 3110993920 + }, + { + "epoch": 0.89, + "learning_rate": 5.7933801404212634e-05, + "loss": 1.2506, + "theoretical_loss": 3.339996527955826, + "tokens_seen": 3111059456 + }, + { + "epoch": 0.89, + "learning_rate": 5.791374122367101e-05, + "loss": 1.3053, + "theoretical_loss": 3.339991204003332, + "tokens_seen": 3111124992 + }, + { + "epoch": 0.89, + "learning_rate": 5.789368104312939e-05, + "loss": 1.2344, + "theoretical_loss": 3.339985880194387, + "tokens_seen": 3111190528 + }, + { + "epoch": 0.89, + "learning_rate": 5.787362086258777e-05, + "loss": 1.2174, + "theoretical_loss": 3.3399805565289844, + "tokens_seen": 3111256064 + }, + { + "epoch": 0.89, + "learning_rate": 5.7853560682046144e-05, + "loss": 1.1783, + "theoretical_loss": 3.339975233007117, + "tokens_seen": 3111321600 + }, + { + "epoch": 0.89, + "learning_rate": 5.783350050150451e-05, + "loss": 1.3798, + "theoretical_loss": 3.3399699096287785, + "tokens_seen": 3111387136 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1130437, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.337208867073059, + "objective/train/theoretical_loss": 3.339964586393961, + "objective/train/tokens_used": 1481911776, + "theoretical_loss": 3.339964586393961, + "tokens_seen": 3111452672 + }, + { + "epoch": 0.89, + "learning_rate": 5.781344032096289e-05, + "loss": 1.2394, + "theoretical_loss": 3.339964586393961, + "tokens_seen": 3111452672 + }, + { + "epoch": 0.89, + "learning_rate": 5.7793380140421264e-05, + "loss": 1.2615, + "theoretical_loss": 3.339959263302659, + "tokens_seen": 3111518208 + }, + { + "epoch": 0.89, + "learning_rate": 5.777331995987964e-05, + "loss": 1.2604, + "theoretical_loss": 3.3399539403548646, + "tokens_seen": 3111583744 + }, + { + "epoch": 0.89, + "learning_rate": 5.7753259779338016e-05, + "loss": 1.2084, + "theoretical_loss": 3.339948617550571, + "tokens_seen": 3111649280 + }, + { + "epoch": 0.89, + "learning_rate": 5.7733199598796385e-05, + "loss": 1.1771, + "theoretical_loss": 3.3399432948897716, + "tokens_seen": 3111714816 + }, + { + "epoch": 0.89, + "learning_rate": 5.771313941825477e-05, + "loss": 1.1843, + "theoretical_loss": 3.339937972372459, + "tokens_seen": 3111780352 + }, + { + "epoch": 0.89, + "learning_rate": 5.769307923771314e-05, + "loss": 1.2796, + "theoretical_loss": 3.339932649998627, + "tokens_seen": 3111845888 + }, + { + "epoch": 0.89, + "learning_rate": 5.767301905717152e-05, + "loss": 1.3057, + "theoretical_loss": 3.3399273277682684, + "tokens_seen": 3111911424 + }, + { + "epoch": 0.89, + "learning_rate": 5.7652958876629895e-05, + "loss": 1.1854, + "theoretical_loss": 3.339922005681376, + "tokens_seen": 3111976960 + }, + { + "epoch": 0.89, + "learning_rate": 5.7632898696088264e-05, + "loss": 1.195, + "theoretical_loss": 3.3399166837379433, + "tokens_seen": 3112042496 + }, + { + "epoch": 0.89, + "learning_rate": 5.761283851554664e-05, + "loss": 1.2883, + "theoretical_loss": 3.339911361937964, + "tokens_seen": 3112108032 + }, + { + "epoch": 0.89, + "learning_rate": 5.7592778335005015e-05, + "loss": 1.217, + "theoretical_loss": 3.33990604028143, + "tokens_seen": 3112173568 + }, + { + "epoch": 0.89, + "learning_rate": 5.757271815446339e-05, + "loss": 1.1899, + "theoretical_loss": 3.3399007187683347, + "tokens_seen": 3112239104 + }, + { + "epoch": 0.89, + "learning_rate": 5.755265797392177e-05, + "loss": 1.3024, + "theoretical_loss": 3.3398953973986716, + "tokens_seen": 3112304640 + }, + { + "epoch": 0.89, + "learning_rate": 5.7532597793380136e-05, + "loss": 1.3839, + "theoretical_loss": 3.3398900761724333, + "tokens_seen": 3112370176 + }, + { + "epoch": 0.89, + "learning_rate": 5.751253761283852e-05, + "loss": 1.2491, + "theoretical_loss": 3.339884755089614, + "tokens_seen": 3112435712 + }, + { + "epoch": 0.89, + "learning_rate": 5.7492477432296894e-05, + "loss": 1.3874, + "theoretical_loss": 3.339879434150206, + "tokens_seen": 3112501248 + }, + { + "epoch": 0.89, + "learning_rate": 5.747241725175527e-05, + "loss": 1.2426, + "theoretical_loss": 3.339874113354202, + "tokens_seen": 3112566784 + }, + { + "epoch": 0.89, + "learning_rate": 5.7452357071213646e-05, + "loss": 1.0943, + "theoretical_loss": 3.339868792701596, + "tokens_seen": 3112632320 + }, + { + "epoch": 0.89, + "learning_rate": 5.7432296890672015e-05, + "loss": 1.25, + "theoretical_loss": 3.3398634721923806, + "tokens_seen": 3112697856 + }, + { + "epoch": 0.89, + "learning_rate": 5.741223671013039e-05, + "loss": 1.3228, + "theoretical_loss": 3.339858151826549, + "tokens_seen": 3112763392 + }, + { + "epoch": 0.89, + "learning_rate": 5.7392176529588767e-05, + "loss": 1.1358, + "theoretical_loss": 3.339852831604095, + "tokens_seen": 3112828928 + }, + { + "epoch": 0.89, + "learning_rate": 5.737211634904714e-05, + "loss": 1.3453, + "theoretical_loss": 3.3398475115250106, + "tokens_seen": 3112894464 + }, + { + "epoch": 0.89, + "learning_rate": 5.735205616850552e-05, + "loss": 1.315, + "theoretical_loss": 3.3398421915892893, + "tokens_seen": 3112960000 + }, + { + "epoch": 0.89, + "learning_rate": 5.7331995987963894e-05, + "loss": 1.3105, + "theoretical_loss": 3.3398368717969245, + "tokens_seen": 3113025536 + }, + { + "epoch": 0.89, + "learning_rate": 5.731193580742227e-05, + "loss": 1.3002, + "theoretical_loss": 3.3398315521479094, + "tokens_seen": 3113091072 + }, + { + "epoch": 0.89, + "learning_rate": 5.7291875626880646e-05, + "loss": 1.2922, + "theoretical_loss": 3.3398262326422365, + "tokens_seen": 3113156608 + }, + { + "epoch": 0.89, + "learning_rate": 5.727181544633902e-05, + "loss": 1.3075, + "theoretical_loss": 3.3398209132798993, + "tokens_seen": 3113222144 + }, + { + "epoch": 0.89, + "learning_rate": 5.725175526579739e-05, + "loss": 1.2579, + "theoretical_loss": 3.3398155940608913, + "tokens_seen": 3113287680 + }, + { + "epoch": 0.89, + "learning_rate": 5.7231695085255766e-05, + "loss": 1.3129, + "theoretical_loss": 3.339810274985205, + "tokens_seen": 3113353216 + }, + { + "epoch": 0.89, + "learning_rate": 5.721163490471414e-05, + "loss": 1.3086, + "theoretical_loss": 3.339804956052834, + "tokens_seen": 3113418752 + }, + { + "epoch": 0.89, + "learning_rate": 5.719157472417252e-05, + "loss": 1.2159, + "theoretical_loss": 3.339799637263771, + "tokens_seen": 3113484288 + }, + { + "epoch": 0.89, + "learning_rate": 5.7171514543630893e-05, + "loss": 1.1763, + "theoretical_loss": 3.339794318618009, + "tokens_seen": 3113549824 + }, + { + "epoch": 0.89, + "learning_rate": 5.715145436308926e-05, + "loss": 1.2982, + "theoretical_loss": 3.3397890001155415, + "tokens_seen": 3113615360 + }, + { + "epoch": 0.89, + "learning_rate": 5.7131394182547645e-05, + "loss": 1.2427, + "theoretical_loss": 3.3397836817563618, + "tokens_seen": 3113680896 + }, + { + "epoch": 0.89, + "learning_rate": 5.711133400200602e-05, + "loss": 1.3064, + "theoretical_loss": 3.3397783635404625, + "tokens_seen": 3113746432 + }, + { + "epoch": 0.89, + "learning_rate": 5.70912738214644e-05, + "loss": 1.271, + "theoretical_loss": 3.339773045467837, + "tokens_seen": 3113811968 + }, + { + "epoch": 0.89, + "learning_rate": 5.707121364092277e-05, + "loss": 1.1153, + "theoretical_loss": 3.339767727538479, + "tokens_seen": 3113877504 + }, + { + "epoch": 0.89, + "learning_rate": 5.705115346038114e-05, + "loss": 1.3252, + "theoretical_loss": 3.339762409752381, + "tokens_seen": 3113943040 + }, + { + "epoch": 0.89, + "learning_rate": 5.703109327983952e-05, + "loss": 1.2571, + "theoretical_loss": 3.3397570921095356, + "tokens_seen": 3114008576 + }, + { + "epoch": 0.89, + "learning_rate": 5.701103309929789e-05, + "loss": 1.4015, + "theoretical_loss": 3.3397517746099363, + "tokens_seen": 3114074112 + }, + { + "epoch": 0.89, + "learning_rate": 5.699097291875627e-05, + "loss": 1.2625, + "theoretical_loss": 3.339746457253577, + "tokens_seen": 3114139648 + }, + { + "epoch": 0.89, + "learning_rate": 5.6970912738214645e-05, + "loss": 1.2543, + "theoretical_loss": 3.33974114004045, + "tokens_seen": 3114205184 + }, + { + "epoch": 0.89, + "learning_rate": 5.695085255767302e-05, + "loss": 1.3036, + "theoretical_loss": 3.3397358229705487, + "tokens_seen": 3114270720 + }, + { + "epoch": 0.89, + "learning_rate": 5.6930792377131396e-05, + "loss": 1.3023, + "theoretical_loss": 3.3397305060438667, + "tokens_seen": 3114336256 + }, + { + "epoch": 0.89, + "learning_rate": 5.691073219658977e-05, + "loss": 1.2986, + "theoretical_loss": 3.339725189260396, + "tokens_seen": 3114401792 + }, + { + "epoch": 0.89, + "learning_rate": 5.689067201604815e-05, + "loss": 1.1943, + "theoretical_loss": 3.3397198726201305, + "tokens_seen": 3114467328 + }, + { + "epoch": 0.89, + "learning_rate": 5.6870611835506524e-05, + "loss": 1.3238, + "theoretical_loss": 3.3397145561230634, + "tokens_seen": 3114532864 + }, + { + "epoch": 0.89, + "learning_rate": 5.685055165496489e-05, + "loss": 1.207, + "theoretical_loss": 3.3397092397691877, + "tokens_seen": 3114598400 + }, + { + "epoch": 0.89, + "learning_rate": 5.683049147442327e-05, + "loss": 1.2736, + "theoretical_loss": 3.339703923558496, + "tokens_seen": 3114663936 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1131665, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2482120990753174, + "objective/train/theoretical_loss": 3.3396986074909822, + "objective/train/tokens_used": 1485188576, + "theoretical_loss": 3.3396986074909822, + "tokens_seen": 3114729472 + }, + { + "epoch": 0.89, + "learning_rate": 5.6810431293881644e-05, + "loss": 1.2979, + "theoretical_loss": 3.3396986074909822, + "tokens_seen": 3114729472 + }, + { + "epoch": 0.89, + "learning_rate": 5.679037111334002e-05, + "loss": 1.1763, + "theoretical_loss": 3.339693291566639, + "tokens_seen": 3114795008 + }, + { + "epoch": 0.89, + "learning_rate": 5.67703109327984e-05, + "loss": 1.2199, + "theoretical_loss": 3.33968797578546, + "tokens_seen": 3114860544 + }, + { + "epoch": 0.89, + "learning_rate": 5.675025075225677e-05, + "loss": 1.2279, + "theoretical_loss": 3.3396826601474374, + "tokens_seen": 3114926080 + }, + { + "epoch": 0.89, + "learning_rate": 5.673019057171515e-05, + "loss": 1.3233, + "theoretical_loss": 3.339677344652565, + "tokens_seen": 3114991616 + }, + { + "epoch": 0.89, + "learning_rate": 5.671013039117352e-05, + "loss": 1.2744, + "theoretical_loss": 3.3396720293008366, + "tokens_seen": 3115057152 + }, + { + "epoch": 0.89, + "learning_rate": 5.66900702106319e-05, + "loss": 1.2741, + "theoretical_loss": 3.339666714092244, + "tokens_seen": 3115122688 + }, + { + "epoch": 0.89, + "learning_rate": 5.6670010030090275e-05, + "loss": 1.2063, + "theoretical_loss": 3.339661399026781, + "tokens_seen": 3115188224 + }, + { + "epoch": 0.89, + "learning_rate": 5.6649949849548644e-05, + "loss": 1.2835, + "theoretical_loss": 3.3396560841044405, + "tokens_seen": 3115253760 + }, + { + "epoch": 0.89, + "learning_rate": 5.662988966900702e-05, + "loss": 1.2886, + "theoretical_loss": 3.3396507693252158, + "tokens_seen": 3115319296 + }, + { + "epoch": 0.89, + "learning_rate": 5.6609829488465395e-05, + "loss": 1.2983, + "theoretical_loss": 3.3396454546891006, + "tokens_seen": 3115384832 + }, + { + "epoch": 0.89, + "learning_rate": 5.658976930792377e-05, + "loss": 1.2183, + "theoretical_loss": 3.339640140196087, + "tokens_seen": 3115450368 + }, + { + "epoch": 0.89, + "learning_rate": 5.6569709127382154e-05, + "loss": 1.3358, + "theoretical_loss": 3.3396348258461686, + "tokens_seen": 3115515904 + }, + { + "epoch": 0.89, + "learning_rate": 5.654964894684052e-05, + "loss": 1.3002, + "theoretical_loss": 3.3396295116393384, + "tokens_seen": 3115581440 + }, + { + "epoch": 0.89, + "learning_rate": 5.65295887662989e-05, + "loss": 1.2854, + "theoretical_loss": 3.33962419757559, + "tokens_seen": 3115646976 + }, + { + "epoch": 0.89, + "learning_rate": 5.6509528585757274e-05, + "loss": 1.2802, + "theoretical_loss": 3.3396188836549157, + "tokens_seen": 3115712512 + }, + { + "epoch": 0.89, + "learning_rate": 5.648946840521565e-05, + "loss": 1.2258, + "theoretical_loss": 3.3396135698773097, + "tokens_seen": 3115778048 + }, + { + "epoch": 0.89, + "learning_rate": 5.646940822467402e-05, + "loss": 1.2355, + "theoretical_loss": 3.339608256242764, + "tokens_seen": 3115843584 + }, + { + "epoch": 0.89, + "learning_rate": 5.6449348044132395e-05, + "loss": 1.3547, + "theoretical_loss": 3.339602942751273, + "tokens_seen": 3115909120 + }, + { + "epoch": 0.89, + "learning_rate": 5.642928786359077e-05, + "loss": 1.2923, + "theoretical_loss": 3.339597629402829, + "tokens_seen": 3115974656 + }, + { + "epoch": 0.89, + "learning_rate": 5.6409227683049147e-05, + "loss": 1.3229, + "theoretical_loss": 3.339592316197425, + "tokens_seen": 3116040192 + }, + { + "epoch": 0.89, + "learning_rate": 5.638916750250753e-05, + "loss": 1.2358, + "theoretical_loss": 3.3395870031350547, + "tokens_seen": 3116105728 + }, + { + "epoch": 0.89, + "learning_rate": 5.63691073219659e-05, + "loss": 1.3138, + "theoretical_loss": 3.3395816902157107, + "tokens_seen": 3116171264 + }, + { + "epoch": 0.89, + "learning_rate": 5.6349047141424274e-05, + "loss": 1.2642, + "theoretical_loss": 3.3395763774393865, + "tokens_seen": 3116236800 + }, + { + "epoch": 0.89, + "learning_rate": 5.632898696088265e-05, + "loss": 1.2901, + "theoretical_loss": 3.3395710648060755, + "tokens_seen": 3116302336 + }, + { + "epoch": 0.89, + "learning_rate": 5.6308926780341025e-05, + "loss": 1.2017, + "theoretical_loss": 3.33956575231577, + "tokens_seen": 3116367872 + }, + { + "epoch": 0.89, + "learning_rate": 5.62888665997994e-05, + "loss": 1.3384, + "theoretical_loss": 3.339560439968464, + "tokens_seen": 3116433408 + }, + { + "epoch": 0.89, + "learning_rate": 5.626880641925777e-05, + "loss": 1.2243, + "theoretical_loss": 3.3395551277641506, + "tokens_seen": 3116498944 + }, + { + "epoch": 0.89, + "learning_rate": 5.6248746238716146e-05, + "loss": 1.3302, + "theoretical_loss": 3.3395498157028225, + "tokens_seen": 3116564480 + }, + { + "epoch": 0.89, + "learning_rate": 5.622868605817452e-05, + "loss": 1.3107, + "theoretical_loss": 3.339544503784473, + "tokens_seen": 3116630016 + }, + { + "epoch": 0.89, + "learning_rate": 5.6208625877632904e-05, + "loss": 1.3954, + "theoretical_loss": 3.3395391920090947, + "tokens_seen": 3116695552 + }, + { + "epoch": 0.89, + "learning_rate": 5.618856569709128e-05, + "loss": 1.2563, + "theoretical_loss": 3.3395338803766816, + "tokens_seen": 3116761088 + }, + { + "epoch": 0.89, + "learning_rate": 5.616850551654965e-05, + "loss": 1.4061, + "theoretical_loss": 3.3395285688872267, + "tokens_seen": 3116826624 + }, + { + "epoch": 0.89, + "learning_rate": 5.6148445336008025e-05, + "loss": 1.267, + "theoretical_loss": 3.3395232575407228, + "tokens_seen": 3116892160 + }, + { + "epoch": 0.89, + "learning_rate": 5.61283851554664e-05, + "loss": 1.1868, + "theoretical_loss": 3.339517946337163, + "tokens_seen": 3116957696 + }, + { + "epoch": 0.89, + "learning_rate": 5.610832497492478e-05, + "loss": 1.3542, + "theoretical_loss": 3.339512635276541, + "tokens_seen": 3117023232 + }, + { + "epoch": 0.89, + "learning_rate": 5.608826479438315e-05, + "loss": 1.3028, + "theoretical_loss": 3.33950732435885, + "tokens_seen": 3117088768 + }, + { + "epoch": 0.89, + "learning_rate": 5.606820461384152e-05, + "loss": 1.2217, + "theoretical_loss": 3.339502013584082, + "tokens_seen": 3117154304 + }, + { + "epoch": 0.89, + "learning_rate": 5.60481444332999e-05, + "loss": 1.2876, + "theoretical_loss": 3.339496702952231, + "tokens_seen": 3117219840 + }, + { + "epoch": 0.89, + "learning_rate": 5.602808425275827e-05, + "loss": 1.2784, + "theoretical_loss": 3.3394913924632905, + "tokens_seen": 3117285376 + }, + { + "epoch": 0.89, + "learning_rate": 5.6008024072216656e-05, + "loss": 1.2231, + "theoretical_loss": 3.3394860821172534, + "tokens_seen": 3117350912 + }, + { + "epoch": 0.89, + "learning_rate": 5.598796389167503e-05, + "loss": 1.2309, + "theoretical_loss": 3.339480771914112, + "tokens_seen": 3117416448 + }, + { + "epoch": 0.89, + "learning_rate": 5.59679037111334e-05, + "loss": 1.3296, + "theoretical_loss": 3.3394754618538607, + "tokens_seen": 3117481984 + }, + { + "epoch": 0.89, + "learning_rate": 5.5947843530591776e-05, + "loss": 1.3286, + "theoretical_loss": 3.3394701519364918, + "tokens_seen": 3117547520 + }, + { + "epoch": 0.89, + "learning_rate": 5.592778335005015e-05, + "loss": 1.3183, + "theoretical_loss": 3.3394648421619983, + "tokens_seen": 3117613056 + }, + { + "epoch": 0.89, + "learning_rate": 5.590772316950853e-05, + "loss": 1.3329, + "theoretical_loss": 3.3394595325303746, + "tokens_seen": 3117678592 + }, + { + "epoch": 0.89, + "learning_rate": 5.5887662988966904e-05, + "loss": 1.3293, + "theoretical_loss": 3.3394542230416127, + "tokens_seen": 3117744128 + }, + { + "epoch": 0.89, + "learning_rate": 5.586760280842527e-05, + "loss": 1.3526, + "theoretical_loss": 3.3394489136957057, + "tokens_seen": 3117809664 + }, + { + "epoch": 0.89, + "learning_rate": 5.584754262788365e-05, + "loss": 1.2459, + "theoretical_loss": 3.3394436044926477, + "tokens_seen": 3117875200 + }, + { + "epoch": 0.89, + "learning_rate": 5.582748244734203e-05, + "loss": 1.3964, + "theoretical_loss": 3.339438295432431, + "tokens_seen": 3117940736 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1132901, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4164438247680664, + "objective/train/theoretical_loss": 3.3394329865150487, + "objective/train/tokens_used": 1488465376, + "theoretical_loss": 3.3394329865150487, + "tokens_seen": 3118006272 + }, + { + "epoch": 0.89, + "learning_rate": 5.580742226680041e-05, + "loss": 1.3957, + "theoretical_loss": 3.3394329865150487, + "tokens_seen": 3118006272 + }, + { + "epoch": 0.89, + "learning_rate": 5.5787362086258776e-05, + "loss": 1.3443, + "theoretical_loss": 3.3394276777404945, + "tokens_seen": 3118071808 + }, + { + "epoch": 0.89, + "learning_rate": 5.576730190571715e-05, + "loss": 1.3855, + "theoretical_loss": 3.3394223691087617, + "tokens_seen": 3118137344 + }, + { + "epoch": 0.89, + "learning_rate": 5.574724172517553e-05, + "loss": 1.3056, + "theoretical_loss": 3.339417060619843, + "tokens_seen": 3118202880 + }, + { + "epoch": 0.89, + "learning_rate": 5.57271815446339e-05, + "loss": 1.2613, + "theoretical_loss": 3.3394117522737314, + "tokens_seen": 3118268416 + }, + { + "epoch": 0.89, + "learning_rate": 5.570712136409228e-05, + "loss": 1.2834, + "theoretical_loss": 3.3394064440704208, + "tokens_seen": 3118333952 + }, + { + "epoch": 0.89, + "learning_rate": 5.568706118355065e-05, + "loss": 1.3286, + "theoretical_loss": 3.3394011360099034, + "tokens_seen": 3118399488 + }, + { + "epoch": 0.89, + "learning_rate": 5.5667001003009024e-05, + "loss": 1.2758, + "theoretical_loss": 3.339395828092173, + "tokens_seen": 3118465024 + }, + { + "epoch": 0.89, + "learning_rate": 5.5646940822467406e-05, + "loss": 1.3395, + "theoretical_loss": 3.339390520317223, + "tokens_seen": 3118530560 + }, + { + "epoch": 0.89, + "learning_rate": 5.562688064192578e-05, + "loss": 1.2574, + "theoretical_loss": 3.3393852126850456, + "tokens_seen": 3118596096 + }, + { + "epoch": 0.89, + "learning_rate": 5.560682046138416e-05, + "loss": 1.3733, + "theoretical_loss": 3.3393799051956345, + "tokens_seen": 3118661632 + }, + { + "epoch": 0.89, + "learning_rate": 5.558676028084253e-05, + "loss": 1.2491, + "theoretical_loss": 3.339374597848983, + "tokens_seen": 3118727168 + }, + { + "epoch": 0.89, + "learning_rate": 5.55667001003009e-05, + "loss": 1.345, + "theoretical_loss": 3.3393692906450845, + "tokens_seen": 3118792704 + }, + { + "epoch": 0.89, + "learning_rate": 5.554663991975928e-05, + "loss": 1.4749, + "theoretical_loss": 3.3393639835839313, + "tokens_seen": 3118858240 + }, + { + "epoch": 0.89, + "learning_rate": 5.5526579739217654e-05, + "loss": 1.2123, + "theoretical_loss": 3.3393586766655172, + "tokens_seen": 3118923776 + }, + { + "epoch": 0.89, + "learning_rate": 5.550651955867603e-05, + "loss": 1.3549, + "theoretical_loss": 3.339353369889835, + "tokens_seen": 3118989312 + }, + { + "epoch": 0.89, + "learning_rate": 5.54864593781344e-05, + "loss": 1.3066, + "theoretical_loss": 3.3393480632568786, + "tokens_seen": 3119054848 + }, + { + "epoch": 0.89, + "learning_rate": 5.5466399197592775e-05, + "loss": 1.2307, + "theoretical_loss": 3.3393427567666403, + "tokens_seen": 3119120384 + }, + { + "epoch": 0.89, + "learning_rate": 5.544633901705116e-05, + "loss": 1.2847, + "theoretical_loss": 3.339337450419114, + "tokens_seen": 3119185920 + }, + { + "epoch": 0.89, + "learning_rate": 5.542627883650953e-05, + "loss": 1.3645, + "theoretical_loss": 3.339332144214292, + "tokens_seen": 3119251456 + }, + { + "epoch": 0.89, + "learning_rate": 5.540621865596791e-05, + "loss": 1.2856, + "theoretical_loss": 3.339326838152168, + "tokens_seen": 3119316992 + }, + { + "epoch": 0.89, + "learning_rate": 5.538615847542628e-05, + "loss": 1.2573, + "theoretical_loss": 3.339321532232735, + "tokens_seen": 3119382528 + }, + { + "epoch": 0.89, + "learning_rate": 5.5366098294884654e-05, + "loss": 1.3143, + "theoretical_loss": 3.3393162264559866, + "tokens_seen": 3119448064 + }, + { + "epoch": 0.89, + "learning_rate": 5.534603811434303e-05, + "loss": 1.1851, + "theoretical_loss": 3.3393109208219154, + "tokens_seen": 3119513600 + }, + { + "epoch": 0.89, + "learning_rate": 5.5325977933801405e-05, + "loss": 1.2783, + "theoretical_loss": 3.339305615330515, + "tokens_seen": 3119579136 + }, + { + "epoch": 0.89, + "learning_rate": 5.530591775325978e-05, + "loss": 1.2836, + "theoretical_loss": 3.339300309981778, + "tokens_seen": 3119644672 + }, + { + "epoch": 0.89, + "learning_rate": 5.528585757271815e-05, + "loss": 1.2631, + "theoretical_loss": 3.3392950047756984, + "tokens_seen": 3119710208 + }, + { + "epoch": 0.89, + "learning_rate": 5.526579739217653e-05, + "loss": 1.2948, + "theoretical_loss": 3.3392896997122685, + "tokens_seen": 3119775744 + }, + { + "epoch": 0.89, + "learning_rate": 5.524573721163491e-05, + "loss": 1.2185, + "theoretical_loss": 3.3392843947914823, + "tokens_seen": 3119841280 + }, + { + "epoch": 0.89, + "learning_rate": 5.5225677031093284e-05, + "loss": 1.2666, + "theoretical_loss": 3.3392790900133322, + "tokens_seen": 3119906816 + }, + { + "epoch": 0.89, + "learning_rate": 5.520561685055166e-05, + "loss": 1.3362, + "theoretical_loss": 3.3392737853778116, + "tokens_seen": 3119972352 + }, + { + "epoch": 0.89, + "learning_rate": 5.518555667001003e-05, + "loss": 1.2567, + "theoretical_loss": 3.339268480884914, + "tokens_seen": 3120037888 + }, + { + "epoch": 0.89, + "learning_rate": 5.5165496489468405e-05, + "loss": 1.3987, + "theoretical_loss": 3.339263176534632, + "tokens_seen": 3120103424 + }, + { + "epoch": 0.89, + "learning_rate": 5.514543630892678e-05, + "loss": 1.337, + "theoretical_loss": 3.3392578723269595, + "tokens_seen": 3120168960 + }, + { + "epoch": 0.89, + "learning_rate": 5.512537612838516e-05, + "loss": 1.3798, + "theoretical_loss": 3.339252568261889, + "tokens_seen": 3120234496 + }, + { + "epoch": 0.89, + "learning_rate": 5.5105315947843526e-05, + "loss": 1.2024, + "theoretical_loss": 3.339247264339414, + "tokens_seen": 3120300032 + }, + { + "epoch": 0.89, + "learning_rate": 5.508525576730191e-05, + "loss": 1.3523, + "theoretical_loss": 3.339241960559528, + "tokens_seen": 3120365568 + }, + { + "epoch": 0.89, + "learning_rate": 5.5065195586760284e-05, + "loss": 1.1827, + "theoretical_loss": 3.339236656922224, + "tokens_seen": 3120431104 + }, + { + "epoch": 0.89, + "learning_rate": 5.504513540621866e-05, + "loss": 1.3958, + "theoretical_loss": 3.339231353427494, + "tokens_seen": 3120496640 + }, + { + "epoch": 0.89, + "learning_rate": 5.5025075225677036e-05, + "loss": 1.3365, + "theoretical_loss": 3.339226050075333, + "tokens_seen": 3120562176 + }, + { + "epoch": 0.89, + "learning_rate": 5.5005015045135405e-05, + "loss": 1.3361, + "theoretical_loss": 3.3392207468657324, + "tokens_seen": 3120627712 + }, + { + "epoch": 0.89, + "learning_rate": 5.498495486459378e-05, + "loss": 1.3132, + "theoretical_loss": 3.339215443798687, + "tokens_seen": 3120693248 + }, + { + "epoch": 0.89, + "learning_rate": 5.4964894684052156e-05, + "loss": 1.3205, + "theoretical_loss": 3.339210140874189, + "tokens_seen": 3120758784 + }, + { + "epoch": 0.89, + "learning_rate": 5.494483450351053e-05, + "loss": 1.3263, + "theoretical_loss": 3.339204838092232, + "tokens_seen": 3120824320 + }, + { + "epoch": 0.89, + "learning_rate": 5.492477432296891e-05, + "loss": 1.3266, + "theoretical_loss": 3.339199535452809, + "tokens_seen": 3120889856 + }, + { + "epoch": 0.89, + "learning_rate": 5.490471414242728e-05, + "loss": 1.3308, + "theoretical_loss": 3.339194232955913, + "tokens_seen": 3120955392 + }, + { + "epoch": 0.89, + "learning_rate": 5.488465396188566e-05, + "loss": 1.23, + "theoretical_loss": 3.3391889306015377, + "tokens_seen": 3121020928 + }, + { + "epoch": 0.89, + "learning_rate": 5.4864593781344035e-05, + "loss": 1.2338, + "theoretical_loss": 3.3391836283896756, + "tokens_seen": 3121086464 + }, + { + "epoch": 0.89, + "learning_rate": 5.484453360080241e-05, + "loss": 1.3373, + "theoretical_loss": 3.3391783263203205, + "tokens_seen": 3121152000 + }, + { + "epoch": 0.89, + "learning_rate": 5.482447342026079e-05, + "loss": 1.3693, + "theoretical_loss": 3.3391730243934648, + "tokens_seen": 3121217536 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1133992, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2980782985687256, + "objective/train/theoretical_loss": 3.339167722609103, + "objective/train/tokens_used": 1491742176, + "theoretical_loss": 3.339167722609103, + "tokens_seen": 3121283072 + }, + { + "epoch": 0.89, + "learning_rate": 5.4804413239719156e-05, + "loss": 1.2572, + "theoretical_loss": 3.339167722609103, + "tokens_seen": 3121283072 + }, + { + "epoch": 0.89, + "learning_rate": 5.478435305917753e-05, + "loss": 1.3695, + "theoretical_loss": 3.339162420967227, + "tokens_seen": 3121348608 + }, + { + "epoch": 0.89, + "learning_rate": 5.476429287863591e-05, + "loss": 1.2458, + "theoretical_loss": 3.33915711946783, + "tokens_seen": 3121414144 + }, + { + "epoch": 0.89, + "learning_rate": 5.474423269809428e-05, + "loss": 1.2867, + "theoretical_loss": 3.3391518181109063, + "tokens_seen": 3121479680 + }, + { + "epoch": 0.89, + "learning_rate": 5.472417251755266e-05, + "loss": 1.3275, + "theoretical_loss": 3.339146516896448, + "tokens_seen": 3121545216 + }, + { + "epoch": 0.89, + "learning_rate": 5.4704112337011035e-05, + "loss": 1.4929, + "theoretical_loss": 3.3391412158244487, + "tokens_seen": 3121610752 + }, + { + "epoch": 0.89, + "learning_rate": 5.468405215646941e-05, + "loss": 1.2621, + "theoretical_loss": 3.339135914894902, + "tokens_seen": 3121676288 + }, + { + "epoch": 0.89, + "learning_rate": 5.4663991975927786e-05, + "loss": 1.3066, + "theoretical_loss": 3.3391306141078, + "tokens_seen": 3121741824 + }, + { + "epoch": 0.89, + "learning_rate": 5.464393179538616e-05, + "loss": 1.2193, + "theoretical_loss": 3.339125313463137, + "tokens_seen": 3121807360 + }, + { + "epoch": 0.89, + "learning_rate": 5.462387161484454e-05, + "loss": 1.2919, + "theoretical_loss": 3.3391200129609055, + "tokens_seen": 3121872896 + }, + { + "epoch": 0.89, + "learning_rate": 5.460381143430291e-05, + "loss": 1.2597, + "theoretical_loss": 3.3391147126010985, + "tokens_seen": 3121938432 + }, + { + "epoch": 0.89, + "learning_rate": 5.458375125376128e-05, + "loss": 1.3236, + "theoretical_loss": 3.33910941238371, + "tokens_seen": 3122003968 + }, + { + "epoch": 0.89, + "learning_rate": 5.456369107321966e-05, + "loss": 1.2004, + "theoretical_loss": 3.339104112308733, + "tokens_seen": 3122069504 + }, + { + "epoch": 0.89, + "learning_rate": 5.4543630892678034e-05, + "loss": 1.2254, + "theoretical_loss": 3.33909881237616, + "tokens_seen": 3122135040 + }, + { + "epoch": 0.89, + "learning_rate": 5.452357071213641e-05, + "loss": 1.4918, + "theoretical_loss": 3.3390935125859844, + "tokens_seen": 3122200576 + }, + { + "epoch": 0.89, + "learning_rate": 5.4503510531594786e-05, + "loss": 1.2966, + "theoretical_loss": 3.3390882129382002, + "tokens_seen": 3122266112 + }, + { + "epoch": 0.89, + "learning_rate": 5.448345035105316e-05, + "loss": 1.2232, + "theoretical_loss": 3.3390829134328, + "tokens_seen": 3122331648 + }, + { + "epoch": 0.89, + "learning_rate": 5.446339017051154e-05, + "loss": 1.3578, + "theoretical_loss": 3.3390776140697764, + "tokens_seen": 3122397184 + }, + { + "epoch": 0.89, + "learning_rate": 5.444332998996991e-05, + "loss": 1.2806, + "theoretical_loss": 3.3390723148491235, + "tokens_seen": 3122462720 + }, + { + "epoch": 0.89, + "learning_rate": 5.442326980942829e-05, + "loss": 1.3346, + "theoretical_loss": 3.3390670157708344, + "tokens_seen": 3122528256 + }, + { + "epoch": 0.89, + "learning_rate": 5.440320962888666e-05, + "loss": 1.3279, + "theoretical_loss": 3.3390617168349017, + "tokens_seen": 3122593792 + }, + { + "epoch": 0.89, + "learning_rate": 5.4383149448345034e-05, + "loss": 1.3347, + "theoretical_loss": 3.3390564180413187, + "tokens_seen": 3122659328 + }, + { + "epoch": 0.89, + "learning_rate": 5.436308926780341e-05, + "loss": 1.2622, + "theoretical_loss": 3.339051119390079, + "tokens_seen": 3122724864 + }, + { + "epoch": 0.89, + "learning_rate": 5.4343029087261785e-05, + "loss": 1.2639, + "theoretical_loss": 3.339045820881176, + "tokens_seen": 3122790400 + }, + { + "epoch": 0.89, + "learning_rate": 5.432296890672016e-05, + "loss": 1.2935, + "theoretical_loss": 3.339040522514602, + "tokens_seen": 3122855936 + }, + { + "epoch": 0.89, + "learning_rate": 5.430290872617854e-05, + "loss": 1.3958, + "theoretical_loss": 3.339035224290351, + "tokens_seen": 3122921472 + }, + { + "epoch": 0.89, + "learning_rate": 5.428284854563691e-05, + "loss": 1.2865, + "theoretical_loss": 3.3390299262084158, + "tokens_seen": 3122987008 + }, + { + "epoch": 0.89, + "learning_rate": 5.426278836509529e-05, + "loss": 1.3166, + "theoretical_loss": 3.3390246282687897, + "tokens_seen": 3123052544 + }, + { + "epoch": 0.89, + "learning_rate": 5.4242728184553664e-05, + "loss": 1.2812, + "theoretical_loss": 3.3390193304714657, + "tokens_seen": 3123118080 + }, + { + "epoch": 0.89, + "learning_rate": 5.4222668004012033e-05, + "loss": 1.3067, + "theoretical_loss": 3.3390140328164373, + "tokens_seen": 3123183616 + }, + { + "epoch": 0.89, + "learning_rate": 5.420260782347041e-05, + "loss": 1.254, + "theoretical_loss": 3.3390087353036972, + "tokens_seen": 3123249152 + }, + { + "epoch": 0.89, + "learning_rate": 5.4182547642928785e-05, + "loss": 1.2016, + "theoretical_loss": 3.3390034379332394, + "tokens_seen": 3123314688 + }, + { + "epoch": 0.89, + "learning_rate": 5.416248746238716e-05, + "loss": 1.3223, + "theoretical_loss": 3.338998140705056, + "tokens_seen": 3123380224 + }, + { + "epoch": 0.89, + "learning_rate": 5.4142427281845543e-05, + "loss": 1.2759, + "theoretical_loss": 3.338992843619142, + "tokens_seen": 3123445760 + }, + { + "epoch": 0.89, + "learning_rate": 5.412236710130391e-05, + "loss": 1.2281, + "theoretical_loss": 3.3389875466754884, + "tokens_seen": 3123511296 + }, + { + "epoch": 0.89, + "learning_rate": 5.410230692076229e-05, + "loss": 1.24, + "theoretical_loss": 3.3389822498740895, + "tokens_seen": 3123576832 + }, + { + "epoch": 0.89, + "learning_rate": 5.4082246740220664e-05, + "loss": 1.2809, + "theoretical_loss": 3.3389769532149387, + "tokens_seen": 3123642368 + }, + { + "epoch": 0.89, + "learning_rate": 5.406218655967904e-05, + "loss": 1.4761, + "theoretical_loss": 3.3389716566980283, + "tokens_seen": 3123707904 + }, + { + "epoch": 0.89, + "learning_rate": 5.4042126379137416e-05, + "loss": 1.2396, + "theoretical_loss": 3.3389663603233526, + "tokens_seen": 3123773440 + }, + { + "epoch": 0.89, + "learning_rate": 5.4022066198595785e-05, + "loss": 1.2691, + "theoretical_loss": 3.3389610640909044, + "tokens_seen": 3123838976 + }, + { + "epoch": 0.89, + "learning_rate": 5.400200601805416e-05, + "loss": 1.2429, + "theoretical_loss": 3.3389557680006767, + "tokens_seen": 3123904512 + }, + { + "epoch": 0.89, + "learning_rate": 5.3981945837512536e-05, + "loss": 1.3699, + "theoretical_loss": 3.338950472052663, + "tokens_seen": 3123970048 + }, + { + "epoch": 0.89, + "learning_rate": 5.396188565697091e-05, + "loss": 1.2821, + "theoretical_loss": 3.3389451762468556, + "tokens_seen": 3124035584 + }, + { + "epoch": 0.89, + "learning_rate": 5.3941825476429295e-05, + "loss": 1.2752, + "theoretical_loss": 3.338939880583249, + "tokens_seen": 3124101120 + }, + { + "epoch": 0.89, + "learning_rate": 5.3921765295887664e-05, + "loss": 1.3569, + "theoretical_loss": 3.3389345850618355, + "tokens_seen": 3124166656 + }, + { + "epoch": 0.89, + "learning_rate": 5.390170511534604e-05, + "loss": 1.1049, + "theoretical_loss": 3.3389292896826084, + "tokens_seen": 3124232192 + }, + { + "epoch": 0.89, + "learning_rate": 5.3881644934804415e-05, + "loss": 1.214, + "theoretical_loss": 3.3389239944455613, + "tokens_seen": 3124297728 + }, + { + "epoch": 0.89, + "learning_rate": 5.386158475426279e-05, + "loss": 1.2059, + "theoretical_loss": 3.338918699350687, + "tokens_seen": 3124363264 + }, + { + "epoch": 0.89, + "learning_rate": 5.384152457372117e-05, + "loss": 1.3401, + "theoretical_loss": 3.3389134043979793, + "tokens_seen": 3124428800 + }, + { + "epoch": 0.89, + "learning_rate": 5.3821464393179536e-05, + "loss": 1.2635, + "theoretical_loss": 3.3389081095874307, + "tokens_seen": 3124494336 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1135223, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2332755327224731, + "objective/train/theoretical_loss": 3.338902814919035, + "objective/train/tokens_used": 1495018976, + "theoretical_loss": 3.338902814919035, + "tokens_seen": 3124559872 + }, + { + "epoch": 0.89, + "learning_rate": 5.380140421263791e-05, + "loss": 1.2549, + "theoretical_loss": 3.338902814919035, + "tokens_seen": 3124559872 + }, + { + "epoch": 0.89, + "learning_rate": 5.378134403209629e-05, + "loss": 1.228, + "theoretical_loss": 3.3388975203927846, + "tokens_seen": 3124625408 + }, + { + "epoch": 0.89, + "learning_rate": 5.376128385155467e-05, + "loss": 1.3342, + "theoretical_loss": 3.3388922260086735, + "tokens_seen": 3124690944 + }, + { + "epoch": 0.89, + "learning_rate": 5.3741223671013046e-05, + "loss": 1.1697, + "theoretical_loss": 3.3388869317666945, + "tokens_seen": 3124756480 + }, + { + "epoch": 0.89, + "learning_rate": 5.3721163490471415e-05, + "loss": 1.2367, + "theoretical_loss": 3.338881637666841, + "tokens_seen": 3124822016 + }, + { + "epoch": 0.89, + "learning_rate": 5.370110330992979e-05, + "loss": 1.2027, + "theoretical_loss": 3.338876343709106, + "tokens_seen": 3124887552 + }, + { + "epoch": 0.89, + "learning_rate": 5.3681043129388166e-05, + "loss": 1.3303, + "theoretical_loss": 3.3388710498934824, + "tokens_seen": 3124953088 + }, + { + "epoch": 0.89, + "learning_rate": 5.366098294884654e-05, + "loss": 1.3132, + "theoretical_loss": 3.338865756219964, + "tokens_seen": 3125018624 + }, + { + "epoch": 0.89, + "learning_rate": 5.364092276830491e-05, + "loss": 1.2024, + "theoretical_loss": 3.3388604626885448, + "tokens_seen": 3125084160 + }, + { + "epoch": 0.89, + "learning_rate": 5.362086258776329e-05, + "loss": 1.2951, + "theoretical_loss": 3.3388551692992157, + "tokens_seen": 3125149696 + }, + { + "epoch": 0.89, + "learning_rate": 5.360080240722166e-05, + "loss": 1.3154, + "theoretical_loss": 3.338849876051972, + "tokens_seen": 3125215232 + }, + { + "epoch": 0.89, + "learning_rate": 5.3580742226680045e-05, + "loss": 1.3403, + "theoretical_loss": 3.338844582946806, + "tokens_seen": 3125280768 + }, + { + "epoch": 0.89, + "learning_rate": 5.356068204613842e-05, + "loss": 1.3437, + "theoretical_loss": 3.338839289983711, + "tokens_seen": 3125346304 + }, + { + "epoch": 0.89, + "learning_rate": 5.354062186559679e-05, + "loss": 1.251, + "theoretical_loss": 3.3388339971626797, + "tokens_seen": 3125411840 + }, + { + "epoch": 0.89, + "learning_rate": 5.3520561685055166e-05, + "loss": 1.2187, + "theoretical_loss": 3.3388287044837064, + "tokens_seen": 3125477376 + }, + { + "epoch": 0.89, + "learning_rate": 5.350050150451354e-05, + "loss": 1.3163, + "theoretical_loss": 3.338823411946784, + "tokens_seen": 3125542912 + }, + { + "epoch": 0.89, + "learning_rate": 5.348044132397192e-05, + "loss": 1.2991, + "theoretical_loss": 3.338818119551905, + "tokens_seen": 3125608448 + }, + { + "epoch": 0.89, + "learning_rate": 5.346038114343029e-05, + "loss": 1.2459, + "theoretical_loss": 3.338812827299063, + "tokens_seen": 3125673984 + }, + { + "epoch": 0.89, + "learning_rate": 5.344032096288866e-05, + "loss": 1.2299, + "theoretical_loss": 3.338807535188252, + "tokens_seen": 3125739520 + }, + { + "epoch": 0.89, + "learning_rate": 5.342026078234704e-05, + "loss": 1.2798, + "theoretical_loss": 3.3388022432194635, + "tokens_seen": 3125805056 + }, + { + "epoch": 0.89, + "learning_rate": 5.3400200601805414e-05, + "loss": 1.243, + "theoretical_loss": 3.3387969513926925, + "tokens_seen": 3125870592 + }, + { + "epoch": 0.89, + "learning_rate": 5.3380140421263796e-05, + "loss": 1.2342, + "theoretical_loss": 3.338791659707931, + "tokens_seen": 3125936128 + }, + { + "epoch": 0.89, + "learning_rate": 5.336008024072217e-05, + "loss": 1.3099, + "theoretical_loss": 3.338786368165173, + "tokens_seen": 3126001664 + }, + { + "epoch": 0.89, + "learning_rate": 5.334002006018054e-05, + "loss": 1.2041, + "theoretical_loss": 3.338781076764411, + "tokens_seen": 3126067200 + }, + { + "epoch": 0.89, + "learning_rate": 5.331995987963892e-05, + "loss": 1.2135, + "theoretical_loss": 3.338775785505639, + "tokens_seen": 3126132736 + }, + { + "epoch": 0.89, + "learning_rate": 5.329989969909729e-05, + "loss": 1.2767, + "theoretical_loss": 3.338770494388849, + "tokens_seen": 3126198272 + }, + { + "epoch": 0.89, + "learning_rate": 5.327983951855567e-05, + "loss": 1.3333, + "theoretical_loss": 3.338765203414036, + "tokens_seen": 3126263808 + }, + { + "epoch": 0.89, + "learning_rate": 5.3259779338014044e-05, + "loss": 1.2329, + "theoretical_loss": 3.3387599125811915, + "tokens_seen": 3126329344 + }, + { + "epoch": 0.89, + "learning_rate": 5.3239719157472413e-05, + "loss": 1.3566, + "theoretical_loss": 3.3387546218903097, + "tokens_seen": 3126394880 + }, + { + "epoch": 0.89, + "learning_rate": 5.321965897693079e-05, + "loss": 1.2451, + "theoretical_loss": 3.338749331341383, + "tokens_seen": 3126460416 + }, + { + "epoch": 0.89, + "learning_rate": 5.319959879638917e-05, + "loss": 1.2823, + "theoretical_loss": 3.338744040934406, + "tokens_seen": 3126525952 + }, + { + "epoch": 0.89, + "learning_rate": 5.317953861584755e-05, + "loss": 1.3009, + "theoretical_loss": 3.3387387506693704, + "tokens_seen": 3126591488 + }, + { + "epoch": 0.89, + "learning_rate": 5.315947843530592e-05, + "loss": 1.2944, + "theoretical_loss": 3.3387334605462704, + "tokens_seen": 3126657024 + }, + { + "epoch": 0.89, + "learning_rate": 5.313941825476429e-05, + "loss": 1.3931, + "theoretical_loss": 3.3387281705650986, + "tokens_seen": 3126722560 + }, + { + "epoch": 0.9, + "learning_rate": 5.311935807422267e-05, + "loss": 1.2772, + "theoretical_loss": 3.3387228807258493, + "tokens_seen": 3126788096 + }, + { + "epoch": 0.9, + "learning_rate": 5.3099297893681044e-05, + "loss": 1.4309, + "theoretical_loss": 3.338717591028514, + "tokens_seen": 3126853632 + }, + { + "epoch": 0.9, + "learning_rate": 5.307923771313942e-05, + "loss": 1.2244, + "theoretical_loss": 3.3387123014730875, + "tokens_seen": 3126919168 + }, + { + "epoch": 0.9, + "learning_rate": 5.3059177532597796e-05, + "loss": 1.2582, + "theoretical_loss": 3.3387070120595617, + "tokens_seen": 3126984704 + }, + { + "epoch": 0.9, + "learning_rate": 5.3039117352056165e-05, + "loss": 1.2502, + "theoretical_loss": 3.338701722787931, + "tokens_seen": 3127050240 + }, + { + "epoch": 0.9, + "learning_rate": 5.301905717151455e-05, + "loss": 1.2751, + "theoretical_loss": 3.338696433658188, + "tokens_seen": 3127115776 + }, + { + "epoch": 0.9, + "learning_rate": 5.299899699097292e-05, + "loss": 1.2583, + "theoretical_loss": 3.3386911446703262, + "tokens_seen": 3127181312 + }, + { + "epoch": 0.9, + "learning_rate": 5.29789368104313e-05, + "loss": 1.3906, + "theoretical_loss": 3.3386858558243384, + "tokens_seen": 3127246848 + }, + { + "epoch": 0.9, + "learning_rate": 5.2958876629889675e-05, + "loss": 1.2248, + "theoretical_loss": 3.3386805671202184, + "tokens_seen": 3127312384 + }, + { + "epoch": 0.9, + "learning_rate": 5.2938816449348044e-05, + "loss": 1.3297, + "theoretical_loss": 3.3386752785579588, + "tokens_seen": 3127377920 + }, + { + "epoch": 0.9, + "learning_rate": 5.291875626880642e-05, + "loss": 1.2604, + "theoretical_loss": 3.338669990137553, + "tokens_seen": 3127443456 + }, + { + "epoch": 0.9, + "learning_rate": 5.2898696088264795e-05, + "loss": 1.2765, + "theoretical_loss": 3.3386647018589946, + "tokens_seen": 3127508992 + }, + { + "epoch": 0.9, + "learning_rate": 5.287863590772317e-05, + "loss": 1.2924, + "theoretical_loss": 3.3386594137222767, + "tokens_seen": 3127574528 + }, + { + "epoch": 0.9, + "learning_rate": 5.285857572718154e-05, + "loss": 1.2733, + "theoretical_loss": 3.338654125727392, + "tokens_seen": 3127640064 + }, + { + "epoch": 0.9, + "learning_rate": 5.2838515546639916e-05, + "loss": 1.3411, + "theoretical_loss": 3.338648837874334, + "tokens_seen": 3127705600 + }, + { + "epoch": 0.9, + "learning_rate": 5.28184553660983e-05, + "loss": 1.2425, + "theoretical_loss": 3.3386435501630967, + "tokens_seen": 3127771136 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1136308, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3864843845367432, + "objective/train/theoretical_loss": 3.338638262593672, + "objective/train/tokens_used": 1498295776, + "theoretical_loss": 3.338638262593672, + "tokens_seen": 3127836672 + }, + { + "epoch": 0.9, + "learning_rate": 5.2798395185556674e-05, + "loss": 1.4017, + "theoretical_loss": 3.338638262593672, + "tokens_seen": 3127836672 + }, + { + "epoch": 0.9, + "learning_rate": 5.277833500501505e-05, + "loss": 1.2845, + "theoretical_loss": 3.338632975166054, + "tokens_seen": 3127902208 + }, + { + "epoch": 0.9, + "learning_rate": 5.275827482447342e-05, + "loss": 1.3054, + "theoretical_loss": 3.338627687880236, + "tokens_seen": 3127967744 + }, + { + "epoch": 0.9, + "learning_rate": 5.2738214643931795e-05, + "loss": 1.1933, + "theoretical_loss": 3.3386224007362104, + "tokens_seen": 3128033280 + }, + { + "epoch": 0.9, + "learning_rate": 5.271815446339017e-05, + "loss": 1.2099, + "theoretical_loss": 3.3386171137339713, + "tokens_seen": 3128098816 + }, + { + "epoch": 0.9, + "learning_rate": 5.2698094282848546e-05, + "loss": 1.2349, + "theoretical_loss": 3.3386118268735117, + "tokens_seen": 3128164352 + }, + { + "epoch": 0.9, + "learning_rate": 5.267803410230692e-05, + "loss": 1.2314, + "theoretical_loss": 3.3386065401548244, + "tokens_seen": 3128229888 + }, + { + "epoch": 0.9, + "learning_rate": 5.265797392176529e-05, + "loss": 1.1569, + "theoretical_loss": 3.3386012535779033, + "tokens_seen": 3128295424 + }, + { + "epoch": 0.9, + "learning_rate": 5.2637913741223674e-05, + "loss": 1.3129, + "theoretical_loss": 3.338595967142741, + "tokens_seen": 3128360960 + }, + { + "epoch": 0.9, + "learning_rate": 5.261785356068205e-05, + "loss": 1.3413, + "theoretical_loss": 3.338590680849331, + "tokens_seen": 3128426496 + }, + { + "epoch": 0.9, + "learning_rate": 5.2597793380140425e-05, + "loss": 1.3588, + "theoretical_loss": 3.3385853946976667, + "tokens_seen": 3128492032 + }, + { + "epoch": 0.9, + "learning_rate": 5.25777331995988e-05, + "loss": 1.3224, + "theoretical_loss": 3.3385801086877414, + "tokens_seen": 3128557568 + }, + { + "epoch": 0.9, + "learning_rate": 5.255767301905717e-05, + "loss": 1.2002, + "theoretical_loss": 3.3385748228195475, + "tokens_seen": 3128623104 + }, + { + "epoch": 0.9, + "learning_rate": 5.2537612838515546e-05, + "loss": 1.2857, + "theoretical_loss": 3.338569537093079, + "tokens_seen": 3128688640 + }, + { + "epoch": 0.9, + "learning_rate": 5.251755265797392e-05, + "loss": 1.1989, + "theoretical_loss": 3.338564251508329, + "tokens_seen": 3128754176 + }, + { + "epoch": 0.9, + "learning_rate": 5.24974924774323e-05, + "loss": 1.2785, + "theoretical_loss": 3.338558966065291, + "tokens_seen": 3128819712 + }, + { + "epoch": 0.9, + "learning_rate": 5.247743229689067e-05, + "loss": 1.2885, + "theoretical_loss": 3.3385536807639578, + "tokens_seen": 3128885248 + }, + { + "epoch": 0.9, + "learning_rate": 5.245737211634905e-05, + "loss": 1.2772, + "theoretical_loss": 3.3385483956043225, + "tokens_seen": 3128950784 + }, + { + "epoch": 0.9, + "learning_rate": 5.2437311935807425e-05, + "loss": 1.2726, + "theoretical_loss": 3.3385431105863788, + "tokens_seen": 3129016320 + }, + { + "epoch": 0.9, + "learning_rate": 5.24172517552658e-05, + "loss": 1.3517, + "theoretical_loss": 3.3385378257101195, + "tokens_seen": 3129081856 + }, + { + "epoch": 0.9, + "learning_rate": 5.2397191574724176e-05, + "loss": 1.258, + "theoretical_loss": 3.3385325409755384, + "tokens_seen": 3129147392 + }, + { + "epoch": 0.9, + "learning_rate": 5.237713139418255e-05, + "loss": 1.1842, + "theoretical_loss": 3.338527256382628, + "tokens_seen": 3129212928 + }, + { + "epoch": 0.9, + "learning_rate": 5.235707121364092e-05, + "loss": 1.2903, + "theoretical_loss": 3.338521971931382, + "tokens_seen": 3129278464 + }, + { + "epoch": 0.9, + "learning_rate": 5.23370110330993e-05, + "loss": 1.1866, + "theoretical_loss": 3.338516687621794, + "tokens_seen": 3129344000 + }, + { + "epoch": 0.9, + "learning_rate": 5.231695085255767e-05, + "loss": 1.2388, + "theoretical_loss": 3.3385114034538566, + "tokens_seen": 3129409536 + }, + { + "epoch": 0.9, + "learning_rate": 5.229689067201605e-05, + "loss": 1.2309, + "theoretical_loss": 3.338506119427563, + "tokens_seen": 3129475072 + }, + { + "epoch": 0.9, + "learning_rate": 5.2276830491474424e-05, + "loss": 1.2792, + "theoretical_loss": 3.3385008355429067, + "tokens_seen": 3129540608 + }, + { + "epoch": 0.9, + "learning_rate": 5.22567703109328e-05, + "loss": 1.3245, + "theoretical_loss": 3.3384955517998813, + "tokens_seen": 3129606144 + }, + { + "epoch": 0.9, + "learning_rate": 5.2236710130391176e-05, + "loss": 1.2108, + "theoretical_loss": 3.338490268198479, + "tokens_seen": 3129671680 + }, + { + "epoch": 0.9, + "learning_rate": 5.221664994984955e-05, + "loss": 1.3192, + "theoretical_loss": 3.3384849847386944, + "tokens_seen": 3129737216 + }, + { + "epoch": 0.9, + "learning_rate": 5.219658976930793e-05, + "loss": 1.3001, + "theoretical_loss": 3.3384797014205194, + "tokens_seen": 3129802752 + }, + { + "epoch": 0.9, + "learning_rate": 5.2176529588766297e-05, + "loss": 1.33, + "theoretical_loss": 3.338474418243948, + "tokens_seen": 3129868288 + }, + { + "epoch": 0.9, + "learning_rate": 5.215646940822467e-05, + "loss": 1.4026, + "theoretical_loss": 3.3384691352089737, + "tokens_seen": 3129933824 + }, + { + "epoch": 0.9, + "learning_rate": 5.213640922768305e-05, + "loss": 1.369, + "theoretical_loss": 3.338463852315589, + "tokens_seen": 3129999360 + }, + { + "epoch": 0.9, + "learning_rate": 5.2116349047141424e-05, + "loss": 1.3477, + "theoretical_loss": 3.3384585695637874, + "tokens_seen": 3130064896 + }, + { + "epoch": 0.9, + "learning_rate": 5.20962888665998e-05, + "loss": 1.2612, + "theoretical_loss": 3.3384532869535626, + "tokens_seen": 3130130432 + }, + { + "epoch": 0.9, + "learning_rate": 5.2076228686058176e-05, + "loss": 1.1733, + "theoretical_loss": 3.338448004484907, + "tokens_seen": 3130195968 + }, + { + "epoch": 0.9, + "learning_rate": 5.205616850551655e-05, + "loss": 1.3776, + "theoretical_loss": 3.338442722157815, + "tokens_seen": 3130261504 + }, + { + "epoch": 0.9, + "learning_rate": 5.203610832497493e-05, + "loss": 1.2528, + "theoretical_loss": 3.3384374399722785, + "tokens_seen": 3130327040 + }, + { + "epoch": 0.9, + "learning_rate": 5.20160481444333e-05, + "loss": 1.2679, + "theoretical_loss": 3.338432157928292, + "tokens_seen": 3130392576 + }, + { + "epoch": 0.9, + "learning_rate": 5.199598796389168e-05, + "loss": 1.4126, + "theoretical_loss": 3.3384268760258475, + "tokens_seen": 3130458112 + }, + { + "epoch": 0.9, + "learning_rate": 5.197592778335005e-05, + "loss": 1.2297, + "theoretical_loss": 3.338421594264939, + "tokens_seen": 3130523648 + }, + { + "epoch": 0.9, + "learning_rate": 5.1955867602808424e-05, + "loss": 1.2534, + "theoretical_loss": 3.3384163126455597, + "tokens_seen": 3130589184 + }, + { + "epoch": 0.9, + "learning_rate": 5.19358074222668e-05, + "loss": 1.2885, + "theoretical_loss": 3.338411031167703, + "tokens_seen": 3130654720 + }, + { + "epoch": 0.9, + "learning_rate": 5.1915747241725175e-05, + "loss": 1.1913, + "theoretical_loss": 3.3384057498313617, + "tokens_seen": 3130720256 + }, + { + "epoch": 0.9, + "learning_rate": 5.189568706118355e-05, + "loss": 1.3617, + "theoretical_loss": 3.338400468636529, + "tokens_seen": 3130785792 + }, + { + "epoch": 0.9, + "learning_rate": 5.187562688064193e-05, + "loss": 1.2078, + "theoretical_loss": 3.338395187583199, + "tokens_seen": 3130851328 + }, + { + "epoch": 0.9, + "learning_rate": 5.18555667001003e-05, + "loss": 1.2324, + "theoretical_loss": 3.338389906671364, + "tokens_seen": 3130916864 + }, + { + "epoch": 0.9, + "learning_rate": 5.183550651955868e-05, + "loss": 1.3074, + "theoretical_loss": 3.3383846259010177, + "tokens_seen": 3130982400 + }, + { + "epoch": 0.9, + "learning_rate": 5.1815446339017054e-05, + "loss": 1.2984, + "theoretical_loss": 3.3383793452721533, + "tokens_seen": 3131047936 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1137606, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1512203216552734, + "objective/train/theoretical_loss": 3.338374064784764, + "objective/train/tokens_used": 1501572576, + "theoretical_loss": 3.338374064784764, + "tokens_seen": 3131113472 + }, + { + "epoch": 0.9, + "learning_rate": 5.179538615847543e-05, + "loss": 1.1797, + "theoretical_loss": 3.338374064784764, + "tokens_seen": 3131113472 + }, + { + "epoch": 0.9, + "learning_rate": 5.17753259779338e-05, + "loss": 1.2284, + "theoretical_loss": 3.3383687844388428, + "tokens_seen": 3131179008 + }, + { + "epoch": 0.9, + "learning_rate": 5.1755265797392175e-05, + "loss": 1.2421, + "theoretical_loss": 3.3383635042343833, + "tokens_seen": 3131244544 + }, + { + "epoch": 0.9, + "learning_rate": 5.173520561685055e-05, + "loss": 1.3055, + "theoretical_loss": 3.3383582241713787, + "tokens_seen": 3131310080 + }, + { + "epoch": 0.9, + "learning_rate": 5.1715145436308926e-05, + "loss": 1.288, + "theoretical_loss": 3.3383529442498228, + "tokens_seen": 3131375616 + }, + { + "epoch": 0.9, + "learning_rate": 5.169508525576731e-05, + "loss": 1.2117, + "theoretical_loss": 3.3383476644697074, + "tokens_seen": 3131441152 + }, + { + "epoch": 0.9, + "learning_rate": 5.167502507522568e-05, + "loss": 1.1477, + "theoretical_loss": 3.338342384831027, + "tokens_seen": 3131506688 + }, + { + "epoch": 0.9, + "learning_rate": 5.1654964894684054e-05, + "loss": 1.3232, + "theoretical_loss": 3.3383371053337743, + "tokens_seen": 3131572224 + }, + { + "epoch": 0.9, + "learning_rate": 5.163490471414243e-05, + "loss": 1.268, + "theoretical_loss": 3.3383318259779426, + "tokens_seen": 3131637760 + }, + { + "epoch": 0.9, + "learning_rate": 5.1614844533600805e-05, + "loss": 1.2622, + "theoretical_loss": 3.3383265467635255, + "tokens_seen": 3131703296 + }, + { + "epoch": 0.9, + "learning_rate": 5.159478435305918e-05, + "loss": 1.2711, + "theoretical_loss": 3.3383212676905156, + "tokens_seen": 3131768832 + }, + { + "epoch": 0.9, + "learning_rate": 5.157472417251755e-05, + "loss": 1.1975, + "theoretical_loss": 3.3383159887589073, + "tokens_seen": 3131834368 + }, + { + "epoch": 0.9, + "learning_rate": 5.1554663991975926e-05, + "loss": 1.2734, + "theoretical_loss": 3.3383107099686926, + "tokens_seen": 3131899904 + }, + { + "epoch": 0.9, + "learning_rate": 5.15346038114343e-05, + "loss": 1.2132, + "theoretical_loss": 3.3383054313198652, + "tokens_seen": 3131965440 + }, + { + "epoch": 0.9, + "learning_rate": 5.1514543630892684e-05, + "loss": 1.1878, + "theoretical_loss": 3.3383001528124185, + "tokens_seen": 3132030976 + }, + { + "epoch": 0.9, + "learning_rate": 5.149448345035106e-05, + "loss": 1.2822, + "theoretical_loss": 3.338294874446346, + "tokens_seen": 3132096512 + }, + { + "epoch": 0.9, + "learning_rate": 5.147442326980943e-05, + "loss": 1.2583, + "theoretical_loss": 3.3382895962216406, + "tokens_seen": 3132162048 + }, + { + "epoch": 0.9, + "learning_rate": 5.1454363089267805e-05, + "loss": 1.2192, + "theoretical_loss": 3.338284318138295, + "tokens_seen": 3132227584 + }, + { + "epoch": 0.9, + "learning_rate": 5.143430290872618e-05, + "loss": 1.2537, + "theoretical_loss": 3.3382790401963036, + "tokens_seen": 3132293120 + }, + { + "epoch": 0.9, + "learning_rate": 5.1414242728184556e-05, + "loss": 1.2361, + "theoretical_loss": 3.3382737623956587, + "tokens_seen": 3132358656 + }, + { + "epoch": 0.9, + "learning_rate": 5.1394182547642925e-05, + "loss": 1.2377, + "theoretical_loss": 3.3382684847363544, + "tokens_seen": 3132424192 + }, + { + "epoch": 0.9, + "learning_rate": 5.13741223671013e-05, + "loss": 1.2989, + "theoretical_loss": 3.3382632072183833, + "tokens_seen": 3132489728 + }, + { + "epoch": 0.9, + "learning_rate": 5.135406218655968e-05, + "loss": 1.2481, + "theoretical_loss": 3.3382579298417387, + "tokens_seen": 3132555264 + }, + { + "epoch": 0.9, + "learning_rate": 5.133400200601805e-05, + "loss": 1.141, + "theoretical_loss": 3.3382526526064145, + "tokens_seen": 3132620800 + }, + { + "epoch": 0.9, + "learning_rate": 5.1313941825476435e-05, + "loss": 1.3588, + "theoretical_loss": 3.338247375512403, + "tokens_seen": 3132686336 + }, + { + "epoch": 0.9, + "learning_rate": 5.1293881644934804e-05, + "loss": 1.1879, + "theoretical_loss": 3.3382420985596983, + "tokens_seen": 3132751872 + }, + { + "epoch": 0.9, + "learning_rate": 5.127382146439318e-05, + "loss": 1.4233, + "theoretical_loss": 3.338236821748293, + "tokens_seen": 3132817408 + }, + { + "epoch": 0.9, + "learning_rate": 5.1253761283851556e-05, + "loss": 1.2385, + "theoretical_loss": 3.338231545078181, + "tokens_seen": 3132882944 + }, + { + "epoch": 0.9, + "learning_rate": 5.123370110330993e-05, + "loss": 1.2706, + "theoretical_loss": 3.338226268549355, + "tokens_seen": 3132948480 + }, + { + "epoch": 0.9, + "learning_rate": 5.121364092276831e-05, + "loss": 1.3112, + "theoretical_loss": 3.338220992161809, + "tokens_seen": 3133014016 + }, + { + "epoch": 0.9, + "learning_rate": 5.1193580742226677e-05, + "loss": 1.3249, + "theoretical_loss": 3.3382157159155352, + "tokens_seen": 3133079552 + }, + { + "epoch": 0.9, + "learning_rate": 5.117352056168505e-05, + "loss": 1.2424, + "theoretical_loss": 3.3382104398105277, + "tokens_seen": 3133145088 + }, + { + "epoch": 0.9, + "learning_rate": 5.115346038114343e-05, + "loss": 1.2118, + "theoretical_loss": 3.338205163846779, + "tokens_seen": 3133210624 + }, + { + "epoch": 0.9, + "learning_rate": 5.113340020060181e-05, + "loss": 1.4645, + "theoretical_loss": 3.3381998880242834, + "tokens_seen": 3133276160 + }, + { + "epoch": 0.9, + "learning_rate": 5.1113340020060187e-05, + "loss": 1.2957, + "theoretical_loss": 3.3381946123430337, + "tokens_seen": 3133341696 + }, + { + "epoch": 0.9, + "learning_rate": 5.1093279839518556e-05, + "loss": 1.332, + "theoretical_loss": 3.3381893368030227, + "tokens_seen": 3133407232 + }, + { + "epoch": 0.9, + "learning_rate": 5.107321965897693e-05, + "loss": 1.2453, + "theoretical_loss": 3.338184061404244, + "tokens_seen": 3133472768 + }, + { + "epoch": 0.9, + "learning_rate": 5.105315947843531e-05, + "loss": 1.1524, + "theoretical_loss": 3.338178786146691, + "tokens_seen": 3133538304 + }, + { + "epoch": 0.9, + "learning_rate": 5.103309929789368e-05, + "loss": 1.2584, + "theoretical_loss": 3.338173511030357, + "tokens_seen": 3133603840 + }, + { + "epoch": 0.9, + "learning_rate": 5.101303911735206e-05, + "loss": 1.3605, + "theoretical_loss": 3.3381682360552354, + "tokens_seen": 3133669376 + }, + { + "epoch": 0.9, + "learning_rate": 5.099297893681043e-05, + "loss": 1.2521, + "theoretical_loss": 3.3381629612213186, + "tokens_seen": 3133734912 + }, + { + "epoch": 0.9, + "learning_rate": 5.0972918756268804e-05, + "loss": 1.4332, + "theoretical_loss": 3.338157686528601, + "tokens_seen": 3133800448 + }, + { + "epoch": 0.9, + "learning_rate": 5.0952858575727186e-05, + "loss": 1.3297, + "theoretical_loss": 3.338152411977075, + "tokens_seen": 3133865984 + }, + { + "epoch": 0.9, + "learning_rate": 5.093279839518556e-05, + "loss": 1.2515, + "theoretical_loss": 3.338147137566734, + "tokens_seen": 3133931520 + }, + { + "epoch": 0.9, + "learning_rate": 5.091273821464394e-05, + "loss": 1.3529, + "theoretical_loss": 3.3381418632975723, + "tokens_seen": 3133997056 + }, + { + "epoch": 0.9, + "learning_rate": 5.089267803410231e-05, + "loss": 1.2739, + "theoretical_loss": 3.3381365891695816, + "tokens_seen": 3134062592 + }, + { + "epoch": 0.9, + "learning_rate": 5.087261785356068e-05, + "loss": 1.2445, + "theoretical_loss": 3.338131315182756, + "tokens_seen": 3134128128 + }, + { + "epoch": 0.9, + "learning_rate": 5.085255767301906e-05, + "loss": 1.3099, + "theoretical_loss": 3.3381260413370892, + "tokens_seen": 3134193664 + }, + { + "epoch": 0.9, + "learning_rate": 5.0832497492477434e-05, + "loss": 1.2533, + "theoretical_loss": 3.3381207676325735, + "tokens_seen": 3134259200 + }, + { + "epoch": 0.9, + "learning_rate": 5.081243731193581e-05, + "loss": 1.3216, + "theoretical_loss": 3.3381154940692026, + "tokens_seen": 3134324736 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1138754, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4435052871704102, + "objective/train/theoretical_loss": 3.3381102206469704, + "objective/train/tokens_used": 1504849376, + "theoretical_loss": 3.3381102206469704, + "tokens_seen": 3134390272 + }, + { + "epoch": 0.9, + "learning_rate": 5.079237713139418e-05, + "loss": 1.3679, + "theoretical_loss": 3.3381102206469704, + "tokens_seen": 3134390272 + }, + { + "epoch": 0.9, + "learning_rate": 5.0772316950852555e-05, + "loss": 1.3588, + "theoretical_loss": 3.338104947365869, + "tokens_seen": 3134455808 + }, + { + "epoch": 0.9, + "learning_rate": 5.075225677031094e-05, + "loss": 1.2505, + "theoretical_loss": 3.338099674225892, + "tokens_seen": 3134521344 + }, + { + "epoch": 0.9, + "learning_rate": 5.073219658976931e-05, + "loss": 1.2055, + "theoretical_loss": 3.3380944012270337, + "tokens_seen": 3134586880 + }, + { + "epoch": 0.9, + "learning_rate": 5.071213640922768e-05, + "loss": 1.2406, + "theoretical_loss": 3.338089128369286, + "tokens_seen": 3134652416 + }, + { + "epoch": 0.9, + "learning_rate": 5.069207622868606e-05, + "loss": 1.3078, + "theoretical_loss": 3.338083855652643, + "tokens_seen": 3134717952 + }, + { + "epoch": 0.9, + "learning_rate": 5.0672016048144434e-05, + "loss": 1.2353, + "theoretical_loss": 3.3380785830770976, + "tokens_seen": 3134783488 + }, + { + "epoch": 0.9, + "learning_rate": 5.065195586760281e-05, + "loss": 1.1747, + "theoretical_loss": 3.3380733106426432, + "tokens_seen": 3134849024 + }, + { + "epoch": 0.9, + "learning_rate": 5.0631895687061185e-05, + "loss": 1.2519, + "theoretical_loss": 3.3380680383492733, + "tokens_seen": 3134914560 + }, + { + "epoch": 0.9, + "learning_rate": 5.0611835506519554e-05, + "loss": 1.4633, + "theoretical_loss": 3.338062766196981, + "tokens_seen": 3134980096 + }, + { + "epoch": 0.9, + "learning_rate": 5.059177532597793e-05, + "loss": 1.2252, + "theoretical_loss": 3.338057494185759, + "tokens_seen": 3135045632 + }, + { + "epoch": 0.9, + "learning_rate": 5.057171514543631e-05, + "loss": 1.3513, + "theoretical_loss": 3.3380522223156017, + "tokens_seen": 3135111168 + }, + { + "epoch": 0.9, + "learning_rate": 5.055165496489469e-05, + "loss": 1.3981, + "theoretical_loss": 3.338046950586502, + "tokens_seen": 3135176704 + }, + { + "epoch": 0.9, + "learning_rate": 5.0531594784353064e-05, + "loss": 1.32, + "theoretical_loss": 3.338041678998452, + "tokens_seen": 3135242240 + }, + { + "epoch": 0.9, + "learning_rate": 5.051153460381143e-05, + "loss": 1.2975, + "theoretical_loss": 3.3380364075514466, + "tokens_seen": 3135307776 + }, + { + "epoch": 0.9, + "learning_rate": 5.049147442326981e-05, + "loss": 1.2276, + "theoretical_loss": 3.3380311362454784, + "tokens_seen": 3135373312 + }, + { + "epoch": 0.9, + "learning_rate": 5.0471414242728185e-05, + "loss": 1.2425, + "theoretical_loss": 3.3380258650805406, + "tokens_seen": 3135438848 + }, + { + "epoch": 0.9, + "learning_rate": 5.045135406218656e-05, + "loss": 1.401, + "theoretical_loss": 3.3380205940566268, + "tokens_seen": 3135504384 + }, + { + "epoch": 0.9, + "learning_rate": 5.0431293881644936e-05, + "loss": 1.2158, + "theoretical_loss": 3.33801532317373, + "tokens_seen": 3135569920 + }, + { + "epoch": 0.9, + "learning_rate": 5.0411233701103305e-05, + "loss": 1.2918, + "theoretical_loss": 3.3380100524318435, + "tokens_seen": 3135635456 + }, + { + "epoch": 0.9, + "learning_rate": 5.039117352056169e-05, + "loss": 1.187, + "theoretical_loss": 3.33800478183096, + "tokens_seen": 3135700992 + }, + { + "epoch": 0.9, + "learning_rate": 5.0371113340020064e-05, + "loss": 1.3466, + "theoretical_loss": 3.3379995113710743, + "tokens_seen": 3135766528 + }, + { + "epoch": 0.9, + "learning_rate": 5.035105315947844e-05, + "loss": 1.1589, + "theoretical_loss": 3.3379942410521783, + "tokens_seen": 3135832064 + }, + { + "epoch": 0.9, + "learning_rate": 5.0330992978936815e-05, + "loss": 1.3644, + "theoretical_loss": 3.337988970874266, + "tokens_seen": 3135897600 + }, + { + "epoch": 0.9, + "learning_rate": 5.0310932798395184e-05, + "loss": 1.1876, + "theoretical_loss": 3.3379837008373303, + "tokens_seen": 3135963136 + }, + { + "epoch": 0.9, + "learning_rate": 5.029087261785356e-05, + "loss": 1.2908, + "theoretical_loss": 3.3379784309413645, + "tokens_seen": 3136028672 + }, + { + "epoch": 0.9, + "learning_rate": 5.0270812437311936e-05, + "loss": 1.1286, + "theoretical_loss": 3.3379731611863623, + "tokens_seen": 3136094208 + }, + { + "epoch": 0.9, + "learning_rate": 5.025075225677031e-05, + "loss": 1.2841, + "theoretical_loss": 3.3379678915723168, + "tokens_seen": 3136159744 + }, + { + "epoch": 0.9, + "learning_rate": 5.023069207622869e-05, + "loss": 1.2129, + "theoretical_loss": 3.337962622099221, + "tokens_seen": 3136225280 + }, + { + "epoch": 0.9, + "learning_rate": 5.0210631895687057e-05, + "loss": 1.3018, + "theoretical_loss": 3.3379573527670683, + "tokens_seen": 3136290816 + }, + { + "epoch": 0.9, + "learning_rate": 5.019057171514544e-05, + "loss": 1.2485, + "theoretical_loss": 3.337952083575852, + "tokens_seen": 3136356352 + }, + { + "epoch": 0.9, + "learning_rate": 5.0170511534603815e-05, + "loss": 1.2041, + "theoretical_loss": 3.3379468145255657, + "tokens_seen": 3136421888 + }, + { + "epoch": 0.9, + "learning_rate": 5.015045135406219e-05, + "loss": 1.3232, + "theoretical_loss": 3.3379415456162023, + "tokens_seen": 3136487424 + }, + { + "epoch": 0.9, + "learning_rate": 5.0130391173520567e-05, + "loss": 1.2693, + "theoretical_loss": 3.337936276847755, + "tokens_seen": 3136552960 + }, + { + "epoch": 0.9, + "learning_rate": 5.0110330992978936e-05, + "loss": 1.2622, + "theoretical_loss": 3.3379310082202176, + "tokens_seen": 3136618496 + }, + { + "epoch": 0.9, + "learning_rate": 5.009027081243731e-05, + "loss": 1.1747, + "theoretical_loss": 3.337925739733583, + "tokens_seen": 3136684032 + }, + { + "epoch": 0.9, + "learning_rate": 5.007021063189569e-05, + "loss": 1.2759, + "theoretical_loss": 3.3379204713878448, + "tokens_seen": 3136749568 + }, + { + "epoch": 0.9, + "learning_rate": 5.005015045135406e-05, + "loss": 1.1638, + "theoretical_loss": 3.3379152031829955, + "tokens_seen": 3136815104 + }, + { + "epoch": 0.9, + "learning_rate": 5.003009027081243e-05, + "loss": 1.1809, + "theoretical_loss": 3.337909935119029, + "tokens_seen": 3136880640 + }, + { + "epoch": 0.9, + "learning_rate": 5.0010030090270814e-05, + "loss": 1.2467, + "theoretical_loss": 3.3379046671959394, + "tokens_seen": 3136946176 + }, + { + "epoch": 0.9, + "learning_rate": 4.998996990972919e-05, + "loss": 1.1885, + "theoretical_loss": 3.3378993994137183, + "tokens_seen": 3137011712 + }, + { + "epoch": 0.9, + "learning_rate": 4.9969909729187566e-05, + "loss": 1.2708, + "theoretical_loss": 3.33789413177236, + "tokens_seen": 3137077248 + }, + { + "epoch": 0.9, + "learning_rate": 4.994984954864594e-05, + "loss": 1.152, + "theoretical_loss": 3.3378888642718576, + "tokens_seen": 3137142784 + }, + { + "epoch": 0.9, + "learning_rate": 4.992978936810431e-05, + "loss": 1.3143, + "theoretical_loss": 3.3378835969122047, + "tokens_seen": 3137208320 + }, + { + "epoch": 0.9, + "learning_rate": 4.990972918756269e-05, + "loss": 1.2261, + "theoretical_loss": 3.337878329693394, + "tokens_seen": 3137273856 + }, + { + "epoch": 0.9, + "learning_rate": 4.988966900702106e-05, + "loss": 1.1775, + "theoretical_loss": 3.337873062615419, + "tokens_seen": 3137339392 + }, + { + "epoch": 0.9, + "learning_rate": 4.986960882647944e-05, + "loss": 1.2312, + "theoretical_loss": 3.337867795678273, + "tokens_seen": 3137404928 + }, + { + "epoch": 0.9, + "learning_rate": 4.9849548645937814e-05, + "loss": 1.3559, + "theoretical_loss": 3.33786252888195, + "tokens_seen": 3137470464 + }, + { + "epoch": 0.9, + "learning_rate": 4.982948846539619e-05, + "loss": 1.1631, + "theoretical_loss": 3.337857262226442, + "tokens_seen": 3137536000 + }, + { + "epoch": 0.9, + "learning_rate": 4.9809428284854566e-05, + "loss": 1.231, + "theoretical_loss": 3.3378519957117434, + "tokens_seen": 3137601536 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1140026, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2733089923858643, + "objective/train/theoretical_loss": 3.337846729337847, + "objective/train/tokens_used": 1508126176, + "theoretical_loss": 3.337846729337847, + "tokens_seen": 3137667072 + }, + { + "epoch": 0.9, + "learning_rate": 4.978936810431294e-05, + "loss": 1.3004, + "theoretical_loss": 3.337846729337847, + "tokens_seen": 3137667072 + }, + { + "epoch": 0.9, + "learning_rate": 4.976930792377132e-05, + "loss": 1.2051, + "theoretical_loss": 3.3378414631047457, + "tokens_seen": 3137732608 + }, + { + "epoch": 0.9, + "learning_rate": 4.974924774322969e-05, + "loss": 1.3251, + "theoretical_loss": 3.3378361970124337, + "tokens_seen": 3137798144 + }, + { + "epoch": 0.9, + "learning_rate": 4.972918756268806e-05, + "loss": 1.273, + "theoretical_loss": 3.337830931060904, + "tokens_seen": 3137863680 + }, + { + "epoch": 0.9, + "learning_rate": 4.970912738214644e-05, + "loss": 1.2596, + "theoretical_loss": 3.3378256652501492, + "tokens_seen": 3137929216 + }, + { + "epoch": 0.9, + "learning_rate": 4.9689067201604814e-05, + "loss": 1.3235, + "theoretical_loss": 3.3378203995801634, + "tokens_seen": 3137994752 + }, + { + "epoch": 0.9, + "learning_rate": 4.966900702106319e-05, + "loss": 1.2236, + "theoretical_loss": 3.3378151340509397, + "tokens_seen": 3138060288 + }, + { + "epoch": 0.9, + "learning_rate": 4.9648946840521565e-05, + "loss": 1.2506, + "theoretical_loss": 3.3378098686624713, + "tokens_seen": 3138125824 + }, + { + "epoch": 0.9, + "learning_rate": 4.962888665997994e-05, + "loss": 1.2146, + "theoretical_loss": 3.3378046034147513, + "tokens_seen": 3138191360 + }, + { + "epoch": 0.9, + "learning_rate": 4.960882647943832e-05, + "loss": 1.2642, + "theoretical_loss": 3.3377993383077733, + "tokens_seen": 3138256896 + }, + { + "epoch": 0.9, + "learning_rate": 4.958876629889669e-05, + "loss": 1.3508, + "theoretical_loss": 3.3377940733415308, + "tokens_seen": 3138322432 + }, + { + "epoch": 0.9, + "learning_rate": 4.956870611835507e-05, + "loss": 1.287, + "theoretical_loss": 3.3377888085160166, + "tokens_seen": 3138387968 + }, + { + "epoch": 0.9, + "learning_rate": 4.9548645937813444e-05, + "loss": 1.1982, + "theoretical_loss": 3.3377835438312244, + "tokens_seen": 3138453504 + }, + { + "epoch": 0.9, + "learning_rate": 4.952858575727181e-05, + "loss": 1.2724, + "theoretical_loss": 3.3377782792871473, + "tokens_seen": 3138519040 + }, + { + "epoch": 0.9, + "learning_rate": 4.950852557673019e-05, + "loss": 1.3182, + "theoretical_loss": 3.337773014883778, + "tokens_seen": 3138584576 + }, + { + "epoch": 0.9, + "learning_rate": 4.9488465396188565e-05, + "loss": 1.2152, + "theoretical_loss": 3.3377677506211114, + "tokens_seen": 3138650112 + }, + { + "epoch": 0.9, + "learning_rate": 4.946840521564694e-05, + "loss": 1.2699, + "theoretical_loss": 3.3377624864991393, + "tokens_seen": 3138715648 + }, + { + "epoch": 0.9, + "learning_rate": 4.944834503510532e-05, + "loss": 1.2687, + "theoretical_loss": 3.3377572225178556, + "tokens_seen": 3138781184 + }, + { + "epoch": 0.9, + "learning_rate": 4.942828485456369e-05, + "loss": 1.2747, + "theoretical_loss": 3.3377519586772535, + "tokens_seen": 3138846720 + }, + { + "epoch": 0.9, + "learning_rate": 4.940822467402207e-05, + "loss": 1.3059, + "theoretical_loss": 3.3377466949773265, + "tokens_seen": 3138912256 + }, + { + "epoch": 0.9, + "learning_rate": 4.9388164493480444e-05, + "loss": 1.2201, + "theoretical_loss": 3.3377414314180673, + "tokens_seen": 3138977792 + }, + { + "epoch": 0.9, + "learning_rate": 4.936810431293882e-05, + "loss": 1.2678, + "theoretical_loss": 3.33773616799947, + "tokens_seen": 3139043328 + }, + { + "epoch": 0.9, + "learning_rate": 4.9348044132397195e-05, + "loss": 1.2635, + "theoretical_loss": 3.3377309047215276, + "tokens_seen": 3139108864 + }, + { + "epoch": 0.9, + "learning_rate": 4.9327983951855564e-05, + "loss": 1.2739, + "theoretical_loss": 3.337725641584233, + "tokens_seen": 3139174400 + }, + { + "epoch": 0.9, + "learning_rate": 4.930792377131394e-05, + "loss": 1.1949, + "theoretical_loss": 3.33772037858758, + "tokens_seen": 3139239936 + }, + { + "epoch": 0.9, + "learning_rate": 4.9287863590772316e-05, + "loss": 1.3155, + "theoretical_loss": 3.337715115731562, + "tokens_seen": 3139305472 + }, + { + "epoch": 0.9, + "learning_rate": 4.92678034102307e-05, + "loss": 1.2822, + "theoretical_loss": 3.3377098530161717, + "tokens_seen": 3139371008 + }, + { + "epoch": 0.9, + "learning_rate": 4.9247743229689074e-05, + "loss": 1.2147, + "theoretical_loss": 3.337704590441403, + "tokens_seen": 3139436544 + }, + { + "epoch": 0.9, + "learning_rate": 4.922768304914744e-05, + "loss": 1.384, + "theoretical_loss": 3.337699328007249, + "tokens_seen": 3139502080 + }, + { + "epoch": 0.9, + "learning_rate": 4.920762286860582e-05, + "loss": 1.3505, + "theoretical_loss": 3.337694065713703, + "tokens_seen": 3139567616 + }, + { + "epoch": 0.9, + "learning_rate": 4.9187562688064195e-05, + "loss": 1.2075, + "theoretical_loss": 3.3376888035607584, + "tokens_seen": 3139633152 + }, + { + "epoch": 0.9, + "learning_rate": 4.916750250752257e-05, + "loss": 1.366, + "theoretical_loss": 3.337683541548408, + "tokens_seen": 3139698688 + }, + { + "epoch": 0.9, + "learning_rate": 4.914744232698094e-05, + "loss": 1.2411, + "theoretical_loss": 3.3376782796766458, + "tokens_seen": 3139764224 + }, + { + "epoch": 0.9, + "learning_rate": 4.9127382146439316e-05, + "loss": 1.2079, + "theoretical_loss": 3.337673017945465, + "tokens_seen": 3139829760 + }, + { + "epoch": 0.9, + "learning_rate": 4.910732196589769e-05, + "loss": 1.2964, + "theoretical_loss": 3.3376677563548585, + "tokens_seen": 3139895296 + }, + { + "epoch": 0.9, + "learning_rate": 4.908726178535607e-05, + "loss": 1.3092, + "theoretical_loss": 3.3376624949048197, + "tokens_seen": 3139960832 + }, + { + "epoch": 0.9, + "learning_rate": 4.906720160481445e-05, + "loss": 1.2976, + "theoretical_loss": 3.3376572335953423, + "tokens_seen": 3140026368 + }, + { + "epoch": 0.9, + "learning_rate": 4.904714142427282e-05, + "loss": 1.1864, + "theoretical_loss": 3.337651972426419, + "tokens_seen": 3140091904 + }, + { + "epoch": 0.9, + "learning_rate": 4.9027081243731194e-05, + "loss": 1.1771, + "theoretical_loss": 3.337646711398044, + "tokens_seen": 3140157440 + }, + { + "epoch": 0.9, + "learning_rate": 4.900702106318957e-05, + "loss": 1.3713, + "theoretical_loss": 3.33764145051021, + "tokens_seen": 3140222976 + }, + { + "epoch": 0.9, + "learning_rate": 4.8986960882647946e-05, + "loss": 1.297, + "theoretical_loss": 3.33763618976291, + "tokens_seen": 3140288512 + }, + { + "epoch": 0.9, + "learning_rate": 4.896690070210632e-05, + "loss": 1.2688, + "theoretical_loss": 3.337630929156138, + "tokens_seen": 3140354048 + }, + { + "epoch": 0.9, + "learning_rate": 4.894684052156469e-05, + "loss": 1.1341, + "theoretical_loss": 3.337625668689887, + "tokens_seen": 3140419584 + }, + { + "epoch": 0.9, + "learning_rate": 4.892678034102307e-05, + "loss": 1.278, + "theoretical_loss": 3.3376204083641507, + "tokens_seen": 3140485120 + }, + { + "epoch": 0.9, + "learning_rate": 4.890672016048144e-05, + "loss": 1.2392, + "theoretical_loss": 3.3376151481789216, + "tokens_seen": 3140550656 + }, + { + "epoch": 0.9, + "learning_rate": 4.8886659979939825e-05, + "loss": 1.2132, + "theoretical_loss": 3.3376098881341933, + "tokens_seen": 3140616192 + }, + { + "epoch": 0.9, + "learning_rate": 4.88665997993982e-05, + "loss": 1.2068, + "theoretical_loss": 3.33760462822996, + "tokens_seen": 3140681728 + }, + { + "epoch": 0.9, + "learning_rate": 4.884653961885657e-05, + "loss": 1.2037, + "theoretical_loss": 3.3375993684662135, + "tokens_seen": 3140747264 + }, + { + "epoch": 0.9, + "learning_rate": 4.8826479438314946e-05, + "loss": 1.269, + "theoretical_loss": 3.3375941088429486, + "tokens_seen": 3140812800 + }, + { + "epoch": 0.9, + "learning_rate": 4.880641925777332e-05, + "loss": 1.318, + "theoretical_loss": 3.3375888493601575, + "tokens_seen": 3140878336 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1141163, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4142014980316162, + "objective/train/theoretical_loss": 3.337583590017834, + "objective/train/tokens_used": 1511402976, + "theoretical_loss": 3.337583590017834, + "tokens_seen": 3140943872 + }, + { + "epoch": 0.9, + "learning_rate": 4.87863590772317e-05, + "loss": 1.3455, + "theoretical_loss": 3.337583590017834, + "tokens_seen": 3140943872 + }, + { + "epoch": 0.9, + "learning_rate": 4.876629889669007e-05, + "loss": 1.2995, + "theoretical_loss": 3.3375783308159717, + "tokens_seen": 3141009408 + }, + { + "epoch": 0.9, + "learning_rate": 4.874623871614844e-05, + "loss": 1.2788, + "theoretical_loss": 3.337573071754563, + "tokens_seen": 3141074944 + }, + { + "epoch": 0.9, + "learning_rate": 4.872617853560682e-05, + "loss": 1.1986, + "theoretical_loss": 3.3375678128336026, + "tokens_seen": 3141140480 + }, + { + "epoch": 0.9, + "learning_rate": 4.8706118355065194e-05, + "loss": 1.3075, + "theoretical_loss": 3.3375625540530827, + "tokens_seen": 3141206016 + }, + { + "epoch": 0.9, + "learning_rate": 4.8686058174523576e-05, + "loss": 1.2499, + "theoretical_loss": 3.3375572954129966, + "tokens_seen": 3141271552 + }, + { + "epoch": 0.9, + "learning_rate": 4.866599799398195e-05, + "loss": 1.3153, + "theoretical_loss": 3.3375520369133382, + "tokens_seen": 3141337088 + }, + { + "epoch": 0.9, + "learning_rate": 4.864593781344032e-05, + "loss": 1.2899, + "theoretical_loss": 3.337546778554101, + "tokens_seen": 3141402624 + }, + { + "epoch": 0.9, + "learning_rate": 4.86258776328987e-05, + "loss": 1.3086, + "theoretical_loss": 3.337541520335277, + "tokens_seen": 3141468160 + }, + { + "epoch": 0.9, + "learning_rate": 4.860581745235707e-05, + "loss": 1.2483, + "theoretical_loss": 3.337536262256861, + "tokens_seen": 3141533696 + }, + { + "epoch": 0.9, + "learning_rate": 4.858575727181545e-05, + "loss": 1.2834, + "theoretical_loss": 3.337531004318846, + "tokens_seen": 3141599232 + }, + { + "epoch": 0.9, + "learning_rate": 4.8565697091273824e-05, + "loss": 1.3788, + "theoretical_loss": 3.3375257465212247, + "tokens_seen": 3141664768 + }, + { + "epoch": 0.9, + "learning_rate": 4.854563691073219e-05, + "loss": 1.291, + "theoretical_loss": 3.3375204888639907, + "tokens_seen": 3141730304 + }, + { + "epoch": 0.9, + "learning_rate": 4.852557673019057e-05, + "loss": 1.2566, + "theoretical_loss": 3.3375152313471377, + "tokens_seen": 3141795840 + }, + { + "epoch": 0.9, + "learning_rate": 4.850551654964895e-05, + "loss": 1.2593, + "theoretical_loss": 3.3375099739706586, + "tokens_seen": 3141861376 + }, + { + "epoch": 0.9, + "learning_rate": 4.848545636910733e-05, + "loss": 1.2794, + "theoretical_loss": 3.337504716734547, + "tokens_seen": 3141926912 + }, + { + "epoch": 0.9, + "learning_rate": 4.8465396188565696e-05, + "loss": 1.2521, + "theoretical_loss": 3.337499459638796, + "tokens_seen": 3141992448 + }, + { + "epoch": 0.9, + "learning_rate": 4.844533600802407e-05, + "loss": 1.1774, + "theoretical_loss": 3.337494202683399, + "tokens_seen": 3142057984 + }, + { + "epoch": 0.9, + "learning_rate": 4.842527582748245e-05, + "loss": 1.3415, + "theoretical_loss": 3.3374889458683494, + "tokens_seen": 3142123520 + }, + { + "epoch": 0.9, + "learning_rate": 4.8405215646940824e-05, + "loss": 1.3214, + "theoretical_loss": 3.3374836891936406, + "tokens_seen": 3142189056 + }, + { + "epoch": 0.9, + "learning_rate": 4.83851554663992e-05, + "loss": 1.1312, + "theoretical_loss": 3.3374784326592657, + "tokens_seen": 3142254592 + }, + { + "epoch": 0.9, + "learning_rate": 4.836509528585757e-05, + "loss": 1.3017, + "theoretical_loss": 3.3374731762652177, + "tokens_seen": 3142320128 + }, + { + "epoch": 0.9, + "learning_rate": 4.8345035105315944e-05, + "loss": 1.3102, + "theoretical_loss": 3.337467920011491, + "tokens_seen": 3142385664 + }, + { + "epoch": 0.9, + "learning_rate": 4.832497492477433e-05, + "loss": 1.2093, + "theoretical_loss": 3.337462663898078, + "tokens_seen": 3142451200 + }, + { + "epoch": 0.9, + "learning_rate": 4.83049147442327e-05, + "loss": 1.1933, + "theoretical_loss": 3.337457407924972, + "tokens_seen": 3142516736 + }, + { + "epoch": 0.9, + "learning_rate": 4.828485456369108e-05, + "loss": 1.249, + "theoretical_loss": 3.337452152092167, + "tokens_seen": 3142582272 + }, + { + "epoch": 0.9, + "learning_rate": 4.826479438314945e-05, + "loss": 1.308, + "theoretical_loss": 3.337446896399656, + "tokens_seen": 3142647808 + }, + { + "epoch": 0.9, + "learning_rate": 4.824473420260782e-05, + "loss": 1.2132, + "theoretical_loss": 3.337441640847432, + "tokens_seen": 3142713344 + }, + { + "epoch": 0.9, + "learning_rate": 4.82246740220662e-05, + "loss": 1.2548, + "theoretical_loss": 3.337436385435489, + "tokens_seen": 3142778880 + }, + { + "epoch": 0.9, + "learning_rate": 4.8204613841524575e-05, + "loss": 1.214, + "theoretical_loss": 3.3374311301638198, + "tokens_seen": 3142844416 + }, + { + "epoch": 0.9, + "learning_rate": 4.818455366098295e-05, + "loss": 1.2522, + "theoretical_loss": 3.337425875032418, + "tokens_seen": 3142909952 + }, + { + "epoch": 0.9, + "learning_rate": 4.816449348044132e-05, + "loss": 1.2783, + "theoretical_loss": 3.337420620041277, + "tokens_seen": 3142975488 + }, + { + "epoch": 0.9, + "learning_rate": 4.8144433299899696e-05, + "loss": 1.1995, + "theoretical_loss": 3.337415365190389, + "tokens_seen": 3143041024 + }, + { + "epoch": 0.9, + "learning_rate": 4.812437311935808e-05, + "loss": 1.414, + "theoretical_loss": 3.337410110479749, + "tokens_seen": 3143106560 + }, + { + "epoch": 0.9, + "learning_rate": 4.8104312938816454e-05, + "loss": 1.3447, + "theoretical_loss": 3.33740485590935, + "tokens_seen": 3143172096 + }, + { + "epoch": 0.9, + "learning_rate": 4.808425275827483e-05, + "loss": 1.2407, + "theoretical_loss": 3.3373996014791842, + "tokens_seen": 3143237632 + }, + { + "epoch": 0.91, + "learning_rate": 4.80641925777332e-05, + "loss": 1.2571, + "theoretical_loss": 3.337394347189246, + "tokens_seen": 3143303168 + }, + { + "epoch": 0.91, + "learning_rate": 4.8044132397191574e-05, + "loss": 1.2274, + "theoretical_loss": 3.3373890930395285, + "tokens_seen": 3143368704 + }, + { + "epoch": 0.91, + "learning_rate": 4.802407221664995e-05, + "loss": 1.2554, + "theoretical_loss": 3.3373838390300246, + "tokens_seen": 3143434240 + }, + { + "epoch": 0.91, + "learning_rate": 4.8004012036108326e-05, + "loss": 1.2451, + "theoretical_loss": 3.337378585160728, + "tokens_seen": 3143499776 + }, + { + "epoch": 0.91, + "learning_rate": 4.79839518555667e-05, + "loss": 1.176, + "theoretical_loss": 3.3373733314316323, + "tokens_seen": 3143565312 + }, + { + "epoch": 0.91, + "learning_rate": 4.796389167502507e-05, + "loss": 1.2415, + "theoretical_loss": 3.3373680778427306, + "tokens_seen": 3143630848 + }, + { + "epoch": 0.91, + "learning_rate": 4.7943831494483453e-05, + "loss": 1.2184, + "theoretical_loss": 3.3373628243940163, + "tokens_seen": 3143696384 + }, + { + "epoch": 0.91, + "learning_rate": 4.792377131394183e-05, + "loss": 1.3028, + "theoretical_loss": 3.337357571085482, + "tokens_seen": 3143761920 + }, + { + "epoch": 0.91, + "learning_rate": 4.7903711133400205e-05, + "loss": 1.3019, + "theoretical_loss": 3.3373523179171225, + "tokens_seen": 3143827456 + }, + { + "epoch": 0.91, + "learning_rate": 4.788365095285858e-05, + "loss": 1.3526, + "theoretical_loss": 3.3373470648889296, + "tokens_seen": 3143892992 + }, + { + "epoch": 0.91, + "learning_rate": 4.786359077231695e-05, + "loss": 1.2532, + "theoretical_loss": 3.3373418120008975, + "tokens_seen": 3143958528 + }, + { + "epoch": 0.91, + "learning_rate": 4.7843530591775326e-05, + "loss": 1.2901, + "theoretical_loss": 3.3373365592530195, + "tokens_seen": 3144024064 + }, + { + "epoch": 0.91, + "learning_rate": 4.78234704112337e-05, + "loss": 1.2985, + "theoretical_loss": 3.337331306645289, + "tokens_seen": 3144089600 + }, + { + "epoch": 0.91, + "learning_rate": 4.780341023069208e-05, + "loss": 1.2921, + "theoretical_loss": 3.3373260541776992, + "tokens_seen": 3144155136 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1142279, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2020081281661987, + "objective/train/theoretical_loss": 3.337320801850243, + "objective/train/tokens_used": 1514679776, + "theoretical_loss": 3.337320801850243, + "tokens_seen": 3144220672 + }, + { + "epoch": 0.91, + "learning_rate": 4.7783350050150446e-05, + "loss": 1.2245, + "theoretical_loss": 3.337320801850243, + "tokens_seen": 3144220672 + }, + { + "epoch": 0.91, + "learning_rate": 4.776328986960883e-05, + "loss": 1.208, + "theoretical_loss": 3.3373155496629145, + "tokens_seen": 3144286208 + }, + { + "epoch": 0.91, + "learning_rate": 4.7743229689067205e-05, + "loss": 1.1283, + "theoretical_loss": 3.3373102976157063, + "tokens_seen": 3144351744 + }, + { + "epoch": 0.91, + "learning_rate": 4.772316950852558e-05, + "loss": 1.2928, + "theoretical_loss": 3.3373050457086126, + "tokens_seen": 3144417280 + }, + { + "epoch": 0.91, + "learning_rate": 4.7703109327983956e-05, + "loss": 1.3078, + "theoretical_loss": 3.337299793941626, + "tokens_seen": 3144482816 + }, + { + "epoch": 0.91, + "learning_rate": 4.7683049147442325e-05, + "loss": 1.1711, + "theoretical_loss": 3.3372945423147398, + "tokens_seen": 3144548352 + }, + { + "epoch": 0.91, + "learning_rate": 4.76629889669007e-05, + "loss": 1.2661, + "theoretical_loss": 3.337289290827948, + "tokens_seen": 3144613888 + }, + { + "epoch": 0.91, + "learning_rate": 4.764292878635908e-05, + "loss": 1.3024, + "theoretical_loss": 3.3372840394812435, + "tokens_seen": 3144679424 + }, + { + "epoch": 0.91, + "learning_rate": 4.762286860581745e-05, + "loss": 1.3013, + "theoretical_loss": 3.33727878827462, + "tokens_seen": 3144744960 + }, + { + "epoch": 0.91, + "learning_rate": 4.760280842527583e-05, + "loss": 1.1943, + "theoretical_loss": 3.3372735372080706, + "tokens_seen": 3144810496 + }, + { + "epoch": 0.91, + "learning_rate": 4.75827482447342e-05, + "loss": 1.3692, + "theoretical_loss": 3.337268286281588, + "tokens_seen": 3144876032 + }, + { + "epoch": 0.91, + "learning_rate": 4.756268806419258e-05, + "loss": 1.2721, + "theoretical_loss": 3.337263035495167, + "tokens_seen": 3144941568 + }, + { + "epoch": 0.91, + "learning_rate": 4.7542627883650956e-05, + "loss": 1.2203, + "theoretical_loss": 3.3372577848487994, + "tokens_seen": 3145007104 + }, + { + "epoch": 0.91, + "learning_rate": 4.752256770310933e-05, + "loss": 1.2806, + "theoretical_loss": 3.3372525343424795, + "tokens_seen": 3145072640 + }, + { + "epoch": 0.91, + "learning_rate": 4.750250752256771e-05, + "loss": 1.3444, + "theoretical_loss": 3.3372472839762004, + "tokens_seen": 3145138176 + }, + { + "epoch": 0.91, + "learning_rate": 4.7482447342026076e-05, + "loss": 1.2555, + "theoretical_loss": 3.3372420337499555, + "tokens_seen": 3145203712 + }, + { + "epoch": 0.91, + "learning_rate": 4.746238716148445e-05, + "loss": 1.2521, + "theoretical_loss": 3.337236783663738, + "tokens_seen": 3145269248 + }, + { + "epoch": 0.91, + "learning_rate": 4.744232698094283e-05, + "loss": 1.1666, + "theoretical_loss": 3.3372315337175418, + "tokens_seen": 3145334784 + }, + { + "epoch": 0.91, + "learning_rate": 4.7422266800401204e-05, + "loss": 1.2848, + "theoretical_loss": 3.3372262839113596, + "tokens_seen": 3145400320 + }, + { + "epoch": 0.91, + "learning_rate": 4.740220661985958e-05, + "loss": 1.1583, + "theoretical_loss": 3.337221034245185, + "tokens_seen": 3145465856 + }, + { + "epoch": 0.91, + "learning_rate": 4.7382146439317955e-05, + "loss": 1.2667, + "theoretical_loss": 3.3372157847190107, + "tokens_seen": 3145531392 + }, + { + "epoch": 0.91, + "learning_rate": 4.736208625877633e-05, + "loss": 1.3222, + "theoretical_loss": 3.337210535332831, + "tokens_seen": 3145596928 + }, + { + "epoch": 0.91, + "learning_rate": 4.734202607823471e-05, + "loss": 1.1587, + "theoretical_loss": 3.3372052860866392, + "tokens_seen": 3145662464 + }, + { + "epoch": 0.91, + "learning_rate": 4.732196589769308e-05, + "loss": 1.2351, + "theoretical_loss": 3.3372000369804278, + "tokens_seen": 3145728000 + }, + { + "epoch": 0.91, + "learning_rate": 4.730190571715146e-05, + "loss": 1.2647, + "theoretical_loss": 3.337194788014191, + "tokens_seen": 3145793536 + }, + { + "epoch": 0.91, + "learning_rate": 4.728184553660983e-05, + "loss": 1.2909, + "theoretical_loss": 3.3371895391879223, + "tokens_seen": 3145859072 + }, + { + "epoch": 0.91, + "learning_rate": 4.72617853560682e-05, + "loss": 1.3241, + "theoretical_loss": 3.337184290501614, + "tokens_seen": 3145924608 + }, + { + "epoch": 0.91, + "learning_rate": 4.724172517552658e-05, + "loss": 1.3148, + "theoretical_loss": 3.3371790419552605, + "tokens_seen": 3145990144 + }, + { + "epoch": 0.91, + "learning_rate": 4.7221664994984955e-05, + "loss": 1.1978, + "theoretical_loss": 3.337173793548854, + "tokens_seen": 3146055680 + }, + { + "epoch": 0.91, + "learning_rate": 4.720160481444334e-05, + "loss": 1.2939, + "theoretical_loss": 3.337168545282389, + "tokens_seen": 3146121216 + }, + { + "epoch": 0.91, + "learning_rate": 4.7181544633901706e-05, + "loss": 1.2446, + "theoretical_loss": 3.3371632971558585, + "tokens_seen": 3146186752 + }, + { + "epoch": 0.91, + "learning_rate": 4.716148445336008e-05, + "loss": 1.311, + "theoretical_loss": 3.337158049169256, + "tokens_seen": 3146252288 + }, + { + "epoch": 0.91, + "learning_rate": 4.714142427281846e-05, + "loss": 1.1678, + "theoretical_loss": 3.337152801322574, + "tokens_seen": 3146317824 + }, + { + "epoch": 0.91, + "learning_rate": 4.7121364092276834e-05, + "loss": 1.1866, + "theoretical_loss": 3.337147553615807, + "tokens_seen": 3146383360 + }, + { + "epoch": 0.91, + "learning_rate": 4.710130391173521e-05, + "loss": 1.2351, + "theoretical_loss": 3.3371423060489476, + "tokens_seen": 3146448896 + }, + { + "epoch": 0.91, + "learning_rate": 4.708124373119358e-05, + "loss": 1.3225, + "theoretical_loss": 3.3371370586219893, + "tokens_seen": 3146514432 + }, + { + "epoch": 0.91, + "learning_rate": 4.7061183550651954e-05, + "loss": 1.2523, + "theoretical_loss": 3.3371318113349258, + "tokens_seen": 3146579968 + }, + { + "epoch": 0.91, + "learning_rate": 4.704112337011033e-05, + "loss": 1.2196, + "theoretical_loss": 3.33712656418775, + "tokens_seen": 3146645504 + }, + { + "epoch": 0.91, + "learning_rate": 4.7021063189568706e-05, + "loss": 1.2186, + "theoretical_loss": 3.3371213171804555, + "tokens_seen": 3146711040 + }, + { + "epoch": 0.91, + "learning_rate": 4.700100300902708e-05, + "loss": 1.1894, + "theoretical_loss": 3.3371160703130354, + "tokens_seen": 3146776576 + }, + { + "epoch": 0.91, + "learning_rate": 4.698094282848546e-05, + "loss": 1.2474, + "theoretical_loss": 3.3371108235854834, + "tokens_seen": 3146842112 + }, + { + "epoch": 0.91, + "learning_rate": 4.6960882647943833e-05, + "loss": 1.2163, + "theoretical_loss": 3.337105576997793, + "tokens_seen": 3146907648 + }, + { + "epoch": 0.91, + "learning_rate": 4.694082246740221e-05, + "loss": 1.2292, + "theoretical_loss": 3.337100330549957, + "tokens_seen": 3146973184 + }, + { + "epoch": 0.91, + "learning_rate": 4.6920762286860585e-05, + "loss": 1.2478, + "theoretical_loss": 3.3370950842419695, + "tokens_seen": 3147038720 + }, + { + "epoch": 0.91, + "learning_rate": 4.6900702106318954e-05, + "loss": 1.2447, + "theoretical_loss": 3.337089838073823, + "tokens_seen": 3147104256 + }, + { + "epoch": 0.91, + "learning_rate": 4.688064192577733e-05, + "loss": 1.3055, + "theoretical_loss": 3.3370845920455112, + "tokens_seen": 3147169792 + }, + { + "epoch": 0.91, + "learning_rate": 4.6860581745235706e-05, + "loss": 1.2198, + "theoretical_loss": 3.337079346157028, + "tokens_seen": 3147235328 + }, + { + "epoch": 0.91, + "learning_rate": 4.684052156469408e-05, + "loss": 1.3074, + "theoretical_loss": 3.337074100408366, + "tokens_seen": 3147300864 + }, + { + "epoch": 0.91, + "learning_rate": 4.6820461384152464e-05, + "loss": 1.2484, + "theoretical_loss": 3.3370688547995186, + "tokens_seen": 3147366400 + }, + { + "epoch": 0.91, + "learning_rate": 4.680040120361083e-05, + "loss": 1.2837, + "theoretical_loss": 3.33706360933048, + "tokens_seen": 3147431936 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1143525, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2574238777160645, + "objective/train/theoretical_loss": 3.337058364001243, + "objective/train/tokens_used": 1517956576, + "theoretical_loss": 3.337058364001243, + "tokens_seen": 3147497472 + }, + { + "epoch": 0.91, + "learning_rate": 4.678034102306921e-05, + "loss": 1.262, + "theoretical_loss": 3.337058364001243, + "tokens_seen": 3147497472 + }, + { + "epoch": 0.91, + "learning_rate": 4.6760280842527585e-05, + "loss": 1.2958, + "theoretical_loss": 3.3370531188118004, + "tokens_seen": 3147563008 + }, + { + "epoch": 0.91, + "learning_rate": 4.674022066198596e-05, + "loss": 1.3488, + "theoretical_loss": 3.3370478737621463, + "tokens_seen": 3147628544 + }, + { + "epoch": 0.91, + "learning_rate": 4.6720160481444336e-05, + "loss": 1.2687, + "theoretical_loss": 3.3370426288522737, + "tokens_seen": 3147694080 + }, + { + "epoch": 0.91, + "learning_rate": 4.6700100300902705e-05, + "loss": 1.2333, + "theoretical_loss": 3.3370373840821763, + "tokens_seen": 3147759616 + }, + { + "epoch": 0.91, + "learning_rate": 4.668004012036108e-05, + "loss": 1.2607, + "theoretical_loss": 3.3370321394518476, + "tokens_seen": 3147825152 + }, + { + "epoch": 0.91, + "learning_rate": 4.665997993981946e-05, + "loss": 1.1735, + "theoretical_loss": 3.3370268949612805, + "tokens_seen": 3147890688 + }, + { + "epoch": 0.91, + "learning_rate": 4.663991975927784e-05, + "loss": 1.1973, + "theoretical_loss": 3.3370216506104686, + "tokens_seen": 3147956224 + }, + { + "epoch": 0.91, + "learning_rate": 4.6619859578736215e-05, + "loss": 1.2872, + "theoretical_loss": 3.337016406399405, + "tokens_seen": 3148021760 + }, + { + "epoch": 0.91, + "learning_rate": 4.6599799398194584e-05, + "loss": 1.2031, + "theoretical_loss": 3.337011162328083, + "tokens_seen": 3148087296 + }, + { + "epoch": 0.91, + "learning_rate": 4.657973921765296e-05, + "loss": 1.2275, + "theoretical_loss": 3.337005918396497, + "tokens_seen": 3148152832 + }, + { + "epoch": 0.91, + "learning_rate": 4.6559679037111336e-05, + "loss": 1.1418, + "theoretical_loss": 3.337000674604639, + "tokens_seen": 3148218368 + }, + { + "epoch": 0.91, + "learning_rate": 4.653961885656971e-05, + "loss": 1.2767, + "theoretical_loss": 3.336995430952503, + "tokens_seen": 3148283904 + }, + { + "epoch": 0.91, + "learning_rate": 4.651955867602809e-05, + "loss": 1.2822, + "theoretical_loss": 3.3369901874400827, + "tokens_seen": 3148349440 + }, + { + "epoch": 0.91, + "learning_rate": 4.6499498495486456e-05, + "loss": 1.1954, + "theoretical_loss": 3.336984944067371, + "tokens_seen": 3148414976 + }, + { + "epoch": 0.91, + "learning_rate": 4.647943831494483e-05, + "loss": 1.2424, + "theoretical_loss": 3.336979700834361, + "tokens_seen": 3148480512 + }, + { + "epoch": 0.91, + "learning_rate": 4.645937813440321e-05, + "loss": 1.2649, + "theoretical_loss": 3.336974457741047, + "tokens_seen": 3148546048 + }, + { + "epoch": 0.91, + "learning_rate": 4.643931795386159e-05, + "loss": 1.2498, + "theoretical_loss": 3.3369692147874215, + "tokens_seen": 3148611584 + }, + { + "epoch": 0.91, + "learning_rate": 4.6419257773319966e-05, + "loss": 1.1909, + "theoretical_loss": 3.336963971973478, + "tokens_seen": 3148677120 + }, + { + "epoch": 0.91, + "learning_rate": 4.6399197592778335e-05, + "loss": 1.204, + "theoretical_loss": 3.3369587292992104, + "tokens_seen": 3148742656 + }, + { + "epoch": 0.91, + "learning_rate": 4.637913741223671e-05, + "loss": 1.2453, + "theoretical_loss": 3.336953486764612, + "tokens_seen": 3148808192 + }, + { + "epoch": 0.91, + "learning_rate": 4.635907723169509e-05, + "loss": 1.2789, + "theoretical_loss": 3.336948244369675, + "tokens_seen": 3148873728 + }, + { + "epoch": 0.91, + "learning_rate": 4.633901705115346e-05, + "loss": 1.3307, + "theoretical_loss": 3.3369430021143947, + "tokens_seen": 3148939264 + }, + { + "epoch": 0.91, + "learning_rate": 4.631895687061183e-05, + "loss": 1.3557, + "theoretical_loss": 3.336937759998763, + "tokens_seen": 3149004800 + }, + { + "epoch": 0.91, + "learning_rate": 4.629889669007021e-05, + "loss": 1.2836, + "theoretical_loss": 3.3369325180227736, + "tokens_seen": 3149070336 + }, + { + "epoch": 0.91, + "learning_rate": 4.627883650952858e-05, + "loss": 1.286, + "theoretical_loss": 3.33692727618642, + "tokens_seen": 3149135872 + }, + { + "epoch": 0.91, + "learning_rate": 4.6258776328986966e-05, + "loss": 1.2702, + "theoretical_loss": 3.336922034489696, + "tokens_seen": 3149201408 + }, + { + "epoch": 0.91, + "learning_rate": 4.623871614844534e-05, + "loss": 1.2521, + "theoretical_loss": 3.3369167929325942, + "tokens_seen": 3149266944 + }, + { + "epoch": 0.91, + "learning_rate": 4.621865596790371e-05, + "loss": 1.3393, + "theoretical_loss": 3.3369115515151084, + "tokens_seen": 3149332480 + }, + { + "epoch": 0.91, + "learning_rate": 4.6198595787362086e-05, + "loss": 1.2407, + "theoretical_loss": 3.336906310237232, + "tokens_seen": 3149398016 + }, + { + "epoch": 0.91, + "learning_rate": 4.617853560682046e-05, + "loss": 1.4112, + "theoretical_loss": 3.3369010690989582, + "tokens_seen": 3149463552 + }, + { + "epoch": 0.91, + "learning_rate": 4.615847542627884e-05, + "loss": 1.2202, + "theoretical_loss": 3.33689582810028, + "tokens_seen": 3149529088 + }, + { + "epoch": 0.91, + "learning_rate": 4.6138415245737214e-05, + "loss": 1.3244, + "theoretical_loss": 3.336890587241192, + "tokens_seen": 3149594624 + }, + { + "epoch": 0.91, + "learning_rate": 4.611835506519558e-05, + "loss": 1.2531, + "theoretical_loss": 3.3368853465216866, + "tokens_seen": 3149660160 + }, + { + "epoch": 0.91, + "learning_rate": 4.609829488465396e-05, + "loss": 1.2194, + "theoretical_loss": 3.336880105941757, + "tokens_seen": 3149725696 + }, + { + "epoch": 0.91, + "learning_rate": 4.6078234704112334e-05, + "loss": 1.2726, + "theoretical_loss": 3.3368748655013976, + "tokens_seen": 3149791232 + }, + { + "epoch": 0.91, + "learning_rate": 4.605817452357072e-05, + "loss": 1.3712, + "theoretical_loss": 3.3368696252006003, + "tokens_seen": 3149856768 + }, + { + "epoch": 0.91, + "learning_rate": 4.603811434302909e-05, + "loss": 1.2365, + "theoretical_loss": 3.33686438503936, + "tokens_seen": 3149922304 + }, + { + "epoch": 0.91, + "learning_rate": 4.601805416248746e-05, + "loss": 1.2406, + "theoretical_loss": 3.336859145017669, + "tokens_seen": 3149987840 + }, + { + "epoch": 0.91, + "learning_rate": 4.599799398194584e-05, + "loss": 1.1566, + "theoretical_loss": 3.3368539051355213, + "tokens_seen": 3150053376 + }, + { + "epoch": 0.91, + "learning_rate": 4.5977933801404213e-05, + "loss": 1.3044, + "theoretical_loss": 3.3368486653929104, + "tokens_seen": 3150118912 + }, + { + "epoch": 0.91, + "learning_rate": 4.595787362086259e-05, + "loss": 1.3376, + "theoretical_loss": 3.3368434257898287, + "tokens_seen": 3150184448 + }, + { + "epoch": 0.91, + "learning_rate": 4.5937813440320965e-05, + "loss": 1.3135, + "theoretical_loss": 3.3368381863262706, + "tokens_seen": 3150249984 + }, + { + "epoch": 0.91, + "learning_rate": 4.5917753259779334e-05, + "loss": 1.2162, + "theoretical_loss": 3.336832947002229, + "tokens_seen": 3150315520 + }, + { + "epoch": 0.91, + "learning_rate": 4.589769307923771e-05, + "loss": 1.1898, + "theoretical_loss": 3.3368277078176973, + "tokens_seen": 3150381056 + }, + { + "epoch": 0.91, + "learning_rate": 4.587763289869609e-05, + "loss": 1.2637, + "theoretical_loss": 3.3368224687726693, + "tokens_seen": 3150446592 + }, + { + "epoch": 0.91, + "learning_rate": 4.585757271815447e-05, + "loss": 1.2479, + "theoretical_loss": 3.336817229867138, + "tokens_seen": 3150512128 + }, + { + "epoch": 0.91, + "learning_rate": 4.5837512537612844e-05, + "loss": 1.3739, + "theoretical_loss": 3.3368119911010967, + "tokens_seen": 3150577664 + }, + { + "epoch": 0.91, + "learning_rate": 4.581745235707121e-05, + "loss": 1.3001, + "theoretical_loss": 3.336806752474539, + "tokens_seen": 3150643200 + }, + { + "epoch": 0.91, + "learning_rate": 4.579739217652959e-05, + "loss": 1.2641, + "theoretical_loss": 3.3368015139874583, + "tokens_seen": 3150708736 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1144741, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2674669027328491, + "objective/train/theoretical_loss": 3.3367962756398475, + "objective/train/tokens_used": 1521233376, + "theoretical_loss": 3.3367962756398475, + "tokens_seen": 3150774272 + }, + { + "epoch": 0.91, + "learning_rate": 4.5777331995987965e-05, + "loss": 1.1916, + "theoretical_loss": 3.3367962756398475, + "tokens_seen": 3150774272 + }, + { + "epoch": 0.91, + "learning_rate": 4.575727181544634e-05, + "loss": 1.2831, + "theoretical_loss": 3.3367910374317007, + "tokens_seen": 3150839808 + }, + { + "epoch": 0.91, + "learning_rate": 4.5737211634904716e-05, + "loss": 1.3161, + "theoretical_loss": 3.3367857993630112, + "tokens_seen": 3150905344 + }, + { + "epoch": 0.91, + "learning_rate": 4.5717151454363085e-05, + "loss": 1.1615, + "theoretical_loss": 3.3367805614337716, + "tokens_seen": 3150970880 + }, + { + "epoch": 0.91, + "learning_rate": 4.569709127382147e-05, + "loss": 1.1384, + "theoretical_loss": 3.3367753236439763, + "tokens_seen": 3151036416 + }, + { + "epoch": 0.91, + "learning_rate": 4.5677031093279844e-05, + "loss": 1.2374, + "theoretical_loss": 3.336770085993618, + "tokens_seen": 3151101952 + }, + { + "epoch": 0.91, + "learning_rate": 4.565697091273822e-05, + "loss": 1.3545, + "theoretical_loss": 3.3367648484826904, + "tokens_seen": 3151167488 + }, + { + "epoch": 0.91, + "learning_rate": 4.5636910732196595e-05, + "loss": 1.2162, + "theoretical_loss": 3.336759611111187, + "tokens_seen": 3151233024 + }, + { + "epoch": 0.91, + "learning_rate": 4.5616850551654964e-05, + "loss": 1.3187, + "theoretical_loss": 3.3367543738791006, + "tokens_seen": 3151298560 + }, + { + "epoch": 0.91, + "learning_rate": 4.559679037111334e-05, + "loss": 1.1118, + "theoretical_loss": 3.336749136786425, + "tokens_seen": 3151364096 + }, + { + "epoch": 0.91, + "learning_rate": 4.5576730190571716e-05, + "loss": 1.2803, + "theoretical_loss": 3.336743899833154, + "tokens_seen": 3151429632 + }, + { + "epoch": 0.91, + "learning_rate": 4.555667001003009e-05, + "loss": 1.2909, + "theoretical_loss": 3.3367386630192803, + "tokens_seen": 3151495168 + }, + { + "epoch": 0.91, + "learning_rate": 4.553660982948846e-05, + "loss": 1.3682, + "theoretical_loss": 3.336733426344798, + "tokens_seen": 3151560704 + }, + { + "epoch": 0.91, + "learning_rate": 4.5516549648946836e-05, + "loss": 1.2141, + "theoretical_loss": 3.3367281898096994, + "tokens_seen": 3151626240 + }, + { + "epoch": 0.91, + "learning_rate": 4.549648946840522e-05, + "loss": 1.3611, + "theoretical_loss": 3.336722953413979, + "tokens_seen": 3151691776 + }, + { + "epoch": 0.91, + "learning_rate": 4.5476429287863595e-05, + "loss": 1.2502, + "theoretical_loss": 3.3367177171576294, + "tokens_seen": 3151757312 + }, + { + "epoch": 0.91, + "learning_rate": 4.545636910732197e-05, + "loss": 1.2531, + "theoretical_loss": 3.3367124810406446, + "tokens_seen": 3151822848 + }, + { + "epoch": 0.91, + "learning_rate": 4.543630892678034e-05, + "loss": 1.2049, + "theoretical_loss": 3.3367072450630175, + "tokens_seen": 3151888384 + }, + { + "epoch": 0.91, + "learning_rate": 4.5416248746238715e-05, + "loss": 1.2621, + "theoretical_loss": 3.3367020092247417, + "tokens_seen": 3151953920 + }, + { + "epoch": 0.91, + "learning_rate": 4.539618856569709e-05, + "loss": 1.202, + "theoretical_loss": 3.336696773525811, + "tokens_seen": 3152019456 + }, + { + "epoch": 0.91, + "learning_rate": 4.537612838515547e-05, + "loss": 1.348, + "theoretical_loss": 3.336691537966218, + "tokens_seen": 3152084992 + }, + { + "epoch": 0.91, + "learning_rate": 4.535606820461384e-05, + "loss": 1.2311, + "theoretical_loss": 3.3366863025459566, + "tokens_seen": 3152150528 + }, + { + "epoch": 0.91, + "learning_rate": 4.533600802407221e-05, + "loss": 1.1958, + "theoretical_loss": 3.3366810672650202, + "tokens_seen": 3152216064 + }, + { + "epoch": 0.91, + "learning_rate": 4.5315947843530594e-05, + "loss": 1.248, + "theoretical_loss": 3.336675832123402, + "tokens_seen": 3152281600 + }, + { + "epoch": 0.91, + "learning_rate": 4.529588766298897e-05, + "loss": 1.2811, + "theoretical_loss": 3.3366705971210955, + "tokens_seen": 3152347136 + }, + { + "epoch": 0.91, + "learning_rate": 4.5275827482447346e-05, + "loss": 1.2417, + "theoretical_loss": 3.3366653622580937, + "tokens_seen": 3152412672 + }, + { + "epoch": 0.91, + "learning_rate": 4.525576730190572e-05, + "loss": 1.2279, + "theoretical_loss": 3.3366601275343912, + "tokens_seen": 3152478208 + }, + { + "epoch": 0.91, + "learning_rate": 4.523570712136409e-05, + "loss": 1.1412, + "theoretical_loss": 3.33665489294998, + "tokens_seen": 3152543744 + }, + { + "epoch": 0.91, + "learning_rate": 4.5215646940822466e-05, + "loss": 1.2648, + "theoretical_loss": 3.336649658504854, + "tokens_seen": 3152609280 + }, + { + "epoch": 0.91, + "learning_rate": 4.519558676028084e-05, + "loss": 1.3128, + "theoretical_loss": 3.336644424199007, + "tokens_seen": 3152674816 + }, + { + "epoch": 0.91, + "learning_rate": 4.517552657973922e-05, + "loss": 1.3406, + "theoretical_loss": 3.3366391900324324, + "tokens_seen": 3152740352 + }, + { + "epoch": 0.91, + "learning_rate": 4.5155466399197594e-05, + "loss": 1.2501, + "theoretical_loss": 3.3366339560051226, + "tokens_seen": 3152805888 + }, + { + "epoch": 0.91, + "learning_rate": 4.513540621865597e-05, + "loss": 1.2561, + "theoretical_loss": 3.336628722117072, + "tokens_seen": 3152871424 + }, + { + "epoch": 0.91, + "learning_rate": 4.5115346038114345e-05, + "loss": 1.3459, + "theoretical_loss": 3.3366234883682733, + "tokens_seen": 3152936960 + }, + { + "epoch": 0.91, + "learning_rate": 4.509528585757272e-05, + "loss": 1.3986, + "theoretical_loss": 3.336618254758721, + "tokens_seen": 3153002496 + }, + { + "epoch": 0.91, + "learning_rate": 4.50752256770311e-05, + "loss": 1.1736, + "theoretical_loss": 3.3366130212884073, + "tokens_seen": 3153068032 + }, + { + "epoch": 0.91, + "learning_rate": 4.505516549648947e-05, + "loss": 1.2123, + "theoretical_loss": 3.336607787957326, + "tokens_seen": 3153133568 + }, + { + "epoch": 0.91, + "learning_rate": 4.503510531594784e-05, + "loss": 1.2911, + "theoretical_loss": 3.3366025547654705, + "tokens_seen": 3153199104 + }, + { + "epoch": 0.91, + "learning_rate": 4.501504513540622e-05, + "loss": 1.2003, + "theoretical_loss": 3.3365973217128344, + "tokens_seen": 3153264640 + }, + { + "epoch": 0.91, + "learning_rate": 4.4994984954864593e-05, + "loss": 1.2711, + "theoretical_loss": 3.3365920887994114, + "tokens_seen": 3153330176 + }, + { + "epoch": 0.91, + "learning_rate": 4.497492477432297e-05, + "loss": 1.247, + "theoretical_loss": 3.336586856025194, + "tokens_seen": 3153395712 + }, + { + "epoch": 0.91, + "learning_rate": 4.4954864593781345e-05, + "loss": 1.248, + "theoretical_loss": 3.336581623390176, + "tokens_seen": 3153461248 + }, + { + "epoch": 0.91, + "learning_rate": 4.493480441323972e-05, + "loss": 1.2997, + "theoretical_loss": 3.3365763908943515, + "tokens_seen": 3153526784 + }, + { + "epoch": 0.91, + "learning_rate": 4.4914744232698097e-05, + "loss": 1.2299, + "theoretical_loss": 3.336571158537713, + "tokens_seen": 3153592320 + }, + { + "epoch": 0.91, + "learning_rate": 4.489468405215647e-05, + "loss": 1.3245, + "theoretical_loss": 3.3365659263202536, + "tokens_seen": 3153657856 + }, + { + "epoch": 0.91, + "learning_rate": 4.487462387161485e-05, + "loss": 1.2915, + "theoretical_loss": 3.3365606942419683, + "tokens_seen": 3153723392 + }, + { + "epoch": 0.91, + "learning_rate": 4.485456369107322e-05, + "loss": 1.2268, + "theoretical_loss": 3.336555462302849, + "tokens_seen": 3153788928 + }, + { + "epoch": 0.91, + "learning_rate": 4.483450351053159e-05, + "loss": 1.359, + "theoretical_loss": 3.3365502305028896, + "tokens_seen": 3153854464 + }, + { + "epoch": 0.91, + "learning_rate": 4.481444332998997e-05, + "loss": 1.2471, + "theoretical_loss": 3.3365449988420837, + "tokens_seen": 3153920000 + }, + { + "epoch": 0.91, + "learning_rate": 4.4794383149448345e-05, + "loss": 1.2608, + "theoretical_loss": 3.3365397673204242, + "tokens_seen": 3153985536 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1145990, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0538361072540283, + "objective/train/theoretical_loss": 3.336534535937905, + "objective/train/tokens_used": 1524510176, + "theoretical_loss": 3.336534535937905, + "tokens_seen": 3154051072 + }, + { + "epoch": 0.91, + "learning_rate": 4.477432296890672e-05, + "loss": 1.1775, + "theoretical_loss": 3.336534535937905, + "tokens_seen": 3154051072 + }, + { + "epoch": 0.91, + "learning_rate": 4.4754262788365096e-05, + "loss": 1.3139, + "theoretical_loss": 3.3365293046945195, + "tokens_seen": 3154116608 + }, + { + "epoch": 0.91, + "learning_rate": 4.473420260782347e-05, + "loss": 1.2761, + "theoretical_loss": 3.3365240735902613, + "tokens_seen": 3154182144 + }, + { + "epoch": 0.91, + "learning_rate": 4.471414242728185e-05, + "loss": 1.1932, + "theoretical_loss": 3.336518842625123, + "tokens_seen": 3154247680 + }, + { + "epoch": 0.91, + "learning_rate": 4.4694082246740224e-05, + "loss": 1.1654, + "theoretical_loss": 3.3365136117990986, + "tokens_seen": 3154313216 + }, + { + "epoch": 0.91, + "learning_rate": 4.46740220661986e-05, + "loss": 1.2718, + "theoretical_loss": 3.336508381112181, + "tokens_seen": 3154378752 + }, + { + "epoch": 0.91, + "learning_rate": 4.465396188565697e-05, + "loss": 1.2226, + "theoretical_loss": 3.3365031505643645, + "tokens_seen": 3154444288 + }, + { + "epoch": 0.91, + "learning_rate": 4.4633901705115344e-05, + "loss": 1.2801, + "theoretical_loss": 3.336497920155642, + "tokens_seen": 3154509824 + }, + { + "epoch": 0.91, + "learning_rate": 4.461384152457372e-05, + "loss": 1.2066, + "theoretical_loss": 3.336492689886007, + "tokens_seen": 3154575360 + }, + { + "epoch": 0.91, + "learning_rate": 4.4593781344032096e-05, + "loss": 1.1455, + "theoretical_loss": 3.3364874597554524, + "tokens_seen": 3154640896 + }, + { + "epoch": 0.91, + "learning_rate": 4.457372116349048e-05, + "loss": 1.3764, + "theoretical_loss": 3.3364822297639725, + "tokens_seen": 3154706432 + }, + { + "epoch": 0.91, + "learning_rate": 4.455366098294885e-05, + "loss": 1.3628, + "theoretical_loss": 3.33647699991156, + "tokens_seen": 3154771968 + }, + { + "epoch": 0.91, + "learning_rate": 4.453360080240722e-05, + "loss": 1.2322, + "theoretical_loss": 3.3364717701982083, + "tokens_seen": 3154837504 + }, + { + "epoch": 0.91, + "learning_rate": 4.45135406218656e-05, + "loss": 1.286, + "theoretical_loss": 3.3364665406239116, + "tokens_seen": 3154903040 + }, + { + "epoch": 0.91, + "learning_rate": 4.4493480441323975e-05, + "loss": 1.3233, + "theoretical_loss": 3.3364613111886627, + "tokens_seen": 3154968576 + }, + { + "epoch": 0.91, + "learning_rate": 4.447342026078235e-05, + "loss": 1.252, + "theoretical_loss": 3.3364560818924547, + "tokens_seen": 3155034112 + }, + { + "epoch": 0.91, + "learning_rate": 4.445336008024072e-05, + "loss": 1.3872, + "theoretical_loss": 3.336450852735282, + "tokens_seen": 3155099648 + }, + { + "epoch": 0.91, + "learning_rate": 4.4433299899699095e-05, + "loss": 1.2238, + "theoretical_loss": 3.3364456237171374, + "tokens_seen": 3155165184 + }, + { + "epoch": 0.91, + "learning_rate": 4.441323971915747e-05, + "loss": 1.1784, + "theoretical_loss": 3.3364403948380144, + "tokens_seen": 3155230720 + }, + { + "epoch": 0.91, + "learning_rate": 4.439317953861585e-05, + "loss": 1.2155, + "theoretical_loss": 3.336435166097906, + "tokens_seen": 3155296256 + }, + { + "epoch": 0.91, + "learning_rate": 4.437311935807423e-05, + "loss": 1.3243, + "theoretical_loss": 3.3364299374968063, + "tokens_seen": 3155361792 + }, + { + "epoch": 0.91, + "learning_rate": 4.43530591775326e-05, + "loss": 1.1857, + "theoretical_loss": 3.3364247090347083, + "tokens_seen": 3155427328 + }, + { + "epoch": 0.91, + "learning_rate": 4.4332998996990974e-05, + "loss": 1.2776, + "theoretical_loss": 3.336419480711605, + "tokens_seen": 3155492864 + }, + { + "epoch": 0.91, + "learning_rate": 4.431293881644935e-05, + "loss": 1.3773, + "theoretical_loss": 3.336414252527491, + "tokens_seen": 3155558400 + }, + { + "epoch": 0.91, + "learning_rate": 4.4292878635907726e-05, + "loss": 1.2883, + "theoretical_loss": 3.336409024482359, + "tokens_seen": 3155623936 + }, + { + "epoch": 0.91, + "learning_rate": 4.42728184553661e-05, + "loss": 1.2252, + "theoretical_loss": 3.3364037965762026, + "tokens_seen": 3155689472 + }, + { + "epoch": 0.91, + "learning_rate": 4.425275827482447e-05, + "loss": 1.2123, + "theoretical_loss": 3.336398568809015, + "tokens_seen": 3155755008 + }, + { + "epoch": 0.91, + "learning_rate": 4.4232698094282846e-05, + "loss": 1.2288, + "theoretical_loss": 3.3363933411807896, + "tokens_seen": 3155820544 + }, + { + "epoch": 0.91, + "learning_rate": 4.421263791374122e-05, + "loss": 1.3102, + "theoretical_loss": 3.33638811369152, + "tokens_seen": 3155886080 + }, + { + "epoch": 0.91, + "learning_rate": 4.4192577733199605e-05, + "loss": 1.2553, + "theoretical_loss": 3.3363828863411995, + "tokens_seen": 3155951616 + }, + { + "epoch": 0.91, + "learning_rate": 4.417251755265798e-05, + "loss": 1.2348, + "theoretical_loss": 3.336377659129822, + "tokens_seen": 3156017152 + }, + { + "epoch": 0.91, + "learning_rate": 4.415245737211635e-05, + "loss": 1.2274, + "theoretical_loss": 3.33637243205738, + "tokens_seen": 3156082688 + }, + { + "epoch": 0.91, + "learning_rate": 4.4132397191574725e-05, + "loss": 1.1872, + "theoretical_loss": 3.3363672051238678, + "tokens_seen": 3156148224 + }, + { + "epoch": 0.91, + "learning_rate": 4.41123370110331e-05, + "loss": 1.1664, + "theoretical_loss": 3.336361978329278, + "tokens_seen": 3156213760 + }, + { + "epoch": 0.91, + "learning_rate": 4.409227683049148e-05, + "loss": 1.23, + "theoretical_loss": 3.336356751673605, + "tokens_seen": 3156279296 + }, + { + "epoch": 0.91, + "learning_rate": 4.4072216649949846e-05, + "loss": 1.2076, + "theoretical_loss": 3.3363515251568416, + "tokens_seen": 3156344832 + }, + { + "epoch": 0.91, + "learning_rate": 4.405215646940822e-05, + "loss": 1.2792, + "theoretical_loss": 3.336346298778981, + "tokens_seen": 3156410368 + }, + { + "epoch": 0.91, + "learning_rate": 4.40320962888666e-05, + "loss": 1.2605, + "theoretical_loss": 3.3363410725400175, + "tokens_seen": 3156475904 + }, + { + "epoch": 0.91, + "learning_rate": 4.401203610832498e-05, + "loss": 1.2029, + "theoretical_loss": 3.3363358464399435, + "tokens_seen": 3156541440 + }, + { + "epoch": 0.91, + "learning_rate": 4.3991975927783356e-05, + "loss": 1.2664, + "theoretical_loss": 3.336330620478753, + "tokens_seen": 3156606976 + }, + { + "epoch": 0.91, + "learning_rate": 4.3971915747241725e-05, + "loss": 1.1964, + "theoretical_loss": 3.3363253946564395, + "tokens_seen": 3156672512 + }, + { + "epoch": 0.91, + "learning_rate": 4.39518555667001e-05, + "loss": 1.2643, + "theoretical_loss": 3.336320168972996, + "tokens_seen": 3156738048 + }, + { + "epoch": 0.91, + "learning_rate": 4.3931795386158477e-05, + "loss": 1.4383, + "theoretical_loss": 3.3363149434284165, + "tokens_seen": 3156803584 + }, + { + "epoch": 0.91, + "learning_rate": 4.391173520561685e-05, + "loss": 1.1866, + "theoretical_loss": 3.3363097180226937, + "tokens_seen": 3156869120 + }, + { + "epoch": 0.91, + "learning_rate": 4.389167502507523e-05, + "loss": 1.2071, + "theoretical_loss": 3.336304492755822, + "tokens_seen": 3156934656 + }, + { + "epoch": 0.91, + "learning_rate": 4.38716148445336e-05, + "loss": 1.277, + "theoretical_loss": 3.336299267627794, + "tokens_seen": 3157000192 + }, + { + "epoch": 0.91, + "learning_rate": 4.385155466399197e-05, + "loss": 1.3284, + "theoretical_loss": 3.3362940426386034, + "tokens_seen": 3157065728 + }, + { + "epoch": 0.91, + "learning_rate": 4.383149448345035e-05, + "loss": 1.2408, + "theoretical_loss": 3.336288817788243, + "tokens_seen": 3157131264 + }, + { + "epoch": 0.91, + "learning_rate": 4.381143430290873e-05, + "loss": 1.355, + "theoretical_loss": 3.3362835930767076, + "tokens_seen": 3157196800 + }, + { + "epoch": 0.91, + "learning_rate": 4.379137412236711e-05, + "loss": 1.2895, + "theoretical_loss": 3.33627836850399, + "tokens_seen": 3157262336 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1147172, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3894332647323608, + "objective/train/theoretical_loss": 3.3362731440700832, + "objective/train/tokens_used": 1527786976, + "theoretical_loss": 3.3362731440700832, + "tokens_seen": 3157327872 + }, + { + "epoch": 0.91, + "learning_rate": 4.3771313941825476e-05, + "loss": 1.3515, + "theoretical_loss": 3.3362731440700832, + "tokens_seen": 3157327872 + }, + { + "epoch": 0.91, + "learning_rate": 4.375125376128385e-05, + "loss": 1.3709, + "theoretical_loss": 3.336267919774981, + "tokens_seen": 3157393408 + }, + { + "epoch": 0.91, + "learning_rate": 4.373119358074223e-05, + "loss": 1.2245, + "theoretical_loss": 3.3362626956186765, + "tokens_seen": 3157458944 + }, + { + "epoch": 0.91, + "learning_rate": 4.3711133400200604e-05, + "loss": 1.2934, + "theoretical_loss": 3.3362574716011637, + "tokens_seen": 3157524480 + }, + { + "epoch": 0.91, + "learning_rate": 4.369107321965898e-05, + "loss": 1.2472, + "theoretical_loss": 3.336252247722436, + "tokens_seen": 3157590016 + }, + { + "epoch": 0.91, + "learning_rate": 4.367101303911735e-05, + "loss": 1.2483, + "theoretical_loss": 3.336247023982486, + "tokens_seen": 3157655552 + }, + { + "epoch": 0.91, + "learning_rate": 4.3650952858575724e-05, + "loss": 1.3176, + "theoretical_loss": 3.336241800381308, + "tokens_seen": 3157721088 + }, + { + "epoch": 0.91, + "learning_rate": 4.363089267803411e-05, + "loss": 1.2789, + "theoretical_loss": 3.3362365769188953, + "tokens_seen": 3157786624 + }, + { + "epoch": 0.91, + "learning_rate": 4.361083249749248e-05, + "loss": 1.2228, + "theoretical_loss": 3.336231353595241, + "tokens_seen": 3157852160 + }, + { + "epoch": 0.91, + "learning_rate": 4.359077231695086e-05, + "loss": 1.1596, + "theoretical_loss": 3.3362261304103384, + "tokens_seen": 3157917696 + }, + { + "epoch": 0.91, + "learning_rate": 4.357071213640923e-05, + "loss": 1.2845, + "theoretical_loss": 3.3362209073641815, + "tokens_seen": 3157983232 + }, + { + "epoch": 0.91, + "learning_rate": 4.35506519558676e-05, + "loss": 1.1985, + "theoretical_loss": 3.3362156844567634, + "tokens_seen": 3158048768 + }, + { + "epoch": 0.91, + "learning_rate": 4.353059177532598e-05, + "loss": 1.3078, + "theoretical_loss": 3.336210461688078, + "tokens_seen": 3158114304 + }, + { + "epoch": 0.91, + "learning_rate": 4.3510531594784355e-05, + "loss": 1.3681, + "theoretical_loss": 3.336205239058118, + "tokens_seen": 3158179840 + }, + { + "epoch": 0.91, + "learning_rate": 4.349047141424273e-05, + "loss": 1.2787, + "theoretical_loss": 3.3362000165668775, + "tokens_seen": 3158245376 + }, + { + "epoch": 0.91, + "learning_rate": 4.34704112337011e-05, + "loss": 1.2356, + "theoretical_loss": 3.3361947942143493, + "tokens_seen": 3158310912 + }, + { + "epoch": 0.91, + "learning_rate": 4.3450351053159475e-05, + "loss": 1.2061, + "theoretical_loss": 3.336189572000527, + "tokens_seen": 3158376448 + }, + { + "epoch": 0.91, + "learning_rate": 4.343029087261786e-05, + "loss": 1.2245, + "theoretical_loss": 3.3361843499254045, + "tokens_seen": 3158441984 + }, + { + "epoch": 0.91, + "learning_rate": 4.3410230692076234e-05, + "loss": 1.1925, + "theoretical_loss": 3.336179127988975, + "tokens_seen": 3158507520 + }, + { + "epoch": 0.91, + "learning_rate": 4.33901705115346e-05, + "loss": 1.1961, + "theoretical_loss": 3.336173906191232, + "tokens_seen": 3158573056 + }, + { + "epoch": 0.91, + "learning_rate": 4.337011033099298e-05, + "loss": 1.3451, + "theoretical_loss": 3.3361686845321685, + "tokens_seen": 3158638592 + }, + { + "epoch": 0.91, + "learning_rate": 4.3350050150451354e-05, + "loss": 1.3091, + "theoretical_loss": 3.3361634630117782, + "tokens_seen": 3158704128 + }, + { + "epoch": 0.91, + "learning_rate": 4.332998996990973e-05, + "loss": 1.1975, + "theoretical_loss": 3.336158241630055, + "tokens_seen": 3158769664 + }, + { + "epoch": 0.91, + "learning_rate": 4.3309929789368106e-05, + "loss": 1.2201, + "theoretical_loss": 3.336153020386991, + "tokens_seen": 3158835200 + }, + { + "epoch": 0.91, + "learning_rate": 4.3289869608826475e-05, + "loss": 1.2286, + "theoretical_loss": 3.3361477992825814, + "tokens_seen": 3158900736 + }, + { + "epoch": 0.91, + "learning_rate": 4.326980942828485e-05, + "loss": 1.1963, + "theoretical_loss": 3.3361425783168186, + "tokens_seen": 3158966272 + }, + { + "epoch": 0.91, + "learning_rate": 4.324974924774323e-05, + "loss": 1.3, + "theoretical_loss": 3.3361373574896964, + "tokens_seen": 3159031808 + }, + { + "epoch": 0.91, + "learning_rate": 4.322968906720161e-05, + "loss": 1.2816, + "theoretical_loss": 3.3361321368012082, + "tokens_seen": 3159097344 + }, + { + "epoch": 0.91, + "learning_rate": 4.3209628886659985e-05, + "loss": 1.2431, + "theoretical_loss": 3.336126916251347, + "tokens_seen": 3159162880 + }, + { + "epoch": 0.91, + "learning_rate": 4.3189568706118354e-05, + "loss": 1.2237, + "theoretical_loss": 3.3361216958401068, + "tokens_seen": 3159228416 + }, + { + "epoch": 0.91, + "learning_rate": 4.316950852557673e-05, + "loss": 1.2858, + "theoretical_loss": 3.336116475567481, + "tokens_seen": 3159293952 + }, + { + "epoch": 0.91, + "learning_rate": 4.3149448345035105e-05, + "loss": 1.1768, + "theoretical_loss": 3.3361112554334627, + "tokens_seen": 3159359488 + }, + { + "epoch": 0.91, + "learning_rate": 4.312938816449348e-05, + "loss": 1.3441, + "theoretical_loss": 3.3361060354380454, + "tokens_seen": 3159425024 + }, + { + "epoch": 0.91, + "learning_rate": 4.310932798395186e-05, + "loss": 1.1866, + "theoretical_loss": 3.3361008155812226, + "tokens_seen": 3159490560 + }, + { + "epoch": 0.91, + "learning_rate": 4.3089267803410226e-05, + "loss": 1.3557, + "theoretical_loss": 3.3360955958629877, + "tokens_seen": 3159556096 + }, + { + "epoch": 0.91, + "learning_rate": 4.306920762286861e-05, + "loss": 1.2246, + "theoretical_loss": 3.3360903762833347, + "tokens_seen": 3159621632 + }, + { + "epoch": 0.91, + "learning_rate": 4.3049147442326984e-05, + "loss": 1.2066, + "theoretical_loss": 3.3360851568422563, + "tokens_seen": 3159687168 + }, + { + "epoch": 0.92, + "learning_rate": 4.302908726178536e-05, + "loss": 1.3282, + "theoretical_loss": 3.3360799375397465, + "tokens_seen": 3159752704 + }, + { + "epoch": 0.92, + "learning_rate": 4.3009027081243736e-05, + "loss": 1.2657, + "theoretical_loss": 3.3360747183757984, + "tokens_seen": 3159818240 + }, + { + "epoch": 0.92, + "learning_rate": 4.2988966900702105e-05, + "loss": 1.2754, + "theoretical_loss": 3.3360694993504056, + "tokens_seen": 3159883776 + }, + { + "epoch": 0.92, + "learning_rate": 4.296890672016048e-05, + "loss": 1.291, + "theoretical_loss": 3.336064280463561, + "tokens_seen": 3159949312 + }, + { + "epoch": 0.92, + "learning_rate": 4.2948846539618857e-05, + "loss": 1.2783, + "theoretical_loss": 3.3360590617152592, + "tokens_seen": 3160014848 + }, + { + "epoch": 0.92, + "learning_rate": 4.292878635907723e-05, + "loss": 1.2711, + "theoretical_loss": 3.3360538431054927, + "tokens_seen": 3160080384 + }, + { + "epoch": 0.92, + "learning_rate": 4.290872617853561e-05, + "loss": 1.249, + "theoretical_loss": 3.3360486246342553, + "tokens_seen": 3160145920 + }, + { + "epoch": 0.92, + "learning_rate": 4.288866599799398e-05, + "loss": 1.1733, + "theoretical_loss": 3.3360434063015405, + "tokens_seen": 3160211456 + }, + { + "epoch": 0.92, + "learning_rate": 4.286860581745236e-05, + "loss": 1.3569, + "theoretical_loss": 3.3360381881073415, + "tokens_seen": 3160276992 + }, + { + "epoch": 0.92, + "learning_rate": 4.2848545636910736e-05, + "loss": 1.3927, + "theoretical_loss": 3.336032970051652, + "tokens_seen": 3160342528 + }, + { + "epoch": 0.92, + "learning_rate": 4.282848545636911e-05, + "loss": 1.187, + "theoretical_loss": 3.336027752134465, + "tokens_seen": 3160408064 + }, + { + "epoch": 0.92, + "learning_rate": 4.280842527582749e-05, + "loss": 1.1135, + "theoretical_loss": 3.3360225343557746, + "tokens_seen": 3160473600 + }, + { + "epoch": 0.92, + "learning_rate": 4.2788365095285856e-05, + "loss": 1.3896, + "theoretical_loss": 3.3360173167155738, + "tokens_seen": 3160539136 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1148479, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1145994663238525, + "objective/train/theoretical_loss": 3.3360120992138564, + "objective/train/tokens_used": 1531063776, + "theoretical_loss": 3.3360120992138564, + "tokens_seen": 3160604672 + }, + { + "epoch": 0.92, + "learning_rate": 4.276830491474423e-05, + "loss": 1.1438, + "theoretical_loss": 3.3360120992138564, + "tokens_seen": 3160604672 + }, + { + "epoch": 0.92, + "learning_rate": 4.274824473420261e-05, + "loss": 1.2758, + "theoretical_loss": 3.3360068818506154, + "tokens_seen": 3160670208 + }, + { + "epoch": 0.92, + "learning_rate": 4.2728184553660983e-05, + "loss": 1.1878, + "theoretical_loss": 3.3360016646258446, + "tokens_seen": 3160735744 + }, + { + "epoch": 0.92, + "learning_rate": 4.270812437311935e-05, + "loss": 1.173, + "theoretical_loss": 3.3359964475395376, + "tokens_seen": 3160801280 + }, + { + "epoch": 0.92, + "learning_rate": 4.2688064192577735e-05, + "loss": 1.2827, + "theoretical_loss": 3.3359912305916875, + "tokens_seen": 3160866816 + }, + { + "epoch": 0.92, + "learning_rate": 4.266800401203611e-05, + "loss": 1.219, + "theoretical_loss": 3.3359860137822874, + "tokens_seen": 3160932352 + }, + { + "epoch": 0.92, + "learning_rate": 4.264794383149449e-05, + "loss": 1.289, + "theoretical_loss": 3.3359807971113318, + "tokens_seen": 3160997888 + }, + { + "epoch": 0.92, + "learning_rate": 4.262788365095286e-05, + "loss": 1.1787, + "theoretical_loss": 3.335975580578814, + "tokens_seen": 3161063424 + }, + { + "epoch": 0.92, + "learning_rate": 4.260782347041123e-05, + "loss": 1.2942, + "theoretical_loss": 3.335970364184726, + "tokens_seen": 3161128960 + }, + { + "epoch": 0.92, + "learning_rate": 4.258776328986961e-05, + "loss": 1.2276, + "theoretical_loss": 3.3359651479290626, + "tokens_seen": 3161194496 + }, + { + "epoch": 0.92, + "learning_rate": 4.256770310932798e-05, + "loss": 1.2196, + "theoretical_loss": 3.3359599318118174, + "tokens_seen": 3161260032 + }, + { + "epoch": 0.92, + "learning_rate": 4.254764292878636e-05, + "loss": 1.2318, + "theoretical_loss": 3.3359547158329828, + "tokens_seen": 3161325568 + }, + { + "epoch": 0.92, + "learning_rate": 4.2527582748244735e-05, + "loss": 1.1965, + "theoretical_loss": 3.3359494999925534, + "tokens_seen": 3161391104 + }, + { + "epoch": 0.92, + "learning_rate": 4.250752256770311e-05, + "loss": 1.2467, + "theoretical_loss": 3.335944284290522, + "tokens_seen": 3161456640 + }, + { + "epoch": 0.92, + "learning_rate": 4.2487462387161486e-05, + "loss": 1.2003, + "theoretical_loss": 3.3359390687268817, + "tokens_seen": 3161522176 + }, + { + "epoch": 0.92, + "learning_rate": 4.246740220661986e-05, + "loss": 1.2843, + "theoretical_loss": 3.335933853301627, + "tokens_seen": 3161587712 + }, + { + "epoch": 0.92, + "learning_rate": 4.244734202607824e-05, + "loss": 1.2995, + "theoretical_loss": 3.3359286380147504, + "tokens_seen": 3161653248 + }, + { + "epoch": 0.92, + "learning_rate": 4.2427281845536614e-05, + "loss": 1.3452, + "theoretical_loss": 3.335923422866246, + "tokens_seen": 3161718784 + }, + { + "epoch": 0.92, + "learning_rate": 4.240722166499498e-05, + "loss": 1.3024, + "theoretical_loss": 3.335918207856107, + "tokens_seen": 3161784320 + }, + { + "epoch": 0.92, + "learning_rate": 4.238716148445336e-05, + "loss": 1.3243, + "theoretical_loss": 3.335912992984327, + "tokens_seen": 3161849856 + }, + { + "epoch": 0.92, + "learning_rate": 4.2367101303911734e-05, + "loss": 1.2425, + "theoretical_loss": 3.3359077782508995, + "tokens_seen": 3161915392 + }, + { + "epoch": 0.92, + "learning_rate": 4.234704112337011e-05, + "loss": 1.3339, + "theoretical_loss": 3.3359025636558175, + "tokens_seen": 3161980928 + }, + { + "epoch": 0.92, + "learning_rate": 4.2326980942828486e-05, + "loss": 1.3079, + "theoretical_loss": 3.3358973491990747, + "tokens_seen": 3162046464 + }, + { + "epoch": 0.92, + "learning_rate": 4.230692076228686e-05, + "loss": 1.2866, + "theoretical_loss": 3.3358921348806647, + "tokens_seen": 3162112000 + }, + { + "epoch": 0.92, + "learning_rate": 4.228686058174524e-05, + "loss": 1.3976, + "theoretical_loss": 3.335886920700581, + "tokens_seen": 3162177536 + }, + { + "epoch": 0.92, + "learning_rate": 4.226680040120361e-05, + "loss": 1.254, + "theoretical_loss": 3.335881706658817, + "tokens_seen": 3162243072 + }, + { + "epoch": 0.92, + "learning_rate": 4.224674022066199e-05, + "loss": 1.2119, + "theoretical_loss": 3.335876492755366, + "tokens_seen": 3162308608 + }, + { + "epoch": 0.92, + "learning_rate": 4.2226680040120365e-05, + "loss": 1.2555, + "theoretical_loss": 3.3358712789902216, + "tokens_seen": 3162374144 + }, + { + "epoch": 0.92, + "learning_rate": 4.2206619859578734e-05, + "loss": 1.2412, + "theoretical_loss": 3.335866065363377, + "tokens_seen": 3162439680 + }, + { + "epoch": 0.92, + "learning_rate": 4.218655967903711e-05, + "loss": 1.2724, + "theoretical_loss": 3.3358608518748265, + "tokens_seen": 3162505216 + }, + { + "epoch": 0.92, + "learning_rate": 4.2166499498495485e-05, + "loss": 1.189, + "theoretical_loss": 3.3358556385245626, + "tokens_seen": 3162570752 + }, + { + "epoch": 0.92, + "learning_rate": 4.214643931795386e-05, + "loss": 1.2102, + "theoretical_loss": 3.3358504253125796, + "tokens_seen": 3162636288 + }, + { + "epoch": 0.92, + "learning_rate": 4.2126379137412244e-05, + "loss": 1.3473, + "theoretical_loss": 3.33584521223887, + "tokens_seen": 3162701824 + }, + { + "epoch": 0.92, + "learning_rate": 4.210631895687061e-05, + "loss": 1.3251, + "theoretical_loss": 3.335839999303428, + "tokens_seen": 3162767360 + }, + { + "epoch": 0.92, + "learning_rate": 4.208625877632899e-05, + "loss": 1.2249, + "theoretical_loss": 3.335834786506247, + "tokens_seen": 3162832896 + }, + { + "epoch": 0.92, + "learning_rate": 4.2066198595787364e-05, + "loss": 1.197, + "theoretical_loss": 3.33582957384732, + "tokens_seen": 3162898432 + }, + { + "epoch": 0.92, + "learning_rate": 4.204613841524574e-05, + "loss": 1.2428, + "theoretical_loss": 3.3358243613266407, + "tokens_seen": 3162963968 + }, + { + "epoch": 0.92, + "learning_rate": 4.2026078234704116e-05, + "loss": 1.2712, + "theoretical_loss": 3.335819148944203, + "tokens_seen": 3163029504 + }, + { + "epoch": 0.92, + "learning_rate": 4.2006018054162485e-05, + "loss": 1.2591, + "theoretical_loss": 3.3358139366999997, + "tokens_seen": 3163095040 + }, + { + "epoch": 0.92, + "learning_rate": 4.198595787362086e-05, + "loss": 1.3888, + "theoretical_loss": 3.335808724594025, + "tokens_seen": 3163160576 + }, + { + "epoch": 0.92, + "learning_rate": 4.1965897693079237e-05, + "loss": 1.2534, + "theoretical_loss": 3.3358035126262715, + "tokens_seen": 3163226112 + }, + { + "epoch": 0.92, + "learning_rate": 4.194583751253762e-05, + "loss": 1.2515, + "theoretical_loss": 3.335798300796734, + "tokens_seen": 3163291648 + }, + { + "epoch": 0.92, + "learning_rate": 4.192577733199599e-05, + "loss": 1.1194, + "theoretical_loss": 3.335793089105404, + "tokens_seen": 3163357184 + }, + { + "epoch": 0.92, + "learning_rate": 4.1905717151454364e-05, + "loss": 1.2015, + "theoretical_loss": 3.335787877552277, + "tokens_seen": 3163422720 + }, + { + "epoch": 0.92, + "learning_rate": 4.188565697091274e-05, + "loss": 1.2591, + "theoretical_loss": 3.3357826661373453, + "tokens_seen": 3163488256 + }, + { + "epoch": 0.92, + "learning_rate": 4.1865596790371115e-05, + "loss": 1.2734, + "theoretical_loss": 3.3357774548606023, + "tokens_seen": 3163553792 + }, + { + "epoch": 0.92, + "learning_rate": 4.184553660982949e-05, + "loss": 1.235, + "theoretical_loss": 3.335772243722042, + "tokens_seen": 3163619328 + }, + { + "epoch": 0.92, + "learning_rate": 4.182547642928786e-05, + "loss": 1.2821, + "theoretical_loss": 3.335767032721658, + "tokens_seen": 3163684864 + }, + { + "epoch": 0.92, + "learning_rate": 4.1805416248746236e-05, + "loss": 1.2522, + "theoretical_loss": 3.335761821859443, + "tokens_seen": 3163750400 + }, + { + "epoch": 0.92, + "learning_rate": 4.178535606820461e-05, + "loss": 1.2549, + "theoretical_loss": 3.335756611135391, + "tokens_seen": 3163815936 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1149760, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3347498178482056, + "objective/train/theoretical_loss": 3.3357514005494955, + "objective/train/tokens_used": 1534340576, + "theoretical_loss": 3.3357514005494955, + "tokens_seen": 3163881472 + }, + { + "epoch": 0.92, + "learning_rate": 4.176529588766299e-05, + "loss": 1.3357, + "theoretical_loss": 3.3357514005494955, + "tokens_seen": 3163881472 + }, + { + "epoch": 0.92, + "learning_rate": 4.174523570712137e-05, + "loss": 1.1763, + "theoretical_loss": 3.3357461901017498, + "tokens_seen": 3163947008 + }, + { + "epoch": 0.92, + "learning_rate": 4.172517552657974e-05, + "loss": 1.2548, + "theoretical_loss": 3.3357409797921473, + "tokens_seen": 3164012544 + }, + { + "epoch": 0.92, + "learning_rate": 4.1705115346038115e-05, + "loss": 1.2481, + "theoretical_loss": 3.335735769620682, + "tokens_seen": 3164078080 + }, + { + "epoch": 0.92, + "learning_rate": 4.168505516549649e-05, + "loss": 1.2568, + "theoretical_loss": 3.335730559587347, + "tokens_seen": 3164143616 + }, + { + "epoch": 0.92, + "learning_rate": 4.166499498495487e-05, + "loss": 1.2242, + "theoretical_loss": 3.3357253496921357, + "tokens_seen": 3164209152 + }, + { + "epoch": 0.92, + "learning_rate": 4.164493480441324e-05, + "loss": 1.176, + "theoretical_loss": 3.3357201399350416, + "tokens_seen": 3164274688 + }, + { + "epoch": 0.92, + "learning_rate": 4.162487462387161e-05, + "loss": 1.2482, + "theoretical_loss": 3.3357149303160583, + "tokens_seen": 3164340224 + }, + { + "epoch": 0.92, + "learning_rate": 4.160481444332999e-05, + "loss": 1.1745, + "theoretical_loss": 3.3357097208351787, + "tokens_seen": 3164405760 + }, + { + "epoch": 0.92, + "learning_rate": 4.158475426278836e-05, + "loss": 1.2409, + "theoretical_loss": 3.3357045114923976, + "tokens_seen": 3164471296 + }, + { + "epoch": 0.92, + "learning_rate": 4.1564694082246746e-05, + "loss": 1.2745, + "theoretical_loss": 3.335699302287707, + "tokens_seen": 3164536832 + }, + { + "epoch": 0.92, + "learning_rate": 4.154463390170512e-05, + "loss": 1.1979, + "theoretical_loss": 3.335694093221101, + "tokens_seen": 3164602368 + }, + { + "epoch": 0.92, + "learning_rate": 4.152457372116349e-05, + "loss": 1.3027, + "theoretical_loss": 3.335688884292574, + "tokens_seen": 3164667904 + }, + { + "epoch": 0.92, + "learning_rate": 4.1504513540621866e-05, + "loss": 1.2482, + "theoretical_loss": 3.3356836755021178, + "tokens_seen": 3164733440 + }, + { + "epoch": 0.92, + "learning_rate": 4.148445336008024e-05, + "loss": 1.1411, + "theoretical_loss": 3.335678466849727, + "tokens_seen": 3164798976 + }, + { + "epoch": 0.92, + "learning_rate": 4.146439317953862e-05, + "loss": 1.228, + "theoretical_loss": 3.3356732583353947, + "tokens_seen": 3164864512 + }, + { + "epoch": 0.92, + "learning_rate": 4.1444332998996994e-05, + "loss": 1.2168, + "theoretical_loss": 3.3356680499591147, + "tokens_seen": 3164930048 + }, + { + "epoch": 0.92, + "learning_rate": 4.142427281845536e-05, + "loss": 1.3234, + "theoretical_loss": 3.33566284172088, + "tokens_seen": 3164995584 + }, + { + "epoch": 0.92, + "learning_rate": 4.140421263791374e-05, + "loss": 1.2297, + "theoretical_loss": 3.3356576336206842, + "tokens_seen": 3165061120 + }, + { + "epoch": 0.92, + "learning_rate": 4.138415245737212e-05, + "loss": 1.2326, + "theoretical_loss": 3.3356524256585214, + "tokens_seen": 3165126656 + }, + { + "epoch": 0.92, + "learning_rate": 4.13640922768305e-05, + "loss": 1.1925, + "theoretical_loss": 3.335647217834384, + "tokens_seen": 3165192192 + }, + { + "epoch": 0.92, + "learning_rate": 4.134403209628887e-05, + "loss": 1.2672, + "theoretical_loss": 3.3356420101482667, + "tokens_seen": 3165257728 + }, + { + "epoch": 0.92, + "learning_rate": 4.132397191574724e-05, + "loss": 1.153, + "theoretical_loss": 3.335636802600162, + "tokens_seen": 3165323264 + }, + { + "epoch": 0.92, + "learning_rate": 4.130391173520562e-05, + "loss": 1.1988, + "theoretical_loss": 3.3356315951900637, + "tokens_seen": 3165388800 + }, + { + "epoch": 0.92, + "learning_rate": 4.128385155466399e-05, + "loss": 1.2385, + "theoretical_loss": 3.3356263879179653, + "tokens_seen": 3165454336 + }, + { + "epoch": 0.92, + "learning_rate": 4.126379137412237e-05, + "loss": 1.1913, + "theoretical_loss": 3.3356211807838605, + "tokens_seen": 3165519872 + }, + { + "epoch": 0.92, + "learning_rate": 4.124373119358074e-05, + "loss": 1.1645, + "theoretical_loss": 3.3356159737877427, + "tokens_seen": 3165585408 + }, + { + "epoch": 0.92, + "learning_rate": 4.1223671013039114e-05, + "loss": 1.2213, + "theoretical_loss": 3.335610766929605, + "tokens_seen": 3165650944 + }, + { + "epoch": 0.92, + "learning_rate": 4.120361083249749e-05, + "loss": 1.2076, + "theoretical_loss": 3.335605560209441, + "tokens_seen": 3165716480 + }, + { + "epoch": 0.92, + "learning_rate": 4.118355065195587e-05, + "loss": 1.2008, + "theoretical_loss": 3.3356003536272447, + "tokens_seen": 3165782016 + }, + { + "epoch": 0.92, + "learning_rate": 4.116349047141425e-05, + "loss": 1.2819, + "theoretical_loss": 3.3355951471830094, + "tokens_seen": 3165847552 + }, + { + "epoch": 0.92, + "learning_rate": 4.114343029087262e-05, + "loss": 1.1804, + "theoretical_loss": 3.335589940876728, + "tokens_seen": 3165913088 + }, + { + "epoch": 0.92, + "learning_rate": 4.112337011033099e-05, + "loss": 1.2733, + "theoretical_loss": 3.335584734708395, + "tokens_seen": 3165978624 + }, + { + "epoch": 0.92, + "learning_rate": 4.110330992978937e-05, + "loss": 1.2726, + "theoretical_loss": 3.335579528678003, + "tokens_seen": 3166044160 + }, + { + "epoch": 0.92, + "learning_rate": 4.1083249749247744e-05, + "loss": 1.2149, + "theoretical_loss": 3.3355743227855457, + "tokens_seen": 3166109696 + }, + { + "epoch": 0.92, + "learning_rate": 4.106318956870612e-05, + "loss": 1.2791, + "theoretical_loss": 3.3355691170310164, + "tokens_seen": 3166175232 + }, + { + "epoch": 0.92, + "learning_rate": 4.104312938816449e-05, + "loss": 1.3315, + "theoretical_loss": 3.3355639114144093, + "tokens_seen": 3166240768 + }, + { + "epoch": 0.92, + "learning_rate": 4.1023069207622865e-05, + "loss": 1.2813, + "theoretical_loss": 3.3355587059357177, + "tokens_seen": 3166306304 + }, + { + "epoch": 0.92, + "learning_rate": 4.100300902708125e-05, + "loss": 1.2571, + "theoretical_loss": 3.3355535005949344, + "tokens_seen": 3166371840 + }, + { + "epoch": 0.92, + "learning_rate": 4.098294884653962e-05, + "loss": 1.2016, + "theoretical_loss": 3.3355482953920537, + "tokens_seen": 3166437376 + }, + { + "epoch": 0.92, + "learning_rate": 4.0962888665998e-05, + "loss": 1.2523, + "theoretical_loss": 3.335543090327069, + "tokens_seen": 3166502912 + }, + { + "epoch": 0.92, + "learning_rate": 4.094282848545637e-05, + "loss": 1.2303, + "theoretical_loss": 3.335537885399973, + "tokens_seen": 3166568448 + }, + { + "epoch": 0.92, + "learning_rate": 4.0922768304914744e-05, + "loss": 1.2607, + "theoretical_loss": 3.33553268061076, + "tokens_seen": 3166633984 + }, + { + "epoch": 0.92, + "learning_rate": 4.090270812437312e-05, + "loss": 1.3269, + "theoretical_loss": 3.335527475959423, + "tokens_seen": 3166699520 + }, + { + "epoch": 0.92, + "learning_rate": 4.0882647943831495e-05, + "loss": 1.3382, + "theoretical_loss": 3.335522271445956, + "tokens_seen": 3166765056 + }, + { + "epoch": 0.92, + "learning_rate": 4.086258776328987e-05, + "loss": 1.2476, + "theoretical_loss": 3.3355170670703522, + "tokens_seen": 3166830592 + }, + { + "epoch": 0.92, + "learning_rate": 4.084252758274824e-05, + "loss": 1.2261, + "theoretical_loss": 3.335511862832605, + "tokens_seen": 3166896128 + }, + { + "epoch": 0.92, + "learning_rate": 4.0822467402206616e-05, + "loss": 1.2053, + "theoretical_loss": 3.335506658732708, + "tokens_seen": 3166961664 + }, + { + "epoch": 0.92, + "learning_rate": 4.0802407221665e-05, + "loss": 1.2507, + "theoretical_loss": 3.3355014547706547, + "tokens_seen": 3167027200 + }, + { + "epoch": 0.92, + "learning_rate": 4.0782347041123374e-05, + "loss": 1.1478, + "theoretical_loss": 3.3354962509464388, + "tokens_seen": 3167092736 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1150963, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2981247901916504, + "objective/train/theoretical_loss": 3.335491047260054, + "objective/train/tokens_used": 1537617376, + "theoretical_loss": 3.335491047260054, + "tokens_seen": 3167158272 + }, + { + "epoch": 0.92, + "learning_rate": 4.076228686058175e-05, + "loss": 1.2483, + "theoretical_loss": 3.335491047260054, + "tokens_seen": 3167158272 + }, + { + "epoch": 0.92, + "learning_rate": 4.074222668004012e-05, + "loss": 1.3542, + "theoretical_loss": 3.335485843711493, + "tokens_seen": 3167223808 + }, + { + "epoch": 0.92, + "learning_rate": 4.0722166499498495e-05, + "loss": 1.2101, + "theoretical_loss": 3.3354806403007493, + "tokens_seen": 3167289344 + }, + { + "epoch": 0.92, + "learning_rate": 4.070210631895687e-05, + "loss": 1.2149, + "theoretical_loss": 3.3354754370278172, + "tokens_seen": 3167354880 + }, + { + "epoch": 0.92, + "learning_rate": 4.068204613841525e-05, + "loss": 1.2491, + "theoretical_loss": 3.3354702338926896, + "tokens_seen": 3167420416 + }, + { + "epoch": 0.92, + "learning_rate": 4.066198595787362e-05, + "loss": 1.2937, + "theoretical_loss": 3.33546503089536, + "tokens_seen": 3167485952 + }, + { + "epoch": 0.92, + "learning_rate": 4.064192577733199e-05, + "loss": 1.3479, + "theoretical_loss": 3.3354598280358227, + "tokens_seen": 3167551488 + }, + { + "epoch": 0.92, + "learning_rate": 4.0621865596790374e-05, + "loss": 1.2743, + "theoretical_loss": 3.33545462531407, + "tokens_seen": 3167617024 + }, + { + "epoch": 0.92, + "learning_rate": 4.060180541624875e-05, + "loss": 1.2649, + "theoretical_loss": 3.335449422730097, + "tokens_seen": 3167682560 + }, + { + "epoch": 0.92, + "learning_rate": 4.0581745235707126e-05, + "loss": 1.2071, + "theoretical_loss": 3.335444220283895, + "tokens_seen": 3167748096 + }, + { + "epoch": 0.92, + "learning_rate": 4.05616850551655e-05, + "loss": 1.2345, + "theoretical_loss": 3.3354390179754594, + "tokens_seen": 3167813632 + }, + { + "epoch": 0.92, + "learning_rate": 4.054162487462387e-05, + "loss": 1.182, + "theoretical_loss": 3.335433815804783, + "tokens_seen": 3167879168 + }, + { + "epoch": 0.92, + "learning_rate": 4.0521564694082246e-05, + "loss": 1.2945, + "theoretical_loss": 3.335428613771859, + "tokens_seen": 3167944704 + }, + { + "epoch": 0.92, + "learning_rate": 4.050150451354062e-05, + "loss": 1.1854, + "theoretical_loss": 3.3354234118766817, + "tokens_seen": 3168010240 + }, + { + "epoch": 0.92, + "learning_rate": 4.0481444332999e-05, + "loss": 1.3149, + "theoretical_loss": 3.3354182101192436, + "tokens_seen": 3168075776 + }, + { + "epoch": 0.92, + "learning_rate": 4.046138415245737e-05, + "loss": 1.3133, + "theoretical_loss": 3.335413008499539, + "tokens_seen": 3168141312 + }, + { + "epoch": 0.92, + "learning_rate": 4.044132397191575e-05, + "loss": 1.2084, + "theoretical_loss": 3.335407807017561, + "tokens_seen": 3168206848 + }, + { + "epoch": 0.92, + "learning_rate": 4.0421263791374125e-05, + "loss": 1.3434, + "theoretical_loss": 3.335402605673303, + "tokens_seen": 3168272384 + }, + { + "epoch": 0.92, + "learning_rate": 4.04012036108325e-05, + "loss": 1.2902, + "theoretical_loss": 3.335397404466759, + "tokens_seen": 3168337920 + }, + { + "epoch": 0.92, + "learning_rate": 4.038114343029088e-05, + "loss": 1.3029, + "theoretical_loss": 3.335392203397922, + "tokens_seen": 3168403456 + }, + { + "epoch": 0.92, + "learning_rate": 4.0361083249749246e-05, + "loss": 1.2531, + "theoretical_loss": 3.3353870024667858, + "tokens_seen": 3168468992 + }, + { + "epoch": 0.92, + "learning_rate": 4.034102306920762e-05, + "loss": 1.2823, + "theoretical_loss": 3.335381801673344, + "tokens_seen": 3168534528 + }, + { + "epoch": 0.92, + "learning_rate": 4.0320962888666e-05, + "loss": 1.4167, + "theoretical_loss": 3.33537660101759, + "tokens_seen": 3168600064 + }, + { + "epoch": 0.92, + "learning_rate": 4.030090270812437e-05, + "loss": 1.3342, + "theoretical_loss": 3.3353714004995174, + "tokens_seen": 3168665600 + }, + { + "epoch": 0.92, + "learning_rate": 4.028084252758275e-05, + "loss": 1.2944, + "theoretical_loss": 3.335366200119119, + "tokens_seen": 3168731136 + }, + { + "epoch": 0.92, + "learning_rate": 4.026078234704112e-05, + "loss": 1.298, + "theoretical_loss": 3.3353609998763893, + "tokens_seen": 3168796672 + }, + { + "epoch": 0.92, + "learning_rate": 4.02407221664995e-05, + "loss": 1.1669, + "theoretical_loss": 3.3353557997713215, + "tokens_seen": 3168862208 + }, + { + "epoch": 0.92, + "learning_rate": 4.0220661985957876e-05, + "loss": 1.2502, + "theoretical_loss": 3.335350599803909, + "tokens_seen": 3168927744 + }, + { + "epoch": 0.92, + "learning_rate": 4.020060180541625e-05, + "loss": 1.3765, + "theoretical_loss": 3.3353453999741447, + "tokens_seen": 3168993280 + }, + { + "epoch": 0.92, + "learning_rate": 4.018054162487463e-05, + "loss": 1.2997, + "theoretical_loss": 3.3353402002820234, + "tokens_seen": 3169058816 + }, + { + "epoch": 0.92, + "learning_rate": 4.0160481444333e-05, + "loss": 1.3257, + "theoretical_loss": 3.3353350007275377, + "tokens_seen": 3169124352 + }, + { + "epoch": 0.92, + "learning_rate": 4.014042126379137e-05, + "loss": 1.3571, + "theoretical_loss": 3.335329801310681, + "tokens_seen": 3169189888 + }, + { + "epoch": 0.92, + "learning_rate": 4.012036108324975e-05, + "loss": 1.3639, + "theoretical_loss": 3.3353246020314478, + "tokens_seen": 3169255424 + }, + { + "epoch": 0.92, + "learning_rate": 4.0100300902708124e-05, + "loss": 1.279, + "theoretical_loss": 3.33531940288983, + "tokens_seen": 3169320960 + }, + { + "epoch": 0.92, + "learning_rate": 4.00802407221665e-05, + "loss": 1.2148, + "theoretical_loss": 3.3353142038858232, + "tokens_seen": 3169386496 + }, + { + "epoch": 0.92, + "learning_rate": 4.0060180541624876e-05, + "loss": 1.2582, + "theoretical_loss": 3.335309005019419, + "tokens_seen": 3169452032 + }, + { + "epoch": 0.92, + "learning_rate": 4.004012036108325e-05, + "loss": 1.2083, + "theoretical_loss": 3.3353038062906117, + "tokens_seen": 3169517568 + }, + { + "epoch": 0.92, + "learning_rate": 4.002006018054163e-05, + "loss": 1.2734, + "theoretical_loss": 3.335298607699395, + "tokens_seen": 3169583104 + }, + { + "epoch": 0.92, + "learning_rate": 4e-05, + "loss": 1.2813, + "theoretical_loss": 3.3352934092457627, + "tokens_seen": 3169648640 + }, + { + "epoch": 0.92, + "learning_rate": 3.997993981945838e-05, + "loss": 1.2224, + "theoretical_loss": 3.3352882109297073, + "tokens_seen": 3169714176 + }, + { + "epoch": 0.92, + "learning_rate": 3.995987963891675e-05, + "loss": 1.2655, + "theoretical_loss": 3.3352830127512227, + "tokens_seen": 3169779712 + }, + { + "epoch": 0.92, + "learning_rate": 3.9939819458375124e-05, + "loss": 1.187, + "theoretical_loss": 3.335277814710303, + "tokens_seen": 3169845248 + }, + { + "epoch": 0.92, + "learning_rate": 3.99197592778335e-05, + "loss": 1.3572, + "theoretical_loss": 3.335272616806941, + "tokens_seen": 3169910784 + }, + { + "epoch": 0.92, + "learning_rate": 3.9899699097291875e-05, + "loss": 1.397, + "theoretical_loss": 3.3352674190411307, + "tokens_seen": 3169976320 + }, + { + "epoch": 0.92, + "learning_rate": 3.987963891675026e-05, + "loss": 1.3049, + "theoretical_loss": 3.335262221412865, + "tokens_seen": 3170041856 + }, + { + "epoch": 0.92, + "learning_rate": 3.985957873620863e-05, + "loss": 1.2751, + "theoretical_loss": 3.3352570239221384, + "tokens_seen": 3170107392 + }, + { + "epoch": 0.92, + "learning_rate": 3.9839518555667e-05, + "loss": 1.1672, + "theoretical_loss": 3.3352518265689435, + "tokens_seen": 3170172928 + }, + { + "epoch": 0.92, + "learning_rate": 3.981945837512538e-05, + "loss": 1.2612, + "theoretical_loss": 3.335246629353274, + "tokens_seen": 3170238464 + }, + { + "epoch": 0.92, + "learning_rate": 3.9799398194583754e-05, + "loss": 1.2787, + "theoretical_loss": 3.335241432275124, + "tokens_seen": 3170304000 + }, + { + "epoch": 0.92, + "learning_rate": 3.9779338014042123e-05, + "loss": 1.2724, + "theoretical_loss": 3.335236235334486, + "tokens_seen": 3170369536 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1152259, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.282792329788208, + "objective/train/theoretical_loss": 3.335231038531355, + "objective/train/tokens_used": 1540894176, + "theoretical_loss": 3.335231038531355, + "tokens_seen": 3170435072 + }, + { + "epoch": 0.92, + "learning_rate": 3.97592778335005e-05, + "loss": 1.2144, + "theoretical_loss": 3.335231038531355, + "tokens_seen": 3170435072 + }, + { + "epoch": 0.92, + "learning_rate": 3.9739217652958875e-05, + "loss": 1.3341, + "theoretical_loss": 3.335225841865723, + "tokens_seen": 3170500608 + }, + { + "epoch": 0.92, + "learning_rate": 3.971915747241725e-05, + "loss": 1.4197, + "theoretical_loss": 3.335220645337584, + "tokens_seen": 3170566144 + }, + { + "epoch": 0.92, + "learning_rate": 3.969909729187563e-05, + "loss": 1.1527, + "theoretical_loss": 3.3352154489469323, + "tokens_seen": 3170631680 + }, + { + "epoch": 0.92, + "learning_rate": 3.9679037111334e-05, + "loss": 1.2158, + "theoretical_loss": 3.33521025269376, + "tokens_seen": 3170697216 + }, + { + "epoch": 0.92, + "learning_rate": 3.965897693079238e-05, + "loss": 1.2928, + "theoretical_loss": 3.3352050565780624, + "tokens_seen": 3170762752 + }, + { + "epoch": 0.92, + "learning_rate": 3.9638916750250754e-05, + "loss": 1.2467, + "theoretical_loss": 3.3351998605998316, + "tokens_seen": 3170828288 + }, + { + "epoch": 0.92, + "learning_rate": 3.961885656970913e-05, + "loss": 1.3147, + "theoretical_loss": 3.3351946647590616, + "tokens_seen": 3170893824 + }, + { + "epoch": 0.92, + "learning_rate": 3.9598796389167506e-05, + "loss": 1.1965, + "theoretical_loss": 3.3351894690557455, + "tokens_seen": 3170959360 + }, + { + "epoch": 0.92, + "learning_rate": 3.9578736208625875e-05, + "loss": 1.3294, + "theoretical_loss": 3.3351842734898773, + "tokens_seen": 3171024896 + }, + { + "epoch": 0.92, + "learning_rate": 3.955867602808425e-05, + "loss": 1.2452, + "theoretical_loss": 3.3351790780614508, + "tokens_seen": 3171090432 + }, + { + "epoch": 0.92, + "learning_rate": 3.9538615847542626e-05, + "loss": 1.2613, + "theoretical_loss": 3.335173882770459, + "tokens_seen": 3171155968 + }, + { + "epoch": 0.92, + "learning_rate": 3.9518555667001e-05, + "loss": 1.2146, + "theoretical_loss": 3.3351686876168958, + "tokens_seen": 3171221504 + }, + { + "epoch": 0.92, + "learning_rate": 3.9498495486459385e-05, + "loss": 1.2612, + "theoretical_loss": 3.335163492600754, + "tokens_seen": 3171287040 + }, + { + "epoch": 0.92, + "learning_rate": 3.9478435305917754e-05, + "loss": 1.3038, + "theoretical_loss": 3.335158297722028, + "tokens_seen": 3171352576 + }, + { + "epoch": 0.92, + "learning_rate": 3.945837512537613e-05, + "loss": 1.2596, + "theoretical_loss": 3.335153102980711, + "tokens_seen": 3171418112 + }, + { + "epoch": 0.92, + "learning_rate": 3.9438314944834505e-05, + "loss": 1.2763, + "theoretical_loss": 3.335147908376796, + "tokens_seen": 3171483648 + }, + { + "epoch": 0.92, + "learning_rate": 3.941825476429288e-05, + "loss": 1.2466, + "theoretical_loss": 3.335142713910278, + "tokens_seen": 3171549184 + }, + { + "epoch": 0.92, + "learning_rate": 3.939819458375126e-05, + "loss": 1.3191, + "theoretical_loss": 3.3351375195811483, + "tokens_seen": 3171614720 + }, + { + "epoch": 0.92, + "learning_rate": 3.9378134403209626e-05, + "loss": 1.2489, + "theoretical_loss": 3.3351323253894027, + "tokens_seen": 3171680256 + }, + { + "epoch": 0.92, + "learning_rate": 3.9358074222668e-05, + "loss": 1.2548, + "theoretical_loss": 3.335127131335033, + "tokens_seen": 3171745792 + }, + { + "epoch": 0.92, + "learning_rate": 3.933801404212638e-05, + "loss": 1.2081, + "theoretical_loss": 3.335121937418034, + "tokens_seen": 3171811328 + }, + { + "epoch": 0.92, + "learning_rate": 3.931795386158476e-05, + "loss": 1.271, + "theoretical_loss": 3.335116743638398, + "tokens_seen": 3171876864 + }, + { + "epoch": 0.92, + "learning_rate": 3.9297893681043136e-05, + "loss": 1.2297, + "theoretical_loss": 3.3351115499961197, + "tokens_seen": 3171942400 + }, + { + "epoch": 0.92, + "learning_rate": 3.9277833500501505e-05, + "loss": 1.1892, + "theoretical_loss": 3.335106356491192, + "tokens_seen": 3172007936 + }, + { + "epoch": 0.92, + "learning_rate": 3.925777331995988e-05, + "loss": 1.2346, + "theoretical_loss": 3.3351011631236083, + "tokens_seen": 3172073472 + }, + { + "epoch": 0.92, + "learning_rate": 3.9237713139418256e-05, + "loss": 1.2885, + "theoretical_loss": 3.3350959698933624, + "tokens_seen": 3172139008 + }, + { + "epoch": 0.92, + "learning_rate": 3.921765295887663e-05, + "loss": 1.3634, + "theoretical_loss": 3.335090776800448, + "tokens_seen": 3172204544 + }, + { + "epoch": 0.92, + "learning_rate": 3.919759277833501e-05, + "loss": 1.1991, + "theoretical_loss": 3.3350855838448585, + "tokens_seen": 3172270080 + }, + { + "epoch": 0.92, + "learning_rate": 3.917753259779338e-05, + "loss": 1.3241, + "theoretical_loss": 3.335080391026587, + "tokens_seen": 3172335616 + }, + { + "epoch": 0.92, + "learning_rate": 3.915747241725175e-05, + "loss": 1.2042, + "theoretical_loss": 3.3350751983456277, + "tokens_seen": 3172401152 + }, + { + "epoch": 0.92, + "learning_rate": 3.913741223671013e-05, + "loss": 1.2522, + "theoretical_loss": 3.3350700058019735, + "tokens_seen": 3172466688 + }, + { + "epoch": 0.92, + "learning_rate": 3.911735205616851e-05, + "loss": 1.1589, + "theoretical_loss": 3.3350648133956184, + "tokens_seen": 3172532224 + }, + { + "epoch": 0.92, + "learning_rate": 3.909729187562689e-05, + "loss": 1.2833, + "theoretical_loss": 3.335059621126556, + "tokens_seen": 3172597760 + }, + { + "epoch": 0.92, + "learning_rate": 3.9077231695085256e-05, + "loss": 1.2333, + "theoretical_loss": 3.3350544289947797, + "tokens_seen": 3172663296 + }, + { + "epoch": 0.92, + "learning_rate": 3.905717151454363e-05, + "loss": 1.3292, + "theoretical_loss": 3.3350492370002827, + "tokens_seen": 3172728832 + }, + { + "epoch": 0.92, + "learning_rate": 3.903711133400201e-05, + "loss": 1.3853, + "theoretical_loss": 3.335044045143059, + "tokens_seen": 3172794368 + }, + { + "epoch": 0.92, + "learning_rate": 3.901705115346038e-05, + "loss": 1.1974, + "theoretical_loss": 3.3350388534231015, + "tokens_seen": 3172859904 + }, + { + "epoch": 0.92, + "learning_rate": 3.899699097291875e-05, + "loss": 1.2925, + "theoretical_loss": 3.335033661840405, + "tokens_seen": 3172925440 + }, + { + "epoch": 0.92, + "learning_rate": 3.897693079237713e-05, + "loss": 1.29, + "theoretical_loss": 3.335028470394961, + "tokens_seen": 3172990976 + }, + { + "epoch": 0.92, + "learning_rate": 3.8956870611835504e-05, + "loss": 1.2213, + "theoretical_loss": 3.3350232790867653, + "tokens_seen": 3173056512 + }, + { + "epoch": 0.92, + "learning_rate": 3.8936810431293886e-05, + "loss": 1.2265, + "theoretical_loss": 3.3350180879158096, + "tokens_seen": 3173122048 + }, + { + "epoch": 0.92, + "learning_rate": 3.891675025075226e-05, + "loss": 1.2382, + "theoretical_loss": 3.335012896882089, + "tokens_seen": 3173187584 + }, + { + "epoch": 0.92, + "learning_rate": 3.889669007021063e-05, + "loss": 1.2474, + "theoretical_loss": 3.335007705985596, + "tokens_seen": 3173253120 + }, + { + "epoch": 0.92, + "learning_rate": 3.887662988966901e-05, + "loss": 1.2511, + "theoretical_loss": 3.335002515226324, + "tokens_seen": 3173318656 + }, + { + "epoch": 0.92, + "learning_rate": 3.885656970912738e-05, + "loss": 1.179, + "theoretical_loss": 3.334997324604267, + "tokens_seen": 3173384192 + }, + { + "epoch": 0.92, + "learning_rate": 3.883650952858576e-05, + "loss": 1.3581, + "theoretical_loss": 3.3349921341194184, + "tokens_seen": 3173449728 + }, + { + "epoch": 0.92, + "learning_rate": 3.8816449348044134e-05, + "loss": 1.2506, + "theoretical_loss": 3.3349869437717725, + "tokens_seen": 3173515264 + }, + { + "epoch": 0.92, + "learning_rate": 3.8796389167502503e-05, + "loss": 1.2539, + "theoretical_loss": 3.3349817535613218, + "tokens_seen": 3173580800 + }, + { + "epoch": 0.92, + "learning_rate": 3.877632898696088e-05, + "loss": 1.2905, + "theoretical_loss": 3.33497656348806, + "tokens_seen": 3173646336 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1153509, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4432129859924316, + "objective/train/theoretical_loss": 3.334971373551981, + "objective/train/tokens_used": 1544170976, + "theoretical_loss": 3.334971373551981, + "tokens_seen": 3173711872 + }, + { + "epoch": 0.92, + "learning_rate": 3.875626880641926e-05, + "loss": 1.2822, + "theoretical_loss": 3.334971373551981, + "tokens_seen": 3173711872 + }, + { + "epoch": 0.92, + "learning_rate": 3.873620862587764e-05, + "loss": 1.22, + "theoretical_loss": 3.3349661837530777, + "tokens_seen": 3173777408 + }, + { + "epoch": 0.92, + "learning_rate": 3.871614844533601e-05, + "loss": 1.2762, + "theoretical_loss": 3.3349609940913445, + "tokens_seen": 3173842944 + }, + { + "epoch": 0.92, + "learning_rate": 3.869608826479438e-05, + "loss": 1.3065, + "theoretical_loss": 3.3349558045667744, + "tokens_seen": 3173908480 + }, + { + "epoch": 0.92, + "learning_rate": 3.867602808425276e-05, + "loss": 1.2465, + "theoretical_loss": 3.3349506151793613, + "tokens_seen": 3173974016 + }, + { + "epoch": 0.92, + "learning_rate": 3.8655967903711134e-05, + "loss": 1.2602, + "theoretical_loss": 3.3349454259290985, + "tokens_seen": 3174039552 + }, + { + "epoch": 0.92, + "learning_rate": 3.863590772316951e-05, + "loss": 1.3327, + "theoretical_loss": 3.3349402368159797, + "tokens_seen": 3174105088 + }, + { + "epoch": 0.92, + "learning_rate": 3.8615847542627886e-05, + "loss": 1.1824, + "theoretical_loss": 3.334935047839998, + "tokens_seen": 3174170624 + }, + { + "epoch": 0.92, + "learning_rate": 3.8595787362086255e-05, + "loss": 1.2658, + "theoretical_loss": 3.3349298590011474, + "tokens_seen": 3174236160 + }, + { + "epoch": 0.92, + "learning_rate": 3.857572718154463e-05, + "loss": 1.2703, + "theoretical_loss": 3.3349246702994213, + "tokens_seen": 3174301696 + }, + { + "epoch": 0.92, + "learning_rate": 3.855566700100301e-05, + "loss": 1.2855, + "theoretical_loss": 3.3349194817348136, + "tokens_seen": 3174367232 + }, + { + "epoch": 0.92, + "learning_rate": 3.853560682046139e-05, + "loss": 1.1609, + "theoretical_loss": 3.334914293307317, + "tokens_seen": 3174432768 + }, + { + "epoch": 0.92, + "learning_rate": 3.8515546639919765e-05, + "loss": 1.2526, + "theoretical_loss": 3.334909105016926, + "tokens_seen": 3174498304 + }, + { + "epoch": 0.92, + "learning_rate": 3.8495486459378134e-05, + "loss": 1.2478, + "theoretical_loss": 3.334903916863633, + "tokens_seen": 3174563840 + }, + { + "epoch": 0.92, + "learning_rate": 3.847542627883651e-05, + "loss": 1.1971, + "theoretical_loss": 3.334898728847433, + "tokens_seen": 3174629376 + }, + { + "epoch": 0.92, + "learning_rate": 3.8455366098294885e-05, + "loss": 1.2087, + "theoretical_loss": 3.3348935409683187, + "tokens_seen": 3174694912 + }, + { + "epoch": 0.92, + "learning_rate": 3.843530591775326e-05, + "loss": 1.2974, + "theoretical_loss": 3.3348883532262836, + "tokens_seen": 3174760448 + }, + { + "epoch": 0.92, + "learning_rate": 3.841524573721164e-05, + "loss": 1.2814, + "theoretical_loss": 3.334883165621321, + "tokens_seen": 3174825984 + }, + { + "epoch": 0.92, + "learning_rate": 3.8395185556670006e-05, + "loss": 1.3162, + "theoretical_loss": 3.3348779781534255, + "tokens_seen": 3174891520 + }, + { + "epoch": 0.92, + "learning_rate": 3.837512537612839e-05, + "loss": 1.262, + "theoretical_loss": 3.3348727908225895, + "tokens_seen": 3174957056 + }, + { + "epoch": 0.92, + "learning_rate": 3.8355065195586764e-05, + "loss": 1.1667, + "theoretical_loss": 3.3348676036288074, + "tokens_seen": 3175022592 + }, + { + "epoch": 0.92, + "learning_rate": 3.833500501504514e-05, + "loss": 1.2396, + "theoretical_loss": 3.334862416572072, + "tokens_seen": 3175088128 + }, + { + "epoch": 0.92, + "learning_rate": 3.8314944834503516e-05, + "loss": 1.1941, + "theoretical_loss": 3.3348572296523775, + "tokens_seen": 3175153664 + }, + { + "epoch": 0.92, + "learning_rate": 3.8294884653961885e-05, + "loss": 1.2491, + "theoretical_loss": 3.3348520428697173, + "tokens_seen": 3175219200 + }, + { + "epoch": 0.92, + "learning_rate": 3.827482447342026e-05, + "loss": 1.212, + "theoretical_loss": 3.3348468562240847, + "tokens_seen": 3175284736 + }, + { + "epoch": 0.92, + "learning_rate": 3.8254764292878636e-05, + "loss": 1.3233, + "theoretical_loss": 3.3348416697154732, + "tokens_seen": 3175350272 + }, + { + "epoch": 0.92, + "learning_rate": 3.823470411233701e-05, + "loss": 1.2434, + "theoretical_loss": 3.334836483343877, + "tokens_seen": 3175415808 + }, + { + "epoch": 0.92, + "learning_rate": 3.821464393179538e-05, + "loss": 1.2508, + "theoretical_loss": 3.334831297109289, + "tokens_seen": 3175481344 + }, + { + "epoch": 0.92, + "learning_rate": 3.819458375125376e-05, + "loss": 1.3311, + "theoretical_loss": 3.3348261110117026, + "tokens_seen": 3175546880 + }, + { + "epoch": 0.92, + "learning_rate": 3.817452357071214e-05, + "loss": 1.2702, + "theoretical_loss": 3.3348209250511123, + "tokens_seen": 3175612416 + }, + { + "epoch": 0.92, + "learning_rate": 3.8154463390170515e-05, + "loss": 1.2059, + "theoretical_loss": 3.3348157392275106, + "tokens_seen": 3175677952 + }, + { + "epoch": 0.92, + "learning_rate": 3.813440320962889e-05, + "loss": 1.3301, + "theoretical_loss": 3.334810553540892, + "tokens_seen": 3175743488 + }, + { + "epoch": 0.92, + "learning_rate": 3.811434302908726e-05, + "loss": 1.3001, + "theoretical_loss": 3.3348053679912493, + "tokens_seen": 3175809024 + }, + { + "epoch": 0.92, + "learning_rate": 3.8094282848545636e-05, + "loss": 1.3618, + "theoretical_loss": 3.334800182578576, + "tokens_seen": 3175874560 + }, + { + "epoch": 0.92, + "learning_rate": 3.807422266800401e-05, + "loss": 1.2222, + "theoretical_loss": 3.334794997302866, + "tokens_seen": 3175940096 + }, + { + "epoch": 0.92, + "learning_rate": 3.805416248746239e-05, + "loss": 1.2676, + "theoretical_loss": 3.3347898121641135, + "tokens_seen": 3176005632 + }, + { + "epoch": 0.92, + "learning_rate": 3.803410230692076e-05, + "loss": 1.2708, + "theoretical_loss": 3.334784627162311, + "tokens_seen": 3176071168 + }, + { + "epoch": 0.92, + "learning_rate": 3.801404212637913e-05, + "loss": 1.3293, + "theoretical_loss": 3.334779442297452, + "tokens_seen": 3176136704 + }, + { + "epoch": 0.92, + "learning_rate": 3.7993981945837515e-05, + "loss": 1.1686, + "theoretical_loss": 3.3347742575695314, + "tokens_seen": 3176202240 + }, + { + "epoch": 0.93, + "learning_rate": 3.797392176529589e-05, + "loss": 1.1868, + "theoretical_loss": 3.334769072978541, + "tokens_seen": 3176267776 + }, + { + "epoch": 0.93, + "learning_rate": 3.7953861584754266e-05, + "loss": 1.164, + "theoretical_loss": 3.334763888524476, + "tokens_seen": 3176333312 + }, + { + "epoch": 0.93, + "learning_rate": 3.793380140421264e-05, + "loss": 1.2851, + "theoretical_loss": 3.334758704207329, + "tokens_seen": 3176398848 + }, + { + "epoch": 0.93, + "learning_rate": 3.791374122367101e-05, + "loss": 1.1949, + "theoretical_loss": 3.334753520027093, + "tokens_seen": 3176464384 + }, + { + "epoch": 0.93, + "learning_rate": 3.789368104312939e-05, + "loss": 1.1931, + "theoretical_loss": 3.334748335983763, + "tokens_seen": 3176529920 + }, + { + "epoch": 0.93, + "learning_rate": 3.787362086258776e-05, + "loss": 1.2806, + "theoretical_loss": 3.3347431520773316, + "tokens_seen": 3176595456 + }, + { + "epoch": 0.93, + "learning_rate": 3.785356068204614e-05, + "loss": 1.2451, + "theoretical_loss": 3.3347379683077927, + "tokens_seen": 3176660992 + }, + { + "epoch": 0.93, + "learning_rate": 3.7833500501504514e-05, + "loss": 1.2177, + "theoretical_loss": 3.33473278467514, + "tokens_seen": 3176726528 + }, + { + "epoch": 0.93, + "learning_rate": 3.781344032096289e-05, + "loss": 1.2541, + "theoretical_loss": 3.3347276011793667, + "tokens_seen": 3176792064 + }, + { + "epoch": 0.93, + "learning_rate": 3.7793380140421266e-05, + "loss": 1.2979, + "theoretical_loss": 3.3347224178204664, + "tokens_seen": 3176857600 + }, + { + "epoch": 0.93, + "learning_rate": 3.777331995987964e-05, + "loss": 1.3063, + "theoretical_loss": 3.3347172345984326, + "tokens_seen": 3176923136 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1154349, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2232264280319214, + "objective/train/theoretical_loss": 3.3347120515132596, + "objective/train/tokens_used": 1547447776, + "theoretical_loss": 3.3347120515132596, + "tokens_seen": 3176988672 + }, + { + "epoch": 0.93, + "learning_rate": 3.775325977933802e-05, + "loss": 1.2685, + "theoretical_loss": 3.3347120515132596, + "tokens_seen": 3176988672 + }, + { + "epoch": 0.93, + "learning_rate": 3.773319959879639e-05, + "loss": 1.2147, + "theoretical_loss": 3.3347068685649397, + "tokens_seen": 3177054208 + }, + { + "epoch": 0.93, + "learning_rate": 3.771313941825476e-05, + "loss": 1.3657, + "theoretical_loss": 3.3347016857534677, + "tokens_seen": 3177119744 + }, + { + "epoch": 0.93, + "learning_rate": 3.769307923771314e-05, + "loss": 1.2965, + "theoretical_loss": 3.3346965030788365, + "tokens_seen": 3177185280 + }, + { + "epoch": 0.93, + "learning_rate": 3.7673019057171514e-05, + "loss": 1.1945, + "theoretical_loss": 3.33469132054104, + "tokens_seen": 3177250816 + }, + { + "epoch": 0.93, + "learning_rate": 3.765295887662989e-05, + "loss": 1.3389, + "theoretical_loss": 3.334686138140071, + "tokens_seen": 3177316352 + }, + { + "epoch": 0.93, + "learning_rate": 3.763289869608826e-05, + "loss": 1.2169, + "theoretical_loss": 3.334680955875924, + "tokens_seen": 3177381888 + }, + { + "epoch": 0.93, + "learning_rate": 3.761283851554664e-05, + "loss": 1.3364, + "theoretical_loss": 3.334675773748592, + "tokens_seen": 3177447424 + }, + { + "epoch": 0.93, + "learning_rate": 3.759277833500502e-05, + "loss": 1.2812, + "theoretical_loss": 3.334670591758069, + "tokens_seen": 3177512960 + }, + { + "epoch": 0.93, + "learning_rate": 3.757271815446339e-05, + "loss": 1.2827, + "theoretical_loss": 3.334665409904348, + "tokens_seen": 3177578496 + }, + { + "epoch": 0.93, + "learning_rate": 3.755265797392177e-05, + "loss": 1.2498, + "theoretical_loss": 3.334660228187423, + "tokens_seen": 3177644032 + }, + { + "epoch": 0.93, + "learning_rate": 3.753259779338014e-05, + "loss": 1.2791, + "theoretical_loss": 3.3346550466072875, + "tokens_seen": 3177709568 + }, + { + "epoch": 0.93, + "learning_rate": 3.7512537612838514e-05, + "loss": 1.2031, + "theoretical_loss": 3.334649865163935, + "tokens_seen": 3177775104 + }, + { + "epoch": 0.93, + "learning_rate": 3.749247743229689e-05, + "loss": 1.3, + "theoretical_loss": 3.334644683857359, + "tokens_seen": 3177840640 + }, + { + "epoch": 0.93, + "learning_rate": 3.7472417251755265e-05, + "loss": 1.2862, + "theoretical_loss": 3.3346395026875535, + "tokens_seen": 3177906176 + }, + { + "epoch": 0.93, + "learning_rate": 3.745235707121364e-05, + "loss": 1.2035, + "theoretical_loss": 3.3346343216545113, + "tokens_seen": 3177971712 + }, + { + "epoch": 0.93, + "learning_rate": 3.743229689067202e-05, + "loss": 1.3484, + "theoretical_loss": 3.334629140758227, + "tokens_seen": 3178037248 + }, + { + "epoch": 0.93, + "learning_rate": 3.741223671013039e-05, + "loss": 1.1984, + "theoretical_loss": 3.3346239599986927, + "tokens_seen": 3178102784 + }, + { + "epoch": 0.93, + "learning_rate": 3.739217652958877e-05, + "loss": 1.3059, + "theoretical_loss": 3.3346187793759032, + "tokens_seen": 3178168320 + }, + { + "epoch": 0.93, + "learning_rate": 3.7372116349047144e-05, + "loss": 1.2096, + "theoretical_loss": 3.334613598889852, + "tokens_seen": 3178233856 + }, + { + "epoch": 0.93, + "learning_rate": 3.735205616850552e-05, + "loss": 1.2815, + "theoretical_loss": 3.3346084185405322, + "tokens_seen": 3178299392 + }, + { + "epoch": 0.93, + "learning_rate": 3.733199598796389e-05, + "loss": 1.3162, + "theoretical_loss": 3.3346032383279374, + "tokens_seen": 3178364928 + }, + { + "epoch": 0.93, + "learning_rate": 3.7311935807422265e-05, + "loss": 1.1661, + "theoretical_loss": 3.3345980582520616, + "tokens_seen": 3178430464 + }, + { + "epoch": 0.93, + "learning_rate": 3.729187562688064e-05, + "loss": 1.2987, + "theoretical_loss": 3.3345928783128977, + "tokens_seen": 3178496000 + }, + { + "epoch": 0.93, + "learning_rate": 3.7271815446339016e-05, + "loss": 1.1627, + "theoretical_loss": 3.33458769851044, + "tokens_seen": 3178561536 + }, + { + "epoch": 0.93, + "learning_rate": 3.72517552657974e-05, + "loss": 1.268, + "theoretical_loss": 3.3345825188446816, + "tokens_seen": 3178627072 + }, + { + "epoch": 0.93, + "learning_rate": 3.723169508525577e-05, + "loss": 1.1921, + "theoretical_loss": 3.3345773393156164, + "tokens_seen": 3178692608 + }, + { + "epoch": 0.93, + "learning_rate": 3.7211634904714144e-05, + "loss": 1.3056, + "theoretical_loss": 3.334572159923238, + "tokens_seen": 3178758144 + }, + { + "epoch": 0.93, + "learning_rate": 3.719157472417252e-05, + "loss": 1.2662, + "theoretical_loss": 3.334566980667539, + "tokens_seen": 3178823680 + }, + { + "epoch": 0.93, + "learning_rate": 3.7171514543630895e-05, + "loss": 1.2799, + "theoretical_loss": 3.334561801548514, + "tokens_seen": 3178889216 + }, + { + "epoch": 0.93, + "learning_rate": 3.715145436308927e-05, + "loss": 1.2912, + "theoretical_loss": 3.334556622566157, + "tokens_seen": 3178954752 + }, + { + "epoch": 0.93, + "learning_rate": 3.713139418254764e-05, + "loss": 1.1587, + "theoretical_loss": 3.33455144372046, + "tokens_seen": 3179020288 + }, + { + "epoch": 0.93, + "learning_rate": 3.7111334002006016e-05, + "loss": 1.2863, + "theoretical_loss": 3.3345462650114177, + "tokens_seen": 3179085824 + }, + { + "epoch": 0.93, + "learning_rate": 3.709127382146439e-05, + "loss": 1.2594, + "theoretical_loss": 3.3345410864390237, + "tokens_seen": 3179151360 + }, + { + "epoch": 0.93, + "learning_rate": 3.707121364092277e-05, + "loss": 1.2831, + "theoretical_loss": 3.3345359080032715, + "tokens_seen": 3179216896 + }, + { + "epoch": 0.93, + "learning_rate": 3.705115346038115e-05, + "loss": 1.2698, + "theoretical_loss": 3.334530729704154, + "tokens_seen": 3179282432 + }, + { + "epoch": 0.93, + "learning_rate": 3.703109327983952e-05, + "loss": 1.2054, + "theoretical_loss": 3.3345255515416654, + "tokens_seen": 3179347968 + }, + { + "epoch": 0.93, + "learning_rate": 3.7011033099297895e-05, + "loss": 1.354, + "theoretical_loss": 3.334520373515799, + "tokens_seen": 3179413504 + }, + { + "epoch": 0.93, + "learning_rate": 3.699097291875627e-05, + "loss": 1.3051, + "theoretical_loss": 3.3345151956265493, + "tokens_seen": 3179479040 + }, + { + "epoch": 0.93, + "learning_rate": 3.6970912738214646e-05, + "loss": 1.2691, + "theoretical_loss": 3.3345100178739084, + "tokens_seen": 3179544576 + }, + { + "epoch": 0.93, + "learning_rate": 3.695085255767302e-05, + "loss": 1.1916, + "theoretical_loss": 3.3345048402578708, + "tokens_seen": 3179610112 + }, + { + "epoch": 0.93, + "learning_rate": 3.693079237713139e-05, + "loss": 1.3089, + "theoretical_loss": 3.33449966277843, + "tokens_seen": 3179675648 + }, + { + "epoch": 0.93, + "learning_rate": 3.691073219658977e-05, + "loss": 1.2863, + "theoretical_loss": 3.334494485435579, + "tokens_seen": 3179741184 + }, + { + "epoch": 0.93, + "learning_rate": 3.689067201604814e-05, + "loss": 1.3048, + "theoretical_loss": 3.3344893082293123, + "tokens_seen": 3179806720 + }, + { + "epoch": 0.93, + "learning_rate": 3.6870611835506525e-05, + "loss": 1.3164, + "theoretical_loss": 3.334484131159623, + "tokens_seen": 3179872256 + }, + { + "epoch": 0.93, + "learning_rate": 3.68505516549649e-05, + "loss": 1.3008, + "theoretical_loss": 3.3344789542265048, + "tokens_seen": 3179937792 + }, + { + "epoch": 0.93, + "learning_rate": 3.683049147442327e-05, + "loss": 1.2802, + "theoretical_loss": 3.334473777429951, + "tokens_seen": 3180003328 + }, + { + "epoch": 0.93, + "learning_rate": 3.6810431293881646e-05, + "loss": 1.2841, + "theoretical_loss": 3.3344686007699553, + "tokens_seen": 3180068864 + }, + { + "epoch": 0.93, + "learning_rate": 3.679037111334002e-05, + "loss": 1.2594, + "theoretical_loss": 3.3344634242465117, + "tokens_seen": 3180134400 + }, + { + "epoch": 0.93, + "learning_rate": 3.67703109327984e-05, + "loss": 1.3712, + "theoretical_loss": 3.334458247859613, + "tokens_seen": 3180199936 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1155539, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3495402336120605, + "objective/train/theoretical_loss": 3.3344530716092535, + "objective/train/tokens_used": 1550724576, + "theoretical_loss": 3.3344530716092535, + "tokens_seen": 3180265472 + }, + { + "epoch": 0.93, + "learning_rate": 3.6750250752256767e-05, + "loss": 1.3193, + "theoretical_loss": 3.3344530716092535, + "tokens_seen": 3180265472 + }, + { + "epoch": 0.93, + "learning_rate": 3.673019057171514e-05, + "loss": 1.343, + "theoretical_loss": 3.334447895495426, + "tokens_seen": 3180331008 + }, + { + "epoch": 0.93, + "learning_rate": 3.671013039117352e-05, + "loss": 1.2165, + "theoretical_loss": 3.334442719518125, + "tokens_seen": 3180396544 + }, + { + "epoch": 0.93, + "learning_rate": 3.66900702106319e-05, + "loss": 1.206, + "theoretical_loss": 3.334437543677344, + "tokens_seen": 3180462080 + }, + { + "epoch": 0.93, + "learning_rate": 3.6670010030090277e-05, + "loss": 1.3021, + "theoretical_loss": 3.334432367973076, + "tokens_seen": 3180527616 + }, + { + "epoch": 0.93, + "learning_rate": 3.6649949849548646e-05, + "loss": 1.3859, + "theoretical_loss": 3.3344271924053146, + "tokens_seen": 3180593152 + }, + { + "epoch": 0.93, + "learning_rate": 3.662988966900702e-05, + "loss": 1.1776, + "theoretical_loss": 3.334422016974054, + "tokens_seen": 3180658688 + }, + { + "epoch": 0.93, + "learning_rate": 3.66098294884654e-05, + "loss": 1.2591, + "theoretical_loss": 3.334416841679287, + "tokens_seen": 3180724224 + }, + { + "epoch": 0.93, + "learning_rate": 3.658976930792377e-05, + "loss": 1.2788, + "theoretical_loss": 3.3344116665210084, + "tokens_seen": 3180789760 + }, + { + "epoch": 0.93, + "learning_rate": 3.656970912738215e-05, + "loss": 1.1545, + "theoretical_loss": 3.33440649149921, + "tokens_seen": 3180855296 + }, + { + "epoch": 0.93, + "learning_rate": 3.654964894684052e-05, + "loss": 1.1856, + "theoretical_loss": 3.3344013166138873, + "tokens_seen": 3180920832 + }, + { + "epoch": 0.93, + "learning_rate": 3.6529588766298894e-05, + "loss": 1.241, + "theoretical_loss": 3.3343961418650325, + "tokens_seen": 3180986368 + }, + { + "epoch": 0.93, + "learning_rate": 3.650952858575727e-05, + "loss": 1.191, + "theoretical_loss": 3.3343909672526397, + "tokens_seen": 3181051904 + }, + { + "epoch": 0.93, + "learning_rate": 3.648946840521565e-05, + "loss": 1.2551, + "theoretical_loss": 3.334385792776702, + "tokens_seen": 3181117440 + }, + { + "epoch": 0.93, + "learning_rate": 3.646940822467403e-05, + "loss": 1.3139, + "theoretical_loss": 3.3343806184372147, + "tokens_seen": 3181182976 + }, + { + "epoch": 0.93, + "learning_rate": 3.64493480441324e-05, + "loss": 1.2336, + "theoretical_loss": 3.334375444234169, + "tokens_seen": 3181248512 + }, + { + "epoch": 0.93, + "learning_rate": 3.642928786359077e-05, + "loss": 1.2899, + "theoretical_loss": 3.33437027016756, + "tokens_seen": 3181314048 + }, + { + "epoch": 0.93, + "learning_rate": 3.640922768304915e-05, + "loss": 1.3485, + "theoretical_loss": 3.334365096237381, + "tokens_seen": 3181379584 + }, + { + "epoch": 0.93, + "learning_rate": 3.6389167502507524e-05, + "loss": 1.206, + "theoretical_loss": 3.3343599224436256, + "tokens_seen": 3181445120 + }, + { + "epoch": 0.93, + "learning_rate": 3.63691073219659e-05, + "loss": 1.2412, + "theoretical_loss": 3.3343547487862875, + "tokens_seen": 3181510656 + }, + { + "epoch": 0.93, + "learning_rate": 3.634904714142427e-05, + "loss": 1.252, + "theoretical_loss": 3.3343495752653594, + "tokens_seen": 3181576192 + }, + { + "epoch": 0.93, + "learning_rate": 3.6328986960882645e-05, + "loss": 1.2794, + "theoretical_loss": 3.334344401880836, + "tokens_seen": 3181641728 + }, + { + "epoch": 0.93, + "learning_rate": 3.630892678034103e-05, + "loss": 1.3079, + "theoretical_loss": 3.334339228632711, + "tokens_seen": 3181707264 + }, + { + "epoch": 0.93, + "learning_rate": 3.62888665997994e-05, + "loss": 1.2271, + "theoretical_loss": 3.3343340555209764, + "tokens_seen": 3181772800 + }, + { + "epoch": 0.93, + "learning_rate": 3.626880641925778e-05, + "loss": 1.2566, + "theoretical_loss": 3.334328882545628, + "tokens_seen": 3181838336 + }, + { + "epoch": 0.93, + "learning_rate": 3.624874623871615e-05, + "loss": 1.221, + "theoretical_loss": 3.3343237097066574, + "tokens_seen": 3181903872 + }, + { + "epoch": 0.93, + "learning_rate": 3.6228686058174524e-05, + "loss": 1.2555, + "theoretical_loss": 3.3343185370040596, + "tokens_seen": 3181969408 + }, + { + "epoch": 0.93, + "learning_rate": 3.62086258776329e-05, + "loss": 1.2415, + "theoretical_loss": 3.3343133644378273, + "tokens_seen": 3182034944 + }, + { + "epoch": 0.93, + "learning_rate": 3.6188565697091275e-05, + "loss": 1.303, + "theoretical_loss": 3.3343081920079545, + "tokens_seen": 3182100480 + }, + { + "epoch": 0.93, + "learning_rate": 3.616850551654965e-05, + "loss": 1.2302, + "theoretical_loss": 3.334303019714435, + "tokens_seen": 3182166016 + }, + { + "epoch": 0.93, + "learning_rate": 3.614844533600802e-05, + "loss": 1.3526, + "theoretical_loss": 3.334297847557262, + "tokens_seen": 3182231552 + }, + { + "epoch": 0.93, + "learning_rate": 3.61283851554664e-05, + "loss": 1.1723, + "theoretical_loss": 3.3342926755364295, + "tokens_seen": 3182297088 + }, + { + "epoch": 0.93, + "learning_rate": 3.610832497492478e-05, + "loss": 1.247, + "theoretical_loss": 3.3342875036519306, + "tokens_seen": 3182362624 + }, + { + "epoch": 0.93, + "learning_rate": 3.6088264794383154e-05, + "loss": 1.2933, + "theoretical_loss": 3.3342823319037596, + "tokens_seen": 3182428160 + }, + { + "epoch": 0.93, + "learning_rate": 3.606820461384152e-05, + "loss": 1.2799, + "theoretical_loss": 3.334277160291909, + "tokens_seen": 3182493696 + }, + { + "epoch": 0.93, + "learning_rate": 3.60481444332999e-05, + "loss": 1.2874, + "theoretical_loss": 3.3342719888163734, + "tokens_seen": 3182559232 + }, + { + "epoch": 0.93, + "learning_rate": 3.6028084252758275e-05, + "loss": 1.3828, + "theoretical_loss": 3.334266817477146, + "tokens_seen": 3182624768 + }, + { + "epoch": 0.93, + "learning_rate": 3.600802407221665e-05, + "loss": 1.3196, + "theoretical_loss": 3.3342616462742205, + "tokens_seen": 3182690304 + }, + { + "epoch": 0.93, + "learning_rate": 3.5987963891675026e-05, + "loss": 1.28, + "theoretical_loss": 3.3342564752075905, + "tokens_seen": 3182755840 + }, + { + "epoch": 0.93, + "learning_rate": 3.5967903711133395e-05, + "loss": 1.2742, + "theoretical_loss": 3.3342513042772493, + "tokens_seen": 3182821376 + }, + { + "epoch": 0.93, + "learning_rate": 3.594784353059177e-05, + "loss": 1.2885, + "theoretical_loss": 3.334246133483191, + "tokens_seen": 3182886912 + }, + { + "epoch": 0.93, + "learning_rate": 3.5927783350050154e-05, + "loss": 1.2238, + "theoretical_loss": 3.334240962825409, + "tokens_seen": 3182952448 + }, + { + "epoch": 0.93, + "learning_rate": 3.590772316950853e-05, + "loss": 1.2385, + "theoretical_loss": 3.3342357923038968, + "tokens_seen": 3183017984 + }, + { + "epoch": 0.93, + "learning_rate": 3.5887662988966905e-05, + "loss": 1.3255, + "theoretical_loss": 3.334230621918648, + "tokens_seen": 3183083520 + }, + { + "epoch": 0.93, + "learning_rate": 3.5867602808425274e-05, + "loss": 1.2388, + "theoretical_loss": 3.3342254516696563, + "tokens_seen": 3183149056 + }, + { + "epoch": 0.93, + "learning_rate": 3.584754262788365e-05, + "loss": 1.22, + "theoretical_loss": 3.3342202815569153, + "tokens_seen": 3183214592 + }, + { + "epoch": 0.93, + "learning_rate": 3.5827482447342026e-05, + "loss": 1.237, + "theoretical_loss": 3.334215111580418, + "tokens_seen": 3183280128 + }, + { + "epoch": 0.93, + "learning_rate": 3.58074222668004e-05, + "loss": 1.2283, + "theoretical_loss": 3.3342099417401596, + "tokens_seen": 3183345664 + }, + { + "epoch": 0.93, + "learning_rate": 3.578736208625878e-05, + "loss": 1.3675, + "theoretical_loss": 3.334204772036132, + "tokens_seen": 3183411200 + }, + { + "epoch": 0.93, + "learning_rate": 3.5767301905717147e-05, + "loss": 1.3235, + "theoretical_loss": 3.33419960246833, + "tokens_seen": 3183476736 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1156704, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2980313301086426, + "objective/train/theoretical_loss": 3.334194433036746, + "objective/train/tokens_used": 1554001376, + "theoretical_loss": 3.334194433036746, + "tokens_seen": 3183542272 + }, + { + "epoch": 0.93, + "learning_rate": 3.574724172517553e-05, + "loss": 1.3093, + "theoretical_loss": 3.334194433036746, + "tokens_seen": 3183542272 + }, + { + "epoch": 0.93, + "learning_rate": 3.5727181544633905e-05, + "loss": 1.2307, + "theoretical_loss": 3.334189263741375, + "tokens_seen": 3183607808 + }, + { + "epoch": 0.93, + "learning_rate": 3.570712136409228e-05, + "loss": 1.2814, + "theoretical_loss": 3.3341840945822097, + "tokens_seen": 3183673344 + }, + { + "epoch": 0.93, + "learning_rate": 3.5687061183550657e-05, + "loss": 1.264, + "theoretical_loss": 3.334178925559244, + "tokens_seen": 3183738880 + }, + { + "epoch": 0.93, + "learning_rate": 3.5667001003009026e-05, + "loss": 1.2727, + "theoretical_loss": 3.334173756672471, + "tokens_seen": 3183804416 + }, + { + "epoch": 0.93, + "learning_rate": 3.56469408224674e-05, + "loss": 1.2419, + "theoretical_loss": 3.3341685879218854, + "tokens_seen": 3183869952 + }, + { + "epoch": 0.93, + "learning_rate": 3.562688064192578e-05, + "loss": 1.1863, + "theoretical_loss": 3.3341634193074796, + "tokens_seen": 3183935488 + }, + { + "epoch": 0.93, + "learning_rate": 3.560682046138415e-05, + "loss": 1.2646, + "theoretical_loss": 3.334158250829248, + "tokens_seen": 3184001024 + }, + { + "epoch": 0.93, + "learning_rate": 3.558676028084253e-05, + "loss": 1.1897, + "theoretical_loss": 3.334153082487184, + "tokens_seen": 3184066560 + }, + { + "epoch": 0.93, + "learning_rate": 3.5566700100300904e-05, + "loss": 1.2812, + "theoretical_loss": 3.3341479142812807, + "tokens_seen": 3184132096 + }, + { + "epoch": 0.93, + "learning_rate": 3.554663991975928e-05, + "loss": 1.1599, + "theoretical_loss": 3.3341427462115325, + "tokens_seen": 3184197632 + }, + { + "epoch": 0.93, + "learning_rate": 3.5526579739217656e-05, + "loss": 1.3222, + "theoretical_loss": 3.334137578277933, + "tokens_seen": 3184263168 + }, + { + "epoch": 0.93, + "learning_rate": 3.550651955867603e-05, + "loss": 1.2741, + "theoretical_loss": 3.3341324104804753, + "tokens_seen": 3184328704 + }, + { + "epoch": 0.93, + "learning_rate": 3.548645937813441e-05, + "loss": 1.3505, + "theoretical_loss": 3.334127242819153, + "tokens_seen": 3184394240 + }, + { + "epoch": 0.93, + "learning_rate": 3.546639919759278e-05, + "loss": 1.2733, + "theoretical_loss": 3.33412207529396, + "tokens_seen": 3184459776 + }, + { + "epoch": 0.93, + "learning_rate": 3.544633901705115e-05, + "loss": 1.2564, + "theoretical_loss": 3.3341169079048902, + "tokens_seen": 3184525312 + }, + { + "epoch": 0.93, + "learning_rate": 3.542627883650953e-05, + "loss": 1.2709, + "theoretical_loss": 3.3341117406519367, + "tokens_seen": 3184590848 + }, + { + "epoch": 0.93, + "learning_rate": 3.5406218655967904e-05, + "loss": 1.2435, + "theoretical_loss": 3.334106573535093, + "tokens_seen": 3184656384 + }, + { + "epoch": 0.93, + "learning_rate": 3.538615847542627e-05, + "loss": 1.2876, + "theoretical_loss": 3.3341014065543533, + "tokens_seen": 3184721920 + }, + { + "epoch": 0.93, + "learning_rate": 3.5366098294884656e-05, + "loss": 1.2395, + "theoretical_loss": 3.3340962397097105, + "tokens_seen": 3184787456 + }, + { + "epoch": 0.93, + "learning_rate": 3.534603811434303e-05, + "loss": 1.2537, + "theoretical_loss": 3.334091073001159, + "tokens_seen": 3184852992 + }, + { + "epoch": 0.93, + "learning_rate": 3.532597793380141e-05, + "loss": 1.24, + "theoretical_loss": 3.334085906428692, + "tokens_seen": 3184918528 + }, + { + "epoch": 0.93, + "learning_rate": 3.530591775325978e-05, + "loss": 1.2621, + "theoretical_loss": 3.334080739992303, + "tokens_seen": 3184984064 + }, + { + "epoch": 0.93, + "learning_rate": 3.528585757271815e-05, + "loss": 1.3221, + "theoretical_loss": 3.334075573691986, + "tokens_seen": 3185049600 + }, + { + "epoch": 0.93, + "learning_rate": 3.526579739217653e-05, + "loss": 1.1736, + "theoretical_loss": 3.334070407527734, + "tokens_seen": 3185115136 + }, + { + "epoch": 0.93, + "learning_rate": 3.5245737211634904e-05, + "loss": 1.204, + "theoretical_loss": 3.334065241499541, + "tokens_seen": 3185180672 + }, + { + "epoch": 0.93, + "learning_rate": 3.522567703109328e-05, + "loss": 1.3741, + "theoretical_loss": 3.3340600756074013, + "tokens_seen": 3185246208 + }, + { + "epoch": 0.93, + "learning_rate": 3.5205616850551655e-05, + "loss": 1.2727, + "theoretical_loss": 3.3340549098513073, + "tokens_seen": 3185311744 + }, + { + "epoch": 0.93, + "learning_rate": 3.518555667001003e-05, + "loss": 1.2723, + "theoretical_loss": 3.334049744231253, + "tokens_seen": 3185377280 + }, + { + "epoch": 0.93, + "learning_rate": 3.516549648946841e-05, + "loss": 1.4063, + "theoretical_loss": 3.3340445787472324, + "tokens_seen": 3185442816 + }, + { + "epoch": 0.93, + "learning_rate": 3.514543630892678e-05, + "loss": 1.2396, + "theoretical_loss": 3.3340394133992386, + "tokens_seen": 3185508352 + }, + { + "epoch": 0.93, + "learning_rate": 3.512537612838516e-05, + "loss": 1.232, + "theoretical_loss": 3.334034248187266, + "tokens_seen": 3185573888 + }, + { + "epoch": 0.93, + "learning_rate": 3.5105315947843534e-05, + "loss": 1.3486, + "theoretical_loss": 3.3340290831113073, + "tokens_seen": 3185639424 + }, + { + "epoch": 0.93, + "learning_rate": 3.50852557673019e-05, + "loss": 1.2853, + "theoretical_loss": 3.334023918171357, + "tokens_seen": 3185704960 + }, + { + "epoch": 0.93, + "learning_rate": 3.506519558676028e-05, + "loss": 1.2522, + "theoretical_loss": 3.3340187533674075, + "tokens_seen": 3185770496 + }, + { + "epoch": 0.93, + "learning_rate": 3.5045135406218655e-05, + "loss": 1.2607, + "theoretical_loss": 3.3340135886994537, + "tokens_seen": 3185836032 + }, + { + "epoch": 0.93, + "learning_rate": 3.502507522567703e-05, + "loss": 1.2641, + "theoretical_loss": 3.3340084241674885, + "tokens_seen": 3185901568 + }, + { + "epoch": 0.93, + "learning_rate": 3.5005015045135406e-05, + "loss": 1.3337, + "theoretical_loss": 3.334003259771506, + "tokens_seen": 3185967104 + }, + { + "epoch": 0.93, + "learning_rate": 3.498495486459378e-05, + "loss": 1.3006, + "theoretical_loss": 3.3339980955114994, + "tokens_seen": 3186032640 + }, + { + "epoch": 0.93, + "learning_rate": 3.496489468405216e-05, + "loss": 1.26, + "theoretical_loss": 3.3339929313874626, + "tokens_seen": 3186098176 + }, + { + "epoch": 0.93, + "learning_rate": 3.4944834503510534e-05, + "loss": 1.2367, + "theoretical_loss": 3.333987767399389, + "tokens_seen": 3186163712 + }, + { + "epoch": 0.93, + "learning_rate": 3.492477432296891e-05, + "loss": 1.2366, + "theoretical_loss": 3.3339826035472724, + "tokens_seen": 3186229248 + }, + { + "epoch": 0.93, + "learning_rate": 3.4904714142427285e-05, + "loss": 1.3484, + "theoretical_loss": 3.333977439831106, + "tokens_seen": 3186294784 + }, + { + "epoch": 0.93, + "learning_rate": 3.4884653961885654e-05, + "loss": 1.325, + "theoretical_loss": 3.3339722762508845, + "tokens_seen": 3186360320 + }, + { + "epoch": 0.93, + "learning_rate": 3.486459378134403e-05, + "loss": 1.3111, + "theoretical_loss": 3.3339671128066004, + "tokens_seen": 3186425856 + }, + { + "epoch": 0.93, + "learning_rate": 3.4844533600802406e-05, + "loss": 1.2599, + "theoretical_loss": 3.3339619494982475, + "tokens_seen": 3186491392 + }, + { + "epoch": 0.93, + "learning_rate": 3.482447342026078e-05, + "loss": 1.3368, + "theoretical_loss": 3.3339567863258197, + "tokens_seen": 3186556928 + }, + { + "epoch": 0.93, + "learning_rate": 3.4804413239719164e-05, + "loss": 1.3137, + "theoretical_loss": 3.3339516232893107, + "tokens_seen": 3186622464 + }, + { + "epoch": 0.93, + "learning_rate": 3.478435305917753e-05, + "loss": 1.076, + "theoretical_loss": 3.3339464603887143, + "tokens_seen": 3186688000 + }, + { + "epoch": 0.93, + "learning_rate": 3.476429287863591e-05, + "loss": 1.2953, + "theoretical_loss": 3.3339412976240235, + "tokens_seen": 3186753536 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1158346, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2709064483642578, + "objective/train/theoretical_loss": 3.3339361349952323, + "objective/train/tokens_used": 1557278176, + "theoretical_loss": 3.3339361349952323, + "tokens_seen": 3186819072 + }, + { + "epoch": 0.93, + "learning_rate": 3.4744232698094285e-05, + "loss": 1.272, + "theoretical_loss": 3.3339361349952323, + "tokens_seen": 3186819072 + }, + { + "epoch": 0.93, + "learning_rate": 3.472417251755266e-05, + "loss": 1.2636, + "theoretical_loss": 3.3339309725023343, + "tokens_seen": 3186884608 + }, + { + "epoch": 0.93, + "learning_rate": 3.4704112337011036e-05, + "loss": 1.3176, + "theoretical_loss": 3.333925810145323, + "tokens_seen": 3186950144 + }, + { + "epoch": 0.93, + "learning_rate": 3.4684052156469406e-05, + "loss": 1.217, + "theoretical_loss": 3.333920647924192, + "tokens_seen": 3187015680 + }, + { + "epoch": 0.93, + "learning_rate": 3.466399197592778e-05, + "loss": 1.3051, + "theoretical_loss": 3.3339154858389355, + "tokens_seen": 3187081216 + }, + { + "epoch": 0.93, + "learning_rate": 3.464393179538616e-05, + "loss": 1.3152, + "theoretical_loss": 3.3339103238895467, + "tokens_seen": 3187146752 + }, + { + "epoch": 0.93, + "learning_rate": 3.462387161484454e-05, + "loss": 1.2361, + "theoretical_loss": 3.333905162076019, + "tokens_seen": 3187212288 + }, + { + "epoch": 0.93, + "learning_rate": 3.460381143430291e-05, + "loss": 1.3361, + "theoretical_loss": 3.333900000398346, + "tokens_seen": 3187277824 + }, + { + "epoch": 0.93, + "learning_rate": 3.4583751253761284e-05, + "loss": 1.2886, + "theoretical_loss": 3.333894838856522, + "tokens_seen": 3187343360 + }, + { + "epoch": 0.93, + "learning_rate": 3.456369107321966e-05, + "loss": 1.2766, + "theoretical_loss": 3.33388967745054, + "tokens_seen": 3187408896 + }, + { + "epoch": 0.93, + "learning_rate": 3.4543630892678036e-05, + "loss": 1.2068, + "theoretical_loss": 3.333884516180394, + "tokens_seen": 3187474432 + }, + { + "epoch": 0.93, + "learning_rate": 3.452357071213641e-05, + "loss": 1.2682, + "theoretical_loss": 3.3338793550460775, + "tokens_seen": 3187539968 + }, + { + "epoch": 0.93, + "learning_rate": 3.450351053159478e-05, + "loss": 1.3271, + "theoretical_loss": 3.333874194047584, + "tokens_seen": 3187605504 + }, + { + "epoch": 0.93, + "learning_rate": 3.448345035105316e-05, + "loss": 1.3246, + "theoretical_loss": 3.3338690331849072, + "tokens_seen": 3187671040 + }, + { + "epoch": 0.93, + "learning_rate": 3.446339017051153e-05, + "loss": 1.3184, + "theoretical_loss": 3.3338638724580414, + "tokens_seen": 3187736576 + }, + { + "epoch": 0.93, + "learning_rate": 3.444332998996991e-05, + "loss": 1.3118, + "theoretical_loss": 3.333858711866979, + "tokens_seen": 3187802112 + }, + { + "epoch": 0.93, + "learning_rate": 3.442326980942829e-05, + "loss": 1.2715, + "theoretical_loss": 3.3338535514117145, + "tokens_seen": 3187867648 + }, + { + "epoch": 0.93, + "learning_rate": 3.440320962888666e-05, + "loss": 1.2541, + "theoretical_loss": 3.333848391092241, + "tokens_seen": 3187933184 + }, + { + "epoch": 0.93, + "learning_rate": 3.4383149448345036e-05, + "loss": 1.3667, + "theoretical_loss": 3.3338432309085526, + "tokens_seen": 3187998720 + }, + { + "epoch": 0.93, + "learning_rate": 3.436308926780341e-05, + "loss": 1.3637, + "theoretical_loss": 3.3338380708606428, + "tokens_seen": 3188064256 + }, + { + "epoch": 0.93, + "learning_rate": 3.434302908726179e-05, + "loss": 1.2311, + "theoretical_loss": 3.333832910948505, + "tokens_seen": 3188129792 + }, + { + "epoch": 0.93, + "learning_rate": 3.432296890672016e-05, + "loss": 1.2438, + "theoretical_loss": 3.3338277511721337, + "tokens_seen": 3188195328 + }, + { + "epoch": 0.93, + "learning_rate": 3.430290872617853e-05, + "loss": 1.2661, + "theoretical_loss": 3.333822591531521, + "tokens_seen": 3188260864 + }, + { + "epoch": 0.93, + "learning_rate": 3.428284854563691e-05, + "loss": 1.2619, + "theoretical_loss": 3.3338174320266623, + "tokens_seen": 3188326400 + }, + { + "epoch": 0.93, + "learning_rate": 3.4262788365095284e-05, + "loss": 1.2885, + "theoretical_loss": 3.3338122726575494, + "tokens_seen": 3188391936 + }, + { + "epoch": 0.93, + "learning_rate": 3.4242728184553666e-05, + "loss": 1.3086, + "theoretical_loss": 3.3338071134241773, + "tokens_seen": 3188457472 + }, + { + "epoch": 0.93, + "learning_rate": 3.422266800401204e-05, + "loss": 1.229, + "theoretical_loss": 3.3338019543265394, + "tokens_seen": 3188523008 + }, + { + "epoch": 0.93, + "learning_rate": 3.420260782347041e-05, + "loss": 1.2089, + "theoretical_loss": 3.333796795364629, + "tokens_seen": 3188588544 + }, + { + "epoch": 0.93, + "learning_rate": 3.418254764292879e-05, + "loss": 1.2151, + "theoretical_loss": 3.33379163653844, + "tokens_seen": 3188654080 + }, + { + "epoch": 0.93, + "learning_rate": 3.416248746238716e-05, + "loss": 1.3197, + "theoretical_loss": 3.333786477847966, + "tokens_seen": 3188719616 + }, + { + "epoch": 0.93, + "learning_rate": 3.414242728184554e-05, + "loss": 1.2131, + "theoretical_loss": 3.3337813192932004, + "tokens_seen": 3188785152 + }, + { + "epoch": 0.93, + "learning_rate": 3.4122367101303914e-05, + "loss": 1.2213, + "theoretical_loss": 3.333776160874137, + "tokens_seen": 3188850688 + }, + { + "epoch": 0.93, + "learning_rate": 3.410230692076228e-05, + "loss": 1.2109, + "theoretical_loss": 3.33377100259077, + "tokens_seen": 3188916224 + }, + { + "epoch": 0.93, + "learning_rate": 3.408224674022066e-05, + "loss": 1.2029, + "theoretical_loss": 3.333765844443092, + "tokens_seen": 3188981760 + }, + { + "epoch": 0.93, + "learning_rate": 3.406218655967904e-05, + "loss": 1.2033, + "theoretical_loss": 3.333760686431097, + "tokens_seen": 3189047296 + }, + { + "epoch": 0.93, + "learning_rate": 3.404212637913742e-05, + "loss": 1.2894, + "theoretical_loss": 3.333755528554779, + "tokens_seen": 3189112832 + }, + { + "epoch": 0.93, + "learning_rate": 3.402206619859579e-05, + "loss": 1.1803, + "theoretical_loss": 3.3337503708141316, + "tokens_seen": 3189178368 + }, + { + "epoch": 0.93, + "learning_rate": 3.400200601805416e-05, + "loss": 1.2397, + "theoretical_loss": 3.333745213209148, + "tokens_seen": 3189243904 + }, + { + "epoch": 0.93, + "learning_rate": 3.398194583751254e-05, + "loss": 1.2817, + "theoretical_loss": 3.3337400557398222, + "tokens_seen": 3189309440 + }, + { + "epoch": 0.93, + "learning_rate": 3.3961885656970914e-05, + "loss": 1.1744, + "theoretical_loss": 3.333734898406148, + "tokens_seen": 3189374976 + }, + { + "epoch": 0.93, + "learning_rate": 3.394182547642929e-05, + "loss": 1.2601, + "theoretical_loss": 3.333729741208119, + "tokens_seen": 3189440512 + }, + { + "epoch": 0.93, + "learning_rate": 3.392176529588766e-05, + "loss": 1.3082, + "theoretical_loss": 3.333724584145728, + "tokens_seen": 3189506048 + }, + { + "epoch": 0.93, + "learning_rate": 3.3901705115346034e-05, + "loss": 1.3395, + "theoretical_loss": 3.3337194272189694, + "tokens_seen": 3189571584 + }, + { + "epoch": 0.93, + "learning_rate": 3.388164493480441e-05, + "loss": 1.2946, + "theoretical_loss": 3.333714270427837, + "tokens_seen": 3189637120 + }, + { + "epoch": 0.93, + "learning_rate": 3.386158475426279e-05, + "loss": 1.2589, + "theoretical_loss": 3.3337091137723243, + "tokens_seen": 3189702656 + }, + { + "epoch": 0.93, + "learning_rate": 3.384152457372117e-05, + "loss": 1.3353, + "theoretical_loss": 3.3337039572524247, + "tokens_seen": 3189768192 + }, + { + "epoch": 0.93, + "learning_rate": 3.382146439317954e-05, + "loss": 1.1622, + "theoretical_loss": 3.333698800868132, + "tokens_seen": 3189833728 + }, + { + "epoch": 0.93, + "learning_rate": 3.380140421263791e-05, + "loss": 1.3705, + "theoretical_loss": 3.33369364461944, + "tokens_seen": 3189899264 + }, + { + "epoch": 0.93, + "learning_rate": 3.378134403209629e-05, + "loss": 1.286, + "theoretical_loss": 3.3336884885063416, + "tokens_seen": 3189964800 + }, + { + "epoch": 0.93, + "learning_rate": 3.3761283851554665e-05, + "loss": 1.2951, + "theoretical_loss": 3.3336833325288318, + "tokens_seen": 3190030336 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1159129, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2354068756103516, + "objective/train/theoretical_loss": 3.3336781766869032, + "objective/train/tokens_used": 1560554976, + "theoretical_loss": 3.3336781766869032, + "tokens_seen": 3190095872 + }, + { + "epoch": 0.93, + "learning_rate": 3.374122367101304e-05, + "loss": 1.3037, + "theoretical_loss": 3.3336781766869032, + "tokens_seen": 3190095872 + }, + { + "epoch": 0.93, + "learning_rate": 3.372116349047141e-05, + "loss": 1.1774, + "theoretical_loss": 3.33367302098055, + "tokens_seen": 3190161408 + }, + { + "epoch": 0.93, + "learning_rate": 3.3701103309929785e-05, + "loss": 1.2721, + "theoretical_loss": 3.333667865409765, + "tokens_seen": 3190226944 + }, + { + "epoch": 0.93, + "learning_rate": 3.368104312938817e-05, + "loss": 1.2423, + "theoretical_loss": 3.3336627099745426, + "tokens_seen": 3190292480 + }, + { + "epoch": 0.93, + "learning_rate": 3.3660982948846544e-05, + "loss": 1.3104, + "theoretical_loss": 3.3336575546748763, + "tokens_seen": 3190358016 + }, + { + "epoch": 0.93, + "learning_rate": 3.364092276830492e-05, + "loss": 1.2745, + "theoretical_loss": 3.33365239951076, + "tokens_seen": 3190423552 + }, + { + "epoch": 0.93, + "learning_rate": 3.362086258776329e-05, + "loss": 1.3171, + "theoretical_loss": 3.333647244482187, + "tokens_seen": 3190489088 + }, + { + "epoch": 0.93, + "learning_rate": 3.3600802407221664e-05, + "loss": 1.2114, + "theoretical_loss": 3.333642089589151, + "tokens_seen": 3190554624 + }, + { + "epoch": 0.93, + "learning_rate": 3.358074222668004e-05, + "loss": 1.225, + "theoretical_loss": 3.3336369348316457, + "tokens_seen": 3190620160 + }, + { + "epoch": 0.93, + "learning_rate": 3.3560682046138416e-05, + "loss": 1.2766, + "theoretical_loss": 3.3336317802096644, + "tokens_seen": 3190685696 + }, + { + "epoch": 0.93, + "learning_rate": 3.354062186559679e-05, + "loss": 1.2729, + "theoretical_loss": 3.3336266257232015, + "tokens_seen": 3190751232 + }, + { + "epoch": 0.93, + "learning_rate": 3.352056168505516e-05, + "loss": 1.2037, + "theoretical_loss": 3.3336214713722505, + "tokens_seen": 3190816768 + }, + { + "epoch": 0.93, + "learning_rate": 3.3500501504513543e-05, + "loss": 1.1866, + "theoretical_loss": 3.333616317156804, + "tokens_seen": 3190882304 + }, + { + "epoch": 0.93, + "learning_rate": 3.348044132397192e-05, + "loss": 1.2633, + "theoretical_loss": 3.3336111630768572, + "tokens_seen": 3190947840 + }, + { + "epoch": 0.93, + "learning_rate": 3.3460381143430295e-05, + "loss": 1.2562, + "theoretical_loss": 3.333606009132403, + "tokens_seen": 3191013376 + }, + { + "epoch": 0.93, + "learning_rate": 3.344032096288867e-05, + "loss": 1.3164, + "theoretical_loss": 3.333600855323435, + "tokens_seen": 3191078912 + }, + { + "epoch": 0.93, + "learning_rate": 3.342026078234704e-05, + "loss": 1.2964, + "theoretical_loss": 3.3335957016499465, + "tokens_seen": 3191144448 + }, + { + "epoch": 0.93, + "learning_rate": 3.3400200601805416e-05, + "loss": 1.1958, + "theoretical_loss": 3.333590548111932, + "tokens_seen": 3191209984 + }, + { + "epoch": 0.93, + "learning_rate": 3.338014042126379e-05, + "loss": 1.3037, + "theoretical_loss": 3.3335853947093845, + "tokens_seen": 3191275520 + }, + { + "epoch": 0.93, + "learning_rate": 3.336008024072217e-05, + "loss": 1.2619, + "theoretical_loss": 3.3335802414422977, + "tokens_seen": 3191341056 + }, + { + "epoch": 0.93, + "learning_rate": 3.334002006018054e-05, + "loss": 1.2993, + "theoretical_loss": 3.333575088310666, + "tokens_seen": 3191406592 + }, + { + "epoch": 0.93, + "learning_rate": 3.331995987963891e-05, + "loss": 1.222, + "theoretical_loss": 3.333569935314482, + "tokens_seen": 3191472128 + }, + { + "epoch": 0.93, + "learning_rate": 3.3299899699097295e-05, + "loss": 1.3095, + "theoretical_loss": 3.3335647824537404, + "tokens_seen": 3191537664 + }, + { + "epoch": 0.93, + "learning_rate": 3.327983951855567e-05, + "loss": 1.187, + "theoretical_loss": 3.333559629728434, + "tokens_seen": 3191603200 + }, + { + "epoch": 0.93, + "learning_rate": 3.3259779338014046e-05, + "loss": 1.2778, + "theoretical_loss": 3.333554477138557, + "tokens_seen": 3191668736 + }, + { + "epoch": 0.93, + "learning_rate": 3.323971915747242e-05, + "loss": 1.2241, + "theoretical_loss": 3.333549324684103, + "tokens_seen": 3191734272 + }, + { + "epoch": 0.93, + "learning_rate": 3.321965897693079e-05, + "loss": 1.2746, + "theoretical_loss": 3.3335441723650647, + "tokens_seen": 3191799808 + }, + { + "epoch": 0.93, + "learning_rate": 3.319959879638917e-05, + "loss": 1.2769, + "theoretical_loss": 3.3335390201814374, + "tokens_seen": 3191865344 + }, + { + "epoch": 0.93, + "learning_rate": 3.317953861584754e-05, + "loss": 1.3967, + "theoretical_loss": 3.3335338681332134, + "tokens_seen": 3191930880 + }, + { + "epoch": 0.93, + "learning_rate": 3.315947843530592e-05, + "loss": 1.1088, + "theoretical_loss": 3.3335287162203873, + "tokens_seen": 3191996416 + }, + { + "epoch": 0.93, + "learning_rate": 3.313941825476429e-05, + "loss": 1.2912, + "theoretical_loss": 3.3335235644429524, + "tokens_seen": 3192061952 + }, + { + "epoch": 0.93, + "learning_rate": 3.311935807422267e-05, + "loss": 1.1715, + "theoretical_loss": 3.333518412800902, + "tokens_seen": 3192127488 + }, + { + "epoch": 0.93, + "learning_rate": 3.3099297893681046e-05, + "loss": 1.2432, + "theoretical_loss": 3.33351326129423, + "tokens_seen": 3192193024 + }, + { + "epoch": 0.93, + "learning_rate": 3.307923771313942e-05, + "loss": 1.2865, + "theoretical_loss": 3.3335081099229305, + "tokens_seen": 3192258560 + }, + { + "epoch": 0.93, + "learning_rate": 3.30591775325978e-05, + "loss": 1.2417, + "theoretical_loss": 3.3335029586869966, + "tokens_seen": 3192324096 + }, + { + "epoch": 0.93, + "learning_rate": 3.3039117352056166e-05, + "loss": 1.2539, + "theoretical_loss": 3.3334978075864226, + "tokens_seen": 3192389632 + }, + { + "epoch": 0.93, + "learning_rate": 3.301905717151454e-05, + "loss": 1.2531, + "theoretical_loss": 3.333492656621201, + "tokens_seen": 3192455168 + }, + { + "epoch": 0.93, + "learning_rate": 3.299899699097292e-05, + "loss": 1.2059, + "theoretical_loss": 3.333487505791327, + "tokens_seen": 3192520704 + }, + { + "epoch": 0.93, + "learning_rate": 3.2978936810431294e-05, + "loss": 1.2704, + "theoretical_loss": 3.333482355096793, + "tokens_seen": 3192586240 + }, + { + "epoch": 0.93, + "learning_rate": 3.295887662988967e-05, + "loss": 1.2672, + "theoretical_loss": 3.3334772045375933, + "tokens_seen": 3192651776 + }, + { + "epoch": 0.93, + "learning_rate": 3.2938816449348045e-05, + "loss": 1.3394, + "theoretical_loss": 3.333472054113721, + "tokens_seen": 3192717312 + }, + { + "epoch": 0.94, + "learning_rate": 3.291875626880642e-05, + "loss": 1.2755, + "theoretical_loss": 3.3334669038251707, + "tokens_seen": 3192782848 + }, + { + "epoch": 0.94, + "learning_rate": 3.28986960882648e-05, + "loss": 1.2919, + "theoretical_loss": 3.3334617536719353, + "tokens_seen": 3192848384 + }, + { + "epoch": 0.94, + "learning_rate": 3.287863590772317e-05, + "loss": 1.2422, + "theoretical_loss": 3.3334566036540085, + "tokens_seen": 3192913920 + }, + { + "epoch": 0.94, + "learning_rate": 3.285857572718155e-05, + "loss": 1.2911, + "theoretical_loss": 3.333451453771384, + "tokens_seen": 3192979456 + }, + { + "epoch": 0.94, + "learning_rate": 3.283851554663992e-05, + "loss": 1.206, + "theoretical_loss": 3.3334463040240565, + "tokens_seen": 3193044992 + }, + { + "epoch": 0.94, + "learning_rate": 3.281845536609829e-05, + "loss": 1.2267, + "theoretical_loss": 3.3334411544120184, + "tokens_seen": 3193110528 + }, + { + "epoch": 0.94, + "learning_rate": 3.279839518555667e-05, + "loss": 1.2779, + "theoretical_loss": 3.3334360049352636, + "tokens_seen": 3193176064 + }, + { + "epoch": 0.94, + "learning_rate": 3.2778335005015045e-05, + "loss": 1.1807, + "theoretical_loss": 3.333430855593786, + "tokens_seen": 3193241600 + }, + { + "epoch": 0.94, + "learning_rate": 3.275827482447342e-05, + "loss": 1.2656, + "theoretical_loss": 3.333425706387579, + "tokens_seen": 3193307136 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1160256, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.259645938873291, + "objective/train/theoretical_loss": 3.333420557316637, + "objective/train/tokens_used": 1563831776, + "theoretical_loss": 3.333420557316637, + "tokens_seen": 3193372672 + }, + { + "epoch": 0.94, + "learning_rate": 3.2738214643931796e-05, + "loss": 1.2385, + "theoretical_loss": 3.333420557316637, + "tokens_seen": 3193372672 + }, + { + "epoch": 0.94, + "learning_rate": 3.271815446339017e-05, + "loss": 1.3075, + "theoretical_loss": 3.3334154083809526, + "tokens_seen": 3193438208 + }, + { + "epoch": 0.94, + "learning_rate": 3.269809428284855e-05, + "loss": 1.3258, + "theoretical_loss": 3.3334102595805204, + "tokens_seen": 3193503744 + }, + { + "epoch": 0.94, + "learning_rate": 3.2678034102306924e-05, + "loss": 1.2857, + "theoretical_loss": 3.3334051109153338, + "tokens_seen": 3193569280 + }, + { + "epoch": 0.94, + "learning_rate": 3.26579739217653e-05, + "loss": 1.3352, + "theoretical_loss": 3.3333999623853865, + "tokens_seen": 3193634816 + }, + { + "epoch": 0.94, + "learning_rate": 3.263791374122367e-05, + "loss": 1.3427, + "theoretical_loss": 3.3333948139906715, + "tokens_seen": 3193700352 + }, + { + "epoch": 0.94, + "learning_rate": 3.2617853560682044e-05, + "loss": 1.1445, + "theoretical_loss": 3.3333896657311834, + "tokens_seen": 3193765888 + }, + { + "epoch": 0.94, + "learning_rate": 3.259779338014042e-05, + "loss": 1.2172, + "theoretical_loss": 3.3333845176069152, + "tokens_seen": 3193831424 + }, + { + "epoch": 0.94, + "learning_rate": 3.2577733199598796e-05, + "loss": 1.2991, + "theoretical_loss": 3.333379369617861, + "tokens_seen": 3193896960 + }, + { + "epoch": 0.94, + "learning_rate": 3.255767301905718e-05, + "loss": 1.2057, + "theoretical_loss": 3.3333742217640143, + "tokens_seen": 3193962496 + }, + { + "epoch": 0.94, + "learning_rate": 3.253761283851555e-05, + "loss": 1.2388, + "theoretical_loss": 3.333369074045369, + "tokens_seen": 3194028032 + }, + { + "epoch": 0.94, + "learning_rate": 3.2517552657973923e-05, + "loss": 1.2882, + "theoretical_loss": 3.3333639264619186, + "tokens_seen": 3194093568 + }, + { + "epoch": 0.94, + "learning_rate": 3.24974924774323e-05, + "loss": 1.2046, + "theoretical_loss": 3.3333587790136567, + "tokens_seen": 3194159104 + }, + { + "epoch": 0.94, + "learning_rate": 3.2477432296890675e-05, + "loss": 1.213, + "theoretical_loss": 3.333353631700577, + "tokens_seen": 3194224640 + }, + { + "epoch": 0.94, + "learning_rate": 3.2457372116349044e-05, + "loss": 1.2268, + "theoretical_loss": 3.333348484522673, + "tokens_seen": 3194290176 + }, + { + "epoch": 0.94, + "learning_rate": 3.243731193580742e-05, + "loss": 1.2195, + "theoretical_loss": 3.333343337479939, + "tokens_seen": 3194355712 + }, + { + "epoch": 0.94, + "learning_rate": 3.2417251755265796e-05, + "loss": 1.2684, + "theoretical_loss": 3.333338190572368, + "tokens_seen": 3194421248 + }, + { + "epoch": 0.94, + "learning_rate": 3.239719157472417e-05, + "loss": 1.2452, + "theoretical_loss": 3.333333043799954, + "tokens_seen": 3194486784 + }, + { + "epoch": 0.94, + "learning_rate": 3.237713139418255e-05, + "loss": 1.2679, + "theoretical_loss": 3.3333278971626905, + "tokens_seen": 3194552320 + }, + { + "epoch": 0.94, + "learning_rate": 3.235707121364092e-05, + "loss": 1.3843, + "theoretical_loss": 3.3333227506605714, + "tokens_seen": 3194617856 + }, + { + "epoch": 0.94, + "learning_rate": 3.23370110330993e-05, + "loss": 1.3511, + "theoretical_loss": 3.3333176042935904, + "tokens_seen": 3194683392 + }, + { + "epoch": 0.94, + "learning_rate": 3.2316950852557675e-05, + "loss": 1.3166, + "theoretical_loss": 3.333312458061741, + "tokens_seen": 3194748928 + }, + { + "epoch": 0.94, + "learning_rate": 3.229689067201605e-05, + "loss": 1.2509, + "theoretical_loss": 3.333307311965017, + "tokens_seen": 3194814464 + }, + { + "epoch": 0.94, + "learning_rate": 3.2276830491474426e-05, + "loss": 1.2086, + "theoretical_loss": 3.3333021660034117, + "tokens_seen": 3194880000 + }, + { + "epoch": 0.94, + "learning_rate": 3.2256770310932795e-05, + "loss": 1.2055, + "theoretical_loss": 3.3332970201769196, + "tokens_seen": 3194945536 + }, + { + "epoch": 0.94, + "learning_rate": 3.223671013039117e-05, + "loss": 1.3144, + "theoretical_loss": 3.3332918744855338, + "tokens_seen": 3195011072 + }, + { + "epoch": 0.94, + "learning_rate": 3.221664994984955e-05, + "loss": 1.266, + "theoretical_loss": 3.333286728929248, + "tokens_seen": 3195076608 + }, + { + "epoch": 0.94, + "learning_rate": 3.219658976930792e-05, + "loss": 1.1838, + "theoretical_loss": 3.3332815835080556, + "tokens_seen": 3195142144 + }, + { + "epoch": 0.94, + "learning_rate": 3.2176529588766305e-05, + "loss": 1.3234, + "theoretical_loss": 3.333276438221951, + "tokens_seen": 3195207680 + }, + { + "epoch": 0.94, + "learning_rate": 3.2156469408224674e-05, + "loss": 1.2338, + "theoretical_loss": 3.3332712930709274, + "tokens_seen": 3195273216 + }, + { + "epoch": 0.94, + "learning_rate": 3.213640922768305e-05, + "loss": 1.2515, + "theoretical_loss": 3.3332661480549786, + "tokens_seen": 3195338752 + }, + { + "epoch": 0.94, + "learning_rate": 3.2116349047141426e-05, + "loss": 1.3061, + "theoretical_loss": 3.333261003174098, + "tokens_seen": 3195404288 + }, + { + "epoch": 0.94, + "learning_rate": 3.20962888665998e-05, + "loss": 1.1317, + "theoretical_loss": 3.3332558584282803, + "tokens_seen": 3195469824 + }, + { + "epoch": 0.94, + "learning_rate": 3.207622868605818e-05, + "loss": 1.2064, + "theoretical_loss": 3.333250713817518, + "tokens_seen": 3195535360 + }, + { + "epoch": 0.94, + "learning_rate": 3.2056168505516546e-05, + "loss": 1.3079, + "theoretical_loss": 3.333245569341805, + "tokens_seen": 3195600896 + }, + { + "epoch": 0.94, + "learning_rate": 3.203610832497492e-05, + "loss": 1.2109, + "theoretical_loss": 3.3332404250011356, + "tokens_seen": 3195666432 + }, + { + "epoch": 0.94, + "learning_rate": 3.20160481444333e-05, + "loss": 1.3276, + "theoretical_loss": 3.333235280795503, + "tokens_seen": 3195731968 + }, + { + "epoch": 0.94, + "learning_rate": 3.199598796389168e-05, + "loss": 1.2517, + "theoretical_loss": 3.333230136724901, + "tokens_seen": 3195797504 + }, + { + "epoch": 0.94, + "learning_rate": 3.1975927783350056e-05, + "loss": 1.3568, + "theoretical_loss": 3.333224992789323, + "tokens_seen": 3195863040 + }, + { + "epoch": 0.94, + "learning_rate": 3.1955867602808425e-05, + "loss": 1.2513, + "theoretical_loss": 3.3332198489887634, + "tokens_seen": 3195928576 + }, + { + "epoch": 0.94, + "learning_rate": 3.19358074222668e-05, + "loss": 1.2124, + "theoretical_loss": 3.3332147053232153, + "tokens_seen": 3195994112 + }, + { + "epoch": 0.94, + "learning_rate": 3.191574724172518e-05, + "loss": 1.2309, + "theoretical_loss": 3.3332095617926725, + "tokens_seen": 3196059648 + }, + { + "epoch": 0.94, + "learning_rate": 3.189568706118355e-05, + "loss": 1.2267, + "theoretical_loss": 3.3332044183971288, + "tokens_seen": 3196125184 + }, + { + "epoch": 0.94, + "learning_rate": 3.187562688064193e-05, + "loss": 1.2994, + "theoretical_loss": 3.333199275136578, + "tokens_seen": 3196190720 + }, + { + "epoch": 0.94, + "learning_rate": 3.18555667001003e-05, + "loss": 1.2225, + "theoretical_loss": 3.333194132011013, + "tokens_seen": 3196256256 + }, + { + "epoch": 0.94, + "learning_rate": 3.183550651955867e-05, + "loss": 1.3153, + "theoretical_loss": 3.3331889890204285, + "tokens_seen": 3196321792 + }, + { + "epoch": 0.94, + "learning_rate": 3.181544633901705e-05, + "loss": 1.2822, + "theoretical_loss": 3.3331838461648178, + "tokens_seen": 3196387328 + }, + { + "epoch": 0.94, + "learning_rate": 3.179538615847543e-05, + "loss": 1.2266, + "theoretical_loss": 3.3331787034441747, + "tokens_seen": 3196452864 + }, + { + "epoch": 0.94, + "learning_rate": 3.177532597793381e-05, + "loss": 1.2894, + "theoretical_loss": 3.3331735608584925, + "tokens_seen": 3196518400 + }, + { + "epoch": 0.94, + "learning_rate": 3.1755265797392176e-05, + "loss": 1.1787, + "theoretical_loss": 3.333168418407765, + "tokens_seen": 3196583936 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1161767, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1227366924285889, + "objective/train/theoretical_loss": 3.3331632760919865, + "objective/train/tokens_used": 1567108576, + "theoretical_loss": 3.3331632760919865, + "tokens_seen": 3196649472 + }, + { + "epoch": 0.94, + "learning_rate": 3.173520561685055e-05, + "loss": 1.2256, + "theoretical_loss": 3.3331632760919865, + "tokens_seen": 3196649472 + }, + { + "epoch": 0.94, + "learning_rate": 3.171514543630893e-05, + "loss": 1.1919, + "theoretical_loss": 3.3331581339111502, + "tokens_seen": 3196715008 + }, + { + "epoch": 0.94, + "learning_rate": 3.1695085255767304e-05, + "loss": 1.1696, + "theoretical_loss": 3.3331529918652496, + "tokens_seen": 3196780544 + }, + { + "epoch": 0.94, + "learning_rate": 3.167502507522567e-05, + "loss": 1.2918, + "theoretical_loss": 3.333147849954279, + "tokens_seen": 3196846080 + }, + { + "epoch": 0.94, + "learning_rate": 3.165496489468405e-05, + "loss": 1.1894, + "theoretical_loss": 3.333142708178231, + "tokens_seen": 3196911616 + }, + { + "epoch": 0.94, + "learning_rate": 3.1634904714142424e-05, + "loss": 1.163, + "theoretical_loss": 3.3331375665371006, + "tokens_seen": 3196977152 + }, + { + "epoch": 0.94, + "learning_rate": 3.161484453360081e-05, + "loss": 1.2177, + "theoretical_loss": 3.3331324250308807, + "tokens_seen": 3197042688 + }, + { + "epoch": 0.94, + "learning_rate": 3.159478435305918e-05, + "loss": 1.3091, + "theoretical_loss": 3.3331272836595653, + "tokens_seen": 3197108224 + }, + { + "epoch": 0.94, + "learning_rate": 3.157472417251755e-05, + "loss": 1.2079, + "theoretical_loss": 3.3331221424231483, + "tokens_seen": 3197173760 + }, + { + "epoch": 0.94, + "learning_rate": 3.155466399197593e-05, + "loss": 1.1817, + "theoretical_loss": 3.3331170013216225, + "tokens_seen": 3197239296 + }, + { + "epoch": 0.94, + "learning_rate": 3.1534603811434303e-05, + "loss": 1.3461, + "theoretical_loss": 3.3331118603549825, + "tokens_seen": 3197304832 + }, + { + "epoch": 0.94, + "learning_rate": 3.151454363089268e-05, + "loss": 1.2436, + "theoretical_loss": 3.333106719523222, + "tokens_seen": 3197370368 + }, + { + "epoch": 0.94, + "learning_rate": 3.1494483450351055e-05, + "loss": 1.1733, + "theoretical_loss": 3.3331015788263336, + "tokens_seen": 3197435904 + }, + { + "epoch": 0.94, + "learning_rate": 3.1474423269809424e-05, + "loss": 1.2231, + "theoretical_loss": 3.3330964382643122, + "tokens_seen": 3197501440 + }, + { + "epoch": 0.94, + "learning_rate": 3.14543630892678e-05, + "loss": 1.2564, + "theoretical_loss": 3.3330912978371514, + "tokens_seen": 3197566976 + }, + { + "epoch": 0.94, + "learning_rate": 3.143430290872618e-05, + "loss": 1.2567, + "theoretical_loss": 3.333086157544844, + "tokens_seen": 3197632512 + }, + { + "epoch": 0.94, + "learning_rate": 3.141424272818456e-05, + "loss": 1.2612, + "theoretical_loss": 3.333081017387385, + "tokens_seen": 3197698048 + }, + { + "epoch": 0.94, + "learning_rate": 3.1394182547642934e-05, + "loss": 1.2837, + "theoretical_loss": 3.3330758773647666, + "tokens_seen": 3197763584 + }, + { + "epoch": 0.94, + "learning_rate": 3.13741223671013e-05, + "loss": 1.2164, + "theoretical_loss": 3.3330707374769837, + "tokens_seen": 3197829120 + }, + { + "epoch": 0.94, + "learning_rate": 3.135406218655968e-05, + "loss": 1.3039, + "theoretical_loss": 3.3330655977240293, + "tokens_seen": 3197894656 + }, + { + "epoch": 0.94, + "learning_rate": 3.1334002006018055e-05, + "loss": 1.2146, + "theoretical_loss": 3.333060458105898, + "tokens_seen": 3197960192 + }, + { + "epoch": 0.94, + "learning_rate": 3.131394182547643e-05, + "loss": 1.3342, + "theoretical_loss": 3.333055318622582, + "tokens_seen": 3198025728 + }, + { + "epoch": 0.94, + "learning_rate": 3.1293881644934806e-05, + "loss": 1.3548, + "theoretical_loss": 3.3330501792740765, + "tokens_seen": 3198091264 + }, + { + "epoch": 0.94, + "learning_rate": 3.1273821464393175e-05, + "loss": 1.232, + "theoretical_loss": 3.3330450400603744, + "tokens_seen": 3198156800 + }, + { + "epoch": 0.94, + "learning_rate": 3.125376128385155e-05, + "loss": 1.2076, + "theoretical_loss": 3.3330399009814693, + "tokens_seen": 3198222336 + }, + { + "epoch": 0.94, + "learning_rate": 3.123370110330993e-05, + "loss": 1.2526, + "theoretical_loss": 3.3330347620373555, + "tokens_seen": 3198287872 + }, + { + "epoch": 0.94, + "learning_rate": 3.12136409227683e-05, + "loss": 1.2514, + "theoretical_loss": 3.3330296232280263, + "tokens_seen": 3198353408 + }, + { + "epoch": 0.94, + "learning_rate": 3.1193580742226685e-05, + "loss": 1.2575, + "theoretical_loss": 3.3330244845534756, + "tokens_seen": 3198418944 + }, + { + "epoch": 0.94, + "learning_rate": 3.1173520561685054e-05, + "loss": 1.2536, + "theoretical_loss": 3.333019346013697, + "tokens_seen": 3198484480 + }, + { + "epoch": 0.94, + "learning_rate": 3.115346038114343e-05, + "loss": 1.2993, + "theoretical_loss": 3.3330142076086844, + "tokens_seen": 3198550016 + }, + { + "epoch": 0.94, + "learning_rate": 3.1133400200601806e-05, + "loss": 1.3024, + "theoretical_loss": 3.3330090693384307, + "tokens_seen": 3198615552 + }, + { + "epoch": 0.94, + "learning_rate": 3.111334002006018e-05, + "loss": 1.1543, + "theoretical_loss": 3.3330039312029305, + "tokens_seen": 3198681088 + }, + { + "epoch": 0.94, + "learning_rate": 3.109327983951856e-05, + "loss": 1.2858, + "theoretical_loss": 3.3329987932021776, + "tokens_seen": 3198746624 + }, + { + "epoch": 0.94, + "learning_rate": 3.107321965897693e-05, + "loss": 1.3004, + "theoretical_loss": 3.332993655336165, + "tokens_seen": 3198812160 + }, + { + "epoch": 0.94, + "learning_rate": 3.105315947843531e-05, + "loss": 1.2507, + "theoretical_loss": 3.3329885176048863, + "tokens_seen": 3198877696 + }, + { + "epoch": 0.94, + "learning_rate": 3.103309929789368e-05, + "loss": 1.1671, + "theoretical_loss": 3.332983380008336, + "tokens_seen": 3198943232 + }, + { + "epoch": 0.94, + "learning_rate": 3.101303911735206e-05, + "loss": 1.2624, + "theoretical_loss": 3.3329782425465075, + "tokens_seen": 3199008768 + }, + { + "epoch": 0.94, + "learning_rate": 3.099297893681043e-05, + "loss": 1.247, + "theoretical_loss": 3.3329731052193945, + "tokens_seen": 3199074304 + }, + { + "epoch": 0.94, + "learning_rate": 3.0972918756268805e-05, + "loss": 1.2433, + "theoretical_loss": 3.3329679680269906, + "tokens_seen": 3199139840 + }, + { + "epoch": 0.94, + "learning_rate": 3.095285857572718e-05, + "loss": 1.2852, + "theoretical_loss": 3.332962830969289, + "tokens_seen": 3199205376 + }, + { + "epoch": 0.94, + "learning_rate": 3.093279839518556e-05, + "loss": 1.1851, + "theoretical_loss": 3.332957694046285, + "tokens_seen": 3199270912 + }, + { + "epoch": 0.94, + "learning_rate": 3.091273821464393e-05, + "loss": 1.2415, + "theoretical_loss": 3.3329525572579706, + "tokens_seen": 3199336448 + }, + { + "epoch": 0.94, + "learning_rate": 3.089267803410231e-05, + "loss": 1.3198, + "theoretical_loss": 3.33294742060434, + "tokens_seen": 3199401984 + }, + { + "epoch": 0.94, + "learning_rate": 3.0872617853560684e-05, + "loss": 1.2211, + "theoretical_loss": 3.3329422840853877, + "tokens_seen": 3199467520 + }, + { + "epoch": 0.94, + "learning_rate": 3.085255767301906e-05, + "loss": 1.1736, + "theoretical_loss": 3.3329371477011067, + "tokens_seen": 3199533056 + }, + { + "epoch": 0.94, + "learning_rate": 3.083249749247743e-05, + "loss": 1.2601, + "theoretical_loss": 3.3329320114514904, + "tokens_seen": 3199598592 + }, + { + "epoch": 0.94, + "learning_rate": 3.081243731193581e-05, + "loss": 1.1871, + "theoretical_loss": 3.332926875336533, + "tokens_seen": 3199664128 + }, + { + "epoch": 0.94, + "learning_rate": 3.079237713139418e-05, + "loss": 1.2228, + "theoretical_loss": 3.3329217393562285, + "tokens_seen": 3199729664 + }, + { + "epoch": 0.94, + "learning_rate": 3.0772316950852556e-05, + "loss": 1.3689, + "theoretical_loss": 3.33291660351057, + "tokens_seen": 3199795200 + }, + { + "epoch": 0.94, + "learning_rate": 3.075225677031093e-05, + "loss": 1.2252, + "theoretical_loss": 3.3329114677995517, + "tokens_seen": 3199860736 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1162972, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.224173665046692, + "objective/train/theoretical_loss": 3.3329063322231667, + "objective/train/tokens_used": 1570385376, + "theoretical_loss": 3.3329063322231667, + "tokens_seen": 3199926272 + }, + { + "epoch": 0.94, + "learning_rate": 3.073219658976931e-05, + "loss": 1.263, + "theoretical_loss": 3.3329063322231667, + "tokens_seen": 3199926272 + }, + { + "epoch": 0.94, + "learning_rate": 3.0712136409227684e-05, + "loss": 1.1724, + "theoretical_loss": 3.3329011967814095, + "tokens_seen": 3199991808 + }, + { + "epoch": 0.94, + "learning_rate": 3.069207622868606e-05, + "loss": 1.2454, + "theoretical_loss": 3.332896061474273, + "tokens_seen": 3200057344 + }, + { + "epoch": 0.94, + "learning_rate": 3.0672016048144435e-05, + "loss": 1.2922, + "theoretical_loss": 3.3328909263017517, + "tokens_seen": 3200122880 + }, + { + "epoch": 0.94, + "learning_rate": 3.0651955867602804e-05, + "loss": 1.3151, + "theoretical_loss": 3.3328857912638385, + "tokens_seen": 3200188416 + }, + { + "epoch": 0.94, + "learning_rate": 3.063189568706119e-05, + "loss": 1.2505, + "theoretical_loss": 3.332880656360528, + "tokens_seen": 3200253952 + }, + { + "epoch": 0.94, + "learning_rate": 3.061183550651956e-05, + "loss": 1.298, + "theoretical_loss": 3.3328755215918133, + "tokens_seen": 3200319488 + }, + { + "epoch": 0.94, + "learning_rate": 3.059177532597793e-05, + "loss": 1.2958, + "theoretical_loss": 3.3328703869576883, + "tokens_seen": 3200385024 + }, + { + "epoch": 0.94, + "learning_rate": 3.057171514543631e-05, + "loss": 1.238, + "theoretical_loss": 3.3328652524581464, + "tokens_seen": 3200450560 + }, + { + "epoch": 0.94, + "learning_rate": 3.055165496489468e-05, + "loss": 1.2934, + "theoretical_loss": 3.332860118093182, + "tokens_seen": 3200516096 + }, + { + "epoch": 0.94, + "learning_rate": 3.053159478435306e-05, + "loss": 1.2758, + "theoretical_loss": 3.3328549838627883, + "tokens_seen": 3200581632 + }, + { + "epoch": 0.94, + "learning_rate": 3.0511534603811435e-05, + "loss": 1.1879, + "theoretical_loss": 3.332849849766959, + "tokens_seen": 3200647168 + }, + { + "epoch": 0.94, + "learning_rate": 3.049147442326981e-05, + "loss": 1.2132, + "theoretical_loss": 3.332844715805688, + "tokens_seen": 3200712704 + }, + { + "epoch": 0.94, + "learning_rate": 3.0471414242728183e-05, + "loss": 1.2519, + "theoretical_loss": 3.3328395819789693, + "tokens_seen": 3200778240 + }, + { + "epoch": 0.94, + "learning_rate": 3.0451354062186562e-05, + "loss": 1.3093, + "theoretical_loss": 3.332834448286796, + "tokens_seen": 3200843776 + }, + { + "epoch": 0.94, + "learning_rate": 3.0431293881644935e-05, + "loss": 1.2797, + "theoretical_loss": 3.332829314729162, + "tokens_seen": 3200909312 + }, + { + "epoch": 0.94, + "learning_rate": 3.041123370110331e-05, + "loss": 1.1728, + "theoretical_loss": 3.3328241813060617, + "tokens_seen": 3200974848 + }, + { + "epoch": 0.94, + "learning_rate": 3.0391173520561686e-05, + "loss": 1.2975, + "theoretical_loss": 3.3328190480174875, + "tokens_seen": 3201040384 + }, + { + "epoch": 0.94, + "learning_rate": 3.037111334002006e-05, + "loss": 1.3148, + "theoretical_loss": 3.3328139148634346, + "tokens_seen": 3201105920 + }, + { + "epoch": 0.94, + "learning_rate": 3.0351053159478438e-05, + "loss": 1.2212, + "theoretical_loss": 3.332808781843896, + "tokens_seen": 3201171456 + }, + { + "epoch": 0.94, + "learning_rate": 3.033099297893681e-05, + "loss": 1.2479, + "theoretical_loss": 3.3328036489588646, + "tokens_seen": 3201236992 + }, + { + "epoch": 0.94, + "learning_rate": 3.0310932798395186e-05, + "loss": 1.2367, + "theoretical_loss": 3.3327985162083356, + "tokens_seen": 3201302528 + }, + { + "epoch": 0.94, + "learning_rate": 3.0290872617853562e-05, + "loss": 1.2439, + "theoretical_loss": 3.332793383592302, + "tokens_seen": 3201368064 + }, + { + "epoch": 0.94, + "learning_rate": 3.0270812437311934e-05, + "loss": 1.2185, + "theoretical_loss": 3.3327882511107574, + "tokens_seen": 3201433600 + }, + { + "epoch": 0.94, + "learning_rate": 3.0250752256770314e-05, + "loss": 1.3755, + "theoretical_loss": 3.332783118763696, + "tokens_seen": 3201499136 + }, + { + "epoch": 0.94, + "learning_rate": 3.0230692076228686e-05, + "loss": 1.2786, + "theoretical_loss": 3.332777986551111, + "tokens_seen": 3201564672 + }, + { + "epoch": 0.94, + "learning_rate": 3.0210631895687062e-05, + "loss": 1.211, + "theoretical_loss": 3.332772854472996, + "tokens_seen": 3201630208 + }, + { + "epoch": 0.94, + "learning_rate": 3.0190571715145438e-05, + "loss": 1.2768, + "theoretical_loss": 3.3327677225293457, + "tokens_seen": 3201695744 + }, + { + "epoch": 0.94, + "learning_rate": 3.0170511534603813e-05, + "loss": 1.2581, + "theoretical_loss": 3.332762590720153, + "tokens_seen": 3201761280 + }, + { + "epoch": 0.94, + "learning_rate": 3.015045135406219e-05, + "loss": 1.1953, + "theoretical_loss": 3.332757459045412, + "tokens_seen": 3201826816 + }, + { + "epoch": 0.94, + "learning_rate": 3.013039117352056e-05, + "loss": 1.2296, + "theoretical_loss": 3.332752327505116, + "tokens_seen": 3201892352 + }, + { + "epoch": 0.94, + "learning_rate": 3.0110330992978937e-05, + "loss": 1.3123, + "theoretical_loss": 3.3327471960992594, + "tokens_seen": 3201957888 + }, + { + "epoch": 0.94, + "learning_rate": 3.009027081243731e-05, + "loss": 1.2276, + "theoretical_loss": 3.332742064827835, + "tokens_seen": 3202023424 + }, + { + "epoch": 0.94, + "learning_rate": 3.007021063189569e-05, + "loss": 1.3513, + "theoretical_loss": 3.3327369336908372, + "tokens_seen": 3202088960 + }, + { + "epoch": 0.94, + "learning_rate": 3.0050150451354065e-05, + "loss": 1.2609, + "theoretical_loss": 3.3327318026882597, + "tokens_seen": 3202154496 + }, + { + "epoch": 0.94, + "learning_rate": 3.0030090270812437e-05, + "loss": 1.3048, + "theoretical_loss": 3.332726671820096, + "tokens_seen": 3202220032 + }, + { + "epoch": 0.94, + "learning_rate": 3.0010030090270813e-05, + "loss": 1.1589, + "theoretical_loss": 3.33272154108634, + "tokens_seen": 3202285568 + }, + { + "epoch": 0.94, + "learning_rate": 2.9989969909729185e-05, + "loss": 1.1905, + "theoretical_loss": 3.3327164104869853, + "tokens_seen": 3202351104 + }, + { + "epoch": 0.94, + "learning_rate": 2.9969909729187564e-05, + "loss": 1.1578, + "theoretical_loss": 3.3327112800220258, + "tokens_seen": 3202416640 + }, + { + "epoch": 0.94, + "learning_rate": 2.994984954864594e-05, + "loss": 1.3288, + "theoretical_loss": 3.3327061496914547, + "tokens_seen": 3202482176 + }, + { + "epoch": 0.94, + "learning_rate": 2.9929789368104313e-05, + "loss": 1.2681, + "theoretical_loss": 3.3327010194952664, + "tokens_seen": 3202547712 + }, + { + "epoch": 0.94, + "learning_rate": 2.990972918756269e-05, + "loss": 1.2673, + "theoretical_loss": 3.332695889433454, + "tokens_seen": 3202613248 + }, + { + "epoch": 0.94, + "learning_rate": 2.988966900702106e-05, + "loss": 1.1942, + "theoretical_loss": 3.3326907595060122, + "tokens_seen": 3202678784 + }, + { + "epoch": 0.94, + "learning_rate": 2.986960882647944e-05, + "loss": 1.0726, + "theoretical_loss": 3.332685629712934, + "tokens_seen": 3202744320 + }, + { + "epoch": 0.94, + "learning_rate": 2.9849548645937816e-05, + "loss": 1.1859, + "theoretical_loss": 3.3326805000542135, + "tokens_seen": 3202809856 + }, + { + "epoch": 0.94, + "learning_rate": 2.9829488465396188e-05, + "loss": 1.3195, + "theoretical_loss": 3.3326753705298438, + "tokens_seen": 3202875392 + }, + { + "epoch": 0.94, + "learning_rate": 2.9809428284854564e-05, + "loss": 1.2584, + "theoretical_loss": 3.332670241139819, + "tokens_seen": 3202940928 + }, + { + "epoch": 0.94, + "learning_rate": 2.978936810431294e-05, + "loss": 1.2537, + "theoretical_loss": 3.332665111884133, + "tokens_seen": 3203006464 + }, + { + "epoch": 0.94, + "learning_rate": 2.9769307923771316e-05, + "loss": 1.2067, + "theoretical_loss": 3.332659982762779, + "tokens_seen": 3203072000 + }, + { + "epoch": 0.94, + "learning_rate": 2.974924774322969e-05, + "loss": 1.2316, + "theoretical_loss": 3.3326548537757517, + "tokens_seen": 3203137536 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1164087, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2418830394744873, + "objective/train/theoretical_loss": 3.3326497249230442, + "objective/train/tokens_used": 1573662176, + "theoretical_loss": 3.3326497249230442, + "tokens_seen": 3203203072 + }, + { + "epoch": 0.94, + "learning_rate": 2.9729187562688064e-05, + "loss": 1.2987, + "theoretical_loss": 3.3326497249230442, + "tokens_seen": 3203203072 + }, + { + "epoch": 0.94, + "learning_rate": 2.970912738214644e-05, + "loss": 1.237, + "theoretical_loss": 3.33264459620465, + "tokens_seen": 3203268608 + }, + { + "epoch": 0.94, + "learning_rate": 2.9689067201604815e-05, + "loss": 1.1192, + "theoretical_loss": 3.332639467620563, + "tokens_seen": 3203334144 + }, + { + "epoch": 0.94, + "learning_rate": 2.966900702106319e-05, + "loss": 1.2422, + "theoretical_loss": 3.3326343391707773, + "tokens_seen": 3203399680 + }, + { + "epoch": 0.94, + "learning_rate": 2.9648946840521564e-05, + "loss": 1.2493, + "theoretical_loss": 3.3326292108552864, + "tokens_seen": 3203465216 + }, + { + "epoch": 0.94, + "learning_rate": 2.962888665997994e-05, + "loss": 1.2324, + "theoretical_loss": 3.332624082674084, + "tokens_seen": 3203530752 + }, + { + "epoch": 0.94, + "learning_rate": 2.9608826479438315e-05, + "loss": 1.1505, + "theoretical_loss": 3.332618954627164, + "tokens_seen": 3203596288 + }, + { + "epoch": 0.94, + "learning_rate": 2.958876629889669e-05, + "loss": 1.2455, + "theoretical_loss": 3.3326138267145202, + "tokens_seen": 3203661824 + }, + { + "epoch": 0.94, + "learning_rate": 2.9568706118355067e-05, + "loss": 1.26, + "theoretical_loss": 3.332608698936146, + "tokens_seen": 3203727360 + }, + { + "epoch": 0.94, + "learning_rate": 2.954864593781344e-05, + "loss": 1.1703, + "theoretical_loss": 3.3326035712920348, + "tokens_seen": 3203792896 + }, + { + "epoch": 0.94, + "learning_rate": 2.9528585757271815e-05, + "loss": 1.2581, + "theoretical_loss": 3.332598443782181, + "tokens_seen": 3203858432 + }, + { + "epoch": 0.94, + "learning_rate": 2.9508525576730194e-05, + "loss": 1.2226, + "theoretical_loss": 3.3325933164065784, + "tokens_seen": 3203923968 + }, + { + "epoch": 0.94, + "learning_rate": 2.9488465396188567e-05, + "loss": 1.2639, + "theoretical_loss": 3.3325881891652207, + "tokens_seen": 3203989504 + }, + { + "epoch": 0.94, + "learning_rate": 2.9468405215646942e-05, + "loss": 1.2379, + "theoretical_loss": 3.332583062058101, + "tokens_seen": 3204055040 + }, + { + "epoch": 0.94, + "learning_rate": 2.9448345035105315e-05, + "loss": 1.3421, + "theoretical_loss": 3.332577935085214, + "tokens_seen": 3204120576 + }, + { + "epoch": 0.94, + "learning_rate": 2.942828485456369e-05, + "loss": 1.278, + "theoretical_loss": 3.332572808246552, + "tokens_seen": 3204186112 + }, + { + "epoch": 0.94, + "learning_rate": 2.940822467402207e-05, + "loss": 1.2472, + "theoretical_loss": 3.332567681542111, + "tokens_seen": 3204251648 + }, + { + "epoch": 0.94, + "learning_rate": 2.9388164493480442e-05, + "loss": 1.2609, + "theoretical_loss": 3.332562554971882, + "tokens_seen": 3204317184 + }, + { + "epoch": 0.94, + "learning_rate": 2.9368104312938818e-05, + "loss": 1.2946, + "theoretical_loss": 3.332557428535861, + "tokens_seen": 3204382720 + }, + { + "epoch": 0.94, + "learning_rate": 2.934804413239719e-05, + "loss": 1.2801, + "theoretical_loss": 3.3325523022340406, + "tokens_seen": 3204448256 + }, + { + "epoch": 0.94, + "learning_rate": 2.9327983951855566e-05, + "loss": 1.2326, + "theoretical_loss": 3.3325471760664147, + "tokens_seen": 3204513792 + }, + { + "epoch": 0.94, + "learning_rate": 2.9307923771313942e-05, + "loss": 1.3357, + "theoretical_loss": 3.3325420500329774, + "tokens_seen": 3204579328 + }, + { + "epoch": 0.94, + "learning_rate": 2.9287863590772318e-05, + "loss": 1.1489, + "theoretical_loss": 3.332536924133722, + "tokens_seen": 3204644864 + }, + { + "epoch": 0.94, + "learning_rate": 2.9267803410230693e-05, + "loss": 1.3658, + "theoretical_loss": 3.3325317983686427, + "tokens_seen": 3204710400 + }, + { + "epoch": 0.94, + "learning_rate": 2.9247743229689066e-05, + "loss": 1.1399, + "theoretical_loss": 3.332526672737733, + "tokens_seen": 3204775936 + }, + { + "epoch": 0.94, + "learning_rate": 2.9227683049147445e-05, + "loss": 1.2736, + "theoretical_loss": 3.3325215472409866, + "tokens_seen": 3204841472 + }, + { + "epoch": 0.94, + "learning_rate": 2.9207622868605817e-05, + "loss": 1.2896, + "theoretical_loss": 3.332516421878397, + "tokens_seen": 3204907008 + }, + { + "epoch": 0.94, + "learning_rate": 2.9187562688064193e-05, + "loss": 1.2777, + "theoretical_loss": 3.332511296649959, + "tokens_seen": 3204972544 + }, + { + "epoch": 0.94, + "learning_rate": 2.916750250752257e-05, + "loss": 1.2867, + "theoretical_loss": 3.3325061715556648, + "tokens_seen": 3205038080 + }, + { + "epoch": 0.94, + "learning_rate": 2.914744232698094e-05, + "loss": 1.2182, + "theoretical_loss": 3.3325010465955094, + "tokens_seen": 3205103616 + }, + { + "epoch": 0.94, + "learning_rate": 2.912738214643932e-05, + "loss": 1.3038, + "theoretical_loss": 3.3324959217694854, + "tokens_seen": 3205169152 + }, + { + "epoch": 0.94, + "learning_rate": 2.9107321965897693e-05, + "loss": 1.2308, + "theoretical_loss": 3.332490797077588, + "tokens_seen": 3205234688 + }, + { + "epoch": 0.94, + "learning_rate": 2.908726178535607e-05, + "loss": 1.197, + "theoretical_loss": 3.3324856725198098, + "tokens_seen": 3205300224 + }, + { + "epoch": 0.94, + "learning_rate": 2.9067201604814445e-05, + "loss": 1.264, + "theoretical_loss": 3.332480548096145, + "tokens_seen": 3205365760 + }, + { + "epoch": 0.94, + "learning_rate": 2.9047141424272817e-05, + "loss": 1.2722, + "theoretical_loss": 3.3324754238065872, + "tokens_seen": 3205431296 + }, + { + "epoch": 0.94, + "learning_rate": 2.9027081243731196e-05, + "loss": 1.3183, + "theoretical_loss": 3.3324702996511304, + "tokens_seen": 3205496832 + }, + { + "epoch": 0.94, + "learning_rate": 2.900702106318957e-05, + "loss": 1.1337, + "theoretical_loss": 3.3324651756297676, + "tokens_seen": 3205562368 + }, + { + "epoch": 0.94, + "learning_rate": 2.8986960882647944e-05, + "loss": 1.2982, + "theoretical_loss": 3.332460051742494, + "tokens_seen": 3205627904 + }, + { + "epoch": 0.94, + "learning_rate": 2.8966900702106317e-05, + "loss": 1.3133, + "theoretical_loss": 3.3324549279893017, + "tokens_seen": 3205693440 + }, + { + "epoch": 0.94, + "learning_rate": 2.8946840521564696e-05, + "loss": 1.2519, + "theoretical_loss": 3.3324498043701856, + "tokens_seen": 3205758976 + }, + { + "epoch": 0.94, + "learning_rate": 2.8926780341023072e-05, + "loss": 1.2266, + "theoretical_loss": 3.3324446808851387, + "tokens_seen": 3205824512 + }, + { + "epoch": 0.94, + "learning_rate": 2.8906720160481444e-05, + "loss": 1.206, + "theoretical_loss": 3.3324395575341557, + "tokens_seen": 3205890048 + }, + { + "epoch": 0.94, + "learning_rate": 2.888665997993982e-05, + "loss": 1.1295, + "theoretical_loss": 3.3324344343172294, + "tokens_seen": 3205955584 + }, + { + "epoch": 0.94, + "learning_rate": 2.8866599799398192e-05, + "loss": 1.2087, + "theoretical_loss": 3.3324293112343537, + "tokens_seen": 3206021120 + }, + { + "epoch": 0.94, + "learning_rate": 2.884653961885657e-05, + "loss": 1.2535, + "theoretical_loss": 3.332424188285523, + "tokens_seen": 3206086656 + }, + { + "epoch": 0.94, + "learning_rate": 2.8826479438314947e-05, + "loss": 1.1982, + "theoretical_loss": 3.3324190654707304, + "tokens_seen": 3206152192 + }, + { + "epoch": 0.94, + "learning_rate": 2.880641925777332e-05, + "loss": 1.2737, + "theoretical_loss": 3.33241394278997, + "tokens_seen": 3206217728 + }, + { + "epoch": 0.94, + "learning_rate": 2.8786359077231696e-05, + "loss": 1.203, + "theoretical_loss": 3.332408820243235, + "tokens_seen": 3206283264 + }, + { + "epoch": 0.94, + "learning_rate": 2.8766298896690068e-05, + "loss": 1.269, + "theoretical_loss": 3.33240369783052, + "tokens_seen": 3206348800 + }, + { + "epoch": 0.94, + "learning_rate": 2.8746238716148447e-05, + "loss": 1.3028, + "theoretical_loss": 3.332398575551818, + "tokens_seen": 3206414336 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1165309, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2051477432250977, + "objective/train/theoretical_loss": 3.3323934534071236, + "objective/train/tokens_used": 1576938976, + "theoretical_loss": 3.3323934534071236, + "tokens_seen": 3206479872 + }, + { + "epoch": 0.94, + "learning_rate": 2.8726178535606823e-05, + "loss": 1.2075, + "theoretical_loss": 3.3323934534071236, + "tokens_seen": 3206479872 + }, + { + "epoch": 0.94, + "learning_rate": 2.8706118355065195e-05, + "loss": 1.1416, + "theoretical_loss": 3.3323883313964298, + "tokens_seen": 3206545408 + }, + { + "epoch": 0.94, + "learning_rate": 2.868605817452357e-05, + "loss": 1.2903, + "theoretical_loss": 3.3323832095197305, + "tokens_seen": 3206610944 + }, + { + "epoch": 0.94, + "learning_rate": 2.8665997993981947e-05, + "loss": 1.2326, + "theoretical_loss": 3.3323780877770197, + "tokens_seen": 3206676480 + }, + { + "epoch": 0.94, + "learning_rate": 2.8645937813440323e-05, + "loss": 1.2224, + "theoretical_loss": 3.332372966168291, + "tokens_seen": 3206742016 + }, + { + "epoch": 0.94, + "learning_rate": 2.8625877632898695e-05, + "loss": 1.2216, + "theoretical_loss": 3.332367844693538, + "tokens_seen": 3206807552 + }, + { + "epoch": 0.94, + "learning_rate": 2.860581745235707e-05, + "loss": 1.1797, + "theoretical_loss": 3.332362723352755, + "tokens_seen": 3206873088 + }, + { + "epoch": 0.94, + "learning_rate": 2.8585757271815447e-05, + "loss": 1.1926, + "theoretical_loss": 3.332357602145935, + "tokens_seen": 3206938624 + }, + { + "epoch": 0.94, + "learning_rate": 2.8565697091273823e-05, + "loss": 1.1943, + "theoretical_loss": 3.3323524810730722, + "tokens_seen": 3207004160 + }, + { + "epoch": 0.94, + "learning_rate": 2.85456369107322e-05, + "loss": 1.2024, + "theoretical_loss": 3.3323473601341607, + "tokens_seen": 3207069696 + }, + { + "epoch": 0.94, + "learning_rate": 2.852557673019057e-05, + "loss": 1.2243, + "theoretical_loss": 3.3323422393291935, + "tokens_seen": 3207135232 + }, + { + "epoch": 0.94, + "learning_rate": 2.8505516549648947e-05, + "loss": 1.2939, + "theoretical_loss": 3.332337118658165, + "tokens_seen": 3207200768 + }, + { + "epoch": 0.94, + "learning_rate": 2.8485456369107322e-05, + "loss": 1.2564, + "theoretical_loss": 3.3323319981210684, + "tokens_seen": 3207266304 + }, + { + "epoch": 0.94, + "learning_rate": 2.8465396188565698e-05, + "loss": 1.1754, + "theoretical_loss": 3.3323268777178976, + "tokens_seen": 3207331840 + }, + { + "epoch": 0.94, + "learning_rate": 2.8445336008024074e-05, + "loss": 1.2122, + "theoretical_loss": 3.332321757448647, + "tokens_seen": 3207397376 + }, + { + "epoch": 0.94, + "learning_rate": 2.8425275827482446e-05, + "loss": 1.2933, + "theoretical_loss": 3.3323166373133093, + "tokens_seen": 3207462912 + }, + { + "epoch": 0.94, + "learning_rate": 2.8405215646940822e-05, + "loss": 1.2718, + "theoretical_loss": 3.3323115173118794, + "tokens_seen": 3207528448 + }, + { + "epoch": 0.94, + "learning_rate": 2.83851554663992e-05, + "loss": 1.2435, + "theoretical_loss": 3.3323063974443503, + "tokens_seen": 3207593984 + }, + { + "epoch": 0.94, + "learning_rate": 2.8365095285857574e-05, + "loss": 1.304, + "theoretical_loss": 3.3323012777107155, + "tokens_seen": 3207659520 + }, + { + "epoch": 0.94, + "learning_rate": 2.834503510531595e-05, + "loss": 1.1445, + "theoretical_loss": 3.3322961581109696, + "tokens_seen": 3207725056 + }, + { + "epoch": 0.94, + "learning_rate": 2.8324974924774322e-05, + "loss": 1.2071, + "theoretical_loss": 3.332291038645106, + "tokens_seen": 3207790592 + }, + { + "epoch": 0.94, + "learning_rate": 2.8304914744232698e-05, + "loss": 1.2105, + "theoretical_loss": 3.3322859193131187, + "tokens_seen": 3207856128 + }, + { + "epoch": 0.94, + "learning_rate": 2.8284854563691077e-05, + "loss": 1.2196, + "theoretical_loss": 3.332280800115001, + "tokens_seen": 3207921664 + }, + { + "epoch": 0.94, + "learning_rate": 2.826479438314945e-05, + "loss": 1.3182, + "theoretical_loss": 3.3322756810507466, + "tokens_seen": 3207987200 + }, + { + "epoch": 0.94, + "learning_rate": 2.8244734202607825e-05, + "loss": 1.2623, + "theoretical_loss": 3.3322705621203497, + "tokens_seen": 3208052736 + }, + { + "epoch": 0.94, + "learning_rate": 2.8224674022066197e-05, + "loss": 1.3058, + "theoretical_loss": 3.332265443323804, + "tokens_seen": 3208118272 + }, + { + "epoch": 0.94, + "learning_rate": 2.8204613841524573e-05, + "loss": 1.2667, + "theoretical_loss": 3.3322603246611027, + "tokens_seen": 3208183808 + }, + { + "epoch": 0.94, + "learning_rate": 2.818455366098295e-05, + "loss": 1.1057, + "theoretical_loss": 3.3322552061322406, + "tokens_seen": 3208249344 + }, + { + "epoch": 0.94, + "learning_rate": 2.8164493480441325e-05, + "loss": 1.2134, + "theoretical_loss": 3.3322500877372105, + "tokens_seen": 3208314880 + }, + { + "epoch": 0.94, + "learning_rate": 2.81444332998997e-05, + "loss": 1.1234, + "theoretical_loss": 3.332244969476007, + "tokens_seen": 3208380416 + }, + { + "epoch": 0.94, + "learning_rate": 2.8124373119358073e-05, + "loss": 1.1379, + "theoretical_loss": 3.332239851348623, + "tokens_seen": 3208445952 + }, + { + "epoch": 0.94, + "learning_rate": 2.8104312938816452e-05, + "loss": 1.247, + "theoretical_loss": 3.332234733355053, + "tokens_seen": 3208511488 + }, + { + "epoch": 0.94, + "learning_rate": 2.8084252758274825e-05, + "loss": 1.2907, + "theoretical_loss": 3.33222961549529, + "tokens_seen": 3208577024 + }, + { + "epoch": 0.94, + "learning_rate": 2.80641925777332e-05, + "loss": 1.3168, + "theoretical_loss": 3.332224497769329, + "tokens_seen": 3208642560 + }, + { + "epoch": 0.94, + "learning_rate": 2.8044132397191576e-05, + "loss": 1.2156, + "theoretical_loss": 3.332219380177163, + "tokens_seen": 3208708096 + }, + { + "epoch": 0.94, + "learning_rate": 2.802407221664995e-05, + "loss": 1.2252, + "theoretical_loss": 3.332214262718785, + "tokens_seen": 3208773632 + }, + { + "epoch": 0.94, + "learning_rate": 2.8004012036108328e-05, + "loss": 1.1185, + "theoretical_loss": 3.3322091453941898, + "tokens_seen": 3208839168 + }, + { + "epoch": 0.94, + "learning_rate": 2.79839518555667e-05, + "loss": 1.2376, + "theoretical_loss": 3.332204028203371, + "tokens_seen": 3208904704 + }, + { + "epoch": 0.94, + "learning_rate": 2.7963891675025076e-05, + "loss": 1.3043, + "theoretical_loss": 3.332198911146323, + "tokens_seen": 3208970240 + }, + { + "epoch": 0.94, + "learning_rate": 2.7943831494483452e-05, + "loss": 1.224, + "theoretical_loss": 3.3321937942230377, + "tokens_seen": 3209035776 + }, + { + "epoch": 0.94, + "learning_rate": 2.7923771313941824e-05, + "loss": 1.234, + "theoretical_loss": 3.3321886774335105, + "tokens_seen": 3209101312 + }, + { + "epoch": 0.94, + "learning_rate": 2.7903711133400203e-05, + "loss": 1.2187, + "theoretical_loss": 3.332183560777735, + "tokens_seen": 3209166848 + }, + { + "epoch": 0.94, + "learning_rate": 2.7883650952858576e-05, + "loss": 1.1539, + "theoretical_loss": 3.3321784442557045, + "tokens_seen": 3209232384 + }, + { + "epoch": 0.95, + "learning_rate": 2.786359077231695e-05, + "loss": 1.2843, + "theoretical_loss": 3.332173327867413, + "tokens_seen": 3209297920 + }, + { + "epoch": 0.95, + "learning_rate": 2.7843530591775324e-05, + "loss": 1.1879, + "theoretical_loss": 3.3321682116128537, + "tokens_seen": 3209363456 + }, + { + "epoch": 0.95, + "learning_rate": 2.7823470411233703e-05, + "loss": 1.1664, + "theoretical_loss": 3.3321630954920214, + "tokens_seen": 3209428992 + }, + { + "epoch": 0.95, + "learning_rate": 2.780341023069208e-05, + "loss": 1.3595, + "theoretical_loss": 3.3321579795049097, + "tokens_seen": 3209494528 + }, + { + "epoch": 0.95, + "learning_rate": 2.778335005015045e-05, + "loss": 1.1858, + "theoretical_loss": 3.3321528636515114, + "tokens_seen": 3209560064 + }, + { + "epoch": 0.95, + "learning_rate": 2.7763289869608827e-05, + "loss": 1.2978, + "theoretical_loss": 3.3321477479318213, + "tokens_seen": 3209625600 + }, + { + "epoch": 0.95, + "learning_rate": 2.77432296890672e-05, + "loss": 1.2576, + "theoretical_loss": 3.3321426323458323, + "tokens_seen": 3209691136 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1166593, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2713661193847656, + "objective/train/theoretical_loss": 3.332137516893539, + "objective/train/tokens_used": 1580215776, + "theoretical_loss": 3.332137516893539, + "tokens_seen": 3209756672 + }, + { + "epoch": 0.95, + "learning_rate": 2.772316950852558e-05, + "loss": 1.2703, + "theoretical_loss": 3.332137516893539, + "tokens_seen": 3209756672 + }, + { + "epoch": 0.95, + "learning_rate": 2.7703109327983955e-05, + "loss": 1.2206, + "theoretical_loss": 3.3321324015749347, + "tokens_seen": 3209822208 + }, + { + "epoch": 0.95, + "learning_rate": 2.7683049147442327e-05, + "loss": 1.2511, + "theoretical_loss": 3.3321272863900138, + "tokens_seen": 3209887744 + }, + { + "epoch": 0.95, + "learning_rate": 2.7662988966900703e-05, + "loss": 1.2916, + "theoretical_loss": 3.332122171338769, + "tokens_seen": 3209953280 + }, + { + "epoch": 0.95, + "learning_rate": 2.7642928786359075e-05, + "loss": 1.1127, + "theoretical_loss": 3.332117056421195, + "tokens_seen": 3210018816 + }, + { + "epoch": 0.95, + "learning_rate": 2.7622868605817454e-05, + "loss": 1.1807, + "theoretical_loss": 3.3321119416372853, + "tokens_seen": 3210084352 + }, + { + "epoch": 0.95, + "learning_rate": 2.760280842527583e-05, + "loss": 1.2449, + "theoretical_loss": 3.3321068269870335, + "tokens_seen": 3210149888 + }, + { + "epoch": 0.95, + "learning_rate": 2.7582748244734203e-05, + "loss": 1.1555, + "theoretical_loss": 3.3321017124704335, + "tokens_seen": 3210215424 + }, + { + "epoch": 0.95, + "learning_rate": 2.756268806419258e-05, + "loss": 1.1704, + "theoretical_loss": 3.3320965980874795, + "tokens_seen": 3210280960 + }, + { + "epoch": 0.95, + "learning_rate": 2.7542627883650954e-05, + "loss": 1.2887, + "theoretical_loss": 3.3320914838381643, + "tokens_seen": 3210346496 + }, + { + "epoch": 0.95, + "learning_rate": 2.752256770310933e-05, + "loss": 1.2056, + "theoretical_loss": 3.3320863697224823, + "tokens_seen": 3210412032 + }, + { + "epoch": 0.95, + "learning_rate": 2.7502507522567702e-05, + "loss": 1.2291, + "theoretical_loss": 3.3320812557404276, + "tokens_seen": 3210477568 + }, + { + "epoch": 0.95, + "learning_rate": 2.7482447342026078e-05, + "loss": 1.3059, + "theoretical_loss": 3.3320761418919935, + "tokens_seen": 3210543104 + }, + { + "epoch": 0.95, + "learning_rate": 2.7462387161484454e-05, + "loss": 1.1847, + "theoretical_loss": 3.332071028177174, + "tokens_seen": 3210608640 + }, + { + "epoch": 0.95, + "learning_rate": 2.744232698094283e-05, + "loss": 1.115, + "theoretical_loss": 3.3320659145959626, + "tokens_seen": 3210674176 + }, + { + "epoch": 0.95, + "learning_rate": 2.7422266800401205e-05, + "loss": 1.3093, + "theoretical_loss": 3.3320608011483532, + "tokens_seen": 3210739712 + }, + { + "epoch": 0.95, + "learning_rate": 2.7402206619859578e-05, + "loss": 1.1393, + "theoretical_loss": 3.3320556878343397, + "tokens_seen": 3210805248 + }, + { + "epoch": 0.95, + "learning_rate": 2.7382146439317954e-05, + "loss": 1.2899, + "theoretical_loss": 3.3320505746539157, + "tokens_seen": 3210870784 + }, + { + "epoch": 0.95, + "learning_rate": 2.736208625877633e-05, + "loss": 1.2978, + "theoretical_loss": 3.332045461607075, + "tokens_seen": 3210936320 + }, + { + "epoch": 0.95, + "learning_rate": 2.7342026078234705e-05, + "loss": 1.2277, + "theoretical_loss": 3.3320403486938117, + "tokens_seen": 3211001856 + }, + { + "epoch": 0.95, + "learning_rate": 2.732196589769308e-05, + "loss": 1.1811, + "theoretical_loss": 3.3320352359141197, + "tokens_seen": 3211067392 + }, + { + "epoch": 0.95, + "learning_rate": 2.7301905717151453e-05, + "loss": 1.1838, + "theoretical_loss": 3.332030123267992, + "tokens_seen": 3211132928 + }, + { + "epoch": 0.95, + "learning_rate": 2.728184553660983e-05, + "loss": 1.2732, + "theoretical_loss": 3.332025010755423, + "tokens_seen": 3211198464 + }, + { + "epoch": 0.95, + "learning_rate": 2.7261785356068205e-05, + "loss": 1.2297, + "theoretical_loss": 3.3320198983764064, + "tokens_seen": 3211264000 + }, + { + "epoch": 0.95, + "learning_rate": 2.724172517552658e-05, + "loss": 1.1829, + "theoretical_loss": 3.332014786130936, + "tokens_seen": 3211329536 + }, + { + "epoch": 0.95, + "learning_rate": 2.7221664994984957e-05, + "loss": 1.3253, + "theoretical_loss": 3.3320096740190053, + "tokens_seen": 3211395072 + }, + { + "epoch": 0.95, + "learning_rate": 2.720160481444333e-05, + "loss": 1.2813, + "theoretical_loss": 3.3320045620406082, + "tokens_seen": 3211460608 + }, + { + "epoch": 0.95, + "learning_rate": 2.7181544633901705e-05, + "loss": 1.2906, + "theoretical_loss": 3.3319994501957386, + "tokens_seen": 3211526144 + }, + { + "epoch": 0.95, + "learning_rate": 2.716148445336008e-05, + "loss": 1.3422, + "theoretical_loss": 3.3319943384843906, + "tokens_seen": 3211591680 + }, + { + "epoch": 0.95, + "learning_rate": 2.7141424272818456e-05, + "loss": 1.196, + "theoretical_loss": 3.331989226906557, + "tokens_seen": 3211657216 + }, + { + "epoch": 0.95, + "learning_rate": 2.7121364092276832e-05, + "loss": 1.2292, + "theoretical_loss": 3.331984115462233, + "tokens_seen": 3211722752 + }, + { + "epoch": 0.95, + "learning_rate": 2.7101303911735205e-05, + "loss": 1.2779, + "theoretical_loss": 3.3319790041514112, + "tokens_seen": 3211788288 + }, + { + "epoch": 0.95, + "learning_rate": 2.708124373119358e-05, + "loss": 1.1779, + "theoretical_loss": 3.331973892974086, + "tokens_seen": 3211853824 + }, + { + "epoch": 0.95, + "learning_rate": 2.7061183550651956e-05, + "loss": 1.264, + "theoretical_loss": 3.331968781930251, + "tokens_seen": 3211919360 + }, + { + "epoch": 0.95, + "learning_rate": 2.7041123370110332e-05, + "loss": 1.2464, + "theoretical_loss": 3.3319636710198997, + "tokens_seen": 3211984896 + }, + { + "epoch": 0.95, + "learning_rate": 2.7021063189568708e-05, + "loss": 1.2371, + "theoretical_loss": 3.3319585602430264, + "tokens_seen": 3212050432 + }, + { + "epoch": 0.95, + "learning_rate": 2.700100300902708e-05, + "loss": 1.2559, + "theoretical_loss": 3.3319534495996246, + "tokens_seen": 3212115968 + }, + { + "epoch": 0.95, + "learning_rate": 2.6980942828485456e-05, + "loss": 1.3026, + "theoretical_loss": 3.3319483390896885, + "tokens_seen": 3212181504 + }, + { + "epoch": 0.95, + "learning_rate": 2.6960882647943832e-05, + "loss": 1.2358, + "theoretical_loss": 3.3319432287132114, + "tokens_seen": 3212247040 + }, + { + "epoch": 0.95, + "learning_rate": 2.6940822467402208e-05, + "loss": 1.2425, + "theoretical_loss": 3.331938118470187, + "tokens_seen": 3212312576 + }, + { + "epoch": 0.95, + "learning_rate": 2.6920762286860583e-05, + "loss": 1.219, + "theoretical_loss": 3.3319330083606093, + "tokens_seen": 3212378112 + }, + { + "epoch": 0.95, + "learning_rate": 2.6900702106318956e-05, + "loss": 1.2619, + "theoretical_loss": 3.3319278983844725, + "tokens_seen": 3212443648 + }, + { + "epoch": 0.95, + "learning_rate": 2.6880641925777335e-05, + "loss": 1.1639, + "theoretical_loss": 3.33192278854177, + "tokens_seen": 3212509184 + }, + { + "epoch": 0.95, + "learning_rate": 2.6860581745235707e-05, + "loss": 1.2732, + "theoretical_loss": 3.3319176788324953, + "tokens_seen": 3212574720 + }, + { + "epoch": 0.95, + "learning_rate": 2.6840521564694083e-05, + "loss": 1.3217, + "theoretical_loss": 3.3319125692566427, + "tokens_seen": 3212640256 + }, + { + "epoch": 0.95, + "learning_rate": 2.6820461384152456e-05, + "loss": 1.2477, + "theoretical_loss": 3.331907459814206, + "tokens_seen": 3212705792 + }, + { + "epoch": 0.95, + "learning_rate": 2.680040120361083e-05, + "loss": 1.2935, + "theoretical_loss": 3.3319023505051786, + "tokens_seen": 3212771328 + }, + { + "epoch": 0.95, + "learning_rate": 2.678034102306921e-05, + "loss": 1.2943, + "theoretical_loss": 3.3318972413295547, + "tokens_seen": 3212836864 + }, + { + "epoch": 0.95, + "learning_rate": 2.6760280842527583e-05, + "loss": 1.2726, + "theoretical_loss": 3.3318921322873276, + "tokens_seen": 3212902400 + }, + { + "epoch": 0.95, + "learning_rate": 2.674022066198596e-05, + "loss": 1.3116, + "theoretical_loss": 3.3318870233784916, + "tokens_seen": 3212967936 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1167631, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.370660662651062, + "objective/train/theoretical_loss": 3.33188191460304, + "objective/train/tokens_used": 1583492576, + "theoretical_loss": 3.33188191460304, + "tokens_seen": 3213033472 + }, + { + "epoch": 0.95, + "learning_rate": 2.672016048144433e-05, + "loss": 1.337, + "theoretical_loss": 3.33188191460304, + "tokens_seen": 3213033472 + }, + { + "epoch": 0.95, + "learning_rate": 2.6700100300902707e-05, + "loss": 1.2832, + "theoretical_loss": 3.3318768059609676, + "tokens_seen": 3213099008 + }, + { + "epoch": 0.95, + "learning_rate": 2.6680040120361086e-05, + "loss": 1.2686, + "theoretical_loss": 3.3318716974522666, + "tokens_seen": 3213164544 + }, + { + "epoch": 0.95, + "learning_rate": 2.665997993981946e-05, + "loss": 1.3907, + "theoretical_loss": 3.331866589076932, + "tokens_seen": 3213230080 + }, + { + "epoch": 0.95, + "learning_rate": 2.6639919759277834e-05, + "loss": 1.2776, + "theoretical_loss": 3.3318614808349576, + "tokens_seen": 3213295616 + }, + { + "epoch": 0.95, + "learning_rate": 2.6619859578736207e-05, + "loss": 1.2363, + "theoretical_loss": 3.3318563727263366, + "tokens_seen": 3213361152 + }, + { + "epoch": 0.95, + "learning_rate": 2.6599799398194586e-05, + "loss": 1.2536, + "theoretical_loss": 3.331851264751063, + "tokens_seen": 3213426688 + }, + { + "epoch": 0.95, + "learning_rate": 2.657973921765296e-05, + "loss": 1.262, + "theoretical_loss": 3.331846156909131, + "tokens_seen": 3213492224 + }, + { + "epoch": 0.95, + "learning_rate": 2.6559679037111334e-05, + "loss": 1.2483, + "theoretical_loss": 3.331841049200534, + "tokens_seen": 3213557760 + }, + { + "epoch": 0.95, + "learning_rate": 2.653961885656971e-05, + "loss": 1.1555, + "theoretical_loss": 3.3318359416252656, + "tokens_seen": 3213623296 + }, + { + "epoch": 0.95, + "learning_rate": 2.6519558676028082e-05, + "loss": 1.229, + "theoretical_loss": 3.3318308341833203, + "tokens_seen": 3213688832 + }, + { + "epoch": 0.95, + "learning_rate": 2.649949849548646e-05, + "loss": 1.2031, + "theoretical_loss": 3.3318257268746914, + "tokens_seen": 3213754368 + }, + { + "epoch": 0.95, + "learning_rate": 2.6479438314944837e-05, + "loss": 1.2461, + "theoretical_loss": 3.3318206196993723, + "tokens_seen": 3213819904 + }, + { + "epoch": 0.95, + "learning_rate": 2.645937813440321e-05, + "loss": 1.2438, + "theoretical_loss": 3.3318155126573576, + "tokens_seen": 3213885440 + }, + { + "epoch": 0.95, + "learning_rate": 2.6439317953861585e-05, + "loss": 1.2304, + "theoretical_loss": 3.331810405748641, + "tokens_seen": 3213950976 + }, + { + "epoch": 0.95, + "learning_rate": 2.6419257773319958e-05, + "loss": 1.2075, + "theoretical_loss": 3.3318052989732156, + "tokens_seen": 3214016512 + }, + { + "epoch": 0.95, + "learning_rate": 2.6399197592778337e-05, + "loss": 1.2814, + "theoretical_loss": 3.3318001923310763, + "tokens_seen": 3214082048 + }, + { + "epoch": 0.95, + "learning_rate": 2.637913741223671e-05, + "loss": 1.2443, + "theoretical_loss": 3.3317950858222156, + "tokens_seen": 3214147584 + }, + { + "epoch": 0.95, + "learning_rate": 2.6359077231695085e-05, + "loss": 1.2158, + "theoretical_loss": 3.331789979446629, + "tokens_seen": 3214213120 + }, + { + "epoch": 0.95, + "learning_rate": 2.633901705115346e-05, + "loss": 1.3457, + "theoretical_loss": 3.3317848732043083, + "tokens_seen": 3214278656 + }, + { + "epoch": 0.95, + "learning_rate": 2.6318956870611837e-05, + "loss": 1.2845, + "theoretical_loss": 3.3317797670952487, + "tokens_seen": 3214344192 + }, + { + "epoch": 0.95, + "learning_rate": 2.6298896690070213e-05, + "loss": 1.3034, + "theoretical_loss": 3.331774661119444, + "tokens_seen": 3214409728 + }, + { + "epoch": 0.95, + "learning_rate": 2.6278836509528585e-05, + "loss": 1.1321, + "theoretical_loss": 3.331769555276887, + "tokens_seen": 3214475264 + }, + { + "epoch": 0.95, + "learning_rate": 2.625877632898696e-05, + "loss": 1.2447, + "theoretical_loss": 3.3317644495675722, + "tokens_seen": 3214540800 + }, + { + "epoch": 0.95, + "learning_rate": 2.6238716148445337e-05, + "loss": 1.2734, + "theoretical_loss": 3.331759343991494, + "tokens_seen": 3214606336 + }, + { + "epoch": 0.95, + "learning_rate": 2.6218655967903712e-05, + "loss": 1.2454, + "theoretical_loss": 3.331754238548645, + "tokens_seen": 3214671872 + }, + { + "epoch": 0.95, + "learning_rate": 2.6198595787362088e-05, + "loss": 1.2356, + "theoretical_loss": 3.3317491332390192, + "tokens_seen": 3214737408 + }, + { + "epoch": 0.95, + "learning_rate": 2.617853560682046e-05, + "loss": 1.2823, + "theoretical_loss": 3.3317440280626114, + "tokens_seen": 3214802944 + }, + { + "epoch": 0.95, + "learning_rate": 2.6158475426278836e-05, + "loss": 1.2481, + "theoretical_loss": 3.3317389230194143, + "tokens_seen": 3214868480 + }, + { + "epoch": 0.95, + "learning_rate": 2.6138415245737212e-05, + "loss": 1.1967, + "theoretical_loss": 3.3317338181094227, + "tokens_seen": 3214934016 + }, + { + "epoch": 0.95, + "learning_rate": 2.6118355065195588e-05, + "loss": 1.1909, + "theoretical_loss": 3.3317287133326294, + "tokens_seen": 3214999552 + }, + { + "epoch": 0.95, + "learning_rate": 2.6098294884653964e-05, + "loss": 1.0884, + "theoretical_loss": 3.331723608689029, + "tokens_seen": 3215065088 + }, + { + "epoch": 0.95, + "learning_rate": 2.6078234704112336e-05, + "loss": 1.1659, + "theoretical_loss": 3.331718504178615, + "tokens_seen": 3215130624 + }, + { + "epoch": 0.95, + "learning_rate": 2.6058174523570712e-05, + "loss": 1.2505, + "theoretical_loss": 3.331713399801381, + "tokens_seen": 3215196160 + }, + { + "epoch": 0.95, + "learning_rate": 2.6038114343029088e-05, + "loss": 1.2156, + "theoretical_loss": 3.331708295557321, + "tokens_seen": 3215261696 + }, + { + "epoch": 0.95, + "learning_rate": 2.6018054162487464e-05, + "loss": 1.1582, + "theoretical_loss": 3.331703191446429, + "tokens_seen": 3215327232 + }, + { + "epoch": 0.95, + "learning_rate": 2.599799398194584e-05, + "loss": 1.2982, + "theoretical_loss": 3.3316980874686988, + "tokens_seen": 3215392768 + }, + { + "epoch": 0.95, + "learning_rate": 2.5977933801404212e-05, + "loss": 1.227, + "theoretical_loss": 3.331692983624124, + "tokens_seen": 3215458304 + }, + { + "epoch": 0.95, + "learning_rate": 2.5957873620862588e-05, + "loss": 1.1656, + "theoretical_loss": 3.331687879912698, + "tokens_seen": 3215523840 + }, + { + "epoch": 0.95, + "learning_rate": 2.5937813440320963e-05, + "loss": 1.2693, + "theoretical_loss": 3.3316827763344157, + "tokens_seen": 3215589376 + }, + { + "epoch": 0.95, + "learning_rate": 2.591775325977934e-05, + "loss": 1.2341, + "theoretical_loss": 3.3316776728892696, + "tokens_seen": 3215654912 + }, + { + "epoch": 0.95, + "learning_rate": 2.5897693079237715e-05, + "loss": 1.1757, + "theoretical_loss": 3.331672569577255, + "tokens_seen": 3215720448 + }, + { + "epoch": 0.95, + "learning_rate": 2.5877632898696087e-05, + "loss": 1.295, + "theoretical_loss": 3.3316674663983648, + "tokens_seen": 3215785984 + }, + { + "epoch": 0.95, + "learning_rate": 2.5857572718154463e-05, + "loss": 1.2113, + "theoretical_loss": 3.331662363352592, + "tokens_seen": 3215851520 + }, + { + "epoch": 0.95, + "learning_rate": 2.583751253761284e-05, + "loss": 1.2717, + "theoretical_loss": 3.3316572604399326, + "tokens_seen": 3215917056 + }, + { + "epoch": 0.95, + "learning_rate": 2.5817452357071215e-05, + "loss": 1.1693, + "theoretical_loss": 3.3316521576603786, + "tokens_seen": 3215982592 + }, + { + "epoch": 0.95, + "learning_rate": 2.579739217652959e-05, + "loss": 1.2978, + "theoretical_loss": 3.3316470550139243, + "tokens_seen": 3216048128 + }, + { + "epoch": 0.95, + "learning_rate": 2.5777331995987963e-05, + "loss": 1.2914, + "theoretical_loss": 3.3316419525005636, + "tokens_seen": 3216113664 + }, + { + "epoch": 0.95, + "learning_rate": 2.5757271815446342e-05, + "loss": 1.2329, + "theoretical_loss": 3.33163685012029, + "tokens_seen": 3216179200 + }, + { + "epoch": 0.95, + "learning_rate": 2.5737211634904715e-05, + "loss": 1.1829, + "theoretical_loss": 3.3316317478730983, + "tokens_seen": 3216244736 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1168814, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0978078842163086, + "objective/train/theoretical_loss": 3.3316266457589814, + "objective/train/tokens_used": 1586769376, + "theoretical_loss": 3.3316266457589814, + "tokens_seen": 3216310272 + }, + { + "epoch": 0.95, + "learning_rate": 2.571715145436309e-05, + "loss": 1.108, + "theoretical_loss": 3.3316266457589814, + "tokens_seen": 3216310272 + }, + { + "epoch": 0.95, + "learning_rate": 2.5697091273821463e-05, + "loss": 1.3228, + "theoretical_loss": 3.3316215437779335, + "tokens_seen": 3216375808 + }, + { + "epoch": 0.95, + "learning_rate": 2.567703109327984e-05, + "loss": 1.3015, + "theoretical_loss": 3.331616441929948, + "tokens_seen": 3216441344 + }, + { + "epoch": 0.95, + "learning_rate": 2.5656970912738218e-05, + "loss": 1.2291, + "theoretical_loss": 3.3316113402150194, + "tokens_seen": 3216506880 + }, + { + "epoch": 0.95, + "learning_rate": 2.563691073219659e-05, + "loss": 1.1842, + "theoretical_loss": 3.3316062386331406, + "tokens_seen": 3216572416 + }, + { + "epoch": 0.95, + "learning_rate": 2.5616850551654966e-05, + "loss": 1.2713, + "theoretical_loss": 3.3316011371843066, + "tokens_seen": 3216637952 + }, + { + "epoch": 0.95, + "learning_rate": 2.5596790371113338e-05, + "loss": 1.2631, + "theoretical_loss": 3.33159603586851, + "tokens_seen": 3216703488 + }, + { + "epoch": 0.95, + "learning_rate": 2.5576730190571714e-05, + "loss": 1.2683, + "theoretical_loss": 3.3315909346857455, + "tokens_seen": 3216769024 + }, + { + "epoch": 0.95, + "learning_rate": 2.5556670010030093e-05, + "loss": 1.315, + "theoretical_loss": 3.3315858336360065, + "tokens_seen": 3216834560 + }, + { + "epoch": 0.95, + "learning_rate": 2.5536609829488466e-05, + "loss": 1.235, + "theoretical_loss": 3.331580732719287, + "tokens_seen": 3216900096 + }, + { + "epoch": 0.95, + "learning_rate": 2.551654964894684e-05, + "loss": 1.2065, + "theoretical_loss": 3.3315756319355807, + "tokens_seen": 3216965632 + }, + { + "epoch": 0.95, + "learning_rate": 2.5496489468405214e-05, + "loss": 1.268, + "theoretical_loss": 3.3315705312848816, + "tokens_seen": 3217031168 + }, + { + "epoch": 0.95, + "learning_rate": 2.5476429287863593e-05, + "loss": 1.2641, + "theoretical_loss": 3.331565430767183, + "tokens_seen": 3217096704 + }, + { + "epoch": 0.95, + "learning_rate": 2.545636910732197e-05, + "loss": 1.3172, + "theoretical_loss": 3.3315603303824792, + "tokens_seen": 3217162240 + }, + { + "epoch": 0.95, + "learning_rate": 2.543630892678034e-05, + "loss": 1.2748, + "theoretical_loss": 3.331555230130764, + "tokens_seen": 3217227776 + }, + { + "epoch": 0.95, + "learning_rate": 2.5416248746238717e-05, + "loss": 1.1847, + "theoretical_loss": 3.3315501300120314, + "tokens_seen": 3217293312 + }, + { + "epoch": 0.95, + "learning_rate": 2.539618856569709e-05, + "loss": 1.2454, + "theoretical_loss": 3.331545030026275, + "tokens_seen": 3217358848 + }, + { + "epoch": 0.95, + "learning_rate": 2.537612838515547e-05, + "loss": 1.1727, + "theoretical_loss": 3.3315399301734887, + "tokens_seen": 3217424384 + }, + { + "epoch": 0.95, + "learning_rate": 2.535606820461384e-05, + "loss": 1.1749, + "theoretical_loss": 3.3315348304536654, + "tokens_seen": 3217489920 + }, + { + "epoch": 0.95, + "learning_rate": 2.5336008024072217e-05, + "loss": 1.233, + "theoretical_loss": 3.3315297308668006, + "tokens_seen": 3217555456 + }, + { + "epoch": 0.95, + "learning_rate": 2.5315947843530593e-05, + "loss": 1.2877, + "theoretical_loss": 3.3315246314128872, + "tokens_seen": 3217620992 + }, + { + "epoch": 0.95, + "learning_rate": 2.5295887662988965e-05, + "loss": 1.2827, + "theoretical_loss": 3.3315195320919186, + "tokens_seen": 3217686528 + }, + { + "epoch": 0.95, + "learning_rate": 2.5275827482447344e-05, + "loss": 1.241, + "theoretical_loss": 3.3315144329038895, + "tokens_seen": 3217752064 + }, + { + "epoch": 0.95, + "learning_rate": 2.5255767301905717e-05, + "loss": 1.3022, + "theoretical_loss": 3.3315093338487936, + "tokens_seen": 3217817600 + }, + { + "epoch": 0.95, + "learning_rate": 2.5235707121364092e-05, + "loss": 1.1386, + "theoretical_loss": 3.331504234926624, + "tokens_seen": 3217883136 + }, + { + "epoch": 0.95, + "learning_rate": 2.5215646940822468e-05, + "loss": 1.3046, + "theoretical_loss": 3.331499136137375, + "tokens_seen": 3217948672 + }, + { + "epoch": 0.95, + "learning_rate": 2.5195586760280844e-05, + "loss": 1.2519, + "theoretical_loss": 3.331494037481041, + "tokens_seen": 3218014208 + }, + { + "epoch": 0.95, + "learning_rate": 2.517552657973922e-05, + "loss": 1.1898, + "theoretical_loss": 3.331488938957615, + "tokens_seen": 3218079744 + }, + { + "epoch": 0.95, + "learning_rate": 2.5155466399197592e-05, + "loss": 1.2472, + "theoretical_loss": 3.331483840567091, + "tokens_seen": 3218145280 + }, + { + "epoch": 0.95, + "learning_rate": 2.5135406218655968e-05, + "loss": 1.1658, + "theoretical_loss": 3.331478742309463, + "tokens_seen": 3218210816 + }, + { + "epoch": 0.95, + "learning_rate": 2.5115346038114344e-05, + "loss": 1.2845, + "theoretical_loss": 3.331473644184725, + "tokens_seen": 3218276352 + }, + { + "epoch": 0.95, + "learning_rate": 2.509528585757272e-05, + "loss": 1.254, + "theoretical_loss": 3.33146854619287, + "tokens_seen": 3218341888 + }, + { + "epoch": 0.95, + "learning_rate": 2.5075225677031095e-05, + "loss": 1.2163, + "theoretical_loss": 3.331463448333893, + "tokens_seen": 3218407424 + }, + { + "epoch": 0.95, + "learning_rate": 2.5055165496489468e-05, + "loss": 1.193, + "theoretical_loss": 3.331458350607787, + "tokens_seen": 3218472960 + }, + { + "epoch": 0.95, + "learning_rate": 2.5035105315947844e-05, + "loss": 1.3643, + "theoretical_loss": 3.331453253014546, + "tokens_seen": 3218538496 + }, + { + "epoch": 0.95, + "learning_rate": 2.5015045135406216e-05, + "loss": 1.2357, + "theoretical_loss": 3.331448155554164, + "tokens_seen": 3218604032 + }, + { + "epoch": 0.95, + "learning_rate": 2.4994984954864595e-05, + "loss": 1.1489, + "theoretical_loss": 3.3314430582266352, + "tokens_seen": 3218669568 + }, + { + "epoch": 0.95, + "learning_rate": 2.497492477432297e-05, + "loss": 1.1916, + "theoretical_loss": 3.3314379610319524, + "tokens_seen": 3218735104 + }, + { + "epoch": 0.95, + "learning_rate": 2.4954864593781343e-05, + "loss": 1.1412, + "theoretical_loss": 3.33143286397011, + "tokens_seen": 3218800640 + }, + { + "epoch": 0.95, + "learning_rate": 2.493480441323972e-05, + "loss": 1.2602, + "theoretical_loss": 3.3314277670411023, + "tokens_seen": 3218866176 + }, + { + "epoch": 0.95, + "learning_rate": 2.4914744232698095e-05, + "loss": 1.3153, + "theoretical_loss": 3.331422670244922, + "tokens_seen": 3218931712 + }, + { + "epoch": 0.95, + "learning_rate": 2.489468405215647e-05, + "loss": 1.251, + "theoretical_loss": 3.3314175735815645, + "tokens_seen": 3218997248 + }, + { + "epoch": 0.95, + "learning_rate": 2.4874623871614847e-05, + "loss": 1.2541, + "theoretical_loss": 3.331412477051022, + "tokens_seen": 3219062784 + }, + { + "epoch": 0.95, + "learning_rate": 2.485456369107322e-05, + "loss": 1.2163, + "theoretical_loss": 3.3314073806532893, + "tokens_seen": 3219128320 + }, + { + "epoch": 0.95, + "learning_rate": 2.4834503510531595e-05, + "loss": 1.1822, + "theoretical_loss": 3.33140228438836, + "tokens_seen": 3219193856 + }, + { + "epoch": 0.95, + "learning_rate": 2.481444332998997e-05, + "loss": 1.4299, + "theoretical_loss": 3.3313971882562283, + "tokens_seen": 3219259392 + }, + { + "epoch": 0.95, + "learning_rate": 2.4794383149448346e-05, + "loss": 1.1915, + "theoretical_loss": 3.3313920922568876, + "tokens_seen": 3219324928 + }, + { + "epoch": 0.95, + "learning_rate": 2.4774322968906722e-05, + "loss": 1.2312, + "theoretical_loss": 3.3313869963903318, + "tokens_seen": 3219390464 + }, + { + "epoch": 0.95, + "learning_rate": 2.4754262788365095e-05, + "loss": 1.1726, + "theoretical_loss": 3.3313819006565546, + "tokens_seen": 3219456000 + }, + { + "epoch": 0.95, + "learning_rate": 2.473420260782347e-05, + "loss": 1.1859, + "theoretical_loss": 3.33137680505555, + "tokens_seen": 3219521536 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1169560, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2335251569747925, + "objective/train/theoretical_loss": 3.331371709587312, + "objective/train/tokens_used": 1590046176, + "theoretical_loss": 3.331371709587312, + "tokens_seen": 3219587072 + }, + { + "epoch": 0.95, + "learning_rate": 2.4714142427281846e-05, + "loss": 1.2528, + "theoretical_loss": 3.331371709587312, + "tokens_seen": 3219587072 + }, + { + "epoch": 0.95, + "learning_rate": 2.4694082246740222e-05, + "loss": 1.1925, + "theoretical_loss": 3.331366614251834, + "tokens_seen": 3219652608 + }, + { + "epoch": 0.95, + "learning_rate": 2.4674022066198598e-05, + "loss": 1.2368, + "theoretical_loss": 3.33136151904911, + "tokens_seen": 3219718144 + }, + { + "epoch": 0.95, + "learning_rate": 2.465396188565697e-05, + "loss": 1.2101, + "theoretical_loss": 3.3313564239791345, + "tokens_seen": 3219783680 + }, + { + "epoch": 0.95, + "learning_rate": 2.463390170511535e-05, + "loss": 1.2169, + "theoretical_loss": 3.3313513290419006, + "tokens_seen": 3219849216 + }, + { + "epoch": 0.95, + "learning_rate": 2.461384152457372e-05, + "loss": 1.2076, + "theoretical_loss": 3.3313462342374023, + "tokens_seen": 3219914752 + }, + { + "epoch": 0.95, + "learning_rate": 2.4593781344032097e-05, + "loss": 1.2444, + "theoretical_loss": 3.3313411395656334, + "tokens_seen": 3219980288 + }, + { + "epoch": 0.95, + "learning_rate": 2.457372116349047e-05, + "loss": 1.2414, + "theoretical_loss": 3.331336045026588, + "tokens_seen": 3220045824 + }, + { + "epoch": 0.95, + "learning_rate": 2.4553660982948846e-05, + "loss": 1.2076, + "theoretical_loss": 3.3313309506202597, + "tokens_seen": 3220111360 + }, + { + "epoch": 0.95, + "learning_rate": 2.4533600802407225e-05, + "loss": 1.1913, + "theoretical_loss": 3.331325856346642, + "tokens_seen": 3220176896 + }, + { + "epoch": 0.95, + "learning_rate": 2.4513540621865597e-05, + "loss": 1.2707, + "theoretical_loss": 3.33132076220573, + "tokens_seen": 3220242432 + }, + { + "epoch": 0.95, + "learning_rate": 2.4493480441323973e-05, + "loss": 1.2394, + "theoretical_loss": 3.331315668197516, + "tokens_seen": 3220307968 + }, + { + "epoch": 0.95, + "learning_rate": 2.4473420260782345e-05, + "loss": 1.1672, + "theoretical_loss": 3.3313105743219946, + "tokens_seen": 3220373504 + }, + { + "epoch": 0.95, + "learning_rate": 2.445336008024072e-05, + "loss": 1.1821, + "theoretical_loss": 3.33130548057916, + "tokens_seen": 3220439040 + }, + { + "epoch": 0.95, + "learning_rate": 2.44332998996991e-05, + "loss": 1.1903, + "theoretical_loss": 3.331300386969005, + "tokens_seen": 3220504576 + }, + { + "epoch": 0.95, + "learning_rate": 2.4413239719157473e-05, + "loss": 1.2706, + "theoretical_loss": 3.3312952934915248, + "tokens_seen": 3220570112 + }, + { + "epoch": 0.95, + "learning_rate": 2.439317953861585e-05, + "loss": 1.353, + "theoretical_loss": 3.3312902001467117, + "tokens_seen": 3220635648 + }, + { + "epoch": 0.95, + "learning_rate": 2.437311935807422e-05, + "loss": 1.2409, + "theoretical_loss": 3.331285106934561, + "tokens_seen": 3220701184 + }, + { + "epoch": 0.95, + "learning_rate": 2.4353059177532597e-05, + "loss": 1.326, + "theoretical_loss": 3.3312800138550656, + "tokens_seen": 3220766720 + }, + { + "epoch": 0.95, + "learning_rate": 2.4332998996990976e-05, + "loss": 1.3305, + "theoretical_loss": 3.3312749209082195, + "tokens_seen": 3220832256 + }, + { + "epoch": 0.95, + "learning_rate": 2.431293881644935e-05, + "loss": 1.3196, + "theoretical_loss": 3.331269828094017, + "tokens_seen": 3220897792 + }, + { + "epoch": 0.95, + "learning_rate": 2.4292878635907724e-05, + "loss": 1.234, + "theoretical_loss": 3.331264735412452, + "tokens_seen": 3220963328 + }, + { + "epoch": 0.95, + "learning_rate": 2.4272818455366097e-05, + "loss": 1.2325, + "theoretical_loss": 3.3312596428635173, + "tokens_seen": 3221028864 + }, + { + "epoch": 0.95, + "learning_rate": 2.4252758274824476e-05, + "loss": 1.1977, + "theoretical_loss": 3.3312545504472073, + "tokens_seen": 3221094400 + }, + { + "epoch": 0.95, + "learning_rate": 2.4232698094282848e-05, + "loss": 1.1636, + "theoretical_loss": 3.3312494581635166, + "tokens_seen": 3221159936 + }, + { + "epoch": 0.95, + "learning_rate": 2.4212637913741224e-05, + "loss": 1.3258, + "theoretical_loss": 3.331244366012438, + "tokens_seen": 3221225472 + }, + { + "epoch": 0.95, + "learning_rate": 2.41925777331996e-05, + "loss": 1.3567, + "theoretical_loss": 3.331239273993966, + "tokens_seen": 3221291008 + }, + { + "epoch": 0.95, + "learning_rate": 2.4172517552657972e-05, + "loss": 1.1214, + "theoretical_loss": 3.331234182108094, + "tokens_seen": 3221356544 + }, + { + "epoch": 0.95, + "learning_rate": 2.415245737211635e-05, + "loss": 1.2211, + "theoretical_loss": 3.331229090354816, + "tokens_seen": 3221422080 + }, + { + "epoch": 0.95, + "learning_rate": 2.4132397191574724e-05, + "loss": 1.2787, + "theoretical_loss": 3.331223998734126, + "tokens_seen": 3221487616 + }, + { + "epoch": 0.95, + "learning_rate": 2.41123370110331e-05, + "loss": 1.2722, + "theoretical_loss": 3.331218907246018, + "tokens_seen": 3221553152 + }, + { + "epoch": 0.95, + "learning_rate": 2.4092276830491475e-05, + "loss": 1.1383, + "theoretical_loss": 3.331213815890486, + "tokens_seen": 3221618688 + }, + { + "epoch": 0.95, + "learning_rate": 2.4072216649949848e-05, + "loss": 1.3732, + "theoretical_loss": 3.3312087246675226, + "tokens_seen": 3221684224 + }, + { + "epoch": 0.95, + "learning_rate": 2.4052156469408227e-05, + "loss": 1.2028, + "theoretical_loss": 3.331203633577123, + "tokens_seen": 3221749760 + }, + { + "epoch": 0.95, + "learning_rate": 2.40320962888666e-05, + "loss": 1.2639, + "theoretical_loss": 3.3311985426192807, + "tokens_seen": 3221815296 + }, + { + "epoch": 0.95, + "learning_rate": 2.4012036108324975e-05, + "loss": 1.1739, + "theoretical_loss": 3.331193451793989, + "tokens_seen": 3221880832 + }, + { + "epoch": 0.95, + "learning_rate": 2.399197592778335e-05, + "loss": 1.3457, + "theoretical_loss": 3.3311883611012423, + "tokens_seen": 3221946368 + }, + { + "epoch": 0.95, + "learning_rate": 2.3971915747241727e-05, + "loss": 1.2242, + "theoretical_loss": 3.3311832705410347, + "tokens_seen": 3222011904 + }, + { + "epoch": 0.95, + "learning_rate": 2.3951855566700103e-05, + "loss": 1.2255, + "theoretical_loss": 3.3311781801133593, + "tokens_seen": 3222077440 + }, + { + "epoch": 0.95, + "learning_rate": 2.3931795386158475e-05, + "loss": 1.3101, + "theoretical_loss": 3.3311730898182104, + "tokens_seen": 3222142976 + }, + { + "epoch": 0.95, + "learning_rate": 2.391173520561685e-05, + "loss": 1.289, + "theoretical_loss": 3.331167999655582, + "tokens_seen": 3222208512 + }, + { + "epoch": 0.95, + "learning_rate": 2.3891675025075223e-05, + "loss": 1.2284, + "theoretical_loss": 3.3311629096254673, + "tokens_seen": 3222274048 + }, + { + "epoch": 0.95, + "learning_rate": 2.3871614844533602e-05, + "loss": 1.2047, + "theoretical_loss": 3.3311578197278613, + "tokens_seen": 3222339584 + }, + { + "epoch": 0.95, + "learning_rate": 2.3851554663991978e-05, + "loss": 1.16, + "theoretical_loss": 3.331152729962757, + "tokens_seen": 3222405120 + }, + { + "epoch": 0.95, + "learning_rate": 2.383149448345035e-05, + "loss": 1.1904, + "theoretical_loss": 3.331147640330148, + "tokens_seen": 3222470656 + }, + { + "epoch": 0.95, + "learning_rate": 2.3811434302908726e-05, + "loss": 1.2483, + "theoretical_loss": 3.3311425508300285, + "tokens_seen": 3222536192 + }, + { + "epoch": 0.95, + "learning_rate": 2.37913741223671e-05, + "loss": 1.1807, + "theoretical_loss": 3.3311374614623928, + "tokens_seen": 3222601728 + }, + { + "epoch": 0.95, + "learning_rate": 2.3771313941825478e-05, + "loss": 1.2372, + "theoretical_loss": 3.3311323722272346, + "tokens_seen": 3222667264 + }, + { + "epoch": 0.95, + "learning_rate": 2.3751253761283854e-05, + "loss": 1.2752, + "theoretical_loss": 3.3311272831245473, + "tokens_seen": 3222732800 + }, + { + "epoch": 0.95, + "learning_rate": 2.3731193580742226e-05, + "loss": 1.2351, + "theoretical_loss": 3.3311221941543248, + "tokens_seen": 3222798336 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1170779, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3256371021270752, + "objective/train/theoretical_loss": 3.3311171053165616, + "objective/train/tokens_used": 1593322976, + "theoretical_loss": 3.3311171053165616, + "tokens_seen": 3222863872 + }, + { + "epoch": 0.95, + "learning_rate": 2.3711133400200602e-05, + "loss": 1.2553, + "theoretical_loss": 3.3311171053165616, + "tokens_seen": 3222863872 + }, + { + "epoch": 0.95, + "learning_rate": 2.3691073219658978e-05, + "loss": 1.2555, + "theoretical_loss": 3.3311120166112507, + "tokens_seen": 3222929408 + }, + { + "epoch": 0.95, + "learning_rate": 2.3671013039117353e-05, + "loss": 1.2298, + "theoretical_loss": 3.3311069280383867, + "tokens_seen": 3222994944 + }, + { + "epoch": 0.95, + "learning_rate": 2.365095285857573e-05, + "loss": 1.2944, + "theoretical_loss": 3.331101839597963, + "tokens_seen": 3223060480 + }, + { + "epoch": 0.95, + "learning_rate": 2.36308926780341e-05, + "loss": 1.2934, + "theoretical_loss": 3.3310967512899734, + "tokens_seen": 3223126016 + }, + { + "epoch": 0.95, + "learning_rate": 2.3610832497492477e-05, + "loss": 1.3311, + "theoretical_loss": 3.331091663114412, + "tokens_seen": 3223191552 + }, + { + "epoch": 0.95, + "learning_rate": 2.3590772316950853e-05, + "loss": 1.3234, + "theoretical_loss": 3.331086575071273, + "tokens_seen": 3223257088 + }, + { + "epoch": 0.95, + "learning_rate": 2.357071213640923e-05, + "loss": 1.3187, + "theoretical_loss": 3.3310814871605494, + "tokens_seen": 3223322624 + }, + { + "epoch": 0.95, + "learning_rate": 2.3550651955867605e-05, + "loss": 1.2346, + "theoretical_loss": 3.331076399382236, + "tokens_seen": 3223388160 + }, + { + "epoch": 0.95, + "learning_rate": 2.3530591775325977e-05, + "loss": 1.3213, + "theoretical_loss": 3.331071311736326, + "tokens_seen": 3223453696 + }, + { + "epoch": 0.95, + "learning_rate": 2.3510531594784353e-05, + "loss": 1.2377, + "theoretical_loss": 3.3310662242228135, + "tokens_seen": 3223519232 + }, + { + "epoch": 0.95, + "learning_rate": 2.349047141424273e-05, + "loss": 1.1226, + "theoretical_loss": 3.3310611368416922, + "tokens_seen": 3223584768 + }, + { + "epoch": 0.95, + "learning_rate": 2.3470411233701105e-05, + "loss": 1.2154, + "theoretical_loss": 3.331056049592956, + "tokens_seen": 3223650304 + }, + { + "epoch": 0.95, + "learning_rate": 2.3450351053159477e-05, + "loss": 1.341, + "theoretical_loss": 3.331050962476599, + "tokens_seen": 3223715840 + }, + { + "epoch": 0.95, + "learning_rate": 2.3430290872617853e-05, + "loss": 1.2315, + "theoretical_loss": 3.3310458754926153, + "tokens_seen": 3223781376 + }, + { + "epoch": 0.95, + "learning_rate": 2.3410230692076232e-05, + "loss": 1.2106, + "theoretical_loss": 3.331040788640998, + "tokens_seen": 3223846912 + }, + { + "epoch": 0.95, + "learning_rate": 2.3390170511534604e-05, + "loss": 1.0443, + "theoretical_loss": 3.3310357019217416, + "tokens_seen": 3223912448 + }, + { + "epoch": 0.95, + "learning_rate": 2.337011033099298e-05, + "loss": 1.2606, + "theoretical_loss": 3.3310306153348397, + "tokens_seen": 3223977984 + }, + { + "epoch": 0.95, + "learning_rate": 2.3350050150451353e-05, + "loss": 1.1925, + "theoretical_loss": 3.331025528880286, + "tokens_seen": 3224043520 + }, + { + "epoch": 0.95, + "learning_rate": 2.332998996990973e-05, + "loss": 1.257, + "theoretical_loss": 3.3310204425580747, + "tokens_seen": 3224109056 + }, + { + "epoch": 0.95, + "learning_rate": 2.3309929789368108e-05, + "loss": 1.2346, + "theoretical_loss": 3.3310153563681992, + "tokens_seen": 3224174592 + }, + { + "epoch": 0.95, + "learning_rate": 2.328986960882648e-05, + "loss": 1.2147, + "theoretical_loss": 3.331010270310654, + "tokens_seen": 3224240128 + }, + { + "epoch": 0.95, + "learning_rate": 2.3269809428284856e-05, + "loss": 1.2232, + "theoretical_loss": 3.3310051843854325, + "tokens_seen": 3224305664 + }, + { + "epoch": 0.95, + "learning_rate": 2.3249749247743228e-05, + "loss": 1.201, + "theoretical_loss": 3.331000098592529, + "tokens_seen": 3224371200 + }, + { + "epoch": 0.95, + "learning_rate": 2.3229689067201604e-05, + "loss": 1.3445, + "theoretical_loss": 3.330995012931937, + "tokens_seen": 3224436736 + }, + { + "epoch": 0.95, + "learning_rate": 2.3209628886659983e-05, + "loss": 1.3363, + "theoretical_loss": 3.33098992740365, + "tokens_seen": 3224502272 + }, + { + "epoch": 0.95, + "learning_rate": 2.3189568706118356e-05, + "loss": 1.2515, + "theoretical_loss": 3.3309848420076626, + "tokens_seen": 3224567808 + }, + { + "epoch": 0.95, + "learning_rate": 2.316950852557673e-05, + "loss": 1.3126, + "theoretical_loss": 3.3309797567439685, + "tokens_seen": 3224633344 + }, + { + "epoch": 0.95, + "learning_rate": 2.3149448345035104e-05, + "loss": 1.3179, + "theoretical_loss": 3.3309746716125614, + "tokens_seen": 3224698880 + }, + { + "epoch": 0.95, + "learning_rate": 2.3129388164493483e-05, + "loss": 1.2071, + "theoretical_loss": 3.3309695866134357, + "tokens_seen": 3224764416 + }, + { + "epoch": 0.95, + "learning_rate": 2.3109327983951855e-05, + "loss": 1.3282, + "theoretical_loss": 3.330964501746584, + "tokens_seen": 3224829952 + }, + { + "epoch": 0.95, + "learning_rate": 2.308926780341023e-05, + "loss": 1.2708, + "theoretical_loss": 3.330959417012002, + "tokens_seen": 3224895488 + }, + { + "epoch": 0.95, + "learning_rate": 2.3069207622868607e-05, + "loss": 1.2635, + "theoretical_loss": 3.3309543324096813, + "tokens_seen": 3224961024 + }, + { + "epoch": 0.95, + "learning_rate": 2.304914744232698e-05, + "loss": 1.125, + "theoretical_loss": 3.3309492479396177, + "tokens_seen": 3225026560 + }, + { + "epoch": 0.95, + "learning_rate": 2.302908726178536e-05, + "loss": 1.3379, + "theoretical_loss": 3.3309441636018047, + "tokens_seen": 3225092096 + }, + { + "epoch": 0.95, + "learning_rate": 2.300902708124373e-05, + "loss": 1.2517, + "theoretical_loss": 3.3309390793962352, + "tokens_seen": 3225157632 + }, + { + "epoch": 0.95, + "learning_rate": 2.2988966900702107e-05, + "loss": 1.1996, + "theoretical_loss": 3.330933995322904, + "tokens_seen": 3225223168 + }, + { + "epoch": 0.95, + "learning_rate": 2.2968906720160482e-05, + "loss": 1.2715, + "theoretical_loss": 3.330928911381805, + "tokens_seen": 3225288704 + }, + { + "epoch": 0.95, + "learning_rate": 2.2948846539618855e-05, + "loss": 1.3601, + "theoretical_loss": 3.3309238275729314, + "tokens_seen": 3225354240 + }, + { + "epoch": 0.95, + "learning_rate": 2.2928786359077234e-05, + "loss": 1.28, + "theoretical_loss": 3.3309187438962775, + "tokens_seen": 3225419776 + }, + { + "epoch": 0.95, + "learning_rate": 2.2908726178535606e-05, + "loss": 1.174, + "theoretical_loss": 3.3309136603518374, + "tokens_seen": 3225485312 + }, + { + "epoch": 0.95, + "learning_rate": 2.2888665997993982e-05, + "loss": 1.2862, + "theoretical_loss": 3.3309085769396045, + "tokens_seen": 3225550848 + }, + { + "epoch": 0.95, + "learning_rate": 2.2868605817452358e-05, + "loss": 1.2035, + "theoretical_loss": 3.330903493659573, + "tokens_seen": 3225616384 + }, + { + "epoch": 0.95, + "learning_rate": 2.2848545636910734e-05, + "loss": 1.2877, + "theoretical_loss": 3.3308984105117365, + "tokens_seen": 3225681920 + }, + { + "epoch": 0.95, + "learning_rate": 2.282848545636911e-05, + "loss": 1.2118, + "theoretical_loss": 3.3308933274960895, + "tokens_seen": 3225747456 + }, + { + "epoch": 0.96, + "learning_rate": 2.2808425275827482e-05, + "loss": 1.3057, + "theoretical_loss": 3.3308882446126247, + "tokens_seen": 3225812992 + }, + { + "epoch": 0.96, + "learning_rate": 2.2788365095285858e-05, + "loss": 1.2642, + "theoretical_loss": 3.3308831618613373, + "tokens_seen": 3225878528 + }, + { + "epoch": 0.96, + "learning_rate": 2.276830491474423e-05, + "loss": 1.2749, + "theoretical_loss": 3.33087807924222, + "tokens_seen": 3225944064 + }, + { + "epoch": 0.96, + "learning_rate": 2.274824473420261e-05, + "loss": 1.2648, + "theoretical_loss": 3.330872996755268, + "tokens_seen": 3226009600 + }, + { + "epoch": 0.96, + "learning_rate": 2.2728184553660985e-05, + "loss": 1.193, + "theoretical_loss": 3.330867914400474, + "tokens_seen": 3226075136 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1172070, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.288637638092041, + "objective/train/theoretical_loss": 3.330862832177832, + "objective/train/tokens_used": 1596599776, + "theoretical_loss": 3.330862832177832, + "tokens_seen": 3226140672 + }, + { + "epoch": 0.96, + "learning_rate": 2.2708124373119358e-05, + "loss": 1.2529, + "theoretical_loss": 3.330862832177832, + "tokens_seen": 3226140672 + }, + { + "epoch": 0.96, + "learning_rate": 2.2688064192577733e-05, + "loss": 1.3025, + "theoretical_loss": 3.3308577500873366, + "tokens_seen": 3226206208 + }, + { + "epoch": 0.96, + "learning_rate": 2.2668004012036106e-05, + "loss": 1.1646, + "theoretical_loss": 3.330852668128981, + "tokens_seen": 3226271744 + }, + { + "epoch": 0.96, + "learning_rate": 2.2647943831494485e-05, + "loss": 1.2932, + "theoretical_loss": 3.3308475863027596, + "tokens_seen": 3226337280 + }, + { + "epoch": 0.96, + "learning_rate": 2.262788365095286e-05, + "loss": 1.3478, + "theoretical_loss": 3.330842504608666, + "tokens_seen": 3226402816 + }, + { + "epoch": 0.96, + "learning_rate": 2.2607823470411233e-05, + "loss": 1.2386, + "theoretical_loss": 3.3308374230466944, + "tokens_seen": 3226468352 + }, + { + "epoch": 0.96, + "learning_rate": 2.258776328986961e-05, + "loss": 1.2556, + "theoretical_loss": 3.330832341616838, + "tokens_seen": 3226533888 + }, + { + "epoch": 0.96, + "learning_rate": 2.2567703109327985e-05, + "loss": 1.3042, + "theoretical_loss": 3.330827260319091, + "tokens_seen": 3226599424 + }, + { + "epoch": 0.96, + "learning_rate": 2.254764292878636e-05, + "loss": 1.3034, + "theoretical_loss": 3.3308221791534476, + "tokens_seen": 3226664960 + }, + { + "epoch": 0.96, + "learning_rate": 2.2527582748244736e-05, + "loss": 1.2261, + "theoretical_loss": 3.3308170981199012, + "tokens_seen": 3226730496 + }, + { + "epoch": 0.96, + "learning_rate": 2.250752256770311e-05, + "loss": 1.2299, + "theoretical_loss": 3.330812017218446, + "tokens_seen": 3226796032 + }, + { + "epoch": 0.96, + "learning_rate": 2.2487462387161485e-05, + "loss": 1.1564, + "theoretical_loss": 3.330806936449076, + "tokens_seen": 3226861568 + }, + { + "epoch": 0.96, + "learning_rate": 2.246740220661986e-05, + "loss": 1.26, + "theoretical_loss": 3.3308018558117847, + "tokens_seen": 3226927104 + }, + { + "epoch": 0.96, + "learning_rate": 2.2447342026078236e-05, + "loss": 1.3442, + "theoretical_loss": 3.3307967753065664, + "tokens_seen": 3226992640 + }, + { + "epoch": 0.96, + "learning_rate": 2.242728184553661e-05, + "loss": 1.2895, + "theoretical_loss": 3.3307916949334144, + "tokens_seen": 3227058176 + }, + { + "epoch": 0.96, + "learning_rate": 2.2407221664994984e-05, + "loss": 1.2644, + "theoretical_loss": 3.330786614692323, + "tokens_seen": 3227123712 + }, + { + "epoch": 0.96, + "learning_rate": 2.238716148445336e-05, + "loss": 1.3059, + "theoretical_loss": 3.330781534583286, + "tokens_seen": 3227189248 + }, + { + "epoch": 0.96, + "learning_rate": 2.2367101303911736e-05, + "loss": 1.3089, + "theoretical_loss": 3.3307764546062977, + "tokens_seen": 3227254784 + }, + { + "epoch": 0.96, + "learning_rate": 2.2347041123370112e-05, + "loss": 1.2569, + "theoretical_loss": 3.330771374761351, + "tokens_seen": 3227320320 + }, + { + "epoch": 0.96, + "learning_rate": 2.2326980942828484e-05, + "loss": 1.2684, + "theoretical_loss": 3.3307662950484413, + "tokens_seen": 3227385856 + }, + { + "epoch": 0.96, + "learning_rate": 2.230692076228686e-05, + "loss": 1.1319, + "theoretical_loss": 3.330761215467561, + "tokens_seen": 3227451392 + }, + { + "epoch": 0.96, + "learning_rate": 2.228686058174524e-05, + "loss": 1.1851, + "theoretical_loss": 3.3307561360187043, + "tokens_seen": 3227516928 + }, + { + "epoch": 0.96, + "learning_rate": 2.226680040120361e-05, + "loss": 1.2837, + "theoretical_loss": 3.3307510567018657, + "tokens_seen": 3227582464 + }, + { + "epoch": 0.96, + "learning_rate": 2.2246740220661987e-05, + "loss": 1.2468, + "theoretical_loss": 3.3307459775170387, + "tokens_seen": 3227648000 + }, + { + "epoch": 0.96, + "learning_rate": 2.222668004012036e-05, + "loss": 1.2416, + "theoretical_loss": 3.3307408984642173, + "tokens_seen": 3227713536 + }, + { + "epoch": 0.96, + "learning_rate": 2.2206619859578736e-05, + "loss": 1.241, + "theoretical_loss": 3.330735819543395, + "tokens_seen": 3227779072 + }, + { + "epoch": 0.96, + "learning_rate": 2.2186559679037115e-05, + "loss": 1.2764, + "theoretical_loss": 3.330730740754566, + "tokens_seen": 3227844608 + }, + { + "epoch": 0.96, + "learning_rate": 2.2166499498495487e-05, + "loss": 1.2791, + "theoretical_loss": 3.330725662097725, + "tokens_seen": 3227910144 + }, + { + "epoch": 0.96, + "learning_rate": 2.2146439317953863e-05, + "loss": 1.3062, + "theoretical_loss": 3.330720583572864, + "tokens_seen": 3227975680 + }, + { + "epoch": 0.96, + "learning_rate": 2.2126379137412235e-05, + "loss": 1.386, + "theoretical_loss": 3.3307155051799784, + "tokens_seen": 3228041216 + }, + { + "epoch": 0.96, + "learning_rate": 2.210631895687061e-05, + "loss": 1.2531, + "theoretical_loss": 3.3307104269190617, + "tokens_seen": 3228106752 + }, + { + "epoch": 0.96, + "learning_rate": 2.208625877632899e-05, + "loss": 1.2367, + "theoretical_loss": 3.330705348790108, + "tokens_seen": 3228172288 + }, + { + "epoch": 0.96, + "learning_rate": 2.2066198595787363e-05, + "loss": 1.213, + "theoretical_loss": 3.330700270793111, + "tokens_seen": 3228237824 + }, + { + "epoch": 0.96, + "learning_rate": 2.204613841524574e-05, + "loss": 1.2761, + "theoretical_loss": 3.330695192928064, + "tokens_seen": 3228303360 + }, + { + "epoch": 0.96, + "learning_rate": 2.202607823470411e-05, + "loss": 1.2637, + "theoretical_loss": 3.3306901151949617, + "tokens_seen": 3228368896 + }, + { + "epoch": 0.96, + "learning_rate": 2.200601805416249e-05, + "loss": 1.1635, + "theoretical_loss": 3.3306850375937977, + "tokens_seen": 3228434432 + }, + { + "epoch": 0.96, + "learning_rate": 2.1985957873620862e-05, + "loss": 1.1789, + "theoretical_loss": 3.330679960124566, + "tokens_seen": 3228499968 + }, + { + "epoch": 0.96, + "learning_rate": 2.1965897693079238e-05, + "loss": 1.244, + "theoretical_loss": 3.3306748827872603, + "tokens_seen": 3228565504 + }, + { + "epoch": 0.96, + "learning_rate": 2.1945837512537614e-05, + "loss": 1.2754, + "theoretical_loss": 3.3306698055818744, + "tokens_seen": 3228631040 + }, + { + "epoch": 0.96, + "learning_rate": 2.1925777331995986e-05, + "loss": 1.3707, + "theoretical_loss": 3.330664728508403, + "tokens_seen": 3228696576 + }, + { + "epoch": 0.96, + "learning_rate": 2.1905717151454366e-05, + "loss": 1.2204, + "theoretical_loss": 3.330659651566839, + "tokens_seen": 3228762112 + }, + { + "epoch": 0.96, + "learning_rate": 2.1885656970912738e-05, + "loss": 1.3536, + "theoretical_loss": 3.330654574757177, + "tokens_seen": 3228827648 + }, + { + "epoch": 0.96, + "learning_rate": 2.1865596790371114e-05, + "loss": 1.3062, + "theoretical_loss": 3.33064949807941, + "tokens_seen": 3228893184 + }, + { + "epoch": 0.96, + "learning_rate": 2.184553660982949e-05, + "loss": 1.2264, + "theoretical_loss": 3.330644421533533, + "tokens_seen": 3228958720 + }, + { + "epoch": 0.96, + "learning_rate": 2.1825476429287862e-05, + "loss": 1.1594, + "theoretical_loss": 3.3306393451195393, + "tokens_seen": 3229024256 + }, + { + "epoch": 0.96, + "learning_rate": 2.180541624874624e-05, + "loss": 1.1646, + "theoretical_loss": 3.330634268837423, + "tokens_seen": 3229089792 + }, + { + "epoch": 0.96, + "learning_rate": 2.1785356068204614e-05, + "loss": 1.2088, + "theoretical_loss": 3.3306291926871774, + "tokens_seen": 3229155328 + }, + { + "epoch": 0.96, + "learning_rate": 2.176529588766299e-05, + "loss": 1.2596, + "theoretical_loss": 3.330624116668797, + "tokens_seen": 3229220864 + }, + { + "epoch": 0.96, + "learning_rate": 2.1745235707121365e-05, + "loss": 1.2711, + "theoretical_loss": 3.3306190407822758, + "tokens_seen": 3229286400 + }, + { + "epoch": 0.96, + "learning_rate": 2.1725175526579738e-05, + "loss": 1.2895, + "theoretical_loss": 3.3306139650276076, + "tokens_seen": 3229351936 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1173331, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1166080236434937, + "objective/train/theoretical_loss": 3.330608889404786, + "objective/train/tokens_used": 1599876576, + "theoretical_loss": 3.330608889404786, + "tokens_seen": 3229417472 + }, + { + "epoch": 0.96, + "learning_rate": 2.1705115346038117e-05, + "loss": 1.1923, + "theoretical_loss": 3.330608889404786, + "tokens_seen": 3229417472 + }, + { + "epoch": 0.96, + "learning_rate": 2.168505516549649e-05, + "loss": 1.2789, + "theoretical_loss": 3.330603813913805, + "tokens_seen": 3229483008 + }, + { + "epoch": 0.96, + "learning_rate": 2.1664994984954865e-05, + "loss": 1.2347, + "theoretical_loss": 3.3305987385546585, + "tokens_seen": 3229548544 + }, + { + "epoch": 0.96, + "learning_rate": 2.1644934804413237e-05, + "loss": 1.2215, + "theoretical_loss": 3.330593663327341, + "tokens_seen": 3229614080 + }, + { + "epoch": 0.96, + "learning_rate": 2.1624874623871617e-05, + "loss": 1.3385, + "theoretical_loss": 3.330588588231845, + "tokens_seen": 3229679616 + }, + { + "epoch": 0.96, + "learning_rate": 2.1604814443329992e-05, + "loss": 1.143, + "theoretical_loss": 3.330583513268166, + "tokens_seen": 3229745152 + }, + { + "epoch": 0.96, + "learning_rate": 2.1584754262788365e-05, + "loss": 1.2193, + "theoretical_loss": 3.330578438436297, + "tokens_seen": 3229810688 + }, + { + "epoch": 0.96, + "learning_rate": 2.156469408224674e-05, + "loss": 1.1068, + "theoretical_loss": 3.3305733637362316, + "tokens_seen": 3229876224 + }, + { + "epoch": 0.96, + "learning_rate": 2.1544633901705113e-05, + "loss": 1.1759, + "theoretical_loss": 3.3305682891679647, + "tokens_seen": 3229941760 + }, + { + "epoch": 0.96, + "learning_rate": 2.1524573721163492e-05, + "loss": 1.2228, + "theoretical_loss": 3.3305632147314896, + "tokens_seen": 3230007296 + }, + { + "epoch": 0.96, + "learning_rate": 2.1504513540621868e-05, + "loss": 1.2695, + "theoretical_loss": 3.3305581404268003, + "tokens_seen": 3230072832 + }, + { + "epoch": 0.96, + "learning_rate": 2.148445336008024e-05, + "loss": 1.271, + "theoretical_loss": 3.330553066253891, + "tokens_seen": 3230138368 + }, + { + "epoch": 0.96, + "learning_rate": 2.1464393179538616e-05, + "loss": 1.1847, + "theoretical_loss": 3.3305479922127548, + "tokens_seen": 3230203904 + }, + { + "epoch": 0.96, + "learning_rate": 2.144433299899699e-05, + "loss": 1.3259, + "theoretical_loss": 3.330542918303386, + "tokens_seen": 3230269440 + }, + { + "epoch": 0.96, + "learning_rate": 2.1424272818455368e-05, + "loss": 1.2689, + "theoretical_loss": 3.3305378445257787, + "tokens_seen": 3230334976 + }, + { + "epoch": 0.96, + "learning_rate": 2.1404212637913744e-05, + "loss": 1.2325, + "theoretical_loss": 3.3305327708799273, + "tokens_seen": 3230400512 + }, + { + "epoch": 0.96, + "learning_rate": 2.1384152457372116e-05, + "loss": 1.1842, + "theoretical_loss": 3.3305276973658247, + "tokens_seen": 3230466048 + }, + { + "epoch": 0.96, + "learning_rate": 2.1364092276830492e-05, + "loss": 1.2463, + "theoretical_loss": 3.330522623983465, + "tokens_seen": 3230531584 + }, + { + "epoch": 0.96, + "learning_rate": 2.1344032096288868e-05, + "loss": 1.2559, + "theoretical_loss": 3.3305175507328424, + "tokens_seen": 3230597120 + }, + { + "epoch": 0.96, + "learning_rate": 2.1323971915747243e-05, + "loss": 1.2282, + "theoretical_loss": 3.330512477613951, + "tokens_seen": 3230662656 + }, + { + "epoch": 0.96, + "learning_rate": 2.1303911735205616e-05, + "loss": 1.2858, + "theoretical_loss": 3.3305074046267844, + "tokens_seen": 3230728192 + }, + { + "epoch": 0.96, + "learning_rate": 2.128385155466399e-05, + "loss": 1.2149, + "theoretical_loss": 3.3305023317713363, + "tokens_seen": 3230793728 + }, + { + "epoch": 0.96, + "learning_rate": 2.1263791374122367e-05, + "loss": 1.1811, + "theoretical_loss": 3.3304972590476014, + "tokens_seen": 3230859264 + }, + { + "epoch": 0.96, + "learning_rate": 2.1243731193580743e-05, + "loss": 1.3527, + "theoretical_loss": 3.3304921864555723, + "tokens_seen": 3230924800 + }, + { + "epoch": 0.96, + "learning_rate": 2.122367101303912e-05, + "loss": 1.2462, + "theoretical_loss": 3.330487113995244, + "tokens_seen": 3230990336 + }, + { + "epoch": 0.96, + "learning_rate": 2.120361083249749e-05, + "loss": 1.2804, + "theoretical_loss": 3.3304820416666105, + "tokens_seen": 3231055872 + }, + { + "epoch": 0.96, + "learning_rate": 2.1183550651955867e-05, + "loss": 1.213, + "theoretical_loss": 3.330476969469665, + "tokens_seen": 3231121408 + }, + { + "epoch": 0.96, + "learning_rate": 2.1163490471414243e-05, + "loss": 1.2923, + "theoretical_loss": 3.3304718974044016, + "tokens_seen": 3231186944 + }, + { + "epoch": 0.96, + "learning_rate": 2.114343029087262e-05, + "loss": 1.2898, + "theoretical_loss": 3.3304668254708143, + "tokens_seen": 3231252480 + }, + { + "epoch": 0.96, + "learning_rate": 2.1123370110330994e-05, + "loss": 1.2517, + "theoretical_loss": 3.3304617536688967, + "tokens_seen": 3231318016 + }, + { + "epoch": 0.96, + "learning_rate": 2.1103309929789367e-05, + "loss": 1.2823, + "theoretical_loss": 3.3304566819986436, + "tokens_seen": 3231383552 + }, + { + "epoch": 0.96, + "learning_rate": 2.1083249749247743e-05, + "loss": 1.2457, + "theoretical_loss": 3.3304516104600483, + "tokens_seen": 3231449088 + }, + { + "epoch": 0.96, + "learning_rate": 2.1063189568706122e-05, + "loss": 1.1985, + "theoretical_loss": 3.3304465390531046, + "tokens_seen": 3231514624 + }, + { + "epoch": 0.96, + "learning_rate": 2.1043129388164494e-05, + "loss": 1.2533, + "theoretical_loss": 3.3304414677778067, + "tokens_seen": 3231580160 + }, + { + "epoch": 0.96, + "learning_rate": 2.102306920762287e-05, + "loss": 1.2413, + "theoretical_loss": 3.3304363966341484, + "tokens_seen": 3231645696 + }, + { + "epoch": 0.96, + "learning_rate": 2.1003009027081242e-05, + "loss": 1.3225, + "theoretical_loss": 3.3304313256221234, + "tokens_seen": 3231711232 + }, + { + "epoch": 0.96, + "learning_rate": 2.0982948846539618e-05, + "loss": 1.2141, + "theoretical_loss": 3.330426254741726, + "tokens_seen": 3231776768 + }, + { + "epoch": 0.96, + "learning_rate": 2.0962888665997994e-05, + "loss": 1.3376, + "theoretical_loss": 3.33042118399295, + "tokens_seen": 3231842304 + }, + { + "epoch": 0.96, + "learning_rate": 2.094282848545637e-05, + "loss": 1.3066, + "theoretical_loss": 3.3304161133757892, + "tokens_seen": 3231907840 + }, + { + "epoch": 0.96, + "learning_rate": 2.0922768304914746e-05, + "loss": 1.3423, + "theoretical_loss": 3.3304110428902374, + "tokens_seen": 3231973376 + }, + { + "epoch": 0.96, + "learning_rate": 2.0902708124373118e-05, + "loss": 1.1437, + "theoretical_loss": 3.3304059725362887, + "tokens_seen": 3232038912 + }, + { + "epoch": 0.96, + "learning_rate": 2.0882647943831494e-05, + "loss": 1.1814, + "theoretical_loss": 3.330400902313937, + "tokens_seen": 3232104448 + }, + { + "epoch": 0.96, + "learning_rate": 2.086258776328987e-05, + "loss": 1.2261, + "theoretical_loss": 3.3303958322231764, + "tokens_seen": 3232169984 + }, + { + "epoch": 0.96, + "learning_rate": 2.0842527582748245e-05, + "loss": 1.1142, + "theoretical_loss": 3.3303907622640003, + "tokens_seen": 3232235520 + }, + { + "epoch": 0.96, + "learning_rate": 2.082246740220662e-05, + "loss": 1.2502, + "theoretical_loss": 3.330385692436403, + "tokens_seen": 3232301056 + }, + { + "epoch": 0.96, + "learning_rate": 2.0802407221664994e-05, + "loss": 1.3445, + "theoretical_loss": 3.3303806227403783, + "tokens_seen": 3232366592 + }, + { + "epoch": 0.96, + "learning_rate": 2.0782347041123373e-05, + "loss": 1.2306, + "theoretical_loss": 3.3303755531759207, + "tokens_seen": 3232432128 + }, + { + "epoch": 0.96, + "learning_rate": 2.0762286860581745e-05, + "loss": 1.3416, + "theoretical_loss": 3.3303704837430232, + "tokens_seen": 3232497664 + }, + { + "epoch": 0.96, + "learning_rate": 2.074222668004012e-05, + "loss": 1.2135, + "theoretical_loss": 3.33036541444168, + "tokens_seen": 3232563200 + }, + { + "epoch": 0.96, + "learning_rate": 2.0722166499498497e-05, + "loss": 1.2767, + "theoretical_loss": 3.3303603452718855, + "tokens_seen": 3232628736 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1174536, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3326020240783691, + "objective/train/theoretical_loss": 3.3303552762336333, + "objective/train/tokens_used": 1603153376, + "theoretical_loss": 3.3303552762336333, + "tokens_seen": 3232694272 + }, + { + "epoch": 0.96, + "learning_rate": 2.070210631895687e-05, + "loss": 1.3162, + "theoretical_loss": 3.3303552762336333, + "tokens_seen": 3232694272 + }, + { + "epoch": 0.96, + "learning_rate": 2.068204613841525e-05, + "loss": 1.2334, + "theoretical_loss": 3.3303502073269167, + "tokens_seen": 3232759808 + }, + { + "epoch": 0.96, + "learning_rate": 2.066198595787362e-05, + "loss": 1.2357, + "theoretical_loss": 3.3303451385517304, + "tokens_seen": 3232825344 + }, + { + "epoch": 0.96, + "learning_rate": 2.0641925777331997e-05, + "loss": 1.3241, + "theoretical_loss": 3.3303400699080683, + "tokens_seen": 3232890880 + }, + { + "epoch": 0.96, + "learning_rate": 2.062186559679037e-05, + "loss": 1.2884, + "theoretical_loss": 3.330335001395924, + "tokens_seen": 3232956416 + }, + { + "epoch": 0.96, + "learning_rate": 2.0601805416248745e-05, + "loss": 1.1879, + "theoretical_loss": 3.3303299330152916, + "tokens_seen": 3233021952 + }, + { + "epoch": 0.96, + "learning_rate": 2.0581745235707124e-05, + "loss": 1.2302, + "theoretical_loss": 3.330324864766165, + "tokens_seen": 3233087488 + }, + { + "epoch": 0.96, + "learning_rate": 2.0561685055165496e-05, + "loss": 1.3018, + "theoretical_loss": 3.330319796648538, + "tokens_seen": 3233153024 + }, + { + "epoch": 0.96, + "learning_rate": 2.0541624874623872e-05, + "loss": 1.2316, + "theoretical_loss": 3.330314728662405, + "tokens_seen": 3233218560 + }, + { + "epoch": 0.96, + "learning_rate": 2.0521564694082245e-05, + "loss": 1.2844, + "theoretical_loss": 3.3303096608077594, + "tokens_seen": 3233284096 + }, + { + "epoch": 0.96, + "learning_rate": 2.0501504513540624e-05, + "loss": 1.201, + "theoretical_loss": 3.3303045930845947, + "tokens_seen": 3233349632 + }, + { + "epoch": 0.96, + "learning_rate": 2.0481444332999e-05, + "loss": 1.3043, + "theoretical_loss": 3.330299525492906, + "tokens_seen": 3233415168 + }, + { + "epoch": 0.96, + "learning_rate": 2.0461384152457372e-05, + "loss": 1.1994, + "theoretical_loss": 3.3302944580326868, + "tokens_seen": 3233480704 + }, + { + "epoch": 0.96, + "learning_rate": 2.0441323971915748e-05, + "loss": 1.2269, + "theoretical_loss": 3.33028939070393, + "tokens_seen": 3233546240 + }, + { + "epoch": 0.96, + "learning_rate": 2.042126379137412e-05, + "loss": 1.243, + "theoretical_loss": 3.330284323506631, + "tokens_seen": 3233611776 + }, + { + "epoch": 0.96, + "learning_rate": 2.04012036108325e-05, + "loss": 1.3085, + "theoretical_loss": 3.3302792564407833, + "tokens_seen": 3233677312 + }, + { + "epoch": 0.96, + "learning_rate": 2.0381143430290875e-05, + "loss": 1.0714, + "theoretical_loss": 3.3302741895063797, + "tokens_seen": 3233742848 + }, + { + "epoch": 0.96, + "learning_rate": 2.0361083249749248e-05, + "loss": 1.2522, + "theoretical_loss": 3.330269122703416, + "tokens_seen": 3233808384 + }, + { + "epoch": 0.96, + "learning_rate": 2.0341023069207623e-05, + "loss": 1.2078, + "theoretical_loss": 3.3302640560318846, + "tokens_seen": 3233873920 + }, + { + "epoch": 0.96, + "learning_rate": 2.0320962888665996e-05, + "loss": 1.2816, + "theoretical_loss": 3.3302589894917802, + "tokens_seen": 3233939456 + }, + { + "epoch": 0.96, + "learning_rate": 2.0300902708124375e-05, + "loss": 1.2415, + "theoretical_loss": 3.3302539230830965, + "tokens_seen": 3234004992 + }, + { + "epoch": 0.96, + "learning_rate": 2.028084252758275e-05, + "loss": 1.2246, + "theoretical_loss": 3.3302488568058277, + "tokens_seen": 3234070528 + }, + { + "epoch": 0.96, + "learning_rate": 2.0260782347041123e-05, + "loss": 1.2353, + "theoretical_loss": 3.330243790659967, + "tokens_seen": 3234136064 + }, + { + "epoch": 0.96, + "learning_rate": 2.02407221664995e-05, + "loss": 1.2617, + "theoretical_loss": 3.3302387246455094, + "tokens_seen": 3234201600 + }, + { + "epoch": 0.96, + "learning_rate": 2.0220661985957875e-05, + "loss": 1.2309, + "theoretical_loss": 3.330233658762448, + "tokens_seen": 3234267136 + }, + { + "epoch": 0.96, + "learning_rate": 2.020060180541625e-05, + "loss": 1.1797, + "theoretical_loss": 3.3302285930107765, + "tokens_seen": 3234332672 + }, + { + "epoch": 0.96, + "learning_rate": 2.0180541624874623e-05, + "loss": 1.2442, + "theoretical_loss": 3.3302235273904897, + "tokens_seen": 3234398208 + }, + { + "epoch": 0.96, + "learning_rate": 2.0160481444333e-05, + "loss": 1.2887, + "theoretical_loss": 3.3302184619015813, + "tokens_seen": 3234463744 + }, + { + "epoch": 0.96, + "learning_rate": 2.0140421263791374e-05, + "loss": 1.2414, + "theoretical_loss": 3.330213396544045, + "tokens_seen": 3234529280 + }, + { + "epoch": 0.96, + "learning_rate": 2.012036108324975e-05, + "loss": 1.1367, + "theoretical_loss": 3.3302083313178743, + "tokens_seen": 3234594816 + }, + { + "epoch": 0.96, + "learning_rate": 2.0100300902708126e-05, + "loss": 1.3469, + "theoretical_loss": 3.330203266223064, + "tokens_seen": 3234660352 + }, + { + "epoch": 0.96, + "learning_rate": 2.00802407221665e-05, + "loss": 1.2163, + "theoretical_loss": 3.330198201259608, + "tokens_seen": 3234725888 + }, + { + "epoch": 0.96, + "learning_rate": 2.0060180541624874e-05, + "loss": 1.3408, + "theoretical_loss": 3.3301931364274995, + "tokens_seen": 3234791424 + }, + { + "epoch": 0.96, + "learning_rate": 2.004012036108325e-05, + "loss": 1.3315, + "theoretical_loss": 3.3301880717267327, + "tokens_seen": 3234856960 + }, + { + "epoch": 0.96, + "learning_rate": 2.0020060180541626e-05, + "loss": 1.1877, + "theoretical_loss": 3.3301830071573018, + "tokens_seen": 3234922496 + }, + { + "epoch": 0.96, + "learning_rate": 2e-05, + "loss": 1.2399, + "theoretical_loss": 3.3301779427192004, + "tokens_seen": 3234988032 + }, + { + "epoch": 0.96, + "learning_rate": 1.9979939819458374e-05, + "loss": 1.3399, + "theoretical_loss": 3.330172878412423, + "tokens_seen": 3235053568 + }, + { + "epoch": 0.96, + "learning_rate": 1.995987963891675e-05, + "loss": 1.2679, + "theoretical_loss": 3.330167814236963, + "tokens_seen": 3235119104 + }, + { + "epoch": 0.96, + "learning_rate": 1.993981945837513e-05, + "loss": 1.2429, + "theoretical_loss": 3.3301627501928146, + "tokens_seen": 3235184640 + }, + { + "epoch": 0.96, + "learning_rate": 1.99197592778335e-05, + "loss": 1.1607, + "theoretical_loss": 3.3301576862799713, + "tokens_seen": 3235250176 + }, + { + "epoch": 0.96, + "learning_rate": 1.9899699097291877e-05, + "loss": 1.2534, + "theoretical_loss": 3.330152622498428, + "tokens_seen": 3235315712 + }, + { + "epoch": 0.96, + "learning_rate": 1.987963891675025e-05, + "loss": 1.3565, + "theoretical_loss": 3.3301475588481773, + "tokens_seen": 3235381248 + }, + { + "epoch": 0.96, + "learning_rate": 1.9859578736208625e-05, + "loss": 1.3022, + "theoretical_loss": 3.3301424953292145, + "tokens_seen": 3235446784 + }, + { + "epoch": 0.96, + "learning_rate": 1.9839518555667e-05, + "loss": 1.1964, + "theoretical_loss": 3.330137431941532, + "tokens_seen": 3235512320 + }, + { + "epoch": 0.96, + "learning_rate": 1.9819458375125377e-05, + "loss": 1.1642, + "theoretical_loss": 3.3301323686851254, + "tokens_seen": 3235577856 + }, + { + "epoch": 0.96, + "learning_rate": 1.9799398194583753e-05, + "loss": 1.2044, + "theoretical_loss": 3.3301273055599876, + "tokens_seen": 3235643392 + }, + { + "epoch": 0.96, + "learning_rate": 1.9779338014042125e-05, + "loss": 1.405, + "theoretical_loss": 3.3301222425661123, + "tokens_seen": 3235708928 + }, + { + "epoch": 0.96, + "learning_rate": 1.97592778335005e-05, + "loss": 1.2417, + "theoretical_loss": 3.3301171797034947, + "tokens_seen": 3235774464 + }, + { + "epoch": 0.96, + "learning_rate": 1.9739217652958877e-05, + "loss": 1.247, + "theoretical_loss": 3.3301121169721277, + "tokens_seen": 3235840000 + }, + { + "epoch": 0.96, + "learning_rate": 1.9719157472417253e-05, + "loss": 1.2357, + "theoretical_loss": 3.3301070543720055, + "tokens_seen": 3235905536 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1175644, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2555317878723145, + "objective/train/theoretical_loss": 3.330101991903122, + "objective/train/tokens_used": 1606430176, + "theoretical_loss": 3.330101991903122, + "tokens_seen": 3235971072 + }, + { + "epoch": 0.96, + "learning_rate": 1.969909729187563e-05, + "loss": 1.2503, + "theoretical_loss": 3.330101991903122, + "tokens_seen": 3235971072 + }, + { + "epoch": 0.96, + "learning_rate": 1.9679037111334e-05, + "loss": 1.2866, + "theoretical_loss": 3.330096929565471, + "tokens_seen": 3236036608 + }, + { + "epoch": 0.96, + "learning_rate": 1.965897693079238e-05, + "loss": 1.1928, + "theoretical_loss": 3.3300918673590467, + "tokens_seen": 3236102144 + }, + { + "epoch": 0.96, + "learning_rate": 1.9638916750250752e-05, + "loss": 1.3017, + "theoretical_loss": 3.3300868052838433, + "tokens_seen": 3236167680 + }, + { + "epoch": 0.96, + "learning_rate": 1.9618856569709128e-05, + "loss": 1.2383, + "theoretical_loss": 3.330081743339854, + "tokens_seen": 3236233216 + }, + { + "epoch": 0.96, + "learning_rate": 1.9598796389167504e-05, + "loss": 1.281, + "theoretical_loss": 3.330076681527073, + "tokens_seen": 3236298752 + }, + { + "epoch": 0.96, + "learning_rate": 1.9578736208625876e-05, + "loss": 1.1203, + "theoretical_loss": 3.3300716198454947, + "tokens_seen": 3236364288 + }, + { + "epoch": 0.96, + "learning_rate": 1.9558676028084256e-05, + "loss": 1.2364, + "theoretical_loss": 3.3300665582951128, + "tokens_seen": 3236429824 + }, + { + "epoch": 0.96, + "learning_rate": 1.9538615847542628e-05, + "loss": 1.3008, + "theoretical_loss": 3.3300614968759215, + "tokens_seen": 3236495360 + }, + { + "epoch": 0.96, + "learning_rate": 1.9518555667001004e-05, + "loss": 1.1865, + "theoretical_loss": 3.3300564355879136, + "tokens_seen": 3236560896 + }, + { + "epoch": 0.96, + "learning_rate": 1.9498495486459376e-05, + "loss": 1.3036, + "theoretical_loss": 3.3300513744310845, + "tokens_seen": 3236626432 + }, + { + "epoch": 0.96, + "learning_rate": 1.9478435305917752e-05, + "loss": 1.1612, + "theoretical_loss": 3.3300463134054272, + "tokens_seen": 3236691968 + }, + { + "epoch": 0.96, + "learning_rate": 1.945837512537613e-05, + "loss": 1.1979, + "theoretical_loss": 3.330041252510936, + "tokens_seen": 3236757504 + }, + { + "epoch": 0.96, + "learning_rate": 1.9438314944834504e-05, + "loss": 1.2317, + "theoretical_loss": 3.330036191747605, + "tokens_seen": 3236823040 + }, + { + "epoch": 0.96, + "learning_rate": 1.941825476429288e-05, + "loss": 1.2584, + "theoretical_loss": 3.330031131115428, + "tokens_seen": 3236888576 + }, + { + "epoch": 0.96, + "learning_rate": 1.9398194583751252e-05, + "loss": 1.2787, + "theoretical_loss": 3.330026070614398, + "tokens_seen": 3236954112 + }, + { + "epoch": 0.96, + "learning_rate": 1.937813440320963e-05, + "loss": 1.2413, + "theoretical_loss": 3.3300210102445105, + "tokens_seen": 3237019648 + }, + { + "epoch": 0.96, + "learning_rate": 1.9358074222668007e-05, + "loss": 1.2869, + "theoretical_loss": 3.330015950005759, + "tokens_seen": 3237085184 + }, + { + "epoch": 0.96, + "learning_rate": 1.933801404212638e-05, + "loss": 1.1795, + "theoretical_loss": 3.330010889898137, + "tokens_seen": 3237150720 + }, + { + "epoch": 0.96, + "learning_rate": 1.9317953861584755e-05, + "loss": 1.1467, + "theoretical_loss": 3.330005829921639, + "tokens_seen": 3237216256 + }, + { + "epoch": 0.96, + "learning_rate": 1.9297893681043127e-05, + "loss": 1.2147, + "theoretical_loss": 3.330000770076258, + "tokens_seen": 3237281792 + }, + { + "epoch": 0.96, + "learning_rate": 1.9277833500501506e-05, + "loss": 1.2502, + "theoretical_loss": 3.3299957103619886, + "tokens_seen": 3237347328 + }, + { + "epoch": 0.96, + "learning_rate": 1.9257773319959882e-05, + "loss": 1.1848, + "theoretical_loss": 3.329990650778825, + "tokens_seen": 3237412864 + }, + { + "epoch": 0.96, + "learning_rate": 1.9237713139418255e-05, + "loss": 1.2578, + "theoretical_loss": 3.329985591326761, + "tokens_seen": 3237478400 + }, + { + "epoch": 0.96, + "learning_rate": 1.921765295887663e-05, + "loss": 1.3356, + "theoretical_loss": 3.3299805320057905, + "tokens_seen": 3237543936 + }, + { + "epoch": 0.96, + "learning_rate": 1.9197592778335003e-05, + "loss": 1.3654, + "theoretical_loss": 3.329975472815907, + "tokens_seen": 3237609472 + }, + { + "epoch": 0.96, + "learning_rate": 1.9177532597793382e-05, + "loss": 1.246, + "theoretical_loss": 3.329970413757105, + "tokens_seen": 3237675008 + }, + { + "epoch": 0.96, + "learning_rate": 1.9157472417251758e-05, + "loss": 1.3285, + "theoretical_loss": 3.329965354829378, + "tokens_seen": 3237740544 + }, + { + "epoch": 0.96, + "learning_rate": 1.913741223671013e-05, + "loss": 1.3147, + "theoretical_loss": 3.3299602960327204, + "tokens_seen": 3237806080 + }, + { + "epoch": 0.96, + "learning_rate": 1.9117352056168506e-05, + "loss": 1.201, + "theoretical_loss": 3.329955237367126, + "tokens_seen": 3237871616 + }, + { + "epoch": 0.96, + "learning_rate": 1.909729187562688e-05, + "loss": 1.1522, + "theoretical_loss": 3.329950178832589, + "tokens_seen": 3237937152 + }, + { + "epoch": 0.96, + "learning_rate": 1.9077231695085258e-05, + "loss": 1.2109, + "theoretical_loss": 3.3299451204291026, + "tokens_seen": 3238002688 + }, + { + "epoch": 0.96, + "learning_rate": 1.905717151454363e-05, + "loss": 1.3205, + "theoretical_loss": 3.3299400621566613, + "tokens_seen": 3238068224 + }, + { + "epoch": 0.96, + "learning_rate": 1.9037111334002006e-05, + "loss": 1.3452, + "theoretical_loss": 3.329935004015259, + "tokens_seen": 3238133760 + }, + { + "epoch": 0.96, + "learning_rate": 1.901705115346038e-05, + "loss": 1.1127, + "theoretical_loss": 3.32992994600489, + "tokens_seen": 3238199296 + }, + { + "epoch": 0.96, + "learning_rate": 1.8996990972918757e-05, + "loss": 1.195, + "theoretical_loss": 3.3299248881255474, + "tokens_seen": 3238264832 + }, + { + "epoch": 0.96, + "learning_rate": 1.8976930792377133e-05, + "loss": 1.2603, + "theoretical_loss": 3.329919830377226, + "tokens_seen": 3238330368 + }, + { + "epoch": 0.96, + "learning_rate": 1.8956870611835506e-05, + "loss": 1.2621, + "theoretical_loss": 3.3299147727599188, + "tokens_seen": 3238395904 + }, + { + "epoch": 0.96, + "learning_rate": 1.893681043129388e-05, + "loss": 1.2256, + "theoretical_loss": 3.329909715273621, + "tokens_seen": 3238461440 + }, + { + "epoch": 0.96, + "learning_rate": 1.8916750250752257e-05, + "loss": 1.1572, + "theoretical_loss": 3.3299046579183256, + "tokens_seen": 3238526976 + }, + { + "epoch": 0.96, + "learning_rate": 1.8896690070210633e-05, + "loss": 1.3873, + "theoretical_loss": 3.329899600694027, + "tokens_seen": 3238592512 + }, + { + "epoch": 0.96, + "learning_rate": 1.887662988966901e-05, + "loss": 1.2839, + "theoretical_loss": 3.329894543600719, + "tokens_seen": 3238658048 + }, + { + "epoch": 0.96, + "learning_rate": 1.885656970912738e-05, + "loss": 1.2948, + "theoretical_loss": 3.3298894866383955, + "tokens_seen": 3238723584 + }, + { + "epoch": 0.96, + "learning_rate": 1.8836509528585757e-05, + "loss": 1.3415, + "theoretical_loss": 3.3298844298070507, + "tokens_seen": 3238789120 + }, + { + "epoch": 0.96, + "learning_rate": 1.881644934804413e-05, + "loss": 1.272, + "theoretical_loss": 3.329879373106678, + "tokens_seen": 3238854656 + }, + { + "epoch": 0.96, + "learning_rate": 1.879638916750251e-05, + "loss": 1.1508, + "theoretical_loss": 3.329874316537272, + "tokens_seen": 3238920192 + }, + { + "epoch": 0.96, + "learning_rate": 1.8776328986960884e-05, + "loss": 1.1829, + "theoretical_loss": 3.3298692600988264, + "tokens_seen": 3238985728 + }, + { + "epoch": 0.96, + "learning_rate": 1.8756268806419257e-05, + "loss": 1.2874, + "theoretical_loss": 3.3298642037913355, + "tokens_seen": 3239051264 + }, + { + "epoch": 0.96, + "learning_rate": 1.8736208625877633e-05, + "loss": 1.2371, + "theoretical_loss": 3.3298591476147923, + "tokens_seen": 3239116800 + }, + { + "epoch": 0.96, + "learning_rate": 1.871614844533601e-05, + "loss": 1.2766, + "theoretical_loss": 3.3298540915691914, + "tokens_seen": 3239182336 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1176615, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3661737442016602, + "objective/train/theoretical_loss": 3.3298490356545276, + "objective/train/tokens_used": 1609706976, + "theoretical_loss": 3.3298490356545276, + "tokens_seen": 3239247872 + }, + { + "epoch": 0.96, + "learning_rate": 1.8696088264794384e-05, + "loss": 1.2656, + "theoretical_loss": 3.3298490356545276, + "tokens_seen": 3239247872 + }, + { + "epoch": 0.96, + "learning_rate": 1.867602808425276e-05, + "loss": 1.295, + "theoretical_loss": 3.3298439798707933, + "tokens_seen": 3239313408 + }, + { + "epoch": 0.96, + "learning_rate": 1.8655967903711132e-05, + "loss": 1.3061, + "theoretical_loss": 3.329838924217983, + "tokens_seen": 3239378944 + }, + { + "epoch": 0.96, + "learning_rate": 1.8635907723169508e-05, + "loss": 1.2553, + "theoretical_loss": 3.3298338686960913, + "tokens_seen": 3239444480 + }, + { + "epoch": 0.96, + "learning_rate": 1.8615847542627884e-05, + "loss": 1.3478, + "theoretical_loss": 3.3298288133051113, + "tokens_seen": 3239510016 + }, + { + "epoch": 0.96, + "learning_rate": 1.859578736208626e-05, + "loss": 1.1873, + "theoretical_loss": 3.3298237580450376, + "tokens_seen": 3239575552 + }, + { + "epoch": 0.96, + "learning_rate": 1.8575727181544636e-05, + "loss": 1.2503, + "theoretical_loss": 3.3298187029158637, + "tokens_seen": 3239641088 + }, + { + "epoch": 0.96, + "learning_rate": 1.8555667001003008e-05, + "loss": 1.2362, + "theoretical_loss": 3.329813647917584, + "tokens_seen": 3239706624 + }, + { + "epoch": 0.96, + "learning_rate": 1.8535606820461384e-05, + "loss": 1.1752, + "theoretical_loss": 3.329808593050192, + "tokens_seen": 3239772160 + }, + { + "epoch": 0.96, + "learning_rate": 1.851554663991976e-05, + "loss": 1.2351, + "theoretical_loss": 3.329803538313682, + "tokens_seen": 3239837696 + }, + { + "epoch": 0.96, + "learning_rate": 1.8495486459378135e-05, + "loss": 1.2023, + "theoretical_loss": 3.329798483708048, + "tokens_seen": 3239903232 + }, + { + "epoch": 0.96, + "learning_rate": 1.847542627883651e-05, + "loss": 1.2257, + "theoretical_loss": 3.3297934292332836, + "tokens_seen": 3239968768 + }, + { + "epoch": 0.96, + "learning_rate": 1.8455366098294884e-05, + "loss": 1.2543, + "theoretical_loss": 3.3297883748893833, + "tokens_seen": 3240034304 + }, + { + "epoch": 0.96, + "learning_rate": 1.8435305917753263e-05, + "loss": 1.2765, + "theoretical_loss": 3.3297833206763405, + "tokens_seen": 3240099840 + }, + { + "epoch": 0.96, + "learning_rate": 1.8415245737211635e-05, + "loss": 1.2904, + "theoretical_loss": 3.3297782665941495, + "tokens_seen": 3240165376 + }, + { + "epoch": 0.96, + "learning_rate": 1.839518555667001e-05, + "loss": 1.3339, + "theoretical_loss": 3.329773212642804, + "tokens_seen": 3240230912 + }, + { + "epoch": 0.96, + "learning_rate": 1.8375125376128383e-05, + "loss": 1.2358, + "theoretical_loss": 3.329768158822298, + "tokens_seen": 3240296448 + }, + { + "epoch": 0.96, + "learning_rate": 1.835506519558676e-05, + "loss": 1.3657, + "theoretical_loss": 3.329763105132626, + "tokens_seen": 3240361984 + }, + { + "epoch": 0.96, + "learning_rate": 1.8335005015045138e-05, + "loss": 1.3197, + "theoretical_loss": 3.3297580515737817, + "tokens_seen": 3240427520 + }, + { + "epoch": 0.96, + "learning_rate": 1.831494483450351e-05, + "loss": 1.3159, + "theoretical_loss": 3.3297529981457585, + "tokens_seen": 3240493056 + }, + { + "epoch": 0.96, + "learning_rate": 1.8294884653961886e-05, + "loss": 1.2419, + "theoretical_loss": 3.329747944848551, + "tokens_seen": 3240558592 + }, + { + "epoch": 0.96, + "learning_rate": 1.827482447342026e-05, + "loss": 1.3488, + "theoretical_loss": 3.3297428916821534, + "tokens_seen": 3240624128 + }, + { + "epoch": 0.96, + "learning_rate": 1.8254764292878635e-05, + "loss": 1.1648, + "theoretical_loss": 3.3297378386465586, + "tokens_seen": 3240689664 + }, + { + "epoch": 0.96, + "learning_rate": 1.8234704112337014e-05, + "loss": 1.2325, + "theoretical_loss": 3.329732785741762, + "tokens_seen": 3240755200 + }, + { + "epoch": 0.96, + "learning_rate": 1.8214643931795386e-05, + "loss": 1.203, + "theoretical_loss": 3.329727732967756, + "tokens_seen": 3240820736 + }, + { + "epoch": 0.96, + "learning_rate": 1.8194583751253762e-05, + "loss": 1.17, + "theoretical_loss": 3.3297226803245357, + "tokens_seen": 3240886272 + }, + { + "epoch": 0.96, + "learning_rate": 1.8174523570712134e-05, + "loss": 1.1967, + "theoretical_loss": 3.329717627812095, + "tokens_seen": 3240951808 + }, + { + "epoch": 0.96, + "learning_rate": 1.8154463390170514e-05, + "loss": 1.2596, + "theoretical_loss": 3.329712575430427, + "tokens_seen": 3241017344 + }, + { + "epoch": 0.96, + "learning_rate": 1.813440320962889e-05, + "loss": 1.2725, + "theoretical_loss": 3.3297075231795263, + "tokens_seen": 3241082880 + }, + { + "epoch": 0.96, + "learning_rate": 1.8114343029087262e-05, + "loss": 1.2369, + "theoretical_loss": 3.329702471059387, + "tokens_seen": 3241148416 + }, + { + "epoch": 0.96, + "learning_rate": 1.8094282848545638e-05, + "loss": 1.3017, + "theoretical_loss": 3.3296974190700035, + "tokens_seen": 3241213952 + }, + { + "epoch": 0.96, + "learning_rate": 1.807422266800401e-05, + "loss": 1.263, + "theoretical_loss": 3.3296923672113685, + "tokens_seen": 3241279488 + }, + { + "epoch": 0.96, + "learning_rate": 1.805416248746239e-05, + "loss": 1.2413, + "theoretical_loss": 3.329687315483477, + "tokens_seen": 3241345024 + }, + { + "epoch": 0.96, + "learning_rate": 1.803410230692076e-05, + "loss": 1.1681, + "theoretical_loss": 3.3296822638863226, + "tokens_seen": 3241410560 + }, + { + "epoch": 0.96, + "learning_rate": 1.8014042126379137e-05, + "loss": 1.1589, + "theoretical_loss": 3.3296772124198992, + "tokens_seen": 3241476096 + }, + { + "epoch": 0.96, + "learning_rate": 1.7993981945837513e-05, + "loss": 1.1314, + "theoretical_loss": 3.329672161084201, + "tokens_seen": 3241541632 + }, + { + "epoch": 0.96, + "learning_rate": 1.7973921765295886e-05, + "loss": 1.1998, + "theoretical_loss": 3.329667109879222, + "tokens_seen": 3241607168 + }, + { + "epoch": 0.96, + "learning_rate": 1.7953861584754265e-05, + "loss": 1.3054, + "theoretical_loss": 3.3296620588049555, + "tokens_seen": 3241672704 + }, + { + "epoch": 0.96, + "learning_rate": 1.7933801404212637e-05, + "loss": 1.236, + "theoretical_loss": 3.3296570078613965, + "tokens_seen": 3241738240 + }, + { + "epoch": 0.96, + "learning_rate": 1.7913741223671013e-05, + "loss": 1.2183, + "theoretical_loss": 3.3296519570485383, + "tokens_seen": 3241803776 + }, + { + "epoch": 0.96, + "learning_rate": 1.789368104312939e-05, + "loss": 1.2893, + "theoretical_loss": 3.329646906366375, + "tokens_seen": 3241869312 + }, + { + "epoch": 0.96, + "learning_rate": 1.7873620862587765e-05, + "loss": 1.1441, + "theoretical_loss": 3.329641855814901, + "tokens_seen": 3241934848 + }, + { + "epoch": 0.96, + "learning_rate": 1.785356068204614e-05, + "loss": 1.2397, + "theoretical_loss": 3.3296368053941094, + "tokens_seen": 3242000384 + }, + { + "epoch": 0.96, + "learning_rate": 1.7833500501504513e-05, + "loss": 1.1861, + "theoretical_loss": 3.329631755103995, + "tokens_seen": 3242065920 + }, + { + "epoch": 0.96, + "learning_rate": 1.781344032096289e-05, + "loss": 1.2324, + "theoretical_loss": 3.3296267049445514, + "tokens_seen": 3242131456 + }, + { + "epoch": 0.96, + "learning_rate": 1.7793380140421264e-05, + "loss": 1.1769, + "theoretical_loss": 3.3296216549157727, + "tokens_seen": 3242196992 + }, + { + "epoch": 0.97, + "learning_rate": 1.777331995987964e-05, + "loss": 1.1948, + "theoretical_loss": 3.3296166050176526, + "tokens_seen": 3242262528 + }, + { + "epoch": 0.97, + "learning_rate": 1.7753259779338016e-05, + "loss": 1.236, + "theoretical_loss": 3.3296115552501853, + "tokens_seen": 3242328064 + }, + { + "epoch": 0.97, + "learning_rate": 1.773319959879639e-05, + "loss": 1.1909, + "theoretical_loss": 3.329606505613365, + "tokens_seen": 3242393600 + }, + { + "epoch": 0.97, + "learning_rate": 1.7713139418254764e-05, + "loss": 1.2025, + "theoretical_loss": 3.3296014561071856, + "tokens_seen": 3242459136 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1177712, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.196829080581665, + "objective/train/theoretical_loss": 3.329596406731641, + "objective/train/tokens_used": 1612983776, + "theoretical_loss": 3.329596406731641, + "tokens_seen": 3242524672 + }, + { + "epoch": 0.97, + "learning_rate": 1.7693079237713137e-05, + "loss": 1.2805, + "theoretical_loss": 3.329596406731641, + "tokens_seen": 3242524672 + }, + { + "epoch": 0.97, + "learning_rate": 1.7673019057171516e-05, + "loss": 1.2134, + "theoretical_loss": 3.3295913574867244, + "tokens_seen": 3242590208 + }, + { + "epoch": 0.97, + "learning_rate": 1.765295887662989e-05, + "loss": 1.3408, + "theoretical_loss": 3.329586308372431, + "tokens_seen": 3242655744 + }, + { + "epoch": 0.97, + "learning_rate": 1.7632898696088264e-05, + "loss": 1.3963, + "theoretical_loss": 3.329581259388754, + "tokens_seen": 3242721280 + }, + { + "epoch": 0.97, + "learning_rate": 1.761283851554664e-05, + "loss": 1.2775, + "theoretical_loss": 3.329576210535688, + "tokens_seen": 3242786816 + }, + { + "epoch": 0.97, + "learning_rate": 1.7592778335005016e-05, + "loss": 1.273, + "theoretical_loss": 3.3295711618132264, + "tokens_seen": 3242852352 + }, + { + "epoch": 0.97, + "learning_rate": 1.757271815446339e-05, + "loss": 1.1919, + "theoretical_loss": 3.3295661132213636, + "tokens_seen": 3242917888 + }, + { + "epoch": 0.97, + "learning_rate": 1.7552657973921767e-05, + "loss": 1.2039, + "theoretical_loss": 3.3295610647600933, + "tokens_seen": 3242983424 + }, + { + "epoch": 0.97, + "learning_rate": 1.753259779338014e-05, + "loss": 1.2257, + "theoretical_loss": 3.32955601642941, + "tokens_seen": 3243048960 + }, + { + "epoch": 0.97, + "learning_rate": 1.7512537612838515e-05, + "loss": 1.3464, + "theoretical_loss": 3.3295509682293067, + "tokens_seen": 3243114496 + }, + { + "epoch": 0.97, + "learning_rate": 1.749247743229689e-05, + "loss": 1.3377, + "theoretical_loss": 3.329545920159778, + "tokens_seen": 3243180032 + }, + { + "epoch": 0.97, + "learning_rate": 1.7472417251755267e-05, + "loss": 1.2603, + "theoretical_loss": 3.329540872220818, + "tokens_seen": 3243245568 + }, + { + "epoch": 0.97, + "learning_rate": 1.7452357071213643e-05, + "loss": 1.2011, + "theoretical_loss": 3.329535824412421, + "tokens_seen": 3243311104 + }, + { + "epoch": 0.97, + "learning_rate": 1.7432296890672015e-05, + "loss": 1.2284, + "theoretical_loss": 3.32953077673458, + "tokens_seen": 3243376640 + }, + { + "epoch": 0.97, + "learning_rate": 1.741223671013039e-05, + "loss": 1.2219, + "theoretical_loss": 3.3295257291872895, + "tokens_seen": 3243442176 + }, + { + "epoch": 0.97, + "learning_rate": 1.7392176529588767e-05, + "loss": 1.2563, + "theoretical_loss": 3.3295206817705436, + "tokens_seen": 3243507712 + }, + { + "epoch": 0.97, + "learning_rate": 1.7372116349047142e-05, + "loss": 1.3705, + "theoretical_loss": 3.329515634484336, + "tokens_seen": 3243573248 + }, + { + "epoch": 0.97, + "learning_rate": 1.7352056168505518e-05, + "loss": 1.2038, + "theoretical_loss": 3.329510587328661, + "tokens_seen": 3243638784 + }, + { + "epoch": 0.97, + "learning_rate": 1.733199598796389e-05, + "loss": 1.1965, + "theoretical_loss": 3.3295055403035123, + "tokens_seen": 3243704320 + }, + { + "epoch": 0.97, + "learning_rate": 1.731193580742227e-05, + "loss": 1.286, + "theoretical_loss": 3.329500493408884, + "tokens_seen": 3243769856 + }, + { + "epoch": 0.97, + "learning_rate": 1.7291875626880642e-05, + "loss": 1.2237, + "theoretical_loss": 3.3294954466447706, + "tokens_seen": 3243835392 + }, + { + "epoch": 0.97, + "learning_rate": 1.7271815446339018e-05, + "loss": 1.2894, + "theoretical_loss": 3.329490400011165, + "tokens_seen": 3243900928 + }, + { + "epoch": 0.97, + "learning_rate": 1.725175526579739e-05, + "loss": 1.2082, + "theoretical_loss": 3.329485353508062, + "tokens_seen": 3243966464 + }, + { + "epoch": 0.97, + "learning_rate": 1.7231695085255766e-05, + "loss": 1.2235, + "theoretical_loss": 3.3294803071354555, + "tokens_seen": 3244032000 + }, + { + "epoch": 0.97, + "learning_rate": 1.7211634904714145e-05, + "loss": 1.1693, + "theoretical_loss": 3.3294752608933393, + "tokens_seen": 3244097536 + }, + { + "epoch": 0.97, + "learning_rate": 1.7191574724172518e-05, + "loss": 1.2504, + "theoretical_loss": 3.3294702147817077, + "tokens_seen": 3244163072 + }, + { + "epoch": 0.97, + "learning_rate": 1.7171514543630894e-05, + "loss": 1.2152, + "theoretical_loss": 3.329465168800554, + "tokens_seen": 3244228608 + }, + { + "epoch": 0.97, + "learning_rate": 1.7151454363089266e-05, + "loss": 1.2596, + "theoretical_loss": 3.3294601229498726, + "tokens_seen": 3244294144 + }, + { + "epoch": 0.97, + "learning_rate": 1.7131394182547642e-05, + "loss": 1.2012, + "theoretical_loss": 3.329455077229658, + "tokens_seen": 3244359680 + }, + { + "epoch": 0.97, + "learning_rate": 1.711133400200602e-05, + "loss": 1.2005, + "theoretical_loss": 3.3294500316399036, + "tokens_seen": 3244425216 + }, + { + "epoch": 0.97, + "learning_rate": 1.7091273821464393e-05, + "loss": 1.2243, + "theoretical_loss": 3.3294449861806035, + "tokens_seen": 3244490752 + }, + { + "epoch": 0.97, + "learning_rate": 1.707121364092277e-05, + "loss": 1.2321, + "theoretical_loss": 3.3294399408517514, + "tokens_seen": 3244556288 + }, + { + "epoch": 0.97, + "learning_rate": 1.705115346038114e-05, + "loss": 1.2133, + "theoretical_loss": 3.329434895653342, + "tokens_seen": 3244621824 + }, + { + "epoch": 0.97, + "learning_rate": 1.703109327983952e-05, + "loss": 1.2198, + "theoretical_loss": 3.3294298505853686, + "tokens_seen": 3244687360 + }, + { + "epoch": 0.97, + "learning_rate": 1.7011033099297897e-05, + "loss": 1.3385, + "theoretical_loss": 3.3294248056478257, + "tokens_seen": 3244752896 + }, + { + "epoch": 0.97, + "learning_rate": 1.699097291875627e-05, + "loss": 1.189, + "theoretical_loss": 3.329419760840707, + "tokens_seen": 3244818432 + }, + { + "epoch": 0.97, + "learning_rate": 1.6970912738214645e-05, + "loss": 1.2857, + "theoretical_loss": 3.3294147161640066, + "tokens_seen": 3244883968 + }, + { + "epoch": 0.97, + "learning_rate": 1.6950852557673017e-05, + "loss": 1.394, + "theoretical_loss": 3.3294096716177184, + "tokens_seen": 3244949504 + }, + { + "epoch": 0.97, + "learning_rate": 1.6930792377131396e-05, + "loss": 1.1718, + "theoretical_loss": 3.3294046272018365, + "tokens_seen": 3245015040 + }, + { + "epoch": 0.97, + "learning_rate": 1.691073219658977e-05, + "loss": 1.3417, + "theoretical_loss": 3.3293995829163547, + "tokens_seen": 3245080576 + }, + { + "epoch": 0.97, + "learning_rate": 1.6890672016048145e-05, + "loss": 1.2176, + "theoretical_loss": 3.3293945387612673, + "tokens_seen": 3245146112 + }, + { + "epoch": 0.97, + "learning_rate": 1.687061183550652e-05, + "loss": 1.2933, + "theoretical_loss": 3.3293894947365685, + "tokens_seen": 3245211648 + }, + { + "epoch": 0.97, + "learning_rate": 1.6850551654964893e-05, + "loss": 1.2661, + "theoretical_loss": 3.3293844508422517, + "tokens_seen": 3245277184 + }, + { + "epoch": 0.97, + "learning_rate": 1.6830491474423272e-05, + "loss": 1.3385, + "theoretical_loss": 3.329379407078311, + "tokens_seen": 3245342720 + }, + { + "epoch": 0.97, + "learning_rate": 1.6810431293881644e-05, + "loss": 1.2323, + "theoretical_loss": 3.3293743634447406, + "tokens_seen": 3245408256 + }, + { + "epoch": 0.97, + "learning_rate": 1.679037111334002e-05, + "loss": 1.249, + "theoretical_loss": 3.3293693199415344, + "tokens_seen": 3245473792 + }, + { + "epoch": 0.97, + "learning_rate": 1.6770310932798396e-05, + "loss": 1.2046, + "theoretical_loss": 3.3293642765686866, + "tokens_seen": 3245539328 + }, + { + "epoch": 0.97, + "learning_rate": 1.6750250752256772e-05, + "loss": 1.1242, + "theoretical_loss": 3.329359233326191, + "tokens_seen": 3245604864 + }, + { + "epoch": 0.97, + "learning_rate": 1.6730190571715148e-05, + "loss": 1.2939, + "theoretical_loss": 3.3293541902140413, + "tokens_seen": 3245670400 + }, + { + "epoch": 0.97, + "learning_rate": 1.671013039117352e-05, + "loss": 1.2214, + "theoretical_loss": 3.3293491472322323, + "tokens_seen": 3245735936 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1178984, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3602168560028076, + "objective/train/theoretical_loss": 3.3293441043807572, + "objective/train/tokens_used": 1616260576, + "theoretical_loss": 3.3293441043807572, + "tokens_seen": 3245801472 + }, + { + "epoch": 0.97, + "learning_rate": 1.6690070210631896e-05, + "loss": 1.2906, + "theoretical_loss": 3.3293441043807572, + "tokens_seen": 3245801472 + }, + { + "epoch": 0.97, + "learning_rate": 1.667001003009027e-05, + "loss": 1.1366, + "theoretical_loss": 3.329339061659611, + "tokens_seen": 3245867008 + }, + { + "epoch": 0.97, + "learning_rate": 1.6649949849548647e-05, + "loss": 1.2749, + "theoretical_loss": 3.329334019068786, + "tokens_seen": 3245932544 + }, + { + "epoch": 0.97, + "learning_rate": 1.6629889669007023e-05, + "loss": 1.2976, + "theoretical_loss": 3.329328976608278, + "tokens_seen": 3245998080 + }, + { + "epoch": 0.97, + "learning_rate": 1.6609829488465395e-05, + "loss": 1.2869, + "theoretical_loss": 3.32932393427808, + "tokens_seen": 3246063616 + }, + { + "epoch": 0.97, + "learning_rate": 1.658976930792377e-05, + "loss": 1.1739, + "theoretical_loss": 3.329318892078186, + "tokens_seen": 3246129152 + }, + { + "epoch": 0.97, + "learning_rate": 1.6569709127382144e-05, + "loss": 1.2049, + "theoretical_loss": 3.3293138500085906, + "tokens_seen": 3246194688 + }, + { + "epoch": 0.97, + "learning_rate": 1.6549648946840523e-05, + "loss": 1.1162, + "theoretical_loss": 3.329308808069287, + "tokens_seen": 3246260224 + }, + { + "epoch": 0.97, + "learning_rate": 1.65295887662989e-05, + "loss": 1.2587, + "theoretical_loss": 3.3293037662602702, + "tokens_seen": 3246325760 + }, + { + "epoch": 0.97, + "learning_rate": 1.650952858575727e-05, + "loss": 1.2555, + "theoretical_loss": 3.3292987245815335, + "tokens_seen": 3246391296 + }, + { + "epoch": 0.97, + "learning_rate": 1.6489468405215647e-05, + "loss": 1.2652, + "theoretical_loss": 3.329293683033071, + "tokens_seen": 3246456832 + }, + { + "epoch": 0.97, + "learning_rate": 1.6469408224674023e-05, + "loss": 1.265, + "theoretical_loss": 3.329288641614877, + "tokens_seen": 3246522368 + }, + { + "epoch": 0.97, + "learning_rate": 1.64493480441324e-05, + "loss": 1.1994, + "theoretical_loss": 3.3292836003269444, + "tokens_seen": 3246587904 + }, + { + "epoch": 0.97, + "learning_rate": 1.6429287863590774e-05, + "loss": 1.3004, + "theoretical_loss": 3.3292785591692686, + "tokens_seen": 3246653440 + }, + { + "epoch": 0.97, + "learning_rate": 1.6409227683049147e-05, + "loss": 1.1224, + "theoretical_loss": 3.3292735181418434, + "tokens_seen": 3246718976 + }, + { + "epoch": 0.97, + "learning_rate": 1.6389167502507522e-05, + "loss": 1.3525, + "theoretical_loss": 3.329268477244662, + "tokens_seen": 3246784512 + }, + { + "epoch": 0.97, + "learning_rate": 1.6369107321965898e-05, + "loss": 1.1878, + "theoretical_loss": 3.329263436477719, + "tokens_seen": 3246850048 + }, + { + "epoch": 0.97, + "learning_rate": 1.6349047141424274e-05, + "loss": 1.3561, + "theoretical_loss": 3.3292583958410082, + "tokens_seen": 3246915584 + }, + { + "epoch": 0.97, + "learning_rate": 1.632898696088265e-05, + "loss": 1.2483, + "theoretical_loss": 3.329253355334524, + "tokens_seen": 3246981120 + }, + { + "epoch": 0.97, + "learning_rate": 1.6308926780341022e-05, + "loss": 1.1972, + "theoretical_loss": 3.32924831495826, + "tokens_seen": 3247046656 + }, + { + "epoch": 0.97, + "learning_rate": 1.6288866599799398e-05, + "loss": 1.276, + "theoretical_loss": 3.3292432747122103, + "tokens_seen": 3247112192 + }, + { + "epoch": 0.97, + "learning_rate": 1.6268806419257774e-05, + "loss": 1.3757, + "theoretical_loss": 3.329238234596369, + "tokens_seen": 3247177728 + }, + { + "epoch": 0.97, + "learning_rate": 1.624874623871615e-05, + "loss": 1.3258, + "theoretical_loss": 3.3292331946107296, + "tokens_seen": 3247243264 + }, + { + "epoch": 0.97, + "learning_rate": 1.6228686058174522e-05, + "loss": 1.1982, + "theoretical_loss": 3.3292281547552873, + "tokens_seen": 3247308800 + }, + { + "epoch": 0.97, + "learning_rate": 1.6208625877632898e-05, + "loss": 1.3658, + "theoretical_loss": 3.3292231150300347, + "tokens_seen": 3247374336 + }, + { + "epoch": 0.97, + "learning_rate": 1.6188565697091274e-05, + "loss": 1.2469, + "theoretical_loss": 3.3292180754349667, + "tokens_seen": 3247439872 + }, + { + "epoch": 0.97, + "learning_rate": 1.616850551654965e-05, + "loss": 1.2519, + "theoretical_loss": 3.329213035970077, + "tokens_seen": 3247505408 + }, + { + "epoch": 0.97, + "learning_rate": 1.6148445336008025e-05, + "loss": 1.2831, + "theoretical_loss": 3.32920799663536, + "tokens_seen": 3247570944 + }, + { + "epoch": 0.97, + "learning_rate": 1.6128385155466398e-05, + "loss": 1.1909, + "theoretical_loss": 3.329202957430809, + "tokens_seen": 3247636480 + }, + { + "epoch": 0.97, + "learning_rate": 1.6108324974924773e-05, + "loss": 1.3081, + "theoretical_loss": 3.329197918356418, + "tokens_seen": 3247702016 + }, + { + "epoch": 0.97, + "learning_rate": 1.6088264794383153e-05, + "loss": 1.3466, + "theoretical_loss": 3.329192879412182, + "tokens_seen": 3247767552 + }, + { + "epoch": 0.97, + "learning_rate": 1.6068204613841525e-05, + "loss": 1.1757, + "theoretical_loss": 3.3291878405980944, + "tokens_seen": 3247833088 + }, + { + "epoch": 0.97, + "learning_rate": 1.60481444332999e-05, + "loss": 1.2516, + "theoretical_loss": 3.3291828019141496, + "tokens_seen": 3247898624 + }, + { + "epoch": 0.97, + "learning_rate": 1.6028084252758273e-05, + "loss": 1.2752, + "theoretical_loss": 3.3291777633603408, + "tokens_seen": 3247964160 + }, + { + "epoch": 0.97, + "learning_rate": 1.600802407221665e-05, + "loss": 1.2573, + "theoretical_loss": 3.329172724936662, + "tokens_seen": 3248029696 + }, + { + "epoch": 0.97, + "learning_rate": 1.5987963891675028e-05, + "loss": 1.2399, + "theoretical_loss": 3.3291676866431086, + "tokens_seen": 3248095232 + }, + { + "epoch": 0.97, + "learning_rate": 1.59679037111334e-05, + "loss": 1.2971, + "theoretical_loss": 3.329162648479673, + "tokens_seen": 3248160768 + }, + { + "epoch": 0.97, + "learning_rate": 1.5947843530591776e-05, + "loss": 1.2711, + "theoretical_loss": 3.32915761044635, + "tokens_seen": 3248226304 + }, + { + "epoch": 0.97, + "learning_rate": 1.592778335005015e-05, + "loss": 1.284, + "theoretical_loss": 3.3291525725431335, + "tokens_seen": 3248291840 + }, + { + "epoch": 0.97, + "learning_rate": 1.5907723169508525e-05, + "loss": 1.2427, + "theoretical_loss": 3.329147534770018, + "tokens_seen": 3248357376 + }, + { + "epoch": 0.97, + "learning_rate": 1.5887662988966904e-05, + "loss": 1.3289, + "theoretical_loss": 3.3291424971269965, + "tokens_seen": 3248422912 + }, + { + "epoch": 0.97, + "learning_rate": 1.5867602808425276e-05, + "loss": 1.248, + "theoretical_loss": 3.329137459614064, + "tokens_seen": 3248488448 + }, + { + "epoch": 0.97, + "learning_rate": 1.5847542627883652e-05, + "loss": 1.2313, + "theoretical_loss": 3.329132422231214, + "tokens_seen": 3248553984 + }, + { + "epoch": 0.97, + "learning_rate": 1.5827482447342024e-05, + "loss": 1.2108, + "theoretical_loss": 3.32912738497844, + "tokens_seen": 3248619520 + }, + { + "epoch": 0.97, + "learning_rate": 1.5807422266800404e-05, + "loss": 1.3355, + "theoretical_loss": 3.3291223478557375, + "tokens_seen": 3248685056 + }, + { + "epoch": 0.97, + "learning_rate": 1.5787362086258776e-05, + "loss": 1.269, + "theoretical_loss": 3.329117310863099, + "tokens_seen": 3248750592 + }, + { + "epoch": 0.97, + "learning_rate": 1.5767301905717152e-05, + "loss": 1.1721, + "theoretical_loss": 3.3291122740005195, + "tokens_seen": 3248816128 + }, + { + "epoch": 0.97, + "learning_rate": 1.5747241725175527e-05, + "loss": 1.3069, + "theoretical_loss": 3.3291072372679924, + "tokens_seen": 3248881664 + }, + { + "epoch": 0.97, + "learning_rate": 1.57271815446339e-05, + "loss": 1.2449, + "theoretical_loss": 3.329102200665512, + "tokens_seen": 3248947200 + }, + { + "epoch": 0.97, + "learning_rate": 1.570712136409228e-05, + "loss": 1.355, + "theoretical_loss": 3.3290971641930724, + "tokens_seen": 3249012736 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1180177, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2066723108291626, + "objective/train/theoretical_loss": 3.3290921278506675, + "objective/train/tokens_used": 1619537376, + "theoretical_loss": 3.3290921278506675, + "tokens_seen": 3249078272 + }, + { + "epoch": 0.97, + "learning_rate": 1.568706118355065e-05, + "loss": 1.214, + "theoretical_loss": 3.3290921278506675, + "tokens_seen": 3249078272 + }, + { + "epoch": 0.97, + "learning_rate": 1.5667001003009027e-05, + "loss": 1.2547, + "theoretical_loss": 3.3290870916382915, + "tokens_seen": 3249143808 + }, + { + "epoch": 0.97, + "learning_rate": 1.5646940822467403e-05, + "loss": 1.3308, + "theoretical_loss": 3.329082055555938, + "tokens_seen": 3249209344 + }, + { + "epoch": 0.97, + "learning_rate": 1.5626880641925775e-05, + "loss": 1.3024, + "theoretical_loss": 3.3290770196036013, + "tokens_seen": 3249274880 + }, + { + "epoch": 0.97, + "learning_rate": 1.560682046138415e-05, + "loss": 1.2139, + "theoretical_loss": 3.329071983781276, + "tokens_seen": 3249340416 + }, + { + "epoch": 0.97, + "learning_rate": 1.5586760280842527e-05, + "loss": 1.3525, + "theoretical_loss": 3.329066948088955, + "tokens_seen": 3249405952 + }, + { + "epoch": 0.97, + "learning_rate": 1.5566700100300903e-05, + "loss": 1.2396, + "theoretical_loss": 3.3290619125266327, + "tokens_seen": 3249471488 + }, + { + "epoch": 0.97, + "learning_rate": 1.554663991975928e-05, + "loss": 1.2964, + "theoretical_loss": 3.3290568770943034, + "tokens_seen": 3249537024 + }, + { + "epoch": 0.97, + "learning_rate": 1.5526579739217654e-05, + "loss": 1.1762, + "theoretical_loss": 3.3290518417919612, + "tokens_seen": 3249602560 + }, + { + "epoch": 0.97, + "learning_rate": 1.550651955867603e-05, + "loss": 1.3426, + "theoretical_loss": 3.3290468066195995, + "tokens_seen": 3249668096 + }, + { + "epoch": 0.97, + "learning_rate": 1.5486459378134403e-05, + "loss": 1.2347, + "theoretical_loss": 3.3290417715772134, + "tokens_seen": 3249733632 + }, + { + "epoch": 0.97, + "learning_rate": 1.546639919759278e-05, + "loss": 1.2457, + "theoretical_loss": 3.3290367366647957, + "tokens_seen": 3249799168 + }, + { + "epoch": 0.97, + "learning_rate": 1.5446339017051154e-05, + "loss": 1.1283, + "theoretical_loss": 3.329031701882341, + "tokens_seen": 3249864704 + }, + { + "epoch": 0.97, + "learning_rate": 1.542627883650953e-05, + "loss": 1.2561, + "theoretical_loss": 3.3290266672298436, + "tokens_seen": 3249930240 + }, + { + "epoch": 0.97, + "learning_rate": 1.5406218655967906e-05, + "loss": 1.3465, + "theoretical_loss": 3.3290216327072972, + "tokens_seen": 3249995776 + }, + { + "epoch": 0.97, + "learning_rate": 1.5386158475426278e-05, + "loss": 1.1933, + "theoretical_loss": 3.3290165983146958, + "tokens_seen": 3250061312 + }, + { + "epoch": 0.97, + "learning_rate": 1.5366098294884654e-05, + "loss": 1.3184, + "theoretical_loss": 3.329011564052034, + "tokens_seen": 3250126848 + }, + { + "epoch": 0.97, + "learning_rate": 1.534603811434303e-05, + "loss": 1.2583, + "theoretical_loss": 3.3290065299193046, + "tokens_seen": 3250192384 + }, + { + "epoch": 0.97, + "learning_rate": 1.5325977933801402e-05, + "loss": 1.2269, + "theoretical_loss": 3.3290014959165024, + "tokens_seen": 3250257920 + }, + { + "epoch": 0.97, + "learning_rate": 1.530591775325978e-05, + "loss": 1.2263, + "theoretical_loss": 3.3289964620436217, + "tokens_seen": 3250323456 + }, + { + "epoch": 0.97, + "learning_rate": 1.5285857572718154e-05, + "loss": 1.2605, + "theoretical_loss": 3.3289914283006556, + "tokens_seen": 3250388992 + }, + { + "epoch": 0.97, + "learning_rate": 1.526579739217653e-05, + "loss": 1.1153, + "theoretical_loss": 3.3289863946875995, + "tokens_seen": 3250454528 + }, + { + "epoch": 0.97, + "learning_rate": 1.5245737211634905e-05, + "loss": 1.1645, + "theoretical_loss": 3.3289813612044465, + "tokens_seen": 3250520064 + }, + { + "epoch": 0.97, + "learning_rate": 1.5225677031093281e-05, + "loss": 1.252, + "theoretical_loss": 3.3289763278511906, + "tokens_seen": 3250585600 + }, + { + "epoch": 0.97, + "learning_rate": 1.5205616850551655e-05, + "loss": 1.2274, + "theoretical_loss": 3.328971294627826, + "tokens_seen": 3250651136 + }, + { + "epoch": 0.97, + "learning_rate": 1.518555667001003e-05, + "loss": 1.2788, + "theoretical_loss": 3.3289662615343465, + "tokens_seen": 3250716672 + }, + { + "epoch": 0.97, + "learning_rate": 1.5165496489468405e-05, + "loss": 1.3312, + "theoretical_loss": 3.328961228570747, + "tokens_seen": 3250782208 + }, + { + "epoch": 0.97, + "learning_rate": 1.5145436308926781e-05, + "loss": 1.2992, + "theoretical_loss": 3.32895619573702, + "tokens_seen": 3250847744 + }, + { + "epoch": 0.97, + "learning_rate": 1.5125376128385157e-05, + "loss": 1.2798, + "theoretical_loss": 3.328951163033161, + "tokens_seen": 3250913280 + }, + { + "epoch": 0.97, + "learning_rate": 1.5105315947843531e-05, + "loss": 1.3633, + "theoretical_loss": 3.3289461304591637, + "tokens_seen": 3250978816 + }, + { + "epoch": 0.97, + "learning_rate": 1.5085255767301907e-05, + "loss": 1.2412, + "theoretical_loss": 3.3289410980150214, + "tokens_seen": 3251044352 + }, + { + "epoch": 0.97, + "learning_rate": 1.506519558676028e-05, + "loss": 1.2607, + "theoretical_loss": 3.328936065700729, + "tokens_seen": 3251109888 + }, + { + "epoch": 0.97, + "learning_rate": 1.5045135406218655e-05, + "loss": 1.2546, + "theoretical_loss": 3.32893103351628, + "tokens_seen": 3251175424 + }, + { + "epoch": 0.97, + "learning_rate": 1.5025075225677032e-05, + "loss": 1.2397, + "theoretical_loss": 3.328926001461668, + "tokens_seen": 3251240960 + }, + { + "epoch": 0.97, + "learning_rate": 1.5005015045135406e-05, + "loss": 1.2235, + "theoretical_loss": 3.3289209695368887, + "tokens_seen": 3251306496 + }, + { + "epoch": 0.97, + "learning_rate": 1.4984954864593782e-05, + "loss": 1.3133, + "theoretical_loss": 3.328915937741934, + "tokens_seen": 3251372032 + }, + { + "epoch": 0.97, + "learning_rate": 1.4964894684052156e-05, + "loss": 1.1961, + "theoretical_loss": 3.3289109060768, + "tokens_seen": 3251437568 + }, + { + "epoch": 0.97, + "learning_rate": 1.494483450351053e-05, + "loss": 1.213, + "theoretical_loss": 3.328905874541479, + "tokens_seen": 3251503104 + }, + { + "epoch": 0.97, + "learning_rate": 1.4924774322968908e-05, + "loss": 1.2025, + "theoretical_loss": 3.3289008431359663, + "tokens_seen": 3251568640 + }, + { + "epoch": 0.97, + "learning_rate": 1.4904714142427282e-05, + "loss": 1.222, + "theoretical_loss": 3.328895811860255, + "tokens_seen": 3251634176 + }, + { + "epoch": 0.97, + "learning_rate": 1.4884653961885658e-05, + "loss": 1.2213, + "theoretical_loss": 3.3288907807143397, + "tokens_seen": 3251699712 + }, + { + "epoch": 0.97, + "learning_rate": 1.4864593781344032e-05, + "loss": 1.2089, + "theoretical_loss": 3.3288857496982143, + "tokens_seen": 3251765248 + }, + { + "epoch": 0.97, + "learning_rate": 1.4844533600802408e-05, + "loss": 1.267, + "theoretical_loss": 3.3288807188118725, + "tokens_seen": 3251830784 + }, + { + "epoch": 0.97, + "learning_rate": 1.4824473420260782e-05, + "loss": 1.3461, + "theoretical_loss": 3.3288756880553088, + "tokens_seen": 3251896320 + }, + { + "epoch": 0.97, + "learning_rate": 1.4804413239719158e-05, + "loss": 1.281, + "theoretical_loss": 3.328870657428517, + "tokens_seen": 3251961856 + }, + { + "epoch": 0.97, + "learning_rate": 1.4784353059177533e-05, + "loss": 1.1909, + "theoretical_loss": 3.3288656269314916, + "tokens_seen": 3252027392 + }, + { + "epoch": 0.97, + "learning_rate": 1.4764292878635907e-05, + "loss": 1.2598, + "theoretical_loss": 3.328860596564226, + "tokens_seen": 3252092928 + }, + { + "epoch": 0.97, + "learning_rate": 1.4744232698094283e-05, + "loss": 1.1586, + "theoretical_loss": 3.3288555663267143, + "tokens_seen": 3252158464 + }, + { + "epoch": 0.97, + "learning_rate": 1.4724172517552657e-05, + "loss": 1.2002, + "theoretical_loss": 3.328850536218951, + "tokens_seen": 3252224000 + }, + { + "epoch": 0.97, + "learning_rate": 1.4704112337011035e-05, + "loss": 1.2045, + "theoretical_loss": 3.32884550624093, + "tokens_seen": 3252289536 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1181412, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1936290264129639, + "objective/train/theoretical_loss": 3.328840476392645, + "objective/train/tokens_used": 1622814176, + "theoretical_loss": 3.328840476392645, + "tokens_seen": 3252355072 + }, + { + "epoch": 0.97, + "learning_rate": 1.4684052156469409e-05, + "loss": 1.2272, + "theoretical_loss": 3.328840476392645, + "tokens_seen": 3252355072 + }, + { + "epoch": 0.97, + "learning_rate": 1.4663991975927783e-05, + "loss": 1.3373, + "theoretical_loss": 3.32883544667409, + "tokens_seen": 3252420608 + }, + { + "epoch": 0.97, + "learning_rate": 1.4643931795386159e-05, + "loss": 1.326, + "theoretical_loss": 3.3288304170852596, + "tokens_seen": 3252486144 + }, + { + "epoch": 0.97, + "learning_rate": 1.4623871614844533e-05, + "loss": 1.25, + "theoretical_loss": 3.3288253876261473, + "tokens_seen": 3252551680 + }, + { + "epoch": 0.97, + "learning_rate": 1.4603811434302909e-05, + "loss": 1.2484, + "theoretical_loss": 3.3288203582967473, + "tokens_seen": 3252617216 + }, + { + "epoch": 0.97, + "learning_rate": 1.4583751253761285e-05, + "loss": 1.345, + "theoretical_loss": 3.328815329097054, + "tokens_seen": 3252682752 + }, + { + "epoch": 0.97, + "learning_rate": 1.456369107321966e-05, + "loss": 1.2932, + "theoretical_loss": 3.328810300027061, + "tokens_seen": 3252748288 + }, + { + "epoch": 0.97, + "learning_rate": 1.4543630892678034e-05, + "loss": 1.1658, + "theoretical_loss": 3.328805271086763, + "tokens_seen": 3252813824 + }, + { + "epoch": 0.97, + "learning_rate": 1.4523570712136409e-05, + "loss": 1.1806, + "theoretical_loss": 3.3288002422761527, + "tokens_seen": 3252879360 + }, + { + "epoch": 0.97, + "learning_rate": 1.4503510531594784e-05, + "loss": 1.2601, + "theoretical_loss": 3.3287952135952255, + "tokens_seen": 3252944896 + }, + { + "epoch": 0.97, + "learning_rate": 1.4483450351053158e-05, + "loss": 1.2152, + "theoretical_loss": 3.3287901850439745, + "tokens_seen": 3253010432 + }, + { + "epoch": 0.97, + "learning_rate": 1.4463390170511536e-05, + "loss": 1.2447, + "theoretical_loss": 3.3287851566223945, + "tokens_seen": 3253075968 + }, + { + "epoch": 0.97, + "learning_rate": 1.444332998996991e-05, + "loss": 1.3417, + "theoretical_loss": 3.3287801283304788, + "tokens_seen": 3253141504 + }, + { + "epoch": 0.97, + "learning_rate": 1.4423269809428286e-05, + "loss": 1.2799, + "theoretical_loss": 3.3287751001682224, + "tokens_seen": 3253207040 + }, + { + "epoch": 0.97, + "learning_rate": 1.440320962888666e-05, + "loss": 1.2577, + "theoretical_loss": 3.3287700721356184, + "tokens_seen": 3253272576 + }, + { + "epoch": 0.97, + "learning_rate": 1.4383149448345034e-05, + "loss": 1.2406, + "theoretical_loss": 3.3287650442326617, + "tokens_seen": 3253338112 + }, + { + "epoch": 0.97, + "learning_rate": 1.4363089267803411e-05, + "loss": 1.2771, + "theoretical_loss": 3.3287600164593454, + "tokens_seen": 3253403648 + }, + { + "epoch": 0.97, + "learning_rate": 1.4343029087261786e-05, + "loss": 1.2273, + "theoretical_loss": 3.328754988815664, + "tokens_seen": 3253469184 + }, + { + "epoch": 0.97, + "learning_rate": 1.4322968906720161e-05, + "loss": 1.212, + "theoretical_loss": 3.328749961301612, + "tokens_seen": 3253534720 + }, + { + "epoch": 0.97, + "learning_rate": 1.4302908726178535e-05, + "loss": 1.288, + "theoretical_loss": 3.3287449339171826, + "tokens_seen": 3253600256 + }, + { + "epoch": 0.97, + "learning_rate": 1.4282848545636911e-05, + "loss": 1.3484, + "theoretical_loss": 3.3287399066623706, + "tokens_seen": 3253665792 + }, + { + "epoch": 0.97, + "learning_rate": 1.4262788365095285e-05, + "loss": 1.2018, + "theoretical_loss": 3.3287348795371696, + "tokens_seen": 3253731328 + }, + { + "epoch": 0.97, + "learning_rate": 1.4242728184553661e-05, + "loss": 1.2637, + "theoretical_loss": 3.328729852541574, + "tokens_seen": 3253796864 + }, + { + "epoch": 0.97, + "learning_rate": 1.4222668004012037e-05, + "loss": 1.2017, + "theoretical_loss": 3.328724825675577, + "tokens_seen": 3253862400 + }, + { + "epoch": 0.97, + "learning_rate": 1.4202607823470411e-05, + "loss": 1.2094, + "theoretical_loss": 3.328719798939174, + "tokens_seen": 3253927936 + }, + { + "epoch": 0.97, + "learning_rate": 1.4182547642928787e-05, + "loss": 1.2522, + "theoretical_loss": 3.328714772332358, + "tokens_seen": 3253993472 + }, + { + "epoch": 0.97, + "learning_rate": 1.4162487462387161e-05, + "loss": 1.1728, + "theoretical_loss": 3.328709745855123, + "tokens_seen": 3254059008 + }, + { + "epoch": 0.97, + "learning_rate": 1.4142427281845538e-05, + "loss": 1.385, + "theoretical_loss": 3.328704719507464, + "tokens_seen": 3254124544 + }, + { + "epoch": 0.97, + "learning_rate": 1.4122367101303913e-05, + "loss": 1.2986, + "theoretical_loss": 3.3286996932893738, + "tokens_seen": 3254190080 + }, + { + "epoch": 0.97, + "learning_rate": 1.4102306920762287e-05, + "loss": 1.3689, + "theoretical_loss": 3.328694667200848, + "tokens_seen": 3254255616 + }, + { + "epoch": 0.97, + "learning_rate": 1.4082246740220662e-05, + "loss": 1.2919, + "theoretical_loss": 3.328689641241879, + "tokens_seen": 3254321152 + }, + { + "epoch": 0.97, + "learning_rate": 1.4062186559679037e-05, + "loss": 1.2017, + "theoretical_loss": 3.328684615412462, + "tokens_seen": 3254386688 + }, + { + "epoch": 0.97, + "learning_rate": 1.4042126379137412e-05, + "loss": 1.3289, + "theoretical_loss": 3.3286795897125905, + "tokens_seen": 3254452224 + }, + { + "epoch": 0.97, + "learning_rate": 1.4022066198595788e-05, + "loss": 1.2563, + "theoretical_loss": 3.3286745641422586, + "tokens_seen": 3254517760 + }, + { + "epoch": 0.97, + "learning_rate": 1.4002006018054164e-05, + "loss": 1.315, + "theoretical_loss": 3.328669538701461, + "tokens_seen": 3254583296 + }, + { + "epoch": 0.97, + "learning_rate": 1.3981945837512538e-05, + "loss": 1.3348, + "theoretical_loss": 3.3286645133901906, + "tokens_seen": 3254648832 + }, + { + "epoch": 0.97, + "learning_rate": 1.3961885656970912e-05, + "loss": 1.1576, + "theoretical_loss": 3.3286594882084426, + "tokens_seen": 3254714368 + }, + { + "epoch": 0.97, + "learning_rate": 1.3941825476429288e-05, + "loss": 1.2104, + "theoretical_loss": 3.32865446315621, + "tokens_seen": 3254779904 + }, + { + "epoch": 0.97, + "learning_rate": 1.3921765295887662e-05, + "loss": 1.3169, + "theoretical_loss": 3.3286494382334877, + "tokens_seen": 3254845440 + }, + { + "epoch": 0.97, + "learning_rate": 1.390170511534604e-05, + "loss": 1.3548, + "theoretical_loss": 3.3286444134402693, + "tokens_seen": 3254910976 + }, + { + "epoch": 0.97, + "learning_rate": 1.3881644934804414e-05, + "loss": 1.3025, + "theoretical_loss": 3.3286393887765495, + "tokens_seen": 3254976512 + }, + { + "epoch": 0.97, + "learning_rate": 1.386158475426279e-05, + "loss": 1.2072, + "theoretical_loss": 3.3286343642423213, + "tokens_seen": 3255042048 + }, + { + "epoch": 0.97, + "learning_rate": 1.3841524573721163e-05, + "loss": 1.2237, + "theoretical_loss": 3.3286293398375792, + "tokens_seen": 3255107584 + }, + { + "epoch": 0.97, + "learning_rate": 1.3821464393179538e-05, + "loss": 1.3324, + "theoretical_loss": 3.328624315562318, + "tokens_seen": 3255173120 + }, + { + "epoch": 0.97, + "learning_rate": 1.3801404212637915e-05, + "loss": 1.2528, + "theoretical_loss": 3.3286192914165307, + "tokens_seen": 3255238656 + }, + { + "epoch": 0.97, + "learning_rate": 1.378134403209629e-05, + "loss": 1.1666, + "theoretical_loss": 3.3286142674002117, + "tokens_seen": 3255304192 + }, + { + "epoch": 0.97, + "learning_rate": 1.3761283851554665e-05, + "loss": 1.2719, + "theoretical_loss": 3.3286092435133554, + "tokens_seen": 3255369728 + }, + { + "epoch": 0.97, + "learning_rate": 1.3741223671013039e-05, + "loss": 1.2528, + "theoretical_loss": 3.328604219755955, + "tokens_seen": 3255435264 + }, + { + "epoch": 0.97, + "learning_rate": 1.3721163490471415e-05, + "loss": 1.1949, + "theoretical_loss": 3.328599196128006, + "tokens_seen": 3255500800 + }, + { + "epoch": 0.97, + "learning_rate": 1.3701103309929789e-05, + "loss": 1.2392, + "theoretical_loss": 3.328594172629501, + "tokens_seen": 3255566336 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1182603, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2460951805114746, + "objective/train/theoretical_loss": 3.328589149260435, + "objective/train/tokens_used": 1626090976, + "theoretical_loss": 3.328589149260435, + "tokens_seen": 3255631872 + }, + { + "epoch": 0.97, + "learning_rate": 1.3681043129388165e-05, + "loss": 1.3049, + "theoretical_loss": 3.328589149260435, + "tokens_seen": 3255631872 + }, + { + "epoch": 0.97, + "learning_rate": 1.366098294884654e-05, + "loss": 1.1836, + "theoretical_loss": 3.3285841260208016, + "tokens_seen": 3255697408 + }, + { + "epoch": 0.97, + "learning_rate": 1.3640922768304915e-05, + "loss": 1.3152, + "theoretical_loss": 3.328579102910595, + "tokens_seen": 3255762944 + }, + { + "epoch": 0.97, + "learning_rate": 1.362086258776329e-05, + "loss": 1.0648, + "theoretical_loss": 3.328574079929809, + "tokens_seen": 3255828480 + }, + { + "epoch": 0.97, + "learning_rate": 1.3600802407221665e-05, + "loss": 1.2075, + "theoretical_loss": 3.3285690570784383, + "tokens_seen": 3255894016 + }, + { + "epoch": 0.97, + "learning_rate": 1.358074222668004e-05, + "loss": 1.2779, + "theoretical_loss": 3.3285640343564764, + "tokens_seen": 3255959552 + }, + { + "epoch": 0.97, + "learning_rate": 1.3560682046138416e-05, + "loss": 1.2518, + "theoretical_loss": 3.3285590117639177, + "tokens_seen": 3256025088 + }, + { + "epoch": 0.97, + "learning_rate": 1.354062186559679e-05, + "loss": 1.293, + "theoretical_loss": 3.328553989300756, + "tokens_seen": 3256090624 + }, + { + "epoch": 0.97, + "learning_rate": 1.3520561685055166e-05, + "loss": 1.3946, + "theoretical_loss": 3.328548966966985, + "tokens_seen": 3256156160 + }, + { + "epoch": 0.97, + "learning_rate": 1.350050150451354e-05, + "loss": 1.2306, + "theoretical_loss": 3.3285439447626, + "tokens_seen": 3256221696 + }, + { + "epoch": 0.97, + "learning_rate": 1.3480441323971916e-05, + "loss": 1.1951, + "theoretical_loss": 3.328538922687594, + "tokens_seen": 3256287232 + }, + { + "epoch": 0.97, + "learning_rate": 1.3460381143430292e-05, + "loss": 1.3086, + "theoretical_loss": 3.3285339007419608, + "tokens_seen": 3256352768 + }, + { + "epoch": 0.97, + "learning_rate": 1.3440320962888667e-05, + "loss": 1.2091, + "theoretical_loss": 3.3285288789256957, + "tokens_seen": 3256418304 + }, + { + "epoch": 0.97, + "learning_rate": 1.3420260782347042e-05, + "loss": 1.279, + "theoretical_loss": 3.3285238572387916, + "tokens_seen": 3256483840 + }, + { + "epoch": 0.97, + "learning_rate": 1.3400200601805416e-05, + "loss": 1.3018, + "theoretical_loss": 3.328518835681243, + "tokens_seen": 3256549376 + }, + { + "epoch": 0.97, + "learning_rate": 1.3380140421263791e-05, + "loss": 1.3645, + "theoretical_loss": 3.3285138142530446, + "tokens_seen": 3256614912 + }, + { + "epoch": 0.97, + "learning_rate": 1.3360080240722166e-05, + "loss": 1.2286, + "theoretical_loss": 3.328508792954189, + "tokens_seen": 3256680448 + }, + { + "epoch": 0.97, + "learning_rate": 1.3340020060180543e-05, + "loss": 1.3713, + "theoretical_loss": 3.328503771784672, + "tokens_seen": 3256745984 + }, + { + "epoch": 0.97, + "learning_rate": 1.3319959879638917e-05, + "loss": 1.3198, + "theoretical_loss": 3.3284987507444863, + "tokens_seen": 3256811520 + }, + { + "epoch": 0.97, + "learning_rate": 1.3299899699097293e-05, + "loss": 1.287, + "theoretical_loss": 3.3284937298336263, + "tokens_seen": 3256877056 + }, + { + "epoch": 0.97, + "learning_rate": 1.3279839518555667e-05, + "loss": 1.2957, + "theoretical_loss": 3.328488709052087, + "tokens_seen": 3256942592 + }, + { + "epoch": 0.97, + "learning_rate": 1.3259779338014041e-05, + "loss": 1.1663, + "theoretical_loss": 3.328483688399861, + "tokens_seen": 3257008128 + }, + { + "epoch": 0.97, + "learning_rate": 1.3239719157472419e-05, + "loss": 1.2483, + "theoretical_loss": 3.328478667876943, + "tokens_seen": 3257073664 + }, + { + "epoch": 0.97, + "learning_rate": 1.3219658976930793e-05, + "loss": 1.2603, + "theoretical_loss": 3.3284736474833276, + "tokens_seen": 3257139200 + }, + { + "epoch": 0.97, + "learning_rate": 1.3199598796389169e-05, + "loss": 1.1923, + "theoretical_loss": 3.3284686272190083, + "tokens_seen": 3257204736 + }, + { + "epoch": 0.97, + "learning_rate": 1.3179538615847543e-05, + "loss": 1.2388, + "theoretical_loss": 3.328463607083979, + "tokens_seen": 3257270272 + }, + { + "epoch": 0.97, + "learning_rate": 1.3159478435305918e-05, + "loss": 1.315, + "theoretical_loss": 3.3284585870782344, + "tokens_seen": 3257335808 + }, + { + "epoch": 0.97, + "learning_rate": 1.3139418254764293e-05, + "loss": 1.1677, + "theoretical_loss": 3.328453567201768, + "tokens_seen": 3257401344 + }, + { + "epoch": 0.97, + "learning_rate": 1.3119358074222668e-05, + "loss": 1.3016, + "theoretical_loss": 3.328448547454574, + "tokens_seen": 3257466880 + }, + { + "epoch": 0.97, + "learning_rate": 1.3099297893681044e-05, + "loss": 1.2404, + "theoretical_loss": 3.3284435278366464, + "tokens_seen": 3257532416 + }, + { + "epoch": 0.97, + "learning_rate": 1.3079237713139418e-05, + "loss": 1.3069, + "theoretical_loss": 3.32843850834798, + "tokens_seen": 3257597952 + }, + { + "epoch": 0.97, + "learning_rate": 1.3059177532597794e-05, + "loss": 1.2728, + "theoretical_loss": 3.328433488988568, + "tokens_seen": 3257663488 + }, + { + "epoch": 0.97, + "learning_rate": 1.3039117352056168e-05, + "loss": 1.2481, + "theoretical_loss": 3.3284284697584043, + "tokens_seen": 3257729024 + }, + { + "epoch": 0.97, + "learning_rate": 1.3019057171514544e-05, + "loss": 1.2479, + "theoretical_loss": 3.3284234506574837, + "tokens_seen": 3257794560 + }, + { + "epoch": 0.97, + "learning_rate": 1.299899699097292e-05, + "loss": 1.2524, + "theoretical_loss": 3.3284184316858, + "tokens_seen": 3257860096 + }, + { + "epoch": 0.97, + "learning_rate": 1.2978936810431294e-05, + "loss": 1.2726, + "theoretical_loss": 3.328413412843348, + "tokens_seen": 3257925632 + }, + { + "epoch": 0.97, + "learning_rate": 1.295887662988967e-05, + "loss": 1.2656, + "theoretical_loss": 3.32840839413012, + "tokens_seen": 3257991168 + }, + { + "epoch": 0.97, + "learning_rate": 1.2938816449348044e-05, + "loss": 1.2977, + "theoretical_loss": 3.3284033755461118, + "tokens_seen": 3258056704 + }, + { + "epoch": 0.97, + "learning_rate": 1.291875626880642e-05, + "loss": 1.2501, + "theoretical_loss": 3.3283983570913165, + "tokens_seen": 3258122240 + }, + { + "epoch": 0.97, + "learning_rate": 1.2898696088264795e-05, + "loss": 1.2104, + "theoretical_loss": 3.3283933387657285, + "tokens_seen": 3258187776 + }, + { + "epoch": 0.97, + "learning_rate": 1.2878635907723171e-05, + "loss": 1.2428, + "theoretical_loss": 3.328388320569342, + "tokens_seen": 3258253312 + }, + { + "epoch": 0.97, + "learning_rate": 1.2858575727181545e-05, + "loss": 1.2353, + "theoretical_loss": 3.3283833025021505, + "tokens_seen": 3258318848 + }, + { + "epoch": 0.97, + "learning_rate": 1.283851554663992e-05, + "loss": 1.2346, + "theoretical_loss": 3.328378284564149, + "tokens_seen": 3258384384 + }, + { + "epoch": 0.97, + "learning_rate": 1.2818455366098295e-05, + "loss": 1.2238, + "theoretical_loss": 3.3283732667553307, + "tokens_seen": 3258449920 + }, + { + "epoch": 0.97, + "learning_rate": 1.2798395185556669e-05, + "loss": 1.2167, + "theoretical_loss": 3.32836824907569, + "tokens_seen": 3258515456 + }, + { + "epoch": 0.97, + "learning_rate": 1.2778335005015047e-05, + "loss": 1.2314, + "theoretical_loss": 3.3283632315252216, + "tokens_seen": 3258580992 + }, + { + "epoch": 0.97, + "learning_rate": 1.275827482447342e-05, + "loss": 1.1805, + "theoretical_loss": 3.3283582141039187, + "tokens_seen": 3258646528 + }, + { + "epoch": 0.97, + "learning_rate": 1.2738214643931797e-05, + "loss": 1.1679, + "theoretical_loss": 3.3283531968117757, + "tokens_seen": 3258712064 + }, + { + "epoch": 0.98, + "learning_rate": 1.271815446339017e-05, + "loss": 1.1496, + "theoretical_loss": 3.3283481796487866, + "tokens_seen": 3258777600 + }, + { + "epoch": 0.98, + "learning_rate": 1.2698094282848545e-05, + "loss": 1.1899, + "theoretical_loss": 3.3283431626149453, + "tokens_seen": 3258843136 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1183737, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.291218638420105, + "objective/train/theoretical_loss": 3.328338145710246, + "objective/train/tokens_used": 1629367776, + "theoretical_loss": 3.328338145710246, + "tokens_seen": 3258908672 + }, + { + "epoch": 0.98, + "learning_rate": 1.267803410230692e-05, + "loss": 1.2191, + "theoretical_loss": 3.328338145710246, + "tokens_seen": 3258908672 + }, + { + "epoch": 0.98, + "learning_rate": 1.2657973921765296e-05, + "loss": 1.1466, + "theoretical_loss": 3.3283331289346836, + "tokens_seen": 3258974208 + }, + { + "epoch": 0.98, + "learning_rate": 1.2637913741223672e-05, + "loss": 1.2605, + "theoretical_loss": 3.328328112288251, + "tokens_seen": 3259039744 + }, + { + "epoch": 0.98, + "learning_rate": 1.2617853560682046e-05, + "loss": 1.2066, + "theoretical_loss": 3.328323095770943, + "tokens_seen": 3259105280 + }, + { + "epoch": 0.98, + "learning_rate": 1.2597793380140422e-05, + "loss": 1.2177, + "theoretical_loss": 3.328318079382753, + "tokens_seen": 3259170816 + }, + { + "epoch": 0.98, + "learning_rate": 1.2577733199598796e-05, + "loss": 1.2354, + "theoretical_loss": 3.3283130631236757, + "tokens_seen": 3259236352 + }, + { + "epoch": 0.98, + "learning_rate": 1.2557673019057172e-05, + "loss": 1.2309, + "theoretical_loss": 3.328308046993705, + "tokens_seen": 3259301888 + }, + { + "epoch": 0.98, + "learning_rate": 1.2537612838515548e-05, + "loss": 1.2423, + "theoretical_loss": 3.328303030992835, + "tokens_seen": 3259367424 + }, + { + "epoch": 0.98, + "learning_rate": 1.2517552657973922e-05, + "loss": 1.2734, + "theoretical_loss": 3.32829801512106, + "tokens_seen": 3259432960 + }, + { + "epoch": 0.98, + "learning_rate": 1.2497492477432298e-05, + "loss": 1.2153, + "theoretical_loss": 3.3282929993783736, + "tokens_seen": 3259498496 + }, + { + "epoch": 0.98, + "learning_rate": 1.2477432296890672e-05, + "loss": 1.1429, + "theoretical_loss": 3.32828798376477, + "tokens_seen": 3259564032 + }, + { + "epoch": 0.98, + "learning_rate": 1.2457372116349047e-05, + "loss": 1.3189, + "theoretical_loss": 3.3282829682802437, + "tokens_seen": 3259629568 + }, + { + "epoch": 0.98, + "learning_rate": 1.2437311935807423e-05, + "loss": 1.2909, + "theoretical_loss": 3.3282779529247883, + "tokens_seen": 3259695104 + }, + { + "epoch": 0.98, + "learning_rate": 1.2417251755265797e-05, + "loss": 1.3375, + "theoretical_loss": 3.328272937698398, + "tokens_seen": 3259760640 + }, + { + "epoch": 0.98, + "learning_rate": 1.2397191574724173e-05, + "loss": 1.2885, + "theoretical_loss": 3.328267922601067, + "tokens_seen": 3259826176 + }, + { + "epoch": 0.98, + "learning_rate": 1.2377131394182547e-05, + "loss": 1.1663, + "theoretical_loss": 3.3282629076327894, + "tokens_seen": 3259891712 + }, + { + "epoch": 0.98, + "learning_rate": 1.2357071213640923e-05, + "loss": 1.2167, + "theoretical_loss": 3.3282578927935593, + "tokens_seen": 3259957248 + }, + { + "epoch": 0.98, + "learning_rate": 1.2337011033099299e-05, + "loss": 1.3034, + "theoretical_loss": 3.3282528780833704, + "tokens_seen": 3260022784 + }, + { + "epoch": 0.98, + "learning_rate": 1.2316950852557675e-05, + "loss": 1.1532, + "theoretical_loss": 3.3282478635022175, + "tokens_seen": 3260088320 + }, + { + "epoch": 0.98, + "learning_rate": 1.2296890672016049e-05, + "loss": 1.148, + "theoretical_loss": 3.328242849050094, + "tokens_seen": 3260153856 + }, + { + "epoch": 0.98, + "learning_rate": 1.2276830491474423e-05, + "loss": 1.1909, + "theoretical_loss": 3.3282378347269943, + "tokens_seen": 3260219392 + }, + { + "epoch": 0.98, + "learning_rate": 1.2256770310932799e-05, + "loss": 1.2728, + "theoretical_loss": 3.3282328205329126, + "tokens_seen": 3260284928 + }, + { + "epoch": 0.98, + "learning_rate": 1.2236710130391173e-05, + "loss": 1.1476, + "theoretical_loss": 3.3282278064678428, + "tokens_seen": 3260350464 + }, + { + "epoch": 0.98, + "learning_rate": 1.221664994984955e-05, + "loss": 1.1461, + "theoretical_loss": 3.3282227925317787, + "tokens_seen": 3260416000 + }, + { + "epoch": 0.98, + "learning_rate": 1.2196589769307924e-05, + "loss": 1.1231, + "theoretical_loss": 3.328217778724715, + "tokens_seen": 3260481536 + }, + { + "epoch": 0.98, + "learning_rate": 1.2176529588766298e-05, + "loss": 1.2041, + "theoretical_loss": 3.3282127650466458, + "tokens_seen": 3260547072 + }, + { + "epoch": 0.98, + "learning_rate": 1.2156469408224674e-05, + "loss": 1.1912, + "theoretical_loss": 3.3282077514975645, + "tokens_seen": 3260612608 + }, + { + "epoch": 0.98, + "learning_rate": 1.2136409227683048e-05, + "loss": 1.3068, + "theoretical_loss": 3.3282027380774655, + "tokens_seen": 3260678144 + }, + { + "epoch": 0.98, + "learning_rate": 1.2116349047141424e-05, + "loss": 1.1813, + "theoretical_loss": 3.3281977247863432, + "tokens_seen": 3260743680 + }, + { + "epoch": 0.98, + "learning_rate": 1.20962888665998e-05, + "loss": 1.2307, + "theoretical_loss": 3.328192711624191, + "tokens_seen": 3260809216 + }, + { + "epoch": 0.98, + "learning_rate": 1.2076228686058176e-05, + "loss": 1.15, + "theoretical_loss": 3.328187698591004, + "tokens_seen": 3260874752 + }, + { + "epoch": 0.98, + "learning_rate": 1.205616850551655e-05, + "loss": 1.1544, + "theoretical_loss": 3.3281826856867758, + "tokens_seen": 3260940288 + }, + { + "epoch": 0.98, + "learning_rate": 1.2036108324974924e-05, + "loss": 1.2035, + "theoretical_loss": 3.3281776729115, + "tokens_seen": 3261005824 + }, + { + "epoch": 0.98, + "learning_rate": 1.20160481444333e-05, + "loss": 1.1989, + "theoretical_loss": 3.3281726602651713, + "tokens_seen": 3261071360 + }, + { + "epoch": 0.98, + "learning_rate": 1.1995987963891675e-05, + "loss": 1.3514, + "theoretical_loss": 3.3281676477477835, + "tokens_seen": 3261136896 + }, + { + "epoch": 0.98, + "learning_rate": 1.1975927783350051e-05, + "loss": 1.1314, + "theoretical_loss": 3.3281626353593308, + "tokens_seen": 3261202432 + }, + { + "epoch": 0.98, + "learning_rate": 1.1955867602808425e-05, + "loss": 1.2421, + "theoretical_loss": 3.3281576230998073, + "tokens_seen": 3261267968 + }, + { + "epoch": 0.98, + "learning_rate": 1.1935807422266801e-05, + "loss": 1.1828, + "theoretical_loss": 3.3281526109692074, + "tokens_seen": 3261333504 + }, + { + "epoch": 0.98, + "learning_rate": 1.1915747241725175e-05, + "loss": 1.1907, + "theoretical_loss": 3.328147598967525, + "tokens_seen": 3261399040 + }, + { + "epoch": 0.98, + "learning_rate": 1.189568706118355e-05, + "loss": 1.2831, + "theoretical_loss": 3.3281425870947534, + "tokens_seen": 3261464576 + }, + { + "epoch": 0.98, + "learning_rate": 1.1875626880641927e-05, + "loss": 1.1705, + "theoretical_loss": 3.3281375753508877, + "tokens_seen": 3261530112 + }, + { + "epoch": 0.98, + "learning_rate": 1.1855566700100301e-05, + "loss": 1.2531, + "theoretical_loss": 3.3281325637359216, + "tokens_seen": 3261595648 + }, + { + "epoch": 0.98, + "learning_rate": 1.1835506519558677e-05, + "loss": 1.188, + "theoretical_loss": 3.3281275522498497, + "tokens_seen": 3261661184 + }, + { + "epoch": 0.98, + "learning_rate": 1.181544633901705e-05, + "loss": 1.1853, + "theoretical_loss": 3.3281225408926653, + "tokens_seen": 3261726720 + }, + { + "epoch": 0.98, + "learning_rate": 1.1795386158475427e-05, + "loss": 1.2576, + "theoretical_loss": 3.328117529664363, + "tokens_seen": 3261792256 + }, + { + "epoch": 0.98, + "learning_rate": 1.1775325977933802e-05, + "loss": 1.2031, + "theoretical_loss": 3.3281125185649363, + "tokens_seen": 3261857792 + }, + { + "epoch": 0.98, + "learning_rate": 1.1755265797392177e-05, + "loss": 1.2801, + "theoretical_loss": 3.32810750759438, + "tokens_seen": 3261923328 + }, + { + "epoch": 0.98, + "learning_rate": 1.1735205616850552e-05, + "loss": 1.1188, + "theoretical_loss": 3.328102496752688, + "tokens_seen": 3261988864 + }, + { + "epoch": 0.98, + "learning_rate": 1.1715145436308926e-05, + "loss": 1.2956, + "theoretical_loss": 3.3280974860398542, + "tokens_seen": 3262054400 + }, + { + "epoch": 0.98, + "learning_rate": 1.1695085255767302e-05, + "loss": 1.2641, + "theoretical_loss": 3.328092475455873, + "tokens_seen": 3262119936 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1184948, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.343144178390503, + "objective/train/theoretical_loss": 3.3280874650007384, + "objective/train/tokens_used": 1632644576, + "theoretical_loss": 3.3280874650007384, + "tokens_seen": 3262185472 + }, + { + "epoch": 0.98, + "learning_rate": 1.1675025075225676e-05, + "loss": 1.3316, + "theoretical_loss": 3.3280874650007384, + "tokens_seen": 3262185472 + }, + { + "epoch": 0.98, + "learning_rate": 1.1654964894684054e-05, + "loss": 1.2526, + "theoretical_loss": 3.3280824546744445, + "tokens_seen": 3262251008 + }, + { + "epoch": 0.98, + "learning_rate": 1.1634904714142428e-05, + "loss": 1.3283, + "theoretical_loss": 3.328077444476985, + "tokens_seen": 3262316544 + }, + { + "epoch": 0.98, + "learning_rate": 1.1614844533600802e-05, + "loss": 1.3862, + "theoretical_loss": 3.3280724344083543, + "tokens_seen": 3262382080 + }, + { + "epoch": 0.98, + "learning_rate": 1.1594784353059178e-05, + "loss": 1.1528, + "theoretical_loss": 3.328067424468547, + "tokens_seen": 3262447616 + }, + { + "epoch": 0.98, + "learning_rate": 1.1574724172517552e-05, + "loss": 1.221, + "theoretical_loss": 3.3280624146575564, + "tokens_seen": 3262513152 + }, + { + "epoch": 0.98, + "learning_rate": 1.1554663991975928e-05, + "loss": 1.2604, + "theoretical_loss": 3.328057404975377, + "tokens_seen": 3262578688 + }, + { + "epoch": 0.98, + "learning_rate": 1.1534603811434303e-05, + "loss": 1.1985, + "theoretical_loss": 3.3280523954220027, + "tokens_seen": 3262644224 + }, + { + "epoch": 0.98, + "learning_rate": 1.151454363089268e-05, + "loss": 1.2146, + "theoretical_loss": 3.3280473859974276, + "tokens_seen": 3262709760 + }, + { + "epoch": 0.98, + "learning_rate": 1.1494483450351053e-05, + "loss": 1.4185, + "theoretical_loss": 3.3280423767016467, + "tokens_seen": 3262775296 + }, + { + "epoch": 0.98, + "learning_rate": 1.1474423269809427e-05, + "loss": 1.2356, + "theoretical_loss": 3.3280373675346526, + "tokens_seen": 3262840832 + }, + { + "epoch": 0.98, + "learning_rate": 1.1454363089267803e-05, + "loss": 1.3503, + "theoretical_loss": 3.3280323584964404, + "tokens_seen": 3262906368 + }, + { + "epoch": 0.98, + "learning_rate": 1.1434302908726179e-05, + "loss": 1.249, + "theoretical_loss": 3.328027349587004, + "tokens_seen": 3262971904 + }, + { + "epoch": 0.98, + "learning_rate": 1.1414242728184555e-05, + "loss": 1.0516, + "theoretical_loss": 3.328022340806337, + "tokens_seen": 3263037440 + }, + { + "epoch": 0.98, + "learning_rate": 1.1394182547642929e-05, + "loss": 1.2017, + "theoretical_loss": 3.328017332154434, + "tokens_seen": 3263102976 + }, + { + "epoch": 0.98, + "learning_rate": 1.1374122367101305e-05, + "loss": 1.1819, + "theoretical_loss": 3.3280123236312895, + "tokens_seen": 3263168512 + }, + { + "epoch": 0.98, + "learning_rate": 1.1354062186559679e-05, + "loss": 1.2312, + "theoretical_loss": 3.328007315236897, + "tokens_seen": 3263234048 + }, + { + "epoch": 0.98, + "learning_rate": 1.1334002006018053e-05, + "loss": 1.2776, + "theoretical_loss": 3.3280023069712508, + "tokens_seen": 3263299584 + }, + { + "epoch": 0.98, + "learning_rate": 1.131394182547643e-05, + "loss": 1.1933, + "theoretical_loss": 3.3279972988343447, + "tokens_seen": 3263365120 + }, + { + "epoch": 0.98, + "learning_rate": 1.1293881644934805e-05, + "loss": 1.0615, + "theoretical_loss": 3.3279922908261734, + "tokens_seen": 3263430656 + }, + { + "epoch": 0.98, + "learning_rate": 1.127382146439318e-05, + "loss": 1.3063, + "theoretical_loss": 3.3279872829467307, + "tokens_seen": 3263496192 + }, + { + "epoch": 0.98, + "learning_rate": 1.1253761283851554e-05, + "loss": 1.2837, + "theoretical_loss": 3.3279822751960104, + "tokens_seen": 3263561728 + }, + { + "epoch": 0.98, + "learning_rate": 1.123370110330993e-05, + "loss": 1.1968, + "theoretical_loss": 3.327977267574007, + "tokens_seen": 3263627264 + }, + { + "epoch": 0.98, + "learning_rate": 1.1213640922768304e-05, + "loss": 1.2229, + "theoretical_loss": 3.3279722600807142, + "tokens_seen": 3263692800 + }, + { + "epoch": 0.98, + "learning_rate": 1.119358074222668e-05, + "loss": 1.2492, + "theoretical_loss": 3.327967252716127, + "tokens_seen": 3263758336 + }, + { + "epoch": 0.98, + "learning_rate": 1.1173520561685056e-05, + "loss": 1.2826, + "theoretical_loss": 3.3279622454802382, + "tokens_seen": 3263823872 + }, + { + "epoch": 0.98, + "learning_rate": 1.115346038114343e-05, + "loss": 1.2047, + "theoretical_loss": 3.3279572383730436, + "tokens_seen": 3263889408 + }, + { + "epoch": 0.98, + "learning_rate": 1.1133400200601806e-05, + "loss": 1.2129, + "theoretical_loss": 3.3279522313945353, + "tokens_seen": 3263954944 + }, + { + "epoch": 0.98, + "learning_rate": 1.111334002006018e-05, + "loss": 1.2951, + "theoretical_loss": 3.327947224544709, + "tokens_seen": 3264020480 + }, + { + "epoch": 0.98, + "learning_rate": 1.1093279839518557e-05, + "loss": 1.2205, + "theoretical_loss": 3.3279422178235585, + "tokens_seen": 3264086016 + }, + { + "epoch": 0.98, + "learning_rate": 1.1073219658976931e-05, + "loss": 1.2923, + "theoretical_loss": 3.327937211231077, + "tokens_seen": 3264151552 + }, + { + "epoch": 0.98, + "learning_rate": 1.1053159478435306e-05, + "loss": 1.2399, + "theoretical_loss": 3.3279322047672597, + "tokens_seen": 3264217088 + }, + { + "epoch": 0.98, + "learning_rate": 1.1033099297893681e-05, + "loss": 1.2497, + "theoretical_loss": 3.3279271984321, + "tokens_seen": 3264282624 + }, + { + "epoch": 0.98, + "learning_rate": 1.1013039117352055e-05, + "loss": 1.2658, + "theoretical_loss": 3.3279221922255924, + "tokens_seen": 3264348160 + }, + { + "epoch": 0.98, + "learning_rate": 1.0992978936810431e-05, + "loss": 1.2592, + "theoretical_loss": 3.327917186147731, + "tokens_seen": 3264413696 + }, + { + "epoch": 0.98, + "learning_rate": 1.0972918756268807e-05, + "loss": 1.2883, + "theoretical_loss": 3.3279121801985094, + "tokens_seen": 3264479232 + }, + { + "epoch": 0.98, + "learning_rate": 1.0952858575727183e-05, + "loss": 1.2484, + "theoretical_loss": 3.3279071743779225, + "tokens_seen": 3264544768 + }, + { + "epoch": 0.98, + "learning_rate": 1.0932798395185557e-05, + "loss": 1.2426, + "theoretical_loss": 3.327902168685964, + "tokens_seen": 3264610304 + }, + { + "epoch": 0.98, + "learning_rate": 1.0912738214643931e-05, + "loss": 1.2467, + "theoretical_loss": 3.327897163122628, + "tokens_seen": 3264675840 + }, + { + "epoch": 0.98, + "learning_rate": 1.0892678034102307e-05, + "loss": 1.3311, + "theoretical_loss": 3.3278921576879084, + "tokens_seen": 3264741376 + }, + { + "epoch": 0.98, + "learning_rate": 1.0872617853560683e-05, + "loss": 1.245, + "theoretical_loss": 3.3278871523818, + "tokens_seen": 3264806912 + }, + { + "epoch": 0.98, + "learning_rate": 1.0852557673019058e-05, + "loss": 1.3083, + "theoretical_loss": 3.3278821472042965, + "tokens_seen": 3264872448 + }, + { + "epoch": 0.98, + "learning_rate": 1.0832497492477433e-05, + "loss": 1.239, + "theoretical_loss": 3.3278771421553914, + "tokens_seen": 3264937984 + }, + { + "epoch": 0.98, + "learning_rate": 1.0812437311935808e-05, + "loss": 1.2552, + "theoretical_loss": 3.32787213723508, + "tokens_seen": 3265003520 + }, + { + "epoch": 0.98, + "learning_rate": 1.0792377131394182e-05, + "loss": 1.1537, + "theoretical_loss": 3.3278671324433557, + "tokens_seen": 3265069056 + }, + { + "epoch": 0.98, + "learning_rate": 1.0772316950852556e-05, + "loss": 1.303, + "theoretical_loss": 3.3278621277802127, + "tokens_seen": 3265134592 + }, + { + "epoch": 0.98, + "learning_rate": 1.0752256770310934e-05, + "loss": 1.1228, + "theoretical_loss": 3.327857123245645, + "tokens_seen": 3265200128 + }, + { + "epoch": 0.98, + "learning_rate": 1.0732196589769308e-05, + "loss": 1.2888, + "theoretical_loss": 3.327852118839647, + "tokens_seen": 3265265664 + }, + { + "epoch": 0.98, + "learning_rate": 1.0712136409227684e-05, + "loss": 1.2225, + "theoretical_loss": 3.327847114562213, + "tokens_seen": 3265331200 + }, + { + "epoch": 0.98, + "learning_rate": 1.0692076228686058e-05, + "loss": 1.3288, + "theoretical_loss": 3.327842110413336, + "tokens_seen": 3265396736 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1186260, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2703136205673218, + "objective/train/theoretical_loss": 3.3278371063930114, + "objective/train/tokens_used": 1635921376, + "theoretical_loss": 3.3278371063930114, + "tokens_seen": 3265462272 + }, + { + "epoch": 0.98, + "learning_rate": 1.0672016048144434e-05, + "loss": 1.2499, + "theoretical_loss": 3.3278371063930114, + "tokens_seen": 3265462272 + }, + { + "epoch": 0.98, + "learning_rate": 1.0651955867602808e-05, + "loss": 1.2397, + "theoretical_loss": 3.327832102501233, + "tokens_seen": 3265527808 + }, + { + "epoch": 0.98, + "learning_rate": 1.0631895687061184e-05, + "loss": 1.3546, + "theoretical_loss": 3.3278270987379948, + "tokens_seen": 3265593344 + }, + { + "epoch": 0.98, + "learning_rate": 1.061183550651956e-05, + "loss": 1.2491, + "theoretical_loss": 3.327822095103291, + "tokens_seen": 3265658880 + }, + { + "epoch": 0.98, + "learning_rate": 1.0591775325977934e-05, + "loss": 1.1893, + "theoretical_loss": 3.3278170915971153, + "tokens_seen": 3265724416 + }, + { + "epoch": 0.98, + "learning_rate": 1.057171514543631e-05, + "loss": 1.2368, + "theoretical_loss": 3.3278120882194617, + "tokens_seen": 3265789952 + }, + { + "epoch": 0.98, + "learning_rate": 1.0551654964894683e-05, + "loss": 1.1226, + "theoretical_loss": 3.3278070849703254, + "tokens_seen": 3265855488 + }, + { + "epoch": 0.98, + "learning_rate": 1.0531594784353061e-05, + "loss": 1.1854, + "theoretical_loss": 3.3278020818496996, + "tokens_seen": 3265921024 + }, + { + "epoch": 0.98, + "learning_rate": 1.0511534603811435e-05, + "loss": 1.2571, + "theoretical_loss": 3.3277970788575786, + "tokens_seen": 3265986560 + }, + { + "epoch": 0.98, + "learning_rate": 1.0491474423269809e-05, + "loss": 1.2759, + "theoretical_loss": 3.327792075993957, + "tokens_seen": 3266052096 + }, + { + "epoch": 0.98, + "learning_rate": 1.0471414242728185e-05, + "loss": 1.3108, + "theoretical_loss": 3.327787073258828, + "tokens_seen": 3266117632 + }, + { + "epoch": 0.98, + "learning_rate": 1.0451354062186559e-05, + "loss": 1.231, + "theoretical_loss": 3.327782070652187, + "tokens_seen": 3266183168 + }, + { + "epoch": 0.98, + "learning_rate": 1.0431293881644935e-05, + "loss": 1.1006, + "theoretical_loss": 3.3277770681740266, + "tokens_seen": 3266248704 + }, + { + "epoch": 0.98, + "learning_rate": 1.041123370110331e-05, + "loss": 1.282, + "theoretical_loss": 3.327772065824342, + "tokens_seen": 3266314240 + }, + { + "epoch": 0.98, + "learning_rate": 1.0391173520561686e-05, + "loss": 1.2876, + "theoretical_loss": 3.327767063603127, + "tokens_seen": 3266379776 + }, + { + "epoch": 0.98, + "learning_rate": 1.037111334002006e-05, + "loss": 1.274, + "theoretical_loss": 3.3277620615103753, + "tokens_seen": 3266445312 + }, + { + "epoch": 0.98, + "learning_rate": 1.0351053159478435e-05, + "loss": 1.2055, + "theoretical_loss": 3.327757059546082, + "tokens_seen": 3266510848 + }, + { + "epoch": 0.98, + "learning_rate": 1.033099297893681e-05, + "loss": 1.207, + "theoretical_loss": 3.3277520577102404, + "tokens_seen": 3266576384 + }, + { + "epoch": 0.98, + "learning_rate": 1.0310932798395184e-05, + "loss": 1.2397, + "theoretical_loss": 3.327747056002845, + "tokens_seen": 3266641920 + }, + { + "epoch": 0.98, + "learning_rate": 1.0290872617853562e-05, + "loss": 1.2611, + "theoretical_loss": 3.32774205442389, + "tokens_seen": 3266707456 + }, + { + "epoch": 0.98, + "learning_rate": 1.0270812437311936e-05, + "loss": 1.2835, + "theoretical_loss": 3.327737052973369, + "tokens_seen": 3266772992 + }, + { + "epoch": 0.98, + "learning_rate": 1.0250752256770312e-05, + "loss": 1.3266, + "theoretical_loss": 3.327732051651277, + "tokens_seen": 3266838528 + }, + { + "epoch": 0.98, + "learning_rate": 1.0230692076228686e-05, + "loss": 1.3428, + "theoretical_loss": 3.3277270504576073, + "tokens_seen": 3266904064 + }, + { + "epoch": 0.98, + "learning_rate": 1.021063189568706e-05, + "loss": 1.2325, + "theoretical_loss": 3.3277220493923543, + "tokens_seen": 3266969600 + }, + { + "epoch": 0.98, + "learning_rate": 1.0190571715145438e-05, + "loss": 1.2649, + "theoretical_loss": 3.3277170484555123, + "tokens_seen": 3267035136 + }, + { + "epoch": 0.98, + "learning_rate": 1.0170511534603812e-05, + "loss": 1.2062, + "theoretical_loss": 3.327712047647075, + "tokens_seen": 3267100672 + }, + { + "epoch": 0.98, + "learning_rate": 1.0150451354062187e-05, + "loss": 1.2556, + "theoretical_loss": 3.327707046967037, + "tokens_seen": 3267166208 + }, + { + "epoch": 0.98, + "learning_rate": 1.0130391173520562e-05, + "loss": 1.1729, + "theoretical_loss": 3.3277020464153924, + "tokens_seen": 3267231744 + }, + { + "epoch": 0.98, + "learning_rate": 1.0110330992978937e-05, + "loss": 1.1693, + "theoretical_loss": 3.327697045992135, + "tokens_seen": 3267297280 + }, + { + "epoch": 0.98, + "learning_rate": 1.0090270812437311e-05, + "loss": 1.2212, + "theoretical_loss": 3.327692045697259, + "tokens_seen": 3267362816 + }, + { + "epoch": 0.98, + "learning_rate": 1.0070210631895687e-05, + "loss": 1.3181, + "theoretical_loss": 3.327687045530759, + "tokens_seen": 3267428352 + }, + { + "epoch": 0.98, + "learning_rate": 1.0050150451354063e-05, + "loss": 1.2405, + "theoretical_loss": 3.327682045492628, + "tokens_seen": 3267493888 + }, + { + "epoch": 0.98, + "learning_rate": 1.0030090270812437e-05, + "loss": 1.2056, + "theoretical_loss": 3.3276770455828615, + "tokens_seen": 3267559424 + }, + { + "epoch": 0.98, + "learning_rate": 1.0010030090270813e-05, + "loss": 1.2869, + "theoretical_loss": 3.327672045801453, + "tokens_seen": 3267624960 + }, + { + "epoch": 0.98, + "learning_rate": 9.989969909729187e-06, + "loss": 1.215, + "theoretical_loss": 3.3276670461483966, + "tokens_seen": 3267690496 + }, + { + "epoch": 0.98, + "learning_rate": 9.969909729187565e-06, + "loss": 1.3275, + "theoretical_loss": 3.3276620466236864, + "tokens_seen": 3267756032 + }, + { + "epoch": 0.98, + "learning_rate": 9.949849548645939e-06, + "loss": 1.1964, + "theoretical_loss": 3.3276570472273166, + "tokens_seen": 3267821568 + }, + { + "epoch": 0.98, + "learning_rate": 9.929789368104313e-06, + "loss": 1.3369, + "theoretical_loss": 3.3276520479592815, + "tokens_seen": 3267887104 + }, + { + "epoch": 0.98, + "learning_rate": 9.909729187562688e-06, + "loss": 1.2774, + "theoretical_loss": 3.327647048819575, + "tokens_seen": 3267952640 + }, + { + "epoch": 0.98, + "learning_rate": 9.889669007021063e-06, + "loss": 1.2673, + "theoretical_loss": 3.327642049808191, + "tokens_seen": 3268018176 + }, + { + "epoch": 0.98, + "learning_rate": 9.869608826479438e-06, + "loss": 1.1772, + "theoretical_loss": 3.3276370509251243, + "tokens_seen": 3268083712 + }, + { + "epoch": 0.98, + "learning_rate": 9.849548645937814e-06, + "loss": 1.312, + "theoretical_loss": 3.3276320521703684, + "tokens_seen": 3268149248 + }, + { + "epoch": 0.98, + "learning_rate": 9.82948846539619e-06, + "loss": 1.1865, + "theoretical_loss": 3.327627053543918, + "tokens_seen": 3268214784 + }, + { + "epoch": 0.98, + "learning_rate": 9.809428284854564e-06, + "loss": 1.2231, + "theoretical_loss": 3.327622055045767, + "tokens_seen": 3268280320 + }, + { + "epoch": 0.98, + "learning_rate": 9.789368104312938e-06, + "loss": 1.3034, + "theoretical_loss": 3.3276170566759093, + "tokens_seen": 3268345856 + }, + { + "epoch": 0.98, + "learning_rate": 9.769307923771314e-06, + "loss": 1.3538, + "theoretical_loss": 3.327612058434339, + "tokens_seen": 3268411392 + }, + { + "epoch": 0.98, + "learning_rate": 9.749247743229688e-06, + "loss": 1.16, + "theoretical_loss": 3.327607060321051, + "tokens_seen": 3268476928 + }, + { + "epoch": 0.98, + "learning_rate": 9.729187562688066e-06, + "loss": 1.1521, + "theoretical_loss": 3.3276020623360383, + "tokens_seen": 3268542464 + }, + { + "epoch": 0.98, + "learning_rate": 9.70912738214644e-06, + "loss": 1.1405, + "theoretical_loss": 3.3275970644792956, + "tokens_seen": 3268608000 + }, + { + "epoch": 0.98, + "learning_rate": 9.689067201604815e-06, + "loss": 1.2841, + "theoretical_loss": 3.3275920667508174, + "tokens_seen": 3268673536 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1187457, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2629066705703735, + "objective/train/theoretical_loss": 3.327587069150597, + "objective/train/tokens_used": 1639198176, + "theoretical_loss": 3.327587069150597, + "tokens_seen": 3268739072 + }, + { + "epoch": 0.98, + "learning_rate": 9.66900702106319e-06, + "loss": 1.2422, + "theoretical_loss": 3.327587069150597, + "tokens_seen": 3268739072 + }, + { + "epoch": 0.98, + "learning_rate": 9.648946840521564e-06, + "loss": 1.2651, + "theoretical_loss": 3.3275820716786297, + "tokens_seen": 3268804608 + }, + { + "epoch": 0.98, + "learning_rate": 9.628886659979941e-06, + "loss": 1.2401, + "theoretical_loss": 3.3275770743349087, + "tokens_seen": 3268870144 + }, + { + "epoch": 0.98, + "learning_rate": 9.608826479438315e-06, + "loss": 1.2943, + "theoretical_loss": 3.327572077119428, + "tokens_seen": 3268935680 + }, + { + "epoch": 0.98, + "learning_rate": 9.588766298896691e-06, + "loss": 1.2836, + "theoretical_loss": 3.327567080032183, + "tokens_seen": 3269001216 + }, + { + "epoch": 0.98, + "learning_rate": 9.568706118355065e-06, + "loss": 1.1451, + "theoretical_loss": 3.327562083073166, + "tokens_seen": 3269066752 + }, + { + "epoch": 0.98, + "learning_rate": 9.54864593781344e-06, + "loss": 1.1994, + "theoretical_loss": 3.327557086242373, + "tokens_seen": 3269132288 + }, + { + "epoch": 0.98, + "learning_rate": 9.528585757271815e-06, + "loss": 1.4072, + "theoretical_loss": 3.3275520895397968, + "tokens_seen": 3269197824 + }, + { + "epoch": 0.98, + "learning_rate": 9.50852557673019e-06, + "loss": 1.3884, + "theoretical_loss": 3.327547092965432, + "tokens_seen": 3269263360 + }, + { + "epoch": 0.98, + "learning_rate": 9.488465396188567e-06, + "loss": 1.3081, + "theoretical_loss": 3.3275420965192724, + "tokens_seen": 3269328896 + }, + { + "epoch": 0.98, + "learning_rate": 9.46840521564694e-06, + "loss": 1.1851, + "theoretical_loss": 3.327537100201313, + "tokens_seen": 3269394432 + }, + { + "epoch": 0.98, + "learning_rate": 9.448345035105316e-06, + "loss": 1.2105, + "theoretical_loss": 3.3275321040115475, + "tokens_seen": 3269459968 + }, + { + "epoch": 0.98, + "learning_rate": 9.42828485456369e-06, + "loss": 1.1922, + "theoretical_loss": 3.3275271079499693, + "tokens_seen": 3269525504 + }, + { + "epoch": 0.98, + "learning_rate": 9.408224674022065e-06, + "loss": 1.227, + "theoretical_loss": 3.327522112016574, + "tokens_seen": 3269591040 + }, + { + "epoch": 0.98, + "learning_rate": 9.388164493480442e-06, + "loss": 1.2702, + "theoretical_loss": 3.3275171162113546, + "tokens_seen": 3269656576 + }, + { + "epoch": 0.98, + "learning_rate": 9.368104312938816e-06, + "loss": 1.3654, + "theoretical_loss": 3.3275121205343052, + "tokens_seen": 3269722112 + }, + { + "epoch": 0.98, + "learning_rate": 9.348044132397192e-06, + "loss": 1.2376, + "theoretical_loss": 3.327507124985421, + "tokens_seen": 3269787648 + }, + { + "epoch": 0.98, + "learning_rate": 9.327983951855566e-06, + "loss": 1.2938, + "theoretical_loss": 3.327502129564695, + "tokens_seen": 3269853184 + }, + { + "epoch": 0.98, + "learning_rate": 9.307923771313942e-06, + "loss": 1.1103, + "theoretical_loss": 3.327497134272122, + "tokens_seen": 3269918720 + }, + { + "epoch": 0.98, + "learning_rate": 9.287863590772318e-06, + "loss": 1.2271, + "theoretical_loss": 3.3274921391076955, + "tokens_seen": 3269984256 + }, + { + "epoch": 0.98, + "learning_rate": 9.267803410230692e-06, + "loss": 1.2453, + "theoretical_loss": 3.327487144071411, + "tokens_seen": 3270049792 + }, + { + "epoch": 0.98, + "learning_rate": 9.247743229689068e-06, + "loss": 1.1314, + "theoretical_loss": 3.327482149163261, + "tokens_seen": 3270115328 + }, + { + "epoch": 0.98, + "learning_rate": 9.227683049147442e-06, + "loss": 1.2628, + "theoretical_loss": 3.3274771543832404, + "tokens_seen": 3270180864 + }, + { + "epoch": 0.98, + "learning_rate": 9.207622868605818e-06, + "loss": 1.225, + "theoretical_loss": 3.327472159731344, + "tokens_seen": 3270246400 + }, + { + "epoch": 0.98, + "learning_rate": 9.187562688064192e-06, + "loss": 1.1356, + "theoretical_loss": 3.327467165207565, + "tokens_seen": 3270311936 + }, + { + "epoch": 0.98, + "learning_rate": 9.167502507522569e-06, + "loss": 1.2122, + "theoretical_loss": 3.3274621708118977, + "tokens_seen": 3270377472 + }, + { + "epoch": 0.98, + "learning_rate": 9.147442326980943e-06, + "loss": 1.1149, + "theoretical_loss": 3.3274571765443364, + "tokens_seen": 3270443008 + }, + { + "epoch": 0.98, + "learning_rate": 9.127382146439317e-06, + "loss": 1.1836, + "theoretical_loss": 3.327452182404875, + "tokens_seen": 3270508544 + }, + { + "epoch": 0.98, + "learning_rate": 9.107321965897693e-06, + "loss": 1.2738, + "theoretical_loss": 3.327447188393508, + "tokens_seen": 3270574080 + }, + { + "epoch": 0.98, + "learning_rate": 9.087261785356067e-06, + "loss": 1.2059, + "theoretical_loss": 3.3274421945102297, + "tokens_seen": 3270639616 + }, + { + "epoch": 0.98, + "learning_rate": 9.067201604814445e-06, + "loss": 1.1482, + "theoretical_loss": 3.3274372007550337, + "tokens_seen": 3270705152 + }, + { + "epoch": 0.98, + "learning_rate": 9.047141424272819e-06, + "loss": 1.1733, + "theoretical_loss": 3.3274322071279143, + "tokens_seen": 3270770688 + }, + { + "epoch": 0.98, + "learning_rate": 9.027081243731195e-06, + "loss": 1.298, + "theoretical_loss": 3.327427213628866, + "tokens_seen": 3270836224 + }, + { + "epoch": 0.98, + "learning_rate": 9.007021063189569e-06, + "loss": 1.3288, + "theoretical_loss": 3.3274222202578825, + "tokens_seen": 3270901760 + }, + { + "epoch": 0.98, + "learning_rate": 8.986960882647943e-06, + "loss": 1.2179, + "theoretical_loss": 3.3274172270149585, + "tokens_seen": 3270967296 + }, + { + "epoch": 0.98, + "learning_rate": 8.966900702106319e-06, + "loss": 1.113, + "theoretical_loss": 3.3274122339000876, + "tokens_seen": 3271032832 + }, + { + "epoch": 0.98, + "learning_rate": 8.946840521564694e-06, + "loss": 1.3052, + "theoretical_loss": 3.3274072409132645, + "tokens_seen": 3271098368 + }, + { + "epoch": 0.98, + "learning_rate": 8.92678034102307e-06, + "loss": 1.1318, + "theoretical_loss": 3.3274022480544825, + "tokens_seen": 3271163904 + }, + { + "epoch": 0.98, + "learning_rate": 8.906720160481444e-06, + "loss": 1.2751, + "theoretical_loss": 3.3273972553237363, + "tokens_seen": 3271229440 + }, + { + "epoch": 0.98, + "learning_rate": 8.88665997993982e-06, + "loss": 1.2206, + "theoretical_loss": 3.3273922627210206, + "tokens_seen": 3271294976 + }, + { + "epoch": 0.98, + "learning_rate": 8.866599799398194e-06, + "loss": 1.2586, + "theoretical_loss": 3.3273872702463283, + "tokens_seen": 3271360512 + }, + { + "epoch": 0.98, + "learning_rate": 8.846539618856568e-06, + "loss": 1.2474, + "theoretical_loss": 3.3273822778996545, + "tokens_seen": 3271426048 + }, + { + "epoch": 0.98, + "learning_rate": 8.826479438314946e-06, + "loss": 1.2413, + "theoretical_loss": 3.327377285680993, + "tokens_seen": 3271491584 + }, + { + "epoch": 0.98, + "learning_rate": 8.80641925777332e-06, + "loss": 1.1794, + "theoretical_loss": 3.3273722935903383, + "tokens_seen": 3271557120 + }, + { + "epoch": 0.98, + "learning_rate": 8.786359077231696e-06, + "loss": 1.2847, + "theoretical_loss": 3.327367301627684, + "tokens_seen": 3271622656 + }, + { + "epoch": 0.98, + "learning_rate": 8.76629889669007e-06, + "loss": 1.2262, + "theoretical_loss": 3.327362309793025, + "tokens_seen": 3271688192 + }, + { + "epoch": 0.98, + "learning_rate": 8.746238716148446e-06, + "loss": 1.3228, + "theoretical_loss": 3.3273573180863543, + "tokens_seen": 3271753728 + }, + { + "epoch": 0.98, + "learning_rate": 8.726178535606821e-06, + "loss": 1.1662, + "theoretical_loss": 3.327352326507667, + "tokens_seen": 3271819264 + }, + { + "epoch": 0.98, + "learning_rate": 8.706118355065195e-06, + "loss": 1.2003, + "theoretical_loss": 3.327347335056957, + "tokens_seen": 3271884800 + }, + { + "epoch": 0.98, + "learning_rate": 8.686058174523571e-06, + "loss": 1.0696, + "theoretical_loss": 3.3273423437342187, + "tokens_seen": 3271950336 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1188646, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1247133016586304, + "objective/train/theoretical_loss": 3.3273373525394456, + "objective/train/tokens_used": 1642474976, + "theoretical_loss": 3.3273373525394456, + "tokens_seen": 3272015872 + }, + { + "epoch": 0.98, + "learning_rate": 8.665997993981945e-06, + "loss": 1.1487, + "theoretical_loss": 3.3273373525394456, + "tokens_seen": 3272015872 + }, + { + "epoch": 0.98, + "learning_rate": 8.645937813440321e-06, + "loss": 1.2024, + "theoretical_loss": 3.327332361472633, + "tokens_seen": 3272081408 + }, + { + "epoch": 0.98, + "learning_rate": 8.625877632898695e-06, + "loss": 1.2993, + "theoretical_loss": 3.327327370533774, + "tokens_seen": 3272146944 + }, + { + "epoch": 0.98, + "learning_rate": 8.605817452357073e-06, + "loss": 1.1568, + "theoretical_loss": 3.3273223797228626, + "tokens_seen": 3272212480 + }, + { + "epoch": 0.98, + "learning_rate": 8.585757271815447e-06, + "loss": 1.2633, + "theoretical_loss": 3.3273173890398935, + "tokens_seen": 3272278016 + }, + { + "epoch": 0.98, + "learning_rate": 8.565697091273821e-06, + "loss": 1.2707, + "theoretical_loss": 3.327312398484861, + "tokens_seen": 3272343552 + }, + { + "epoch": 0.98, + "learning_rate": 8.545636910732197e-06, + "loss": 1.3644, + "theoretical_loss": 3.3273074080577594, + "tokens_seen": 3272409088 + }, + { + "epoch": 0.98, + "learning_rate": 8.52557673019057e-06, + "loss": 1.347, + "theoretical_loss": 3.3273024177585824, + "tokens_seen": 3272474624 + }, + { + "epoch": 0.98, + "learning_rate": 8.505516549648948e-06, + "loss": 1.2442, + "theoretical_loss": 3.327297427587324, + "tokens_seen": 3272540160 + }, + { + "epoch": 0.98, + "learning_rate": 8.485456369107322e-06, + "loss": 1.1388, + "theoretical_loss": 3.3272924375439787, + "tokens_seen": 3272605696 + }, + { + "epoch": 0.98, + "learning_rate": 8.465396188565698e-06, + "loss": 1.3811, + "theoretical_loss": 3.3272874476285406, + "tokens_seen": 3272671232 + }, + { + "epoch": 0.98, + "learning_rate": 8.445336008024072e-06, + "loss": 1.2336, + "theoretical_loss": 3.3272824578410036, + "tokens_seen": 3272736768 + }, + { + "epoch": 0.98, + "learning_rate": 8.425275827482446e-06, + "loss": 1.3284, + "theoretical_loss": 3.3272774681813626, + "tokens_seen": 3272802304 + }, + { + "epoch": 0.98, + "learning_rate": 8.405215646940822e-06, + "loss": 1.0991, + "theoretical_loss": 3.3272724786496113, + "tokens_seen": 3272867840 + }, + { + "epoch": 0.98, + "learning_rate": 8.385155466399198e-06, + "loss": 1.3947, + "theoretical_loss": 3.3272674892457434, + "tokens_seen": 3272933376 + }, + { + "epoch": 0.98, + "learning_rate": 8.365095285857574e-06, + "loss": 1.1369, + "theoretical_loss": 3.3272624999697538, + "tokens_seen": 3272998912 + }, + { + "epoch": 0.98, + "learning_rate": 8.345035105315948e-06, + "loss": 1.2269, + "theoretical_loss": 3.3272575108216365, + "tokens_seen": 3273064448 + }, + { + "epoch": 0.98, + "learning_rate": 8.324974924774324e-06, + "loss": 1.1551, + "theoretical_loss": 3.327252521801385, + "tokens_seen": 3273129984 + }, + { + "epoch": 0.98, + "learning_rate": 8.304914744232698e-06, + "loss": 1.2579, + "theoretical_loss": 3.3272475329089946, + "tokens_seen": 3273195520 + }, + { + "epoch": 0.98, + "learning_rate": 8.284854563691072e-06, + "loss": 1.2495, + "theoretical_loss": 3.3272425441444584, + "tokens_seen": 3273261056 + }, + { + "epoch": 0.98, + "learning_rate": 8.26479438314945e-06, + "loss": 1.2456, + "theoretical_loss": 3.327237555507771, + "tokens_seen": 3273326592 + }, + { + "epoch": 0.98, + "learning_rate": 8.244734202607823e-06, + "loss": 1.3888, + "theoretical_loss": 3.327232566998927, + "tokens_seen": 3273392128 + }, + { + "epoch": 0.98, + "learning_rate": 8.2246740220662e-06, + "loss": 1.1258, + "theoretical_loss": 3.3272275786179195, + "tokens_seen": 3273457664 + }, + { + "epoch": 0.98, + "learning_rate": 8.204613841524573e-06, + "loss": 1.3112, + "theoretical_loss": 3.3272225903647437, + "tokens_seen": 3273523200 + }, + { + "epoch": 0.98, + "learning_rate": 8.184553660982949e-06, + "loss": 1.2697, + "theoretical_loss": 3.3272176022393936, + "tokens_seen": 3273588736 + }, + { + "epoch": 0.98, + "learning_rate": 8.164493480441325e-06, + "loss": 1.2033, + "theoretical_loss": 3.327212614241863, + "tokens_seen": 3273654272 + }, + { + "epoch": 0.98, + "learning_rate": 8.144433299899699e-06, + "loss": 1.2083, + "theoretical_loss": 3.3272076263721457, + "tokens_seen": 3273719808 + }, + { + "epoch": 0.98, + "learning_rate": 8.124373119358075e-06, + "loss": 1.3542, + "theoretical_loss": 3.327202638630237, + "tokens_seen": 3273785344 + }, + { + "epoch": 0.98, + "learning_rate": 8.104312938816449e-06, + "loss": 1.3098, + "theoretical_loss": 3.32719765101613, + "tokens_seen": 3273850880 + }, + { + "epoch": 0.98, + "learning_rate": 8.084252758274825e-06, + "loss": 1.2142, + "theoretical_loss": 3.3271926635298197, + "tokens_seen": 3273916416 + }, + { + "epoch": 0.98, + "learning_rate": 8.064192577733199e-06, + "loss": 1.3258, + "theoretical_loss": 3.3271876761712993, + "tokens_seen": 3273981952 + }, + { + "epoch": 0.98, + "learning_rate": 8.044132397191576e-06, + "loss": 1.2625, + "theoretical_loss": 3.3271826889405642, + "tokens_seen": 3274047488 + }, + { + "epoch": 0.98, + "learning_rate": 8.02407221664995e-06, + "loss": 1.1777, + "theoretical_loss": 3.3271777018376074, + "tokens_seen": 3274113024 + }, + { + "epoch": 0.98, + "learning_rate": 8.004012036108324e-06, + "loss": 1.1308, + "theoretical_loss": 3.327172714862424, + "tokens_seen": 3274178560 + }, + { + "epoch": 0.98, + "learning_rate": 7.9839518555667e-06, + "loss": 1.221, + "theoretical_loss": 3.3271677280150076, + "tokens_seen": 3274244096 + }, + { + "epoch": 0.98, + "learning_rate": 7.963891675025074e-06, + "loss": 1.2728, + "theoretical_loss": 3.327162741295352, + "tokens_seen": 3274309632 + }, + { + "epoch": 0.98, + "learning_rate": 7.943831494483452e-06, + "loss": 1.346, + "theoretical_loss": 3.3271577547034523, + "tokens_seen": 3274375168 + }, + { + "epoch": 0.98, + "learning_rate": 7.923771313941826e-06, + "loss": 1.286, + "theoretical_loss": 3.3271527682393023, + "tokens_seen": 3274440704 + }, + { + "epoch": 0.98, + "learning_rate": 7.903711133400202e-06, + "loss": 1.1868, + "theoretical_loss": 3.3271477819028963, + "tokens_seen": 3274506240 + }, + { + "epoch": 0.98, + "learning_rate": 7.883650952858576e-06, + "loss": 1.2461, + "theoretical_loss": 3.327142795694228, + "tokens_seen": 3274571776 + }, + { + "epoch": 0.98, + "learning_rate": 7.86359077231695e-06, + "loss": 1.2521, + "theoretical_loss": 3.327137809613292, + "tokens_seen": 3274637312 + }, + { + "epoch": 0.98, + "learning_rate": 7.843530591775326e-06, + "loss": 1.2259, + "theoretical_loss": 3.327132823660082, + "tokens_seen": 3274702848 + }, + { + "epoch": 0.98, + "learning_rate": 7.823470411233702e-06, + "loss": 1.1532, + "theoretical_loss": 3.3271278378345928, + "tokens_seen": 3274768384 + }, + { + "epoch": 0.98, + "learning_rate": 7.803410230692076e-06, + "loss": 1.2058, + "theoretical_loss": 3.3271228521368186, + "tokens_seen": 3274833920 + }, + { + "epoch": 0.98, + "learning_rate": 7.783350050150451e-06, + "loss": 1.3265, + "theoretical_loss": 3.327117866566753, + "tokens_seen": 3274899456 + }, + { + "epoch": 0.98, + "learning_rate": 7.763289869608827e-06, + "loss": 1.2839, + "theoretical_loss": 3.32711288112439, + "tokens_seen": 3274964992 + }, + { + "epoch": 0.98, + "learning_rate": 7.743229689067201e-06, + "loss": 1.2782, + "theoretical_loss": 3.327107895809725, + "tokens_seen": 3275030528 + }, + { + "epoch": 0.98, + "learning_rate": 7.723169508525577e-06, + "loss": 1.3296, + "theoretical_loss": 3.3271029106227505, + "tokens_seen": 3275096064 + }, + { + "epoch": 0.98, + "learning_rate": 7.703109327983953e-06, + "loss": 1.2955, + "theoretical_loss": 3.327097925563462, + "tokens_seen": 3275161600 + }, + { + "epoch": 0.98, + "learning_rate": 7.683049147442327e-06, + "loss": 1.1839, + "theoretical_loss": 3.3270929406318532, + "tokens_seen": 3275227136 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1189805, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2356234788894653, + "objective/train/theoretical_loss": 3.3270879558279183, + "objective/train/tokens_used": 1645751776, + "theoretical_loss": 3.3270879558279183, + "tokens_seen": 3275292672 + }, + { + "epoch": 0.99, + "learning_rate": 7.662988966900701e-06, + "loss": 1.2169, + "theoretical_loss": 3.3270879558279183, + "tokens_seen": 3275292672 + }, + { + "epoch": 0.99, + "learning_rate": 7.642928786359077e-06, + "loss": 1.1747, + "theoretical_loss": 3.3270829711516514, + "tokens_seen": 3275358208 + }, + { + "epoch": 0.99, + "learning_rate": 7.622868605817453e-06, + "loss": 1.3522, + "theoretical_loss": 3.327077986603047, + "tokens_seen": 3275423744 + }, + { + "epoch": 0.99, + "learning_rate": 7.602808425275828e-06, + "loss": 1.1514, + "theoretical_loss": 3.3270730021820984, + "tokens_seen": 3275489280 + }, + { + "epoch": 0.99, + "learning_rate": 7.582748244734203e-06, + "loss": 1.3263, + "theoretical_loss": 3.3270680178888012, + "tokens_seen": 3275554816 + }, + { + "epoch": 0.99, + "learning_rate": 7.562688064192578e-06, + "loss": 1.2838, + "theoretical_loss": 3.3270630337231486, + "tokens_seen": 3275620352 + }, + { + "epoch": 0.99, + "learning_rate": 7.542627883650953e-06, + "loss": 1.1717, + "theoretical_loss": 3.3270580496851347, + "tokens_seen": 3275685888 + }, + { + "epoch": 0.99, + "learning_rate": 7.522567703109327e-06, + "loss": 1.2191, + "theoretical_loss": 3.327053065774754, + "tokens_seen": 3275751424 + }, + { + "epoch": 0.99, + "learning_rate": 7.502507522567703e-06, + "loss": 1.2855, + "theoretical_loss": 3.3270480819920003, + "tokens_seen": 3275816960 + }, + { + "epoch": 0.99, + "learning_rate": 7.482447342026078e-06, + "loss": 1.2846, + "theoretical_loss": 3.3270430983368686, + "tokens_seen": 3275882496 + }, + { + "epoch": 0.99, + "learning_rate": 7.462387161484454e-06, + "loss": 1.2448, + "theoretical_loss": 3.327038114809352, + "tokens_seen": 3275948032 + }, + { + "epoch": 0.99, + "learning_rate": 7.442326980942829e-06, + "loss": 1.2377, + "theoretical_loss": 3.327033131409446, + "tokens_seen": 3276013568 + }, + { + "epoch": 0.99, + "learning_rate": 7.422266800401204e-06, + "loss": 1.3179, + "theoretical_loss": 3.3270281481371438, + "tokens_seen": 3276079104 + }, + { + "epoch": 0.99, + "learning_rate": 7.402206619859579e-06, + "loss": 1.1789, + "theoretical_loss": 3.3270231649924398, + "tokens_seen": 3276144640 + }, + { + "epoch": 0.99, + "learning_rate": 7.382146439317954e-06, + "loss": 1.2516, + "theoretical_loss": 3.327018181975328, + "tokens_seen": 3276210176 + }, + { + "epoch": 0.99, + "learning_rate": 7.362086258776329e-06, + "loss": 1.4164, + "theoretical_loss": 3.3270131990858025, + "tokens_seen": 3276275712 + }, + { + "epoch": 0.99, + "learning_rate": 7.3420260782347045e-06, + "loss": 1.2033, + "theoretical_loss": 3.3270082163238586, + "tokens_seen": 3276341248 + }, + { + "epoch": 0.99, + "learning_rate": 7.3219658976930794e-06, + "loss": 1.2229, + "theoretical_loss": 3.327003233689489, + "tokens_seen": 3276406784 + }, + { + "epoch": 0.99, + "learning_rate": 7.301905717151454e-06, + "loss": 0.9995, + "theoretical_loss": 3.3269982511826885, + "tokens_seen": 3276472320 + }, + { + "epoch": 0.99, + "learning_rate": 7.28184553660983e-06, + "loss": 1.1979, + "theoretical_loss": 3.3269932688034514, + "tokens_seen": 3276537856 + }, + { + "epoch": 0.99, + "learning_rate": 7.261785356068204e-06, + "loss": 1.3219, + "theoretical_loss": 3.3269882865517717, + "tokens_seen": 3276603392 + }, + { + "epoch": 0.99, + "learning_rate": 7.241725175526579e-06, + "loss": 1.1667, + "theoretical_loss": 3.326983304427644, + "tokens_seen": 3276668928 + }, + { + "epoch": 0.99, + "learning_rate": 7.221664994984955e-06, + "loss": 1.2598, + "theoretical_loss": 3.326978322431062, + "tokens_seen": 3276734464 + }, + { + "epoch": 0.99, + "learning_rate": 7.20160481444333e-06, + "loss": 1.1848, + "theoretical_loss": 3.3269733405620197, + "tokens_seen": 3276800000 + }, + { + "epoch": 0.99, + "learning_rate": 7.181544633901706e-06, + "loss": 1.166, + "theoretical_loss": 3.326968358820512, + "tokens_seen": 3276865536 + }, + { + "epoch": 0.99, + "learning_rate": 7.161484453360081e-06, + "loss": 1.1907, + "theoretical_loss": 3.326963377206533, + "tokens_seen": 3276931072 + }, + { + "epoch": 0.99, + "learning_rate": 7.141424272818456e-06, + "loss": 1.2616, + "theoretical_loss": 3.3269583957200757, + "tokens_seen": 3276996608 + }, + { + "epoch": 0.99, + "learning_rate": 7.121364092276831e-06, + "loss": 1.2074, + "theoretical_loss": 3.3269534143611357, + "tokens_seen": 3277062144 + }, + { + "epoch": 0.99, + "learning_rate": 7.1013039117352055e-06, + "loss": 1.2233, + "theoretical_loss": 3.3269484331297066, + "tokens_seen": 3277127680 + }, + { + "epoch": 0.99, + "learning_rate": 7.0812437311935805e-06, + "loss": 1.2435, + "theoretical_loss": 3.3269434520257826, + "tokens_seen": 3277193216 + }, + { + "epoch": 0.99, + "learning_rate": 7.061183550651956e-06, + "loss": 1.2749, + "theoretical_loss": 3.326938471049358, + "tokens_seen": 3277258752 + }, + { + "epoch": 0.99, + "learning_rate": 7.041123370110331e-06, + "loss": 1.2484, + "theoretical_loss": 3.326933490200427, + "tokens_seen": 3277324288 + }, + { + "epoch": 0.99, + "learning_rate": 7.021063189568706e-06, + "loss": 1.1987, + "theoretical_loss": 3.326928509478983, + "tokens_seen": 3277389824 + }, + { + "epoch": 0.99, + "learning_rate": 7.001003009027082e-06, + "loss": 1.1654, + "theoretical_loss": 3.326923528885022, + "tokens_seen": 3277455360 + }, + { + "epoch": 0.99, + "learning_rate": 6.980942828485456e-06, + "loss": 1.3083, + "theoretical_loss": 3.3269185484185364, + "tokens_seen": 3277520896 + }, + { + "epoch": 0.99, + "learning_rate": 6.960882647943831e-06, + "loss": 1.1691, + "theoretical_loss": 3.326913568079521, + "tokens_seen": 3277586432 + }, + { + "epoch": 0.99, + "learning_rate": 6.940822467402207e-06, + "loss": 1.2001, + "theoretical_loss": 3.3269085878679703, + "tokens_seen": 3277651968 + }, + { + "epoch": 0.99, + "learning_rate": 6.920762286860582e-06, + "loss": 1.1655, + "theoretical_loss": 3.326903607783878, + "tokens_seen": 3277717504 + }, + { + "epoch": 0.99, + "learning_rate": 6.9007021063189575e-06, + "loss": 1.2866, + "theoretical_loss": 3.326898627827239, + "tokens_seen": 3277783040 + }, + { + "epoch": 0.99, + "learning_rate": 6.8806419257773325e-06, + "loss": 1.2919, + "theoretical_loss": 3.326893647998047, + "tokens_seen": 3277848576 + }, + { + "epoch": 0.99, + "learning_rate": 6.860581745235707e-06, + "loss": 1.2182, + "theoretical_loss": 3.326888668296296, + "tokens_seen": 3277914112 + }, + { + "epoch": 0.99, + "learning_rate": 6.840521564694082e-06, + "loss": 1.2619, + "theoretical_loss": 3.3268836887219804, + "tokens_seen": 3277979648 + }, + { + "epoch": 0.99, + "learning_rate": 6.820461384152457e-06, + "loss": 1.298, + "theoretical_loss": 3.326878709275094, + "tokens_seen": 3278045184 + }, + { + "epoch": 0.99, + "learning_rate": 6.800401203610832e-06, + "loss": 1.25, + "theoretical_loss": 3.326873729955632, + "tokens_seen": 3278110720 + }, + { + "epoch": 0.99, + "learning_rate": 6.780341023069208e-06, + "loss": 1.2109, + "theoretical_loss": 3.3268687507635875, + "tokens_seen": 3278176256 + }, + { + "epoch": 0.99, + "learning_rate": 6.760280842527583e-06, + "loss": 1.2643, + "theoretical_loss": 3.3268637716989558, + "tokens_seen": 3278241792 + }, + { + "epoch": 0.99, + "learning_rate": 6.740220661985958e-06, + "loss": 1.2653, + "theoretical_loss": 3.3268587927617297, + "tokens_seen": 3278307328 + }, + { + "epoch": 0.99, + "learning_rate": 6.720160481444334e-06, + "loss": 1.2195, + "theoretical_loss": 3.326853813951905, + "tokens_seen": 3278372864 + }, + { + "epoch": 0.99, + "learning_rate": 6.700100300902708e-06, + "loss": 1.2569, + "theoretical_loss": 3.326848835269474, + "tokens_seen": 3278438400 + }, + { + "epoch": 0.99, + "learning_rate": 6.680040120361083e-06, + "loss": 1.2175, + "theoretical_loss": 3.326843856714433, + "tokens_seen": 3278503936 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1191058, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.281537413597107, + "objective/train/theoretical_loss": 3.3268388782867744, + "objective/train/tokens_used": 1649028576, + "theoretical_loss": 3.3268388782867744, + "tokens_seen": 3278569472 + }, + { + "epoch": 0.99, + "learning_rate": 6.659979939819459e-06, + "loss": 1.2865, + "theoretical_loss": 3.3268388782867744, + "tokens_seen": 3278569472 + }, + { + "epoch": 0.99, + "learning_rate": 6.6399197592778335e-06, + "loss": 1.3637, + "theoretical_loss": 3.3268338999864935, + "tokens_seen": 3278635008 + }, + { + "epoch": 0.99, + "learning_rate": 6.619859578736209e-06, + "loss": 1.1381, + "theoretical_loss": 3.326828921813584, + "tokens_seen": 3278700544 + }, + { + "epoch": 0.99, + "learning_rate": 6.599799398194584e-06, + "loss": 1.2147, + "theoretical_loss": 3.3268239437680402, + "tokens_seen": 3278766080 + }, + { + "epoch": 0.99, + "learning_rate": 6.579739217652959e-06, + "loss": 1.2252, + "theoretical_loss": 3.3268189658498564, + "tokens_seen": 3278831616 + }, + { + "epoch": 0.99, + "learning_rate": 6.559679037111334e-06, + "loss": 1.3043, + "theoretical_loss": 3.326813988059027, + "tokens_seen": 3278897152 + }, + { + "epoch": 0.99, + "learning_rate": 6.539618856569709e-06, + "loss": 1.1832, + "theoretical_loss": 3.3268090103955457, + "tokens_seen": 3278962688 + }, + { + "epoch": 0.99, + "learning_rate": 6.519558676028084e-06, + "loss": 1.232, + "theoretical_loss": 3.326804032859407, + "tokens_seen": 3279028224 + }, + { + "epoch": 0.99, + "learning_rate": 6.49949849548646e-06, + "loss": 1.2406, + "theoretical_loss": 3.3267990554506044, + "tokens_seen": 3279093760 + }, + { + "epoch": 0.99, + "learning_rate": 6.479438314944835e-06, + "loss": 1.3241, + "theoretical_loss": 3.326794078169133, + "tokens_seen": 3279159296 + }, + { + "epoch": 0.99, + "learning_rate": 6.45937813440321e-06, + "loss": 1.286, + "theoretical_loss": 3.3267891010149873, + "tokens_seen": 3279224832 + }, + { + "epoch": 0.99, + "learning_rate": 6.4393179538615855e-06, + "loss": 1.3377, + "theoretical_loss": 3.32678412398816, + "tokens_seen": 3279290368 + }, + { + "epoch": 0.99, + "learning_rate": 6.41925777331996e-06, + "loss": 1.2111, + "theoretical_loss": 3.3267791470886467, + "tokens_seen": 3279355904 + }, + { + "epoch": 0.99, + "learning_rate": 6.3991975927783346e-06, + "loss": 1.2186, + "theoretical_loss": 3.3267741703164413, + "tokens_seen": 3279421440 + }, + { + "epoch": 0.99, + "learning_rate": 6.37913741223671e-06, + "loss": 1.2705, + "theoretical_loss": 3.326769193671537, + "tokens_seen": 3279486976 + }, + { + "epoch": 0.99, + "learning_rate": 6.359077231695085e-06, + "loss": 1.153, + "theoretical_loss": 3.32676421715393, + "tokens_seen": 3279552512 + }, + { + "epoch": 0.99, + "learning_rate": 6.33901705115346e-06, + "loss": 1.1856, + "theoretical_loss": 3.326759240763612, + "tokens_seen": 3279618048 + }, + { + "epoch": 0.99, + "learning_rate": 6.318956870611836e-06, + "loss": 1.1462, + "theoretical_loss": 3.3267542645005794, + "tokens_seen": 3279683584 + }, + { + "epoch": 0.99, + "learning_rate": 6.298896690070211e-06, + "loss": 1.1243, + "theoretical_loss": 3.3267492883648253, + "tokens_seen": 3279749120 + }, + { + "epoch": 0.99, + "learning_rate": 6.278836509528586e-06, + "loss": 1.2597, + "theoretical_loss": 3.326744312356344, + "tokens_seen": 3279814656 + }, + { + "epoch": 0.99, + "learning_rate": 6.258776328986961e-06, + "loss": 1.4619, + "theoretical_loss": 3.32673933647513, + "tokens_seen": 3279880192 + }, + { + "epoch": 0.99, + "learning_rate": 6.238716148445336e-06, + "loss": 1.2687, + "theoretical_loss": 3.326734360721177, + "tokens_seen": 3279945728 + }, + { + "epoch": 0.99, + "learning_rate": 6.218655967903712e-06, + "loss": 1.1921, + "theoretical_loss": 3.3267293850944797, + "tokens_seen": 3280011264 + }, + { + "epoch": 0.99, + "learning_rate": 6.1985957873620866e-06, + "loss": 1.3959, + "theoretical_loss": 3.326724409595032, + "tokens_seen": 3280076800 + }, + { + "epoch": 0.99, + "learning_rate": 6.1785356068204615e-06, + "loss": 1.2609, + "theoretical_loss": 3.326719434222828, + "tokens_seen": 3280142336 + }, + { + "epoch": 0.99, + "learning_rate": 6.158475426278837e-06, + "loss": 1.3338, + "theoretical_loss": 3.3267144589778628, + "tokens_seen": 3280207872 + }, + { + "epoch": 0.99, + "learning_rate": 6.138415245737211e-06, + "loss": 1.2287, + "theoretical_loss": 3.326709483860129, + "tokens_seen": 3280273408 + }, + { + "epoch": 0.99, + "learning_rate": 6.118355065195586e-06, + "loss": 1.1483, + "theoretical_loss": 3.3267045088696223, + "tokens_seen": 3280338944 + }, + { + "epoch": 0.99, + "learning_rate": 6.098294884653962e-06, + "loss": 1.1787, + "theoretical_loss": 3.3266995340063366, + "tokens_seen": 3280404480 + }, + { + "epoch": 0.99, + "learning_rate": 6.078234704112337e-06, + "loss": 1.3678, + "theoretical_loss": 3.3266945592702655, + "tokens_seen": 3280470016 + }, + { + "epoch": 0.99, + "learning_rate": 6.058174523570712e-06, + "loss": 1.2575, + "theoretical_loss": 3.3266895846614033, + "tokens_seen": 3280535552 + }, + { + "epoch": 0.99, + "learning_rate": 6.038114343029088e-06, + "loss": 1.1839, + "theoretical_loss": 3.3266846101797447, + "tokens_seen": 3280601088 + }, + { + "epoch": 0.99, + "learning_rate": 6.018054162487462e-06, + "loss": 1.2354, + "theoretical_loss": 3.3266796358252835, + "tokens_seen": 3280666624 + }, + { + "epoch": 0.99, + "learning_rate": 5.997993981945838e-06, + "loss": 1.2435, + "theoretical_loss": 3.3266746615980143, + "tokens_seen": 3280732160 + }, + { + "epoch": 0.99, + "learning_rate": 5.977933801404213e-06, + "loss": 1.2373, + "theoretical_loss": 3.3266696874979313, + "tokens_seen": 3280797696 + }, + { + "epoch": 0.99, + "learning_rate": 5.957873620862588e-06, + "loss": 1.2502, + "theoretical_loss": 3.326664713525028, + "tokens_seen": 3280863232 + }, + { + "epoch": 0.99, + "learning_rate": 5.937813440320963e-06, + "loss": 1.0898, + "theoretical_loss": 3.326659739679299, + "tokens_seen": 3280928768 + }, + { + "epoch": 0.99, + "learning_rate": 5.917753259779338e-06, + "loss": 1.2984, + "theoretical_loss": 3.326654765960739, + "tokens_seen": 3280994304 + }, + { + "epoch": 0.99, + "learning_rate": 5.897693079237713e-06, + "loss": 1.2581, + "theoretical_loss": 3.3266497923693414, + "tokens_seen": 3281059840 + }, + { + "epoch": 0.99, + "learning_rate": 5.877632898696088e-06, + "loss": 1.2397, + "theoretical_loss": 3.326644818905101, + "tokens_seen": 3281125376 + }, + { + "epoch": 0.99, + "learning_rate": 5.857572718154463e-06, + "loss": 1.3176, + "theoretical_loss": 3.326639845568012, + "tokens_seen": 3281190912 + }, + { + "epoch": 0.99, + "learning_rate": 5.837512537612838e-06, + "loss": 1.1153, + "theoretical_loss": 3.3266348723580683, + "tokens_seen": 3281256448 + }, + { + "epoch": 0.99, + "learning_rate": 5.817452357071214e-06, + "loss": 1.2397, + "theoretical_loss": 3.326629899275264, + "tokens_seen": 3281321984 + }, + { + "epoch": 0.99, + "learning_rate": 5.797392176529589e-06, + "loss": 1.2683, + "theoretical_loss": 3.326624926319594, + "tokens_seen": 3281387520 + }, + { + "epoch": 0.99, + "learning_rate": 5.777331995987964e-06, + "loss": 1.2463, + "theoretical_loss": 3.326619953491052, + "tokens_seen": 3281453056 + }, + { + "epoch": 0.99, + "learning_rate": 5.75727181544634e-06, + "loss": 1.2091, + "theoretical_loss": 3.326614980789632, + "tokens_seen": 3281518592 + }, + { + "epoch": 0.99, + "learning_rate": 5.737211634904714e-06, + "loss": 1.245, + "theoretical_loss": 3.3266100082153285, + "tokens_seen": 3281584128 + }, + { + "epoch": 0.99, + "learning_rate": 5.7171514543630895e-06, + "loss": 1.283, + "theoretical_loss": 3.326605035768136, + "tokens_seen": 3281649664 + }, + { + "epoch": 0.99, + "learning_rate": 5.6970912738214645e-06, + "loss": 1.2049, + "theoretical_loss": 3.3266000634480477, + "tokens_seen": 3281715200 + }, + { + "epoch": 0.99, + "learning_rate": 5.677031093279839e-06, + "loss": 1.2097, + "theoretical_loss": 3.326595091255059, + "tokens_seen": 3281780736 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1192226, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3532216548919678, + "objective/train/theoretical_loss": 3.326590119189164, + "objective/train/tokens_used": 1652305376, + "theoretical_loss": 3.326590119189164, + "tokens_seen": 3281846272 + }, + { + "epoch": 0.99, + "learning_rate": 5.656970912738215e-06, + "loss": 1.2546, + "theoretical_loss": 3.326590119189164, + "tokens_seen": 3281846272 + }, + { + "epoch": 0.99, + "learning_rate": 5.63691073219659e-06, + "loss": 1.2939, + "theoretical_loss": 3.3265851472503556, + "tokens_seen": 3281911808 + }, + { + "epoch": 0.99, + "learning_rate": 5.616850551654965e-06, + "loss": 1.3417, + "theoretical_loss": 3.3265801754386297, + "tokens_seen": 3281977344 + }, + { + "epoch": 0.99, + "learning_rate": 5.59679037111334e-06, + "loss": 1.2101, + "theoretical_loss": 3.3265752037539795, + "tokens_seen": 3282042880 + }, + { + "epoch": 0.99, + "learning_rate": 5.576730190571715e-06, + "loss": 1.2092, + "theoretical_loss": 3.3265702321963997, + "tokens_seen": 3282108416 + }, + { + "epoch": 0.99, + "learning_rate": 5.55667001003009e-06, + "loss": 1.2494, + "theoretical_loss": 3.326565260765884, + "tokens_seen": 3282173952 + }, + { + "epoch": 0.99, + "learning_rate": 5.536609829488466e-06, + "loss": 1.2048, + "theoretical_loss": 3.326560289462427, + "tokens_seen": 3282239488 + }, + { + "epoch": 0.99, + "learning_rate": 5.516549648946841e-06, + "loss": 1.2325, + "theoretical_loss": 3.3265553182860232, + "tokens_seen": 3282305024 + }, + { + "epoch": 0.99, + "learning_rate": 5.496489468405216e-06, + "loss": 1.245, + "theoretical_loss": 3.326550347236666, + "tokens_seen": 3282370560 + }, + { + "epoch": 0.99, + "learning_rate": 5.476429287863591e-06, + "loss": 1.1769, + "theoretical_loss": 3.32654537631435, + "tokens_seen": 3282436096 + }, + { + "epoch": 0.99, + "learning_rate": 5.4563691073219655e-06, + "loss": 1.3459, + "theoretical_loss": 3.3265404055190704, + "tokens_seen": 3282501632 + }, + { + "epoch": 0.99, + "learning_rate": 5.436308926780341e-06, + "loss": 1.1513, + "theoretical_loss": 3.3265354348508196, + "tokens_seen": 3282567168 + }, + { + "epoch": 0.99, + "learning_rate": 5.416248746238716e-06, + "loss": 1.2469, + "theoretical_loss": 3.326530464309593, + "tokens_seen": 3282632704 + }, + { + "epoch": 0.99, + "learning_rate": 5.396188565697091e-06, + "loss": 1.2873, + "theoretical_loss": 3.3265254938953843, + "tokens_seen": 3282698240 + }, + { + "epoch": 0.99, + "learning_rate": 5.376128385155467e-06, + "loss": 1.2867, + "theoretical_loss": 3.3265205236081883, + "tokens_seen": 3282763776 + }, + { + "epoch": 0.99, + "learning_rate": 5.356068204613842e-06, + "loss": 1.2819, + "theoretical_loss": 3.3265155534479987, + "tokens_seen": 3282829312 + }, + { + "epoch": 0.99, + "learning_rate": 5.336008024072217e-06, + "loss": 1.2396, + "theoretical_loss": 3.3265105834148097, + "tokens_seen": 3282894848 + }, + { + "epoch": 0.99, + "learning_rate": 5.315947843530592e-06, + "loss": 1.3211, + "theoretical_loss": 3.326505613508616, + "tokens_seen": 3282960384 + }, + { + "epoch": 0.99, + "learning_rate": 5.295887662988967e-06, + "loss": 1.2013, + "theoretical_loss": 3.3265006437294113, + "tokens_seen": 3283025920 + }, + { + "epoch": 0.99, + "learning_rate": 5.275827482447342e-06, + "loss": 1.2709, + "theoretical_loss": 3.3264956740771905, + "tokens_seen": 3283091456 + }, + { + "epoch": 0.99, + "learning_rate": 5.2557673019057175e-06, + "loss": 1.2177, + "theoretical_loss": 3.326490704551947, + "tokens_seen": 3283156992 + }, + { + "epoch": 0.99, + "learning_rate": 5.2357071213640925e-06, + "loss": 1.2369, + "theoretical_loss": 3.3264857351536756, + "tokens_seen": 3283222528 + }, + { + "epoch": 0.99, + "learning_rate": 5.215646940822467e-06, + "loss": 1.2898, + "theoretical_loss": 3.32648076588237, + "tokens_seen": 3283288064 + }, + { + "epoch": 0.99, + "learning_rate": 5.195586760280843e-06, + "loss": 1.1685, + "theoretical_loss": 3.326475796738025, + "tokens_seen": 3283353600 + }, + { + "epoch": 0.99, + "learning_rate": 5.175526579739217e-06, + "loss": 1.2923, + "theoretical_loss": 3.3264708277206347, + "tokens_seen": 3283419136 + }, + { + "epoch": 0.99, + "learning_rate": 5.155466399197592e-06, + "loss": 1.3577, + "theoretical_loss": 3.326465858830193, + "tokens_seen": 3283484672 + }, + { + "epoch": 0.99, + "learning_rate": 5.135406218655968e-06, + "loss": 1.1845, + "theoretical_loss": 3.326460890066694, + "tokens_seen": 3283550208 + }, + { + "epoch": 0.99, + "learning_rate": 5.115346038114343e-06, + "loss": 1.1306, + "theoretical_loss": 3.3264559214301324, + "tokens_seen": 3283615744 + }, + { + "epoch": 0.99, + "learning_rate": 5.095285857572719e-06, + "loss": 1.26, + "theoretical_loss": 3.3264509529205024, + "tokens_seen": 3283681280 + }, + { + "epoch": 0.99, + "learning_rate": 5.075225677031094e-06, + "loss": 1.2087, + "theoretical_loss": 3.3264459845377985, + "tokens_seen": 3283746816 + }, + { + "epoch": 0.99, + "learning_rate": 5.055165496489469e-06, + "loss": 1.2744, + "theoretical_loss": 3.326441016282014, + "tokens_seen": 3283812352 + }, + { + "epoch": 0.99, + "learning_rate": 5.035105315947844e-06, + "loss": 1.2052, + "theoretical_loss": 3.3264360481531434, + "tokens_seen": 3283877888 + }, + { + "epoch": 0.99, + "learning_rate": 5.0150451354062186e-06, + "loss": 1.2628, + "theoretical_loss": 3.3264310801511816, + "tokens_seen": 3283943424 + }, + { + "epoch": 0.99, + "learning_rate": 4.9949849548645935e-06, + "loss": 1.1248, + "theoretical_loss": 3.3264261122761223, + "tokens_seen": 3284008960 + }, + { + "epoch": 0.99, + "learning_rate": 4.974924774322969e-06, + "loss": 1.145, + "theoretical_loss": 3.3264211445279592, + "tokens_seen": 3284074496 + }, + { + "epoch": 0.99, + "learning_rate": 4.954864593781344e-06, + "loss": 1.1055, + "theoretical_loss": 3.326416176906688, + "tokens_seen": 3284140032 + }, + { + "epoch": 0.99, + "learning_rate": 4.934804413239719e-06, + "loss": 1.093, + "theoretical_loss": 3.3264112094123015, + "tokens_seen": 3284205568 + }, + { + "epoch": 0.99, + "learning_rate": 4.914744232698095e-06, + "loss": 1.197, + "theoretical_loss": 3.326406242044795, + "tokens_seen": 3284271104 + }, + { + "epoch": 0.99, + "learning_rate": 4.894684052156469e-06, + "loss": 1.161, + "theoretical_loss": 3.326401274804162, + "tokens_seen": 3284336640 + }, + { + "epoch": 0.99, + "learning_rate": 4.874623871614844e-06, + "loss": 1.1979, + "theoretical_loss": 3.3263963076903966, + "tokens_seen": 3284402176 + }, + { + "epoch": 0.99, + "learning_rate": 4.85456369107322e-06, + "loss": 1.1886, + "theoretical_loss": 3.3263913407034935, + "tokens_seen": 3284467712 + }, + { + "epoch": 0.99, + "learning_rate": 4.834503510531595e-06, + "loss": 1.2801, + "theoretical_loss": 3.326386373843447, + "tokens_seen": 3284533248 + }, + { + "epoch": 0.99, + "learning_rate": 4.8144433299899706e-06, + "loss": 1.238, + "theoretical_loss": 3.326381407110251, + "tokens_seen": 3284598784 + }, + { + "epoch": 0.99, + "learning_rate": 4.7943831494483455e-06, + "loss": 1.2238, + "theoretical_loss": 3.3263764405039, + "tokens_seen": 3284664320 + }, + { + "epoch": 0.99, + "learning_rate": 4.77432296890672e-06, + "loss": 1.3227, + "theoretical_loss": 3.3263714740243877, + "tokens_seen": 3284729856 + }, + { + "epoch": 0.99, + "learning_rate": 4.754262788365095e-06, + "loss": 1.1752, + "theoretical_loss": 3.3263665076717093, + "tokens_seen": 3284795392 + }, + { + "epoch": 0.99, + "learning_rate": 4.73420260782347e-06, + "loss": 1.3283, + "theoretical_loss": 3.326361541445858, + "tokens_seen": 3284860928 + }, + { + "epoch": 0.99, + "learning_rate": 4.714142427281845e-06, + "loss": 1.1776, + "theoretical_loss": 3.3263565753468285, + "tokens_seen": 3284926464 + }, + { + "epoch": 0.99, + "learning_rate": 4.694082246740221e-06, + "loss": 1.0673, + "theoretical_loss": 3.3263516093746155, + "tokens_seen": 3284992000 + }, + { + "epoch": 0.99, + "learning_rate": 4.674022066198596e-06, + "loss": 1.2555, + "theoretical_loss": 3.326346643529212, + "tokens_seen": 3285057536 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1193279, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.219916582107544, + "objective/train/theoretical_loss": 3.3263416778106136, + "objective/train/tokens_used": 1655582176, + "theoretical_loss": 3.3263416778106136, + "tokens_seen": 3285123072 + }, + { + "epoch": 0.99, + "learning_rate": 4.653961885656971e-06, + "loss": 1.2843, + "theoretical_loss": 3.3263416778106136, + "tokens_seen": 3285123072 + }, + { + "epoch": 0.99, + "learning_rate": 4.633901705115346e-06, + "loss": 1.1641, + "theoretical_loss": 3.3263367122188137, + "tokens_seen": 3285188608 + }, + { + "epoch": 0.99, + "learning_rate": 4.613841524573721e-06, + "loss": 1.213, + "theoretical_loss": 3.3263317467538065, + "tokens_seen": 3285254144 + }, + { + "epoch": 0.99, + "learning_rate": 4.593781344032096e-06, + "loss": 1.205, + "theoretical_loss": 3.3263267814155864, + "tokens_seen": 3285319680 + }, + { + "epoch": 0.99, + "learning_rate": 4.573721163490472e-06, + "loss": 1.3009, + "theoretical_loss": 3.3263218162041484, + "tokens_seen": 3285385216 + }, + { + "epoch": 0.99, + "learning_rate": 4.5536609829488466e-06, + "loss": 1.2798, + "theoretical_loss": 3.3263168511194854, + "tokens_seen": 3285450752 + }, + { + "epoch": 0.99, + "learning_rate": 4.533600802407222e-06, + "loss": 1.2068, + "theoretical_loss": 3.3263118861615926, + "tokens_seen": 3285516288 + }, + { + "epoch": 0.99, + "learning_rate": 4.513540621865597e-06, + "loss": 1.3391, + "theoretical_loss": 3.3263069213304637, + "tokens_seen": 3285581824 + }, + { + "epoch": 0.99, + "learning_rate": 4.493480441323971e-06, + "loss": 1.2679, + "theoretical_loss": 3.326301956626093, + "tokens_seen": 3285647360 + }, + { + "epoch": 0.99, + "learning_rate": 4.473420260782347e-06, + "loss": 1.2227, + "theoretical_loss": 3.326296992048475, + "tokens_seen": 3285712896 + }, + { + "epoch": 0.99, + "learning_rate": 4.453360080240722e-06, + "loss": 1.2722, + "theoretical_loss": 3.326292027597604, + "tokens_seen": 3285778432 + }, + { + "epoch": 0.99, + "learning_rate": 4.433299899699097e-06, + "loss": 1.2676, + "theoretical_loss": 3.3262870632734742, + "tokens_seen": 3285843968 + }, + { + "epoch": 0.99, + "learning_rate": 4.413239719157473e-06, + "loss": 1.2457, + "theoretical_loss": 3.3262820990760793, + "tokens_seen": 3285909504 + }, + { + "epoch": 0.99, + "learning_rate": 4.393179538615848e-06, + "loss": 1.2175, + "theoretical_loss": 3.326277135005414, + "tokens_seen": 3285975040 + }, + { + "epoch": 0.99, + "learning_rate": 4.373119358074223e-06, + "loss": 1.1674, + "theoretical_loss": 3.3262721710614724, + "tokens_seen": 3286040576 + }, + { + "epoch": 0.99, + "learning_rate": 4.353059177532598e-06, + "loss": 1.1571, + "theoretical_loss": 3.326267207244249, + "tokens_seen": 3286106112 + }, + { + "epoch": 0.99, + "learning_rate": 4.332998996990973e-06, + "loss": 1.2456, + "theoretical_loss": 3.326262243553738, + "tokens_seen": 3286171648 + }, + { + "epoch": 0.99, + "learning_rate": 4.312938816449348e-06, + "loss": 1.2905, + "theoretical_loss": 3.326257279989933, + "tokens_seen": 3286237184 + }, + { + "epoch": 0.99, + "learning_rate": 4.292878635907723e-06, + "loss": 1.2521, + "theoretical_loss": 3.326252316552829, + "tokens_seen": 3286302720 + }, + { + "epoch": 0.99, + "learning_rate": 4.272818455366098e-06, + "loss": 1.2651, + "theoretical_loss": 3.3262473532424197, + "tokens_seen": 3286368256 + }, + { + "epoch": 0.99, + "learning_rate": 4.252758274824474e-06, + "loss": 1.1995, + "theoretical_loss": 3.3262423900586997, + "tokens_seen": 3286433792 + }, + { + "epoch": 0.99, + "learning_rate": 4.232698094282849e-06, + "loss": 1.2482, + "theoretical_loss": 3.3262374270016632, + "tokens_seen": 3286499328 + }, + { + "epoch": 0.99, + "learning_rate": 4.212637913741223e-06, + "loss": 1.185, + "theoretical_loss": 3.3262324640713046, + "tokens_seen": 3286564864 + }, + { + "epoch": 0.99, + "learning_rate": 4.192577733199599e-06, + "loss": 1.247, + "theoretical_loss": 3.326227501267618, + "tokens_seen": 3286630400 + }, + { + "epoch": 0.99, + "learning_rate": 4.172517552657974e-06, + "loss": 1.2239, + "theoretical_loss": 3.326222538590597, + "tokens_seen": 3286695936 + }, + { + "epoch": 0.99, + "learning_rate": 4.152457372116349e-06, + "loss": 1.1768, + "theoretical_loss": 3.3262175760402366, + "tokens_seen": 3286761472 + }, + { + "epoch": 0.99, + "learning_rate": 4.132397191574725e-06, + "loss": 1.2245, + "theoretical_loss": 3.326212613616531, + "tokens_seen": 3286827008 + }, + { + "epoch": 0.99, + "learning_rate": 4.1123370110331e-06, + "loss": 1.3168, + "theoretical_loss": 3.326207651319474, + "tokens_seen": 3286892544 + }, + { + "epoch": 0.99, + "learning_rate": 4.0922768304914746e-06, + "loss": 1.301, + "theoretical_loss": 3.3262026891490604, + "tokens_seen": 3286958080 + }, + { + "epoch": 0.99, + "learning_rate": 4.0722166499498495e-06, + "loss": 1.1921, + "theoretical_loss": 3.326197727105284, + "tokens_seen": 3287023616 + }, + { + "epoch": 0.99, + "learning_rate": 4.0521564694082245e-06, + "loss": 1.2488, + "theoretical_loss": 3.3261927651881393, + "tokens_seen": 3287089152 + }, + { + "epoch": 0.99, + "learning_rate": 4.032096288866599e-06, + "loss": 1.2205, + "theoretical_loss": 3.3261878033976204, + "tokens_seen": 3287154688 + }, + { + "epoch": 0.99, + "learning_rate": 4.012036108324975e-06, + "loss": 1.2707, + "theoretical_loss": 3.3261828417337216, + "tokens_seen": 3287220224 + }, + { + "epoch": 0.99, + "learning_rate": 3.99197592778335e-06, + "loss": 1.1292, + "theoretical_loss": 3.326177880196437, + "tokens_seen": 3287285760 + }, + { + "epoch": 0.99, + "learning_rate": 3.971915747241726e-06, + "loss": 1.2631, + "theoretical_loss": 3.3261729187857614, + "tokens_seen": 3287351296 + }, + { + "epoch": 0.99, + "learning_rate": 3.951855566700101e-06, + "loss": 1.1814, + "theoretical_loss": 3.3261679575016885, + "tokens_seen": 3287416832 + }, + { + "epoch": 0.99, + "learning_rate": 3.931795386158475e-06, + "loss": 1.158, + "theoretical_loss": 3.3261629963442125, + "tokens_seen": 3287482368 + }, + { + "epoch": 0.99, + "learning_rate": 3.911735205616851e-06, + "loss": 1.1857, + "theoretical_loss": 3.3261580353133278, + "tokens_seen": 3287547904 + }, + { + "epoch": 0.99, + "learning_rate": 3.891675025075226e-06, + "loss": 1.2503, + "theoretical_loss": 3.326153074409029, + "tokens_seen": 3287613440 + }, + { + "epoch": 0.99, + "learning_rate": 3.871614844533601e-06, + "loss": 1.2186, + "theoretical_loss": 3.3261481136313096, + "tokens_seen": 3287678976 + }, + { + "epoch": 0.99, + "learning_rate": 3.8515546639919765e-06, + "loss": 1.1396, + "theoretical_loss": 3.326143152980164, + "tokens_seen": 3287744512 + }, + { + "epoch": 0.99, + "learning_rate": 3.8314944834503506e-06, + "loss": 1.1752, + "theoretical_loss": 3.3261381924555877, + "tokens_seen": 3287810048 + }, + { + "epoch": 0.99, + "learning_rate": 3.8114343029087263e-06, + "loss": 1.2233, + "theoretical_loss": 3.326133232057573, + "tokens_seen": 3287875584 + }, + { + "epoch": 0.99, + "learning_rate": 3.7913741223671013e-06, + "loss": 1.2527, + "theoretical_loss": 3.3261282717861156, + "tokens_seen": 3287941120 + }, + { + "epoch": 0.99, + "learning_rate": 3.7713139418254767e-06, + "loss": 1.1023, + "theoretical_loss": 3.3261233116412092, + "tokens_seen": 3288006656 + }, + { + "epoch": 0.99, + "learning_rate": 3.7512537612838516e-06, + "loss": 1.2622, + "theoretical_loss": 3.3261183516228483, + "tokens_seen": 3288072192 + }, + { + "epoch": 0.99, + "learning_rate": 3.731193580742227e-06, + "loss": 1.2262, + "theoretical_loss": 3.3261133917310266, + "tokens_seen": 3288137728 + }, + { + "epoch": 0.99, + "learning_rate": 3.711133400200602e-06, + "loss": 1.1073, + "theoretical_loss": 3.326108431965739, + "tokens_seen": 3288203264 + }, + { + "epoch": 0.99, + "learning_rate": 3.691073219658977e-06, + "loss": 1.232, + "theoretical_loss": 3.326103472326979, + "tokens_seen": 3288268800 + }, + { + "epoch": 0.99, + "learning_rate": 3.6710130391173522e-06, + "loss": 1.1887, + "theoretical_loss": 3.326098512814742, + "tokens_seen": 3288334336 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.7477778926818737, + "debugging/entropy-1-grams": 4.957593128654467, + "debugging/length": 609.625, + "debugging/num_segments": 8, + "debugging/score": 0.006138877505802302, + "debugging/score_std": 0.007441179575288503, + "epoch": 0.99, + "objective/train/docs_used": 1194491, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.221968650817871, + "objective/train/theoretical_loss": 3.3260935534290206, + "objective/train/tokens_used": 1658858976, + "theoretical_loss": 3.3260935534290206, + "tokens_seen": 3288399872 + }, + { + "epoch": 0.99, + "learning_rate": 3.650952858575727e-06, + "loss": 1.2151, + "theoretical_loss": 3.3260935534290206, + "tokens_seen": 3288399872 + }, + { + "epoch": 0.99, + "learning_rate": 3.630892678034102e-06, + "loss": 1.1775, + "theoretical_loss": 3.3260885941698106, + "tokens_seen": 3288465408 + }, + { + "epoch": 0.99, + "learning_rate": 3.6108324974924775e-06, + "loss": 1.2175, + "theoretical_loss": 3.3260836350371057, + "tokens_seen": 3288530944 + }, + { + "epoch": 0.99, + "learning_rate": 3.590772316950853e-06, + "loss": 1.2625, + "theoretical_loss": 3.3260786760309, + "tokens_seen": 3288596480 + }, + { + "epoch": 0.99, + "learning_rate": 3.570712136409228e-06, + "loss": 1.3513, + "theoretical_loss": 3.326073717151188, + "tokens_seen": 3288662016 + }, + { + "epoch": 0.99, + "learning_rate": 3.5506519558676028e-06, + "loss": 1.3692, + "theoretical_loss": 3.326068758397964, + "tokens_seen": 3288727552 + }, + { + "epoch": 0.99, + "learning_rate": 3.530591775325978e-06, + "loss": 1.329, + "theoretical_loss": 3.3260637997712212, + "tokens_seen": 3288793088 + }, + { + "epoch": 0.99, + "learning_rate": 3.510531594784353e-06, + "loss": 1.2477, + "theoretical_loss": 3.3260588412709553, + "tokens_seen": 3288858624 + }, + { + "epoch": 0.99, + "learning_rate": 3.490471414242728e-06, + "loss": 1.2413, + "theoretical_loss": 3.32605388289716, + "tokens_seen": 3288924160 + }, + { + "epoch": 0.99, + "learning_rate": 3.4704112337011034e-06, + "loss": 1.197, + "theoretical_loss": 3.3260489246498297, + "tokens_seen": 3288989696 + }, + { + "epoch": 0.99, + "learning_rate": 3.4503510531594788e-06, + "loss": 1.2549, + "theoretical_loss": 3.326043966528958, + "tokens_seen": 3289055232 + }, + { + "epoch": 0.99, + "learning_rate": 3.4302908726178537e-06, + "loss": 1.2034, + "theoretical_loss": 3.32603900853454, + "tokens_seen": 3289120768 + }, + { + "epoch": 0.99, + "learning_rate": 3.4102306920762287e-06, + "loss": 1.3714, + "theoretical_loss": 3.3260340506665695, + "tokens_seen": 3289186304 + }, + { + "epoch": 0.99, + "learning_rate": 3.390170511534604e-06, + "loss": 1.216, + "theoretical_loss": 3.326029092925041, + "tokens_seen": 3289251840 + }, + { + "epoch": 0.99, + "learning_rate": 3.370110330992979e-06, + "loss": 1.1607, + "theoretical_loss": 3.3260241353099484, + "tokens_seen": 3289317376 + }, + { + "epoch": 0.99, + "learning_rate": 3.350050150451354e-06, + "loss": 1.1728, + "theoretical_loss": 3.326019177821286, + "tokens_seen": 3289382912 + }, + { + "epoch": 0.99, + "learning_rate": 3.3299899699097293e-06, + "loss": 1.2671, + "theoretical_loss": 3.3260142204590486, + "tokens_seen": 3289448448 + }, + { + "epoch": 0.99, + "learning_rate": 3.3099297893681047e-06, + "loss": 1.1539, + "theoretical_loss": 3.32600926322323, + "tokens_seen": 3289513984 + }, + { + "epoch": 0.99, + "learning_rate": 3.2898696088264796e-06, + "loss": 1.2499, + "theoretical_loss": 3.3260043061138247, + "tokens_seen": 3289579520 + }, + { + "epoch": 0.99, + "learning_rate": 3.2698094282848546e-06, + "loss": 1.2284, + "theoretical_loss": 3.3259993491308264, + "tokens_seen": 3289645056 + }, + { + "epoch": 0.99, + "learning_rate": 3.24974924774323e-06, + "loss": 1.3379, + "theoretical_loss": 3.32599439227423, + "tokens_seen": 3289710592 + }, + { + "epoch": 0.99, + "learning_rate": 3.229689067201605e-06, + "loss": 1.1807, + "theoretical_loss": 3.325989435544029, + "tokens_seen": 3289776128 + }, + { + "epoch": 0.99, + "learning_rate": 3.20962888665998e-06, + "loss": 1.186, + "theoretical_loss": 3.325984478940219, + "tokens_seen": 3289841664 + }, + { + "epoch": 0.99, + "learning_rate": 3.189568706118355e-06, + "loss": 1.24, + "theoretical_loss": 3.3259795224627933, + "tokens_seen": 3289907200 + }, + { + "epoch": 0.99, + "learning_rate": 3.16950852557673e-06, + "loss": 1.256, + "theoretical_loss": 3.325974566111746, + "tokens_seen": 3289972736 + }, + { + "epoch": 0.99, + "learning_rate": 3.1494483450351055e-06, + "loss": 1.1997, + "theoretical_loss": 3.3259696098870712, + "tokens_seen": 3290038272 + }, + { + "epoch": 0.99, + "learning_rate": 3.1293881644934804e-06, + "loss": 1.2308, + "theoretical_loss": 3.3259646537887644, + "tokens_seen": 3290103808 + }, + { + "epoch": 0.99, + "learning_rate": 3.109327983951856e-06, + "loss": 1.3138, + "theoretical_loss": 3.3259596978168187, + "tokens_seen": 3290169344 + }, + { + "epoch": 0.99, + "learning_rate": 3.0892678034102308e-06, + "loss": 1.2015, + "theoretical_loss": 3.3259547419712288, + "tokens_seen": 3290234880 + }, + { + "epoch": 0.99, + "learning_rate": 3.0692076228686057e-06, + "loss": 1.2385, + "theoretical_loss": 3.325949786251989, + "tokens_seen": 3290300416 + }, + { + "epoch": 0.99, + "learning_rate": 3.049147442326981e-06, + "loss": 1.1651, + "theoretical_loss": 3.3259448306590933, + "tokens_seen": 3290365952 + }, + { + "epoch": 0.99, + "learning_rate": 3.029087261785356e-06, + "loss": 1.2099, + "theoretical_loss": 3.325939875192536, + "tokens_seen": 3290431488 + }, + { + "epoch": 0.99, + "learning_rate": 3.009027081243731e-06, + "loss": 1.2478, + "theoretical_loss": 3.325934919852312, + "tokens_seen": 3290497024 + }, + { + "epoch": 0.99, + "learning_rate": 2.9889669007021063e-06, + "loss": 1.2222, + "theoretical_loss": 3.3259299646384144, + "tokens_seen": 3290562560 + }, + { + "epoch": 0.99, + "learning_rate": 2.9689067201604817e-06, + "loss": 1.245, + "theoretical_loss": 3.325925009550838, + "tokens_seen": 3290628096 + }, + { + "epoch": 0.99, + "learning_rate": 2.9488465396188567e-06, + "loss": 1.2625, + "theoretical_loss": 3.3259200545895777, + "tokens_seen": 3290693632 + }, + { + "epoch": 0.99, + "learning_rate": 2.9287863590772316e-06, + "loss": 1.2822, + "theoretical_loss": 3.3259150997546274, + "tokens_seen": 3290759168 + }, + { + "epoch": 0.99, + "learning_rate": 2.908726178535607e-06, + "loss": 1.1807, + "theoretical_loss": 3.325910145045981, + "tokens_seen": 3290824704 + }, + { + "epoch": 0.99, + "learning_rate": 2.888665997993982e-06, + "loss": 1.2411, + "theoretical_loss": 3.3259051904636325, + "tokens_seen": 3290890240 + }, + { + "epoch": 0.99, + "learning_rate": 2.868605817452357e-06, + "loss": 1.2048, + "theoretical_loss": 3.3259002360075773, + "tokens_seen": 3290955776 + }, + { + "epoch": 0.99, + "learning_rate": 2.8485456369107322e-06, + "loss": 1.2314, + "theoretical_loss": 3.3258952816778087, + "tokens_seen": 3291021312 + }, + { + "epoch": 0.99, + "learning_rate": 2.8284854563691076e-06, + "loss": 1.1849, + "theoretical_loss": 3.3258903274743212, + "tokens_seen": 3291086848 + }, + { + "epoch": 0.99, + "learning_rate": 2.8084252758274825e-06, + "loss": 1.2415, + "theoretical_loss": 3.325885373397109, + "tokens_seen": 3291152384 + }, + { + "epoch": 0.99, + "learning_rate": 2.7883650952858575e-06, + "loss": 1.2716, + "theoretical_loss": 3.325880419446167, + "tokens_seen": 3291217920 + }, + { + "epoch": 0.99, + "learning_rate": 2.768304914744233e-06, + "loss": 1.1803, + "theoretical_loss": 3.325875465621489, + "tokens_seen": 3291283456 + }, + { + "epoch": 0.99, + "learning_rate": 2.748244734202608e-06, + "loss": 1.2979, + "theoretical_loss": 3.3258705119230685, + "tokens_seen": 3291348992 + }, + { + "epoch": 0.99, + "learning_rate": 2.7281845536609828e-06, + "loss": 1.2292, + "theoretical_loss": 3.325865558350901, + "tokens_seen": 3291414528 + }, + { + "epoch": 0.99, + "learning_rate": 2.708124373119358e-06, + "loss": 1.1924, + "theoretical_loss": 3.32586060490498, + "tokens_seen": 3291480064 + }, + { + "epoch": 0.99, + "learning_rate": 2.6880641925777335e-06, + "loss": 1.2563, + "theoretical_loss": 3.3258556515853, + "tokens_seen": 3291545600 + }, + { + "epoch": 0.99, + "learning_rate": 2.6680040120361084e-06, + "loss": 1.2346, + "theoretical_loss": 3.325850698391856, + "tokens_seen": 3291611136 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1195662, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2195073366165161, + "objective/train/theoretical_loss": 3.325845745324641, + "objective/train/tokens_used": 1662135776, + "theoretical_loss": 3.325845745324641, + "tokens_seen": 3291676672 + }, + { + "epoch": 0.99, + "learning_rate": 2.6479438314944834e-06, + "loss": 1.201, + "theoretical_loss": 3.325845745324641, + "tokens_seen": 3291676672 + }, + { + "epoch": 0.99, + "learning_rate": 2.6278836509528588e-06, + "loss": 1.1945, + "theoretical_loss": 3.3258407923836497, + "tokens_seen": 3291742208 + }, + { + "epoch": 1.0, + "learning_rate": 2.6078234704112337e-06, + "loss": 1.2287, + "theoretical_loss": 3.325835839568877, + "tokens_seen": 3291807744 + }, + { + "epoch": 1.0, + "learning_rate": 2.5877632898696087e-06, + "loss": 1.2077, + "theoretical_loss": 3.3258308868803166, + "tokens_seen": 3291873280 + }, + { + "epoch": 1.0, + "learning_rate": 2.567703109327984e-06, + "loss": 1.1069, + "theoretical_loss": 3.3258259343179626, + "tokens_seen": 3291938816 + }, + { + "epoch": 1.0, + "learning_rate": 2.5476429287863594e-06, + "loss": 1.2305, + "theoretical_loss": 3.3258209818818094, + "tokens_seen": 3292004352 + }, + { + "epoch": 1.0, + "learning_rate": 2.5275827482447343e-06, + "loss": 1.2144, + "theoretical_loss": 3.3258160295718517, + "tokens_seen": 3292069888 + }, + { + "epoch": 1.0, + "learning_rate": 2.5075225677031093e-06, + "loss": 1.2231, + "theoretical_loss": 3.3258110773880833, + "tokens_seen": 3292135424 + }, + { + "epoch": 1.0, + "learning_rate": 2.4874623871614847e-06, + "loss": 1.214, + "theoretical_loss": 3.325806125330499, + "tokens_seen": 3292200960 + }, + { + "epoch": 1.0, + "learning_rate": 2.4674022066198596e-06, + "loss": 1.3652, + "theoretical_loss": 3.3258011733990926, + "tokens_seen": 3292266496 + }, + { + "epoch": 1.0, + "learning_rate": 2.4473420260782345e-06, + "loss": 1.1702, + "theoretical_loss": 3.3257962215938583, + "tokens_seen": 3292332032 + }, + { + "epoch": 1.0, + "learning_rate": 2.42728184553661e-06, + "loss": 1.2422, + "theoretical_loss": 3.3257912699147907, + "tokens_seen": 3292397568 + }, + { + "epoch": 1.0, + "learning_rate": 2.4072216649949853e-06, + "loss": 1.247, + "theoretical_loss": 3.325786318361884, + "tokens_seen": 3292463104 + }, + { + "epoch": 1.0, + "learning_rate": 2.38716148445336e-06, + "loss": 1.4013, + "theoretical_loss": 3.325781366935132, + "tokens_seen": 3292528640 + }, + { + "epoch": 1.0, + "learning_rate": 2.367101303911735e-06, + "loss": 1.2293, + "theoretical_loss": 3.32577641563453, + "tokens_seen": 3292594176 + }, + { + "epoch": 1.0, + "learning_rate": 2.3470411233701105e-06, + "loss": 1.2132, + "theoretical_loss": 3.3257714644600713, + "tokens_seen": 3292659712 + }, + { + "epoch": 1.0, + "learning_rate": 2.3269809428284855e-06, + "loss": 1.1991, + "theoretical_loss": 3.3257665134117507, + "tokens_seen": 3292725248 + }, + { + "epoch": 1.0, + "learning_rate": 2.3069207622868604e-06, + "loss": 1.1438, + "theoretical_loss": 3.3257615624895625, + "tokens_seen": 3292790784 + }, + { + "epoch": 1.0, + "learning_rate": 2.286860581745236e-06, + "loss": 1.2986, + "theoretical_loss": 3.3257566116935005, + "tokens_seen": 3292856320 + }, + { + "epoch": 1.0, + "learning_rate": 2.266800401203611e-06, + "loss": 1.1599, + "theoretical_loss": 3.3257516610235593, + "tokens_seen": 3292921856 + }, + { + "epoch": 1.0, + "learning_rate": 2.2467402206619857e-06, + "loss": 1.2845, + "theoretical_loss": 3.325746710479733, + "tokens_seen": 3292987392 + }, + { + "epoch": 1.0, + "learning_rate": 2.226680040120361e-06, + "loss": 1.1509, + "theoretical_loss": 3.3257417600620163, + "tokens_seen": 3293052928 + }, + { + "epoch": 1.0, + "learning_rate": 2.2066198595787364e-06, + "loss": 1.0711, + "theoretical_loss": 3.325736809770403, + "tokens_seen": 3293118464 + }, + { + "epoch": 1.0, + "learning_rate": 2.1865596790371114e-06, + "loss": 1.2054, + "theoretical_loss": 3.325731859604888, + "tokens_seen": 3293184000 + }, + { + "epoch": 1.0, + "learning_rate": 2.1664994984954863e-06, + "loss": 1.2846, + "theoretical_loss": 3.3257269095654647, + "tokens_seen": 3293249536 + }, + { + "epoch": 1.0, + "learning_rate": 2.1464393179538617e-06, + "loss": 1.1699, + "theoretical_loss": 3.325721959652128, + "tokens_seen": 3293315072 + }, + { + "epoch": 1.0, + "learning_rate": 2.126379137412237e-06, + "loss": 1.1783, + "theoretical_loss": 3.325717009864872, + "tokens_seen": 3293380608 + }, + { + "epoch": 1.0, + "learning_rate": 2.1063189568706116e-06, + "loss": 1.2862, + "theoretical_loss": 3.325712060203691, + "tokens_seen": 3293446144 + }, + { + "epoch": 1.0, + "learning_rate": 2.086258776328987e-06, + "loss": 1.1791, + "theoretical_loss": 3.325707110668579, + "tokens_seen": 3293511680 + }, + { + "epoch": 1.0, + "learning_rate": 2.0661985957873623e-06, + "loss": 1.2801, + "theoretical_loss": 3.325702161259531, + "tokens_seen": 3293577216 + }, + { + "epoch": 1.0, + "learning_rate": 2.0461384152457373e-06, + "loss": 1.1293, + "theoretical_loss": 3.325697211976541, + "tokens_seen": 3293642752 + }, + { + "epoch": 1.0, + "learning_rate": 2.0260782347041122e-06, + "loss": 1.3476, + "theoretical_loss": 3.3256922628196026, + "tokens_seen": 3293708288 + }, + { + "epoch": 1.0, + "learning_rate": 2.0060180541624876e-06, + "loss": 1.2636, + "theoretical_loss": 3.325687313788711, + "tokens_seen": 3293773824 + }, + { + "epoch": 1.0, + "learning_rate": 1.985957873620863e-06, + "loss": 1.1937, + "theoretical_loss": 3.32568236488386, + "tokens_seen": 3293839360 + }, + { + "epoch": 1.0, + "learning_rate": 1.9658976930792375e-06, + "loss": 1.2964, + "theoretical_loss": 3.325677416105044, + "tokens_seen": 3293904896 + }, + { + "epoch": 1.0, + "learning_rate": 1.945837512537613e-06, + "loss": 1.1777, + "theoretical_loss": 3.325672467452257, + "tokens_seen": 3293970432 + }, + { + "epoch": 1.0, + "learning_rate": 1.9257773319959882e-06, + "loss": 1.1471, + "theoretical_loss": 3.325667518925494, + "tokens_seen": 3294035968 + }, + { + "epoch": 1.0, + "learning_rate": 1.9057171514543632e-06, + "loss": 1.1783, + "theoretical_loss": 3.3256625705247482, + "tokens_seen": 3294101504 + }, + { + "epoch": 1.0, + "learning_rate": 1.8856569709127383e-06, + "loss": 1.2824, + "theoretical_loss": 3.3256576222500147, + "tokens_seen": 3294167040 + }, + { + "epoch": 1.0, + "learning_rate": 1.8655967903711135e-06, + "loss": 1.233, + "theoretical_loss": 3.3256526741012875, + "tokens_seen": 3294232576 + }, + { + "epoch": 1.0, + "learning_rate": 1.8455366098294884e-06, + "loss": 1.2736, + "theoretical_loss": 3.3256477260785613, + "tokens_seen": 3294298112 + }, + { + "epoch": 1.0, + "learning_rate": 1.8254764292878636e-06, + "loss": 1.2584, + "theoretical_loss": 3.3256427781818303, + "tokens_seen": 3294363648 + }, + { + "epoch": 1.0, + "learning_rate": 1.8054162487462388e-06, + "loss": 1.322, + "theoretical_loss": 3.325637830411088, + "tokens_seen": 3294429184 + }, + { + "epoch": 1.0, + "learning_rate": 1.785356068204614e-06, + "loss": 1.2184, + "theoretical_loss": 3.325632882766329, + "tokens_seen": 3294494720 + }, + { + "epoch": 1.0, + "learning_rate": 1.765295887662989e-06, + "loss": 1.1413, + "theoretical_loss": 3.3256279352475486, + "tokens_seen": 3294560256 + }, + { + "epoch": 1.0, + "learning_rate": 1.745235707121364e-06, + "loss": 1.0844, + "theoretical_loss": 3.32562298785474, + "tokens_seen": 3294625792 + }, + { + "epoch": 1.0, + "learning_rate": 1.7251755265797394e-06, + "loss": 1.2721, + "theoretical_loss": 3.3256180405878975, + "tokens_seen": 3294691328 + }, + { + "epoch": 1.0, + "learning_rate": 1.7051153460381143e-06, + "loss": 1.2063, + "theoretical_loss": 3.3256130934470156, + "tokens_seen": 3294756864 + }, + { + "epoch": 1.0, + "learning_rate": 1.6850551654964895e-06, + "loss": 1.2552, + "theoretical_loss": 3.325608146432089, + "tokens_seen": 3294822400 + }, + { + "epoch": 1.0, + "learning_rate": 1.6649949849548646e-06, + "loss": 1.3633, + "theoretical_loss": 3.3256031995431115, + "tokens_seen": 3294887936 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 1196765, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2364988327026367, + "objective/train/theoretical_loss": 3.325598252780078, + "objective/train/tokens_used": 1665412576, + "theoretical_loss": 3.325598252780078, + "tokens_seen": 3294953472 + }, + { + "epoch": 1.0, + "learning_rate": 1.6449348044132398e-06, + "loss": 1.1791, + "theoretical_loss": 3.325598252780078, + "tokens_seen": 3294953472 + }, + { + "epoch": 1.0, + "learning_rate": 1.624874623871615e-06, + "loss": 1.3507, + "theoretical_loss": 3.3255933061429817, + "tokens_seen": 3295019008 + }, + { + "epoch": 1.0, + "learning_rate": 1.60481444332999e-06, + "loss": 1.1827, + "theoretical_loss": 3.325588359631818, + "tokens_seen": 3295084544 + }, + { + "epoch": 1.0, + "learning_rate": 1.584754262788365e-06, + "loss": 1.093, + "theoretical_loss": 3.32558341324658, + "tokens_seen": 3295150080 + }, + { + "epoch": 1.0, + "learning_rate": 1.5646940822467402e-06, + "loss": 1.2426, + "theoretical_loss": 3.3255784669872632, + "tokens_seen": 3295215616 + }, + { + "epoch": 1.0, + "learning_rate": 1.5446339017051154e-06, + "loss": 1.212, + "theoretical_loss": 3.3255735208538613, + "tokens_seen": 3295281152 + }, + { + "epoch": 1.0, + "learning_rate": 1.5245737211634905e-06, + "loss": 1.2461, + "theoretical_loss": 3.325568574846369, + "tokens_seen": 3295346688 + }, + { + "epoch": 1.0, + "learning_rate": 1.5045135406218655e-06, + "loss": 1.2043, + "theoretical_loss": 3.3255636289647796, + "tokens_seen": 3295412224 + }, + { + "epoch": 1.0, + "learning_rate": 1.4844533600802409e-06, + "loss": 1.2506, + "theoretical_loss": 3.3255586832090884, + "tokens_seen": 3295477760 + }, + { + "epoch": 1.0, + "learning_rate": 1.4643931795386158e-06, + "loss": 1.188, + "theoretical_loss": 3.3255537375792894, + "tokens_seen": 3295543296 + }, + { + "epoch": 1.0, + "learning_rate": 1.444332998996991e-06, + "loss": 1.2367, + "theoretical_loss": 3.3255487920753772, + "tokens_seen": 3295608832 + }, + { + "epoch": 1.0, + "learning_rate": 1.4242728184553661e-06, + "loss": 1.1824, + "theoretical_loss": 3.3255438466973453, + "tokens_seen": 3295674368 + }, + { + "epoch": 1.0, + "learning_rate": 1.4042126379137413e-06, + "loss": 1.2212, + "theoretical_loss": 3.325538901445188, + "tokens_seen": 3295739904 + }, + { + "epoch": 1.0, + "learning_rate": 1.3841524573721164e-06, + "loss": 1.2779, + "theoretical_loss": 3.3255339563189006, + "tokens_seen": 3295805440 + }, + { + "epoch": 1.0, + "learning_rate": 1.3640922768304914e-06, + "loss": 1.1365, + "theoretical_loss": 3.325529011318477, + "tokens_seen": 3295870976 + }, + { + "epoch": 1.0, + "learning_rate": 1.3440320962888667e-06, + "loss": 1.2008, + "theoretical_loss": 3.325524066443911, + "tokens_seen": 3295936512 + }, + { + "epoch": 1.0, + "learning_rate": 1.3239719157472417e-06, + "loss": 1.3011, + "theoretical_loss": 3.325519121695197, + "tokens_seen": 3296002048 + }, + { + "epoch": 1.0, + "learning_rate": 1.3039117352056169e-06, + "loss": 1.3223, + "theoretical_loss": 3.3255141770723298, + "tokens_seen": 3296067584 + }, + { + "epoch": 1.0, + "learning_rate": 1.283851554663992e-06, + "loss": 1.1882, + "theoretical_loss": 3.3255092325753033, + "tokens_seen": 3296133120 + }, + { + "epoch": 1.0, + "learning_rate": 1.2637913741223672e-06, + "loss": 1.2385, + "theoretical_loss": 3.3255042882041117, + "tokens_seen": 3296198656 + }, + { + "epoch": 1.0, + "learning_rate": 1.2437311935807423e-06, + "loss": 1.2445, + "theoretical_loss": 3.32549934395875, + "tokens_seen": 3296264192 + }, + { + "epoch": 1.0, + "learning_rate": 1.2236710130391173e-06, + "loss": 1.2896, + "theoretical_loss": 3.3254943998392115, + "tokens_seen": 3296329728 + }, + { + "epoch": 1.0, + "learning_rate": 1.2036108324974926e-06, + "loss": 1.225, + "theoretical_loss": 3.3254894558454913, + "tokens_seen": 3296395264 + }, + { + "epoch": 1.0, + "learning_rate": 1.1835506519558676e-06, + "loss": 1.2776, + "theoretical_loss": 3.3254845119775833, + "tokens_seen": 3296460800 + }, + { + "epoch": 1.0, + "learning_rate": 1.1634904714142427e-06, + "loss": 1.2901, + "theoretical_loss": 3.3254795682354814, + "tokens_seen": 3296526336 + }, + { + "epoch": 1.0, + "learning_rate": 1.143430290872618e-06, + "loss": 1.2501, + "theoretical_loss": 3.325474624619181, + "tokens_seen": 3296591872 + }, + { + "epoch": 1.0, + "learning_rate": 1.1233701103309928e-06, + "loss": 1.2448, + "theoretical_loss": 3.3254696811286752, + "tokens_seen": 3296657408 + }, + { + "epoch": 1.0, + "learning_rate": 1.1033099297893682e-06, + "loss": 1.2308, + "theoretical_loss": 3.3254647377639595, + "tokens_seen": 3296722944 + }, + { + "epoch": 1.0, + "learning_rate": 1.0832497492477432e-06, + "loss": 1.2936, + "theoretical_loss": 3.325459794525027, + "tokens_seen": 3296788480 + }, + { + "epoch": 1.0, + "learning_rate": 1.0631895687061185e-06, + "loss": 1.2781, + "theoretical_loss": 3.3254548514118727, + "tokens_seen": 3296854016 + }, + { + "epoch": 1.0, + "learning_rate": 1.0431293881644935e-06, + "loss": 1.283, + "theoretical_loss": 3.325449908424491, + "tokens_seen": 3296919552 + }, + { + "epoch": 1.0, + "learning_rate": 1.0230692076228686e-06, + "loss": 1.2684, + "theoretical_loss": 3.325444965562876, + "tokens_seen": 3296985088 + }, + { + "epoch": 1.0, + "learning_rate": 1.0030090270812438e-06, + "loss": 1.1707, + "theoretical_loss": 3.3254400228270216, + "tokens_seen": 3297050624 + }, + { + "epoch": 1.0, + "learning_rate": 9.829488465396187e-07, + "loss": 1.325, + "theoretical_loss": 3.3254350802169226, + "tokens_seen": 3297116160 + }, + { + "epoch": 1.0, + "learning_rate": 9.628886659979941e-07, + "loss": 1.1086, + "theoretical_loss": 3.3254301377325732, + "tokens_seen": 3297181696 + }, + { + "epoch": 1.0, + "learning_rate": 9.428284854563692e-07, + "loss": 1.2355, + "theoretical_loss": 3.3254251953739677, + "tokens_seen": 3297247232 + }, + { + "epoch": 1.0, + "learning_rate": 9.227683049147442e-07, + "loss": 1.2311, + "theoretical_loss": 3.3254202531411003, + "tokens_seen": 3297312768 + }, + { + "epoch": 1.0, + "learning_rate": 9.027081243731194e-07, + "loss": 1.1702, + "theoretical_loss": 3.3254153110339653, + "tokens_seen": 3297378304 + }, + { + "epoch": 1.0, + "learning_rate": 8.826479438314945e-07, + "loss": 1.3253, + "theoretical_loss": 3.325410369052557, + "tokens_seen": 3297443840 + }, + { + "epoch": 1.0, + "learning_rate": 8.625877632898697e-07, + "loss": 1.2886, + "theoretical_loss": 3.32540542719687, + "tokens_seen": 3297509376 + }, + { + "epoch": 1.0, + "learning_rate": 8.425275827482447e-07, + "loss": 1.1456, + "theoretical_loss": 3.3254004854668984, + "tokens_seen": 3297574912 + }, + { + "epoch": 1.0, + "learning_rate": 8.224674022066199e-07, + "loss": 1.1835, + "theoretical_loss": 3.325395543862636, + "tokens_seen": 3297640448 + }, + { + "epoch": 1.0, + "learning_rate": 8.02407221664995e-07, + "loss": 1.2055, + "theoretical_loss": 3.3253906023840782, + "tokens_seen": 3297705984 + }, + { + "epoch": 1.0, + "learning_rate": 7.823470411233701e-07, + "loss": 1.2654, + "theoretical_loss": 3.3253856610312185, + "tokens_seen": 3297771520 + }, + { + "epoch": 1.0, + "learning_rate": 7.622868605817453e-07, + "loss": 1.0963, + "theoretical_loss": 3.3253807198040515, + "tokens_seen": 3297837056 + }, + { + "epoch": 1.0, + "learning_rate": 7.422266800401204e-07, + "loss": 1.2342, + "theoretical_loss": 3.325375778702571, + "tokens_seen": 3297902592 + }, + { + "epoch": 1.0, + "learning_rate": 7.221664994984955e-07, + "loss": 1.2497, + "theoretical_loss": 3.325370837726772, + "tokens_seen": 3297968128 + }, + { + "epoch": 1.0, + "learning_rate": 7.021063189568706e-07, + "loss": 1.2119, + "theoretical_loss": 3.325365896876648, + "tokens_seen": 3298033664 + }, + { + "epoch": 1.0, + "learning_rate": 6.820461384152457e-07, + "loss": 1.2671, + "theoretical_loss": 3.3253609561521946, + "tokens_seen": 3298099200 + }, + { + "epoch": 1.0, + "learning_rate": 6.619859578736208e-07, + "loss": 1.2433, + "theoretical_loss": 3.325356015553405, + "tokens_seen": 3298164736 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 1197928, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.270293951034546, + "objective/train/theoretical_loss": 3.325351075080274, + "objective/train/tokens_used": 1668689376, + "theoretical_loss": 3.325351075080274, + "tokens_seen": 3298230272 + }, + { + "epoch": 1.0, + "learning_rate": 6.41925777331996e-07, + "loss": 1.2633, + "theoretical_loss": 3.325351075080274, + "tokens_seen": 3298230272 + }, + { + "epoch": 1.0, + "learning_rate": 6.218655967903712e-07, + "loss": 1.1949, + "theoretical_loss": 3.3253461347327953, + "tokens_seen": 3298295808 + }, + { + "epoch": 1.0, + "learning_rate": 6.018054162487463e-07, + "loss": 1.1414, + "theoretical_loss": 3.325341194510964, + "tokens_seen": 3298361344 + }, + { + "epoch": 1.0, + "learning_rate": 5.817452357071214e-07, + "loss": 1.2067, + "theoretical_loss": 3.3253362544147738, + "tokens_seen": 3298426880 + }, + { + "epoch": 1.0, + "learning_rate": 5.616850551654964e-07, + "loss": 1.2156, + "theoretical_loss": 3.3253313144442194, + "tokens_seen": 3298492416 + }, + { + "epoch": 1.0, + "learning_rate": 5.416248746238716e-07, + "loss": 1.3, + "theoretical_loss": 3.325326374599295, + "tokens_seen": 3298557952 + }, + { + "epoch": 1.0, + "learning_rate": 5.215646940822467e-07, + "loss": 1.3118, + "theoretical_loss": 3.325321434879995, + "tokens_seen": 3298623488 + }, + { + "epoch": 1.0, + "learning_rate": 5.015045135406219e-07, + "loss": 1.3131, + "theoretical_loss": 3.3253164952863132, + "tokens_seen": 3298689024 + }, + { + "epoch": 1.0, + "learning_rate": 4.814443329989971e-07, + "loss": 1.1684, + "theoretical_loss": 3.325311555818245, + "tokens_seen": 3298754560 + }, + { + "epoch": 1.0, + "learning_rate": 4.613841524573721e-07, + "loss": 1.2351, + "theoretical_loss": 3.325306616475783, + "tokens_seen": 3298820096 + }, + { + "epoch": 1.0, + "learning_rate": 4.4132397191574727e-07, + "loss": 1.2381, + "theoretical_loss": 3.3253016772589232, + "tokens_seen": 3298885632 + }, + { + "epoch": 1.0, + "learning_rate": 4.2126379137412237e-07, + "loss": 1.1597, + "theoretical_loss": 3.3252967381676592, + "tokens_seen": 3298951168 + }, + { + "epoch": 1.0, + "learning_rate": 4.012036108324975e-07, + "loss": 1.2651, + "theoretical_loss": 3.325291799201985, + "tokens_seen": 3299016704 + }, + { + "epoch": 1.0, + "learning_rate": 3.8114343029087263e-07, + "loss": 1.224, + "theoretical_loss": 3.3252868603618957, + "tokens_seen": 3299082240 + }, + { + "epoch": 1.0, + "learning_rate": 3.6108324974924774e-07, + "loss": 1.2995, + "theoretical_loss": 3.3252819216473846, + "tokens_seen": 3299147776 + }, + { + "epoch": 1.0, + "learning_rate": 3.4102306920762284e-07, + "loss": 1.2241, + "theoretical_loss": 3.325276983058447, + "tokens_seen": 3299213312 + }, + { + "epoch": 1.0, + "learning_rate": 3.20962888665998e-07, + "loss": 1.1384, + "theoretical_loss": 3.3252720445950765, + "tokens_seen": 3299278848 + }, + { + "epoch": 1.0, + "learning_rate": 3.0090270812437316e-07, + "loss": 1.31, + "theoretical_loss": 3.325267106257268, + "tokens_seen": 3299344384 + }, + { + "epoch": 1.0, + "learning_rate": 2.808425275827482e-07, + "loss": 1.2418, + "theoretical_loss": 3.3252621680450156, + "tokens_seen": 3299409920 + }, + { + "epoch": 1.0, + "learning_rate": 2.6078234704112337e-07, + "loss": 1.2549, + "theoretical_loss": 3.3252572299583134, + "tokens_seen": 3299475456 + }, + { + "epoch": 1.0, + "learning_rate": 2.4072216649949853e-07, + "loss": 1.228, + "theoretical_loss": 3.3252522919971557, + "tokens_seen": 3299540992 + }, + { + "epoch": 1.0, + "learning_rate": 2.2066198595787363e-07, + "loss": 1.3123, + "theoretical_loss": 3.325247354161537, + "tokens_seen": 3299606528 + }, + { + "epoch": 1.0, + "learning_rate": 2.0060180541624874e-07, + "loss": 1.199, + "theoretical_loss": 3.3252424164514514, + "tokens_seen": 3299672064 + }, + { + "epoch": 1.0, + "learning_rate": 1.8054162487462387e-07, + "loss": 1.2633, + "theoretical_loss": 3.3252374788668937, + "tokens_seen": 3299737600 + }, + { + "epoch": 1.0, + "learning_rate": 1.60481444332999e-07, + "loss": 1.1791, + "theoretical_loss": 3.325232541407858, + "tokens_seen": 3299803136 + }, + { + "epoch": 1.0, + "learning_rate": 1.404212637913741e-07, + "loss": 1.1447, + "theoretical_loss": 3.325227604074338, + "tokens_seen": 3299868672 + }, + { + "epoch": 1.0, + "learning_rate": 1.2036108324974926e-07, + "loss": 1.26, + "theoretical_loss": 3.325222666866329, + "tokens_seen": 3299934208 + }, + { + "epoch": 1.0, + "learning_rate": 1.0030090270812437e-07, + "loss": 1.2552, + "theoretical_loss": 3.325217729783825, + "tokens_seen": 3299999744 + } + ], + "max_steps": 25177, + "num_train_epochs": 9223372036854775807, + "total_flos": 8.42054334676992e+17, + "trial_name": null, + "trial_params": null +}