diff --git "a/checkpoint-22815/trainer_state.json" "b/checkpoint-22815/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-22815/trainer_state.json" @@ -0,0 +1,169816 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9126, + "global_step": 22815, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 2e-06, + "loss": 3.1659, + "theoretical_loss": 3.484796891144814, + "tokens_seen": 1661665280 + }, + { + "epoch": 0.0, + "learning_rate": 4e-06, + "loss": 3.0486, + "theoretical_loss": 3.484785010063602, + "tokens_seen": 1661730816 + }, + { + "epoch": 0.0, + "learning_rate": 6e-06, + "loss": 2.5634, + "theoretical_loss": 3.4847731295821447, + "tokens_seen": 1661796352 + }, + { + "epoch": 0.0, + "learning_rate": 8e-06, + "loss": 3.0242, + "theoretical_loss": 3.48476124970039, + "tokens_seen": 1661861888 + }, + { + "epoch": 0.0, + "learning_rate": 1e-05, + "loss": 2.685, + "theoretical_loss": 3.4847493704182817, + "tokens_seen": 1661927424 + }, + { + "epoch": 0.0, + "learning_rate": 1.2e-05, + "loss": 2.7646, + "theoretical_loss": 3.4847374917357676, + "tokens_seen": 1661992960 + }, + { + "epoch": 0.0, + "learning_rate": 1.4e-05, + "loss": 2.8002, + "theoretical_loss": 3.4847256136527935, + "tokens_seen": 1662058496 + }, + { + "epoch": 0.0, + "learning_rate": 1.6e-05, + "loss": 3.0059, + "theoretical_loss": 3.4847137361693052, + "tokens_seen": 1662124032 + }, + { + "epoch": 0.0, + "learning_rate": 1.8e-05, + "loss": 3.0361, + "theoretical_loss": 3.4847018592852494, + "tokens_seen": 1662189568 + }, + { + "epoch": 0.0, + "learning_rate": 2e-05, + "loss": 2.79, + "theoretical_loss": 3.484689983000571, + "tokens_seen": 1662255104 + }, + { + "epoch": 0.0, + "learning_rate": 2.2e-05, + "loss": 2.7581, + "theoretical_loss": 3.4846781073152173, + "tokens_seen": 1662320640 + }, + { + "epoch": 0.0, + "learning_rate": 2.4e-05, + "loss": 2.6647, + "theoretical_loss": 3.484666232229134, + "tokens_seen": 1662386176 + }, + { + "epoch": 0.0, + "learning_rate": 2.6e-05, + "loss": 2.8627, + "theoretical_loss": 3.484654357742267, + "tokens_seen": 1662451712 + }, + { + "epoch": 0.0, + "learning_rate": 2.8e-05, + "loss": 2.8585, + "theoretical_loss": 3.4846424838545627, + "tokens_seen": 1662517248 + }, + { + "epoch": 0.0, + "learning_rate": 3e-05, + "loss": 2.7554, + "theoretical_loss": 3.4846306105659677, + "tokens_seen": 1662582784 + }, + { + "epoch": 0.0, + "learning_rate": 3.2e-05, + "loss": 2.5777, + "theoretical_loss": 3.484618737876427, + "tokens_seen": 1662648320 + }, + { + "epoch": 0.0, + "learning_rate": 3.4000000000000007e-05, + "loss": 2.5762, + "theoretical_loss": 3.4846068657858877, + "tokens_seen": 1662713856 + }, + { + "epoch": 0.0, + "learning_rate": 3.6e-05, + "loss": 2.6419, + "theoretical_loss": 3.484594994294296, + "tokens_seen": 1662779392 + }, + { + "epoch": 0.0, + "learning_rate": 3.8e-05, + "loss": 2.6613, + "theoretical_loss": 3.4845831234015976, + "tokens_seen": 1662844928 + }, + { + "epoch": 0.0, + "learning_rate": 4e-05, + "loss": 2.6489, + "theoretical_loss": 3.484571253107739, + "tokens_seen": 1662910464 + }, + { + "epoch": 0.0, + "learning_rate": 4.2000000000000004e-05, + "loss": 2.8326, + "theoretical_loss": 3.484559383412666, + "tokens_seen": 1662976000 + }, + { + "epoch": 0.0, + "learning_rate": 4.4e-05, + "loss": 2.771, + "theoretical_loss": 3.4845475143163256, + "tokens_seen": 1663041536 + }, + { + "epoch": 0.0, + "learning_rate": 4.6e-05, + "loss": 2.8005, + "theoretical_loss": 3.4845356458186627, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.0, + "learning_rate": 4.8e-05, + "loss": 2.6649, + "theoretical_loss": 3.484523777919625, + "tokens_seen": 1663172608 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 918485, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4342758655548096, + "objective/train/theoretical_loss": 3.4845119106191578, + "objective/train/tokens_used": 22097376, + "theoretical_loss": 3.4845119106191578, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.0, + "learning_rate": 5e-05, + "loss": 2.6636, + "theoretical_loss": 3.4845119106191578, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.0, + "learning_rate": 5.2e-05, + "loss": 2.8955, + "theoretical_loss": 3.4845000439172074, + "tokens_seen": 1663303680 + }, + { + "epoch": 0.0, + "learning_rate": 5.4e-05, + "loss": 2.5046, + "theoretical_loss": 3.48448817781372, + "tokens_seen": 1663369216 + }, + { + "epoch": 0.0, + "learning_rate": 5.6e-05, + "loss": 2.709, + "theoretical_loss": 3.484476312308643, + "tokens_seen": 1663434752 + }, + { + "epoch": 0.0, + "learning_rate": 5.800000000000001e-05, + "loss": 2.4005, + "theoretical_loss": 3.484464447401921, + "tokens_seen": 1663500288 + }, + { + "epoch": 0.0, + "learning_rate": 6e-05, + "loss": 2.5362, + "theoretical_loss": 3.4844525830935007, + "tokens_seen": 1663565824 + }, + { + "epoch": 0.0, + "learning_rate": 6.2e-05, + "loss": 2.5606, + "theoretical_loss": 3.4844407193833282, + "tokens_seen": 1663631360 + }, + { + "epoch": 0.0, + "learning_rate": 6.4e-05, + "loss": 2.6669, + "theoretical_loss": 3.484428856271351, + "tokens_seen": 1663696896 + }, + { + "epoch": 0.0, + "learning_rate": 6.6e-05, + "loss": 2.626, + "theoretical_loss": 3.484416993757514, + "tokens_seen": 1663762432 + }, + { + "epoch": 0.0, + "learning_rate": 6.800000000000001e-05, + "loss": 2.5635, + "theoretical_loss": 3.484405131841765, + "tokens_seen": 1663827968 + }, + { + "epoch": 0.0, + "learning_rate": 7.000000000000001e-05, + "loss": 2.454, + "theoretical_loss": 3.4843932705240483, + "tokens_seen": 1663893504 + }, + { + "epoch": 0.0, + "learning_rate": 7.2e-05, + "loss": 2.5559, + "theoretical_loss": 3.4843814098043113, + "tokens_seen": 1663959040 + }, + { + "epoch": 0.0, + "learning_rate": 7.4e-05, + "loss": 2.1608, + "theoretical_loss": 3.4843695496825005, + "tokens_seen": 1664024576 + }, + { + "epoch": 0.0, + "learning_rate": 7.6e-05, + "loss": 2.6816, + "theoretical_loss": 3.4843576901585616, + "tokens_seen": 1664090112 + }, + { + "epoch": 0.0, + "learning_rate": 7.8e-05, + "loss": 2.5955, + "theoretical_loss": 3.4843458312324413, + "tokens_seen": 1664155648 + }, + { + "epoch": 0.0, + "learning_rate": 8e-05, + "loss": 2.7015, + "theoretical_loss": 3.484333972904086, + "tokens_seen": 1664221184 + }, + { + "epoch": 0.0, + "learning_rate": 8.2e-05, + "loss": 2.5895, + "theoretical_loss": 3.484322115173442, + "tokens_seen": 1664286720 + }, + { + "epoch": 0.0, + "learning_rate": 8.400000000000001e-05, + "loss": 2.6653, + "theoretical_loss": 3.4843102580404555, + "tokens_seen": 1664352256 + }, + { + "epoch": 0.0, + "learning_rate": 8.599999999999999e-05, + "loss": 2.4374, + "theoretical_loss": 3.4842984015050726, + "tokens_seen": 1664417792 + }, + { + "epoch": 0.0, + "learning_rate": 8.8e-05, + "loss": 2.6655, + "theoretical_loss": 3.4842865455672403, + "tokens_seen": 1664483328 + }, + { + "epoch": 0.0, + "learning_rate": 8.999999999999999e-05, + "loss": 2.1725, + "theoretical_loss": 3.484274690226904, + "tokens_seen": 1664548864 + }, + { + "epoch": 0.0, + "learning_rate": 9.2e-05, + "loss": 2.6074, + "theoretical_loss": 3.484262835484011, + "tokens_seen": 1664614400 + }, + { + "epoch": 0.0, + "learning_rate": 9.400000000000001e-05, + "loss": 2.2801, + "theoretical_loss": 3.484250981338507, + "tokens_seen": 1664679936 + }, + { + "epoch": 0.0, + "learning_rate": 9.6e-05, + "loss": 2.6196, + "theoretical_loss": 3.484239127790339, + "tokens_seen": 1664745472 + }, + { + "epoch": 0.0, + "learning_rate": 9.800000000000001e-05, + "loss": 2.349, + "theoretical_loss": 3.484227274839453, + "tokens_seen": 1664811008 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 919051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5782582759857178, + "objective/train/theoretical_loss": 3.4842154224857955, + "objective/train/tokens_used": 23735776, + "theoretical_loss": 3.4842154224857955, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001, + "loss": 2.5304, + "theoretical_loss": 3.4842154224857955, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.0, + "learning_rate": 0.000102, + "loss": 2.5036, + "theoretical_loss": 3.484203570729313, + "tokens_seen": 1664942080 + }, + { + "epoch": 0.0, + "learning_rate": 0.000104, + "loss": 2.4537, + "theoretical_loss": 3.4841917195699517, + "tokens_seen": 1665007616 + }, + { + "epoch": 0.0, + "learning_rate": 0.000106, + "loss": 2.2072, + "theoretical_loss": 3.484179869007658, + "tokens_seen": 1665073152 + }, + { + "epoch": 0.0, + "learning_rate": 0.000108, + "loss": 2.5087, + "theoretical_loss": 3.4841680190423787, + "tokens_seen": 1665138688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011, + "loss": 2.2714, + "theoretical_loss": 3.4841561696740597, + "tokens_seen": 1665204224 + }, + { + "epoch": 0.0, + "learning_rate": 0.000112, + "loss": 2.4838, + "theoretical_loss": 3.4841443209026477, + "tokens_seen": 1665269760 + }, + { + "epoch": 0.0, + "learning_rate": 0.000114, + "loss": 2.6179, + "theoretical_loss": 3.4841324727280893, + "tokens_seen": 1665335296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011600000000000001, + "loss": 2.5005, + "theoretical_loss": 3.4841206251503305, + "tokens_seen": 1665400832 + }, + { + "epoch": 0.0, + "learning_rate": 0.000118, + "loss": 2.562, + "theoretical_loss": 3.484108778169318, + "tokens_seen": 1665466368 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012, + "loss": 2.5914, + "theoretical_loss": 3.4840969317849986, + "tokens_seen": 1665531904 + }, + { + "epoch": 0.0, + "learning_rate": 0.000122, + "loss": 2.3786, + "theoretical_loss": 3.484085085997318, + "tokens_seen": 1665597440 + }, + { + "epoch": 0.0, + "learning_rate": 0.000124, + "loss": 2.3953, + "theoretical_loss": 3.4840732408062234, + "tokens_seen": 1665662976 + }, + { + "epoch": 0.0, + "learning_rate": 0.000126, + "loss": 2.5667, + "theoretical_loss": 3.484061396211661, + "tokens_seen": 1665728512 + }, + { + "epoch": 0.0, + "learning_rate": 0.000128, + "loss": 2.5867, + "theoretical_loss": 3.4840495522135777, + "tokens_seen": 1665794048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013000000000000002, + "loss": 2.4648, + "theoretical_loss": 3.4840377088119188, + "tokens_seen": 1665859584 + }, + { + "epoch": 0.0, + "learning_rate": 0.000132, + "loss": 2.3746, + "theoretical_loss": 3.484025866006632, + "tokens_seen": 1665925120 + }, + { + "epoch": 0.0, + "learning_rate": 0.000134, + "loss": 2.4735, + "theoretical_loss": 3.4840140237976636, + "tokens_seen": 1665990656 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013600000000000003, + "loss": 2.3041, + "theoretical_loss": 3.48400218218496, + "tokens_seen": 1666056192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013800000000000002, + "loss": 2.5027, + "theoretical_loss": 3.483990341168467, + "tokens_seen": 1666121728 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014000000000000001, + "loss": 2.3187, + "theoretical_loss": 3.4839785007481323, + "tokens_seen": 1666187264 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014199999999999998, + "loss": 2.6289, + "theoretical_loss": 3.483966660923902, + "tokens_seen": 1666252800 + }, + { + "epoch": 0.0, + "learning_rate": 0.000144, + "loss": 2.5713, + "theoretical_loss": 3.483954821695723, + "tokens_seen": 1666318336 + }, + { + "epoch": 0.0, + "learning_rate": 0.000146, + "loss": 2.5538, + "theoretical_loss": 3.4839429830635407, + "tokens_seen": 1666383872 + }, + { + "epoch": 0.0, + "learning_rate": 0.000148, + "loss": 2.4974, + "theoretical_loss": 3.4839311450273023, + "tokens_seen": 1666449408 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 920018, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4347381591796875, + "objective/train/theoretical_loss": 3.4839193075869543, + "objective/train/tokens_used": 25374176, + "theoretical_loss": 3.4839193075869543, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015, + "loss": 2.4437, + "theoretical_loss": 3.4839193075869543, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.0, + "learning_rate": 0.000152, + "loss": 2.3852, + "theoretical_loss": 3.4839074707424444, + "tokens_seen": 1666580480 + }, + { + "epoch": 0.0, + "learning_rate": 0.000154, + "loss": 2.5793, + "theoretical_loss": 3.4838956344937175, + "tokens_seen": 1666646016 + }, + { + "epoch": 0.0, + "learning_rate": 0.000156, + "loss": 2.4539, + "theoretical_loss": 3.483883798840721, + "tokens_seen": 1666711552 + }, + { + "epoch": 0.0, + "learning_rate": 0.000158, + "loss": 2.466, + "theoretical_loss": 3.4838719637834012, + "tokens_seen": 1666777088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016, + "loss": 2.4158, + "theoretical_loss": 3.4838601293217053, + "tokens_seen": 1666842624 + }, + { + "epoch": 0.0, + "learning_rate": 0.000162, + "loss": 2.3978, + "theoretical_loss": 3.483848295455579, + "tokens_seen": 1666908160 + }, + { + "epoch": 0.0, + "learning_rate": 0.000164, + "loss": 2.3159, + "theoretical_loss": 3.4838364621849705, + "tokens_seen": 1666973696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016600000000000002, + "loss": 2.5254, + "theoretical_loss": 3.483824629509824, + "tokens_seen": 1667039232 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016800000000000002, + "loss": 2.2786, + "theoretical_loss": 3.483812797430088, + "tokens_seen": 1667104768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017, + "loss": 2.4052, + "theoretical_loss": 3.4838009659457088, + "tokens_seen": 1667170304 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017199999999999998, + "loss": 2.5102, + "theoretical_loss": 3.4837891350566323, + "tokens_seen": 1667235840 + }, + { + "epoch": 0.0, + "learning_rate": 0.000174, + "loss": 2.4416, + "theoretical_loss": 3.4837773047628064, + "tokens_seen": 1667301376 + }, + { + "epoch": 0.0, + "learning_rate": 0.000176, + "loss": 2.2819, + "theoretical_loss": 3.4837654750641764, + "tokens_seen": 1667366912 + }, + { + "epoch": 0.0, + "learning_rate": 0.000178, + "loss": 2.4608, + "theoretical_loss": 3.4837536459606904, + "tokens_seen": 1667432448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017999999999999998, + "loss": 2.367, + "theoretical_loss": 3.4837418174522936, + "tokens_seen": 1667497984 + }, + { + "epoch": 0.0, + "learning_rate": 0.000182, + "loss": 2.4133, + "theoretical_loss": 3.4837299895389333, + "tokens_seen": 1667563520 + }, + { + "epoch": 0.0, + "learning_rate": 0.000184, + "loss": 2.515, + "theoretical_loss": 3.4837181622205566, + "tokens_seen": 1667629056 + }, + { + "epoch": 0.0, + "learning_rate": 0.000186, + "loss": 2.4365, + "theoretical_loss": 3.48370633549711, + "tokens_seen": 1667694592 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018800000000000002, + "loss": 2.4476, + "theoretical_loss": 3.483694509368539, + "tokens_seen": 1667760128 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019, + "loss": 2.6584, + "theoretical_loss": 3.4836826838347923, + "tokens_seen": 1667825664 + }, + { + "epoch": 0.0, + "learning_rate": 0.000192, + "loss": 2.2917, + "theoretical_loss": 3.483670858895815, + "tokens_seen": 1667891200 + }, + { + "epoch": 0.0, + "learning_rate": 0.000194, + "loss": 2.5954, + "theoretical_loss": 3.483659034551555, + "tokens_seen": 1667956736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019600000000000002, + "loss": 2.4191, + "theoretical_loss": 3.4836472108019585, + "tokens_seen": 1668022272 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019800000000000002, + "loss": 2.4538, + "theoretical_loss": 3.483635387646972, + "tokens_seen": 1668087808 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 921237, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6976654529571533, + "objective/train/theoretical_loss": 3.483623565086542, + "objective/train/tokens_used": 27012576, + "theoretical_loss": 3.483623565086542, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002, + "loss": 2.6155, + "theoretical_loss": 3.483623565086542, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.0, + "learning_rate": 0.000202, + "loss": 2.5577, + "theoretical_loss": 3.4836117431206164, + "tokens_seen": 1668218880 + }, + { + "epoch": 0.0, + "learning_rate": 0.000204, + "loss": 2.397, + "theoretical_loss": 3.483599921749141, + "tokens_seen": 1668284416 + }, + { + "epoch": 0.0, + "learning_rate": 0.000206, + "loss": 2.5303, + "theoretical_loss": 3.483588100972063, + "tokens_seen": 1668349952 + }, + { + "epoch": 0.0, + "learning_rate": 0.000208, + "loss": 2.3655, + "theoretical_loss": 3.483576280789328, + "tokens_seen": 1668415488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021, + "loss": 2.3679, + "theoretical_loss": 3.483564461200885, + "tokens_seen": 1668481024 + }, + { + "epoch": 0.0, + "learning_rate": 0.000212, + "loss": 2.3781, + "theoretical_loss": 3.4835526422066785, + "tokens_seen": 1668546560 + }, + { + "epoch": 0.0, + "learning_rate": 0.000214, + "loss": 2.444, + "theoretical_loss": 3.4835408238066567, + "tokens_seen": 1668612096 + }, + { + "epoch": 0.0, + "learning_rate": 0.000216, + "loss": 2.3319, + "theoretical_loss": 3.483529006000766, + "tokens_seen": 1668677632 + }, + { + "epoch": 0.0, + "learning_rate": 0.000218, + "loss": 2.2344, + "theoretical_loss": 3.4835171887889533, + "tokens_seen": 1668743168 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022, + "loss": 2.4793, + "theoretical_loss": 3.483505372171165, + "tokens_seen": 1668808704 + }, + { + "epoch": 0.0, + "learning_rate": 0.000222, + "loss": 2.4368, + "theoretical_loss": 3.483493556147349, + "tokens_seen": 1668874240 + }, + { + "epoch": 0.0, + "learning_rate": 0.000224, + "loss": 2.4653, + "theoretical_loss": 3.4834817407174503, + "tokens_seen": 1668939776 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022600000000000002, + "loss": 2.2769, + "theoretical_loss": 3.4834699258814172, + "tokens_seen": 1669005312 + }, + { + "epoch": 0.0, + "learning_rate": 0.000228, + "loss": 2.5012, + "theoretical_loss": 3.4834581116391963, + "tokens_seen": 1669070848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023, + "loss": 2.7566, + "theoretical_loss": 3.483446297990734, + "tokens_seen": 1669136384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023200000000000003, + "loss": 2.427, + "theoretical_loss": 3.4834344849359775, + "tokens_seen": 1669201920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023400000000000002, + "loss": 2.5868, + "theoretical_loss": 3.4834226724748736, + "tokens_seen": 1669267456 + }, + { + "epoch": 0.0, + "learning_rate": 0.000236, + "loss": 2.5531, + "theoretical_loss": 3.483410860607369, + "tokens_seen": 1669332992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023799999999999998, + "loss": 2.5605, + "theoretical_loss": 3.4833990493334106, + "tokens_seen": 1669398528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024, + "loss": 2.2757, + "theoretical_loss": 3.4833872386529454, + "tokens_seen": 1669464064 + }, + { + "epoch": 0.0, + "learning_rate": 0.000242, + "loss": 2.5149, + "theoretical_loss": 3.48337542856592, + "tokens_seen": 1669529600 + }, + { + "epoch": 0.0, + "learning_rate": 0.000244, + "loss": 2.2792, + "theoretical_loss": 3.483363619072282, + "tokens_seen": 1669595136 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246, + "loss": 2.4745, + "theoretical_loss": 3.4833518101719774, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248, + "loss": 2.37, + "theoretical_loss": 3.4833400018649536, + "tokens_seen": 1669726208 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 921959, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.675856590270996, + "objective/train/theoretical_loss": 3.4833281941511576, + "objective/train/tokens_used": 28650976, + "theoretical_loss": 3.4833281941511576, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 2.8538, + "theoretical_loss": 3.4833281941511576, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000252, + "loss": 2.3873, + "theoretical_loss": 3.483316387030536, + "tokens_seen": 1669857280 + }, + { + "epoch": 0.01, + "learning_rate": 0.000254, + "loss": 2.3762, + "theoretical_loss": 3.4833045805030363, + "tokens_seen": 1669922816 + }, + { + "epoch": 0.01, + "learning_rate": 0.000256, + "loss": 2.5253, + "theoretical_loss": 3.4832927745686044, + "tokens_seen": 1669988352 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025800000000000004, + "loss": 2.489, + "theoretical_loss": 3.483280969227188, + "tokens_seen": 1670053888 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026000000000000003, + "loss": 2.3014, + "theoretical_loss": 3.4832691644787337, + "tokens_seen": 1670119424 + }, + { + "epoch": 0.01, + "learning_rate": 0.000262, + "loss": 2.2745, + "theoretical_loss": 3.4832573603231887, + "tokens_seen": 1670184960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000264, + "loss": 2.6538, + "theoretical_loss": 3.4832455567605, + "tokens_seen": 1670250496 + }, + { + "epoch": 0.01, + "learning_rate": 0.000266, + "loss": 2.3405, + "theoretical_loss": 3.4832337537906146, + "tokens_seen": 1670316032 + }, + { + "epoch": 0.01, + "learning_rate": 0.000268, + "loss": 2.4211, + "theoretical_loss": 3.483221951413479, + "tokens_seen": 1670381568 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027, + "loss": 2.5642, + "theoretical_loss": 3.4832101496290404, + "tokens_seen": 1670447104 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027200000000000005, + "loss": 2.3653, + "theoretical_loss": 3.483198348437246, + "tokens_seen": 1670512640 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027400000000000005, + "loss": 2.4734, + "theoretical_loss": 3.4831865478380433, + "tokens_seen": 1670578176 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027600000000000004, + "loss": 2.41, + "theoretical_loss": 3.483174747831378, + "tokens_seen": 1670643712 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027800000000000004, + "loss": 2.2828, + "theoretical_loss": 3.4831629484171978, + "tokens_seen": 1670709248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028000000000000003, + "loss": 2.5743, + "theoretical_loss": 3.4831511495954497, + "tokens_seen": 1670774784 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028199999999999997, + "loss": 2.4478, + "theoretical_loss": 3.4831393513660807, + "tokens_seen": 1670840320 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028399999999999996, + "loss": 2.5524, + "theoretical_loss": 3.483127553729038, + "tokens_seen": 1670905856 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028599999999999996, + "loss": 2.6412, + "theoretical_loss": 3.483115756684268, + "tokens_seen": 1670971392 + }, + { + "epoch": 0.01, + "learning_rate": 0.000288, + "loss": 2.3564, + "theoretical_loss": 3.483103960231719, + "tokens_seen": 1671036928 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029, + "loss": 2.5918, + "theoretical_loss": 3.4830921643713366, + "tokens_seen": 1671102464 + }, + { + "epoch": 0.01, + "learning_rate": 0.000292, + "loss": 2.2021, + "theoretical_loss": 3.4830803691030683, + "tokens_seen": 1671168000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000294, + "loss": 2.3097, + "theoretical_loss": 3.483068574426862, + "tokens_seen": 1671233536 + }, + { + "epoch": 0.01, + "learning_rate": 0.000296, + "loss": 2.4126, + "theoretical_loss": 3.4830567803426637, + "tokens_seen": 1671299072 + }, + { + "epoch": 0.01, + "learning_rate": 0.000298, + "loss": 2.4376, + "theoretical_loss": 3.483044986850421, + "tokens_seen": 1671364608 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 922598, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7050137519836426, + "objective/train/theoretical_loss": 3.4830331939500807, + "objective/train/tokens_used": 30289376, + "theoretical_loss": 3.4830331939500807, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003, + "loss": 2.5272, + "theoretical_loss": 3.4830331939500807, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.01, + "learning_rate": 0.000302, + "loss": 2.4159, + "theoretical_loss": 3.48302140164159, + "tokens_seen": 1671495680 + }, + { + "epoch": 0.01, + "learning_rate": 0.000304, + "loss": 2.3413, + "theoretical_loss": 3.483009609924897, + "tokens_seen": 1671561216 + }, + { + "epoch": 0.01, + "learning_rate": 0.000306, + "loss": 2.7255, + "theoretical_loss": 3.482997818799947, + "tokens_seen": 1671626752 + }, + { + "epoch": 0.01, + "learning_rate": 0.000308, + "loss": 2.4114, + "theoretical_loss": 3.482986028266688, + "tokens_seen": 1671692288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031, + "loss": 2.3985, + "theoretical_loss": 3.4829742383250673, + "tokens_seen": 1671757824 + }, + { + "epoch": 0.01, + "learning_rate": 0.000312, + "loss": 2.4447, + "theoretical_loss": 3.4829624489750315, + "tokens_seen": 1671823360 + }, + { + "epoch": 0.01, + "learning_rate": 0.000314, + "loss": 2.1292, + "theoretical_loss": 3.4829506602165283, + "tokens_seen": 1671888896 + }, + { + "epoch": 0.01, + "learning_rate": 0.000316, + "loss": 2.4829, + "theoretical_loss": 3.4829388720495045, + "tokens_seen": 1671954432 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031800000000000003, + "loss": 2.3397, + "theoretical_loss": 3.482927084473907, + "tokens_seen": 1672019968 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032, + "loss": 2.5797, + "theoretical_loss": 3.4829152974896838, + "tokens_seen": 1672085504 + }, + { + "epoch": 0.01, + "learning_rate": 0.000322, + "loss": 2.6122, + "theoretical_loss": 3.482903511096781, + "tokens_seen": 1672151040 + }, + { + "epoch": 0.01, + "learning_rate": 0.000324, + "loss": 2.3373, + "theoretical_loss": 3.482891725295146, + "tokens_seen": 1672216576 + }, + { + "epoch": 0.01, + "learning_rate": 0.000326, + "loss": 2.2181, + "theoretical_loss": 3.482879940084727, + "tokens_seen": 1672282112 + }, + { + "epoch": 0.01, + "learning_rate": 0.000328, + "loss": 2.5586, + "theoretical_loss": 3.48286815546547, + "tokens_seen": 1672347648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033, + "loss": 2.4485, + "theoretical_loss": 3.4828563714373226, + "tokens_seen": 1672413184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033200000000000005, + "loss": 2.6228, + "theoretical_loss": 3.482844588000232, + "tokens_seen": 1672478720 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033400000000000004, + "loss": 2.2578, + "theoretical_loss": 3.4828328051541453, + "tokens_seen": 1672544256 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033600000000000004, + "loss": 2.5647, + "theoretical_loss": 3.48282102289901, + "tokens_seen": 1672609792 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033800000000000003, + "loss": 2.27, + "theoretical_loss": 3.482809241234773, + "tokens_seen": 1672675328 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034, + "loss": 2.8122, + "theoretical_loss": 3.4827974601613816, + "tokens_seen": 1672740864 + }, + { + "epoch": 0.01, + "learning_rate": 0.000342, + "loss": 2.6542, + "theoretical_loss": 3.482785679678783, + "tokens_seen": 1672806400 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034399999999999996, + "loss": 2.4875, + "theoretical_loss": 3.482773899786924, + "tokens_seen": 1672871936 + }, + { + "epoch": 0.01, + "learning_rate": 0.000346, + "loss": 2.5134, + "theoretical_loss": 3.482762120485753, + "tokens_seen": 1672937472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000348, + "loss": 2.5356, + "theoretical_loss": 3.4827503417752164, + "tokens_seen": 1673003008 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 923228, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5057578086853027, + "objective/train/theoretical_loss": 3.482738563655261, + "objective/train/tokens_used": 31927776, + "theoretical_loss": 3.482738563655261, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035, + "loss": 2.7017, + "theoretical_loss": 3.482738563655261, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000352, + "loss": 2.3692, + "theoretical_loss": 3.482726786125835, + "tokens_seen": 1673134080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000354, + "loss": 2.4674, + "theoretical_loss": 3.4827150091868853, + "tokens_seen": 1673199616 + }, + { + "epoch": 0.01, + "learning_rate": 0.000356, + "loss": 2.3015, + "theoretical_loss": 3.482703232838359, + "tokens_seen": 1673265152 + }, + { + "epoch": 0.01, + "learning_rate": 0.000358, + "loss": 2.4719, + "theoretical_loss": 3.482691457080204, + "tokens_seen": 1673330688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035999999999999997, + "loss": 2.3731, + "theoretical_loss": 3.4826796819123667, + "tokens_seen": 1673396224 + }, + { + "epoch": 0.01, + "learning_rate": 0.000362, + "loss": 2.3984, + "theoretical_loss": 3.482667907334795, + "tokens_seen": 1673461760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000364, + "loss": 2.5499, + "theoretical_loss": 3.4826561333474357, + "tokens_seen": 1673527296 + }, + { + "epoch": 0.01, + "learning_rate": 0.000366, + "loss": 2.4005, + "theoretical_loss": 3.482644359950237, + "tokens_seen": 1673592832 + }, + { + "epoch": 0.01, + "learning_rate": 0.000368, + "loss": 2.5822, + "theoretical_loss": 3.482632587143145, + "tokens_seen": 1673658368 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037, + "loss": 2.5947, + "theoretical_loss": 3.4826208149261078, + "tokens_seen": 1673723904 + }, + { + "epoch": 0.01, + "learning_rate": 0.000372, + "loss": 2.4242, + "theoretical_loss": 3.482609043299073, + "tokens_seen": 1673789440 + }, + { + "epoch": 0.01, + "learning_rate": 0.000374, + "loss": 2.5893, + "theoretical_loss": 3.482597272261987, + "tokens_seen": 1673854976 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037600000000000003, + "loss": 2.7811, + "theoretical_loss": 3.482585501814798, + "tokens_seen": 1673920512 + }, + { + "epoch": 0.01, + "learning_rate": 0.000378, + "loss": 2.6416, + "theoretical_loss": 3.4825737319574523, + "tokens_seen": 1673986048 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038, + "loss": 2.3689, + "theoretical_loss": 3.482561962689899, + "tokens_seen": 1674051584 + }, + { + "epoch": 0.01, + "learning_rate": 0.000382, + "loss": 2.4532, + "theoretical_loss": 3.4825501940120835, + "tokens_seen": 1674117120 + }, + { + "epoch": 0.01, + "learning_rate": 0.000384, + "loss": 2.2893, + "theoretical_loss": 3.4825384259239547, + "tokens_seen": 1674182656 + }, + { + "epoch": 0.01, + "learning_rate": 0.000386, + "loss": 2.4148, + "theoretical_loss": 3.4825266584254586, + "tokens_seen": 1674248192 + }, + { + "epoch": 0.01, + "learning_rate": 0.000388, + "loss": 2.4807, + "theoretical_loss": 3.482514891516544, + "tokens_seen": 1674313728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039000000000000005, + "loss": 2.5216, + "theoretical_loss": 3.482503125197157, + "tokens_seen": 1674379264 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039200000000000004, + "loss": 2.2569, + "theoretical_loss": 3.4824913594672458, + "tokens_seen": 1674444800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039400000000000004, + "loss": 2.2831, + "theoretical_loss": 3.4824795943267577, + "tokens_seen": 1674510336 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039600000000000003, + "loss": 2.5188, + "theoretical_loss": 3.48246782977564, + "tokens_seen": 1674575872 + }, + { + "epoch": 0.01, + "learning_rate": 0.000398, + "loss": 2.576, + "theoretical_loss": 3.4824560658138397, + "tokens_seen": 1674641408 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 924425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.702085256576538, + "objective/train/theoretical_loss": 3.482444302441305, + "objective/train/tokens_used": 33566176, + "theoretical_loss": 3.482444302441305, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004, + "loss": 2.6435, + "theoretical_loss": 3.482444302441305, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.01, + "learning_rate": 0.000402, + "loss": 2.4602, + "theoretical_loss": 3.482432539657983, + "tokens_seen": 1674772480 + }, + { + "epoch": 0.01, + "learning_rate": 0.000404, + "loss": 2.3951, + "theoretical_loss": 3.4824207774638207, + "tokens_seen": 1674838016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040600000000000006, + "loss": 2.4707, + "theoretical_loss": 3.4824090158587664, + "tokens_seen": 1674903552 + }, + { + "epoch": 0.01, + "learning_rate": 0.000408, + "loss": 2.4597, + "theoretical_loss": 3.482397254842767, + "tokens_seen": 1674969088 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041, + "loss": 2.5413, + "theoretical_loss": 3.4823854944157695, + "tokens_seen": 1675034624 + }, + { + "epoch": 0.01, + "learning_rate": 0.000412, + "loss": 2.4902, + "theoretical_loss": 3.482373734577722, + "tokens_seen": 1675100160 + }, + { + "epoch": 0.01, + "learning_rate": 0.000414, + "loss": 2.3291, + "theoretical_loss": 3.482361975328572, + "tokens_seen": 1675165696 + }, + { + "epoch": 0.01, + "learning_rate": 0.000416, + "loss": 2.4068, + "theoretical_loss": 3.4823502166682667, + "tokens_seen": 1675231232 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041799999999999997, + "loss": 2.4366, + "theoretical_loss": 3.482338458596754, + "tokens_seen": 1675296768 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042, + "loss": 2.5455, + "theoretical_loss": 3.4823267011139807, + "tokens_seen": 1675362304 + }, + { + "epoch": 0.01, + "learning_rate": 0.000422, + "loss": 2.6574, + "theoretical_loss": 3.482314944219895, + "tokens_seen": 1675427840 + }, + { + "epoch": 0.01, + "learning_rate": 0.000424, + "loss": 2.3464, + "theoretical_loss": 3.482303187914444, + "tokens_seen": 1675493376 + }, + { + "epoch": 0.01, + "learning_rate": 0.000426, + "loss": 2.5067, + "theoretical_loss": 3.482291432197575, + "tokens_seen": 1675558912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000428, + "loss": 2.4157, + "theoretical_loss": 3.4822796770692364, + "tokens_seen": 1675624448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043, + "loss": 2.5832, + "theoretical_loss": 3.482267922529374, + "tokens_seen": 1675689984 + }, + { + "epoch": 0.01, + "learning_rate": 0.000432, + "loss": 2.8137, + "theoretical_loss": 3.4822561685779374, + "tokens_seen": 1675755520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043400000000000003, + "loss": 2.3953, + "theoretical_loss": 3.482244415214873, + "tokens_seen": 1675821056 + }, + { + "epoch": 0.01, + "learning_rate": 0.000436, + "loss": 2.6011, + "theoretical_loss": 3.482232662440128, + "tokens_seen": 1675886592 + }, + { + "epoch": 0.01, + "learning_rate": 0.000438, + "loss": 2.2128, + "theoretical_loss": 3.482220910253651, + "tokens_seen": 1675952128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044, + "loss": 2.4778, + "theoretical_loss": 3.4822091586553885, + "tokens_seen": 1676017664 + }, + { + "epoch": 0.01, + "learning_rate": 0.000442, + "loss": 2.2702, + "theoretical_loss": 3.482197407645289, + "tokens_seen": 1676083200 + }, + { + "epoch": 0.01, + "learning_rate": 0.000444, + "loss": 2.5679, + "theoretical_loss": 3.482185657223299, + "tokens_seen": 1676148736 + }, + { + "epoch": 0.01, + "learning_rate": 0.000446, + "loss": 2.4653, + "theoretical_loss": 3.4821739073893676, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000448, + "loss": 2.27, + "theoretical_loss": 3.482162158143441, + "tokens_seen": 1676279808 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 924948, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.733900785446167, + "objective/train/theoretical_loss": 3.482150409485467, + "objective/train/tokens_used": 35204576, + "theoretical_loss": 3.482150409485467, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045000000000000004, + "loss": 2.5402, + "theoretical_loss": 3.482150409485467, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045200000000000004, + "loss": 2.5286, + "theoretical_loss": 3.4821386614153944, + "tokens_seen": 1676410880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045400000000000003, + "loss": 2.6377, + "theoretical_loss": 3.482126913933169, + "tokens_seen": 1676476416 + }, + { + "epoch": 0.01, + "learning_rate": 0.000456, + "loss": 2.5819, + "theoretical_loss": 3.4821151670387396, + "tokens_seen": 1676541952 + }, + { + "epoch": 0.01, + "learning_rate": 0.000458, + "loss": 2.5565, + "theoretical_loss": 3.482103420732053, + "tokens_seen": 1676607488 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046, + "loss": 2.5833, + "theoretical_loss": 3.482091675013058, + "tokens_seen": 1676673024 + }, + { + "epoch": 0.01, + "learning_rate": 0.000462, + "loss": 2.6621, + "theoretical_loss": 3.482079929881701, + "tokens_seen": 1676738560 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046400000000000006, + "loss": 2.3564, + "theoretical_loss": 3.4820681853379307, + "tokens_seen": 1676804096 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046600000000000005, + "loss": 2.4111, + "theoretical_loss": 3.482056441381694, + "tokens_seen": 1676869632 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046800000000000005, + "loss": 2.5978, + "theoretical_loss": 3.4820446980129383, + "tokens_seen": 1676935168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047, + "loss": 2.5111, + "theoretical_loss": 3.4820329552316123, + "tokens_seen": 1677000704 + }, + { + "epoch": 0.01, + "learning_rate": 0.000472, + "loss": 2.5309, + "theoretical_loss": 3.4820212130376627, + "tokens_seen": 1677066240 + }, + { + "epoch": 0.01, + "learning_rate": 0.000474, + "loss": 2.4254, + "theoretical_loss": 3.482009471431038, + "tokens_seen": 1677131776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047599999999999997, + "loss": 2.3422, + "theoretical_loss": 3.4819977304116847, + "tokens_seen": 1677197312 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047799999999999996, + "loss": 2.5434, + "theoretical_loss": 3.4819859899795516, + "tokens_seen": 1677262848 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048, + "loss": 2.5123, + "theoretical_loss": 3.481974250134586, + "tokens_seen": 1677328384 + }, + { + "epoch": 0.01, + "learning_rate": 0.000482, + "loss": 2.6815, + "theoretical_loss": 3.481962510876736, + "tokens_seen": 1677393920 + }, + { + "epoch": 0.01, + "learning_rate": 0.000484, + "loss": 2.3318, + "theoretical_loss": 3.481950772205948, + "tokens_seen": 1677459456 + }, + { + "epoch": 0.01, + "learning_rate": 0.000486, + "loss": 2.4658, + "theoretical_loss": 3.481939034122171, + "tokens_seen": 1677524992 + }, + { + "epoch": 0.01, + "learning_rate": 0.000488, + "loss": 2.3931, + "theoretical_loss": 3.4819272966253525, + "tokens_seen": 1677590528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049, + "loss": 2.5888, + "theoretical_loss": 3.4819155597154396, + "tokens_seen": 1677656064 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492, + "loss": 2.5776, + "theoretical_loss": 3.481903823392381, + "tokens_seen": 1677721600 + }, + { + "epoch": 0.01, + "learning_rate": 0.000494, + "loss": 2.3611, + "theoretical_loss": 3.4818920876561235, + "tokens_seen": 1677787136 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496, + "loss": 2.501, + "theoretical_loss": 3.4818803525066153, + "tokens_seen": 1677852672 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498, + "loss": 2.6304, + "theoretical_loss": 3.481868617943804, + "tokens_seen": 1677918208 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 926151, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.59950590133667, + "objective/train/theoretical_loss": 3.4818568839676374, + "objective/train/tokens_used": 36842976, + "theoretical_loss": 3.4818568839676374, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 2.582, + "theoretical_loss": 3.4818568839676374, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499979797979798, + "loss": 2.3636, + "theoretical_loss": 3.481845150578063, + "tokens_seen": 1678049280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999595959595959, + "loss": 2.4696, + "theoretical_loss": 3.4818334177750296, + "tokens_seen": 1678114816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999393939393939, + "loss": 2.5003, + "theoretical_loss": 3.481821685558484, + "tokens_seen": 1678180352 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499919191919192, + "loss": 2.6498, + "theoretical_loss": 3.481809953928374, + "tokens_seen": 1678245888 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049989898989899, + "loss": 2.5553, + "theoretical_loss": 3.4817982228846476, + "tokens_seen": 1678311424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998787878787879, + "loss": 2.5759, + "theoretical_loss": 3.4817864924272524, + "tokens_seen": 1678376960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998585858585859, + "loss": 2.7333, + "theoretical_loss": 3.481774762556137, + "tokens_seen": 1678442496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998383838383839, + "loss": 2.6417, + "theoretical_loss": 3.4817630332712484, + "tokens_seen": 1678508032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998181818181818, + "loss": 2.8269, + "theoretical_loss": 3.4817513045725343, + "tokens_seen": 1678573568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997979797979798, + "loss": 2.4396, + "theoretical_loss": 3.481739576459943, + "tokens_seen": 1678639104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997777777777778, + "loss": 2.6778, + "theoretical_loss": 3.4817278489334225, + "tokens_seen": 1678704640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997575757575757, + "loss": 2.4876, + "theoretical_loss": 3.48171612199292, + "tokens_seen": 1678770176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997373737373737, + "loss": 2.5085, + "theoretical_loss": 3.481704395638383, + "tokens_seen": 1678835712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997171717171717, + "loss": 2.523, + "theoretical_loss": 3.4816926698697612, + "tokens_seen": 1678901248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996969696969697, + "loss": 2.5983, + "theoretical_loss": 3.4816809446870005, + "tokens_seen": 1678966784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996767676767676, + "loss": 2.5985, + "theoretical_loss": 3.48166922009005, + "tokens_seen": 1679032320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996565656565657, + "loss": 2.6887, + "theoretical_loss": 3.4816574960788564, + "tokens_seen": 1679097856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996363636363637, + "loss": 2.6201, + "theoretical_loss": 3.481645772653369, + "tokens_seen": 1679163392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996161616161617, + "loss": 2.6373, + "theoretical_loss": 3.4816340498135343, + "tokens_seen": 1679228928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995959595959596, + "loss": 2.5179, + "theoretical_loss": 3.4816223275593012, + "tokens_seen": 1679294464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995757575757576, + "loss": 2.514, + "theoretical_loss": 3.4816106058906175, + "tokens_seen": 1679360000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995555555555555, + "loss": 2.5432, + "theoretical_loss": 3.48159888480743, + "tokens_seen": 1679425536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995353535353536, + "loss": 2.5299, + "theoretical_loss": 3.481587164309688, + "tokens_seen": 1679491072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995151515151515, + "loss": 2.3886, + "theoretical_loss": 3.481575444397339, + "tokens_seen": 1679556608 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 927219, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.202238082885742, + "objective/train/theoretical_loss": 3.4815637250703304, + "objective/train/tokens_used": 38481376, + "theoretical_loss": 3.4815637250703304, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994949494949495, + "loss": 2.5746, + "theoretical_loss": 3.4815637250703304, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994747474747474, + "loss": 2.6998, + "theoretical_loss": 3.4815520063286103, + "tokens_seen": 1679687680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994545454545454, + "loss": 2.4526, + "theoretical_loss": 3.4815402881721274, + "tokens_seen": 1679753216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994343434343435, + "loss": 2.6013, + "theoretical_loss": 3.4815285706008288, + "tokens_seen": 1679818752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994141414141415, + "loss": 2.2639, + "theoretical_loss": 3.4815168536146626, + "tokens_seen": 1679884288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993939393939394, + "loss": 2.5337, + "theoretical_loss": 3.481505137213577, + "tokens_seen": 1679949824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993737373737374, + "loss": 2.6064, + "theoretical_loss": 3.48149342139752, + "tokens_seen": 1680015360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993535353535354, + "loss": 2.4297, + "theoretical_loss": 3.4814817061664396, + "tokens_seen": 1680080896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993333333333334, + "loss": 2.548, + "theoretical_loss": 3.481469991520283, + "tokens_seen": 1680146432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993131313131313, + "loss": 2.5833, + "theoretical_loss": 3.481458277458999, + "tokens_seen": 1680211968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992929292929293, + "loss": 2.6257, + "theoretical_loss": 3.4814465639825354, + "tokens_seen": 1680277504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992727272727272, + "loss": 2.2464, + "theoretical_loss": 3.4814348510908397, + "tokens_seen": 1680343040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992525252525252, + "loss": 2.526, + "theoretical_loss": 3.481423138783861, + "tokens_seen": 1680408576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992323232323232, + "loss": 2.5034, + "theoretical_loss": 3.4814114270615466, + "tokens_seen": 1680474112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992121212121212, + "loss": 2.5144, + "theoretical_loss": 3.4813997159238443, + "tokens_seen": 1680539648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991919191919192, + "loss": 2.7348, + "theoretical_loss": 3.481388005370702, + "tokens_seen": 1680605184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991717171717172, + "loss": 2.3679, + "theoretical_loss": 3.481376295402069, + "tokens_seen": 1680670720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991515151515152, + "loss": 2.5985, + "theoretical_loss": 3.481364586017892, + "tokens_seen": 1680736256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991313131313132, + "loss": 2.6316, + "theoretical_loss": 3.4813528772181193, + "tokens_seen": 1680801792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991111111111111, + "loss": 2.601, + "theoretical_loss": 3.4813411690026994, + "tokens_seen": 1680867328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990909090909091, + "loss": 2.57, + "theoretical_loss": 3.4813294613715797, + "tokens_seen": 1680932864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990707070707071, + "loss": 2.4923, + "theoretical_loss": 3.4813177543247096, + "tokens_seen": 1680998400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990505050505051, + "loss": 2.5084, + "theoretical_loss": 3.4813060478620352, + "tokens_seen": 1681063936 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499030303030303, + "loss": 2.6574, + "theoretical_loss": 3.4812943419835065, + "tokens_seen": 1681129472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499010101010101, + "loss": 2.7915, + "theoretical_loss": 3.48128263668907, + "tokens_seen": 1681195008 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 927968, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5613365173339844, + "objective/train/theoretical_loss": 3.481270931978675, + "objective/train/tokens_used": 40119776, + "theoretical_loss": 3.481270931978675, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498989898989899, + "loss": 2.4737, + "theoretical_loss": 3.481270931978675, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498969696969697, + "loss": 2.6981, + "theoretical_loss": 3.4812592278522687, + "tokens_seen": 1681326080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498949494949495, + "loss": 2.6027, + "theoretical_loss": 3.4812475243098, + "tokens_seen": 1681391616 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498929292929293, + "loss": 2.6348, + "theoretical_loss": 3.4812358213512162, + "tokens_seen": 1681457152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989090909090909, + "loss": 2.6082, + "theoretical_loss": 3.4812241189764657, + "tokens_seen": 1681522688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988888888888889, + "loss": 2.6419, + "theoretical_loss": 3.4812124171854966, + "tokens_seen": 1681588224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988686868686869, + "loss": 2.6168, + "theoretical_loss": 3.4812007159782574, + "tokens_seen": 1681653760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988484848484849, + "loss": 2.5625, + "theoretical_loss": 3.481189015354696, + "tokens_seen": 1681719296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988282828282828, + "loss": 2.6334, + "theoretical_loss": 3.4811773153147607, + "tokens_seen": 1681784832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988080808080808, + "loss": 2.6787, + "theoretical_loss": 3.481165615858399, + "tokens_seen": 1681850368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987878787878788, + "loss": 2.8212, + "theoretical_loss": 3.48115391698556, + "tokens_seen": 1681915904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987676767676767, + "loss": 2.5745, + "theoretical_loss": 3.481142218696191, + "tokens_seen": 1681981440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987474747474747, + "loss": 2.848, + "theoretical_loss": 3.4811305209902406, + "tokens_seen": 1682046976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987272727272728, + "loss": 2.4886, + "theoretical_loss": 3.481118823867657, + "tokens_seen": 1682112512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987070707070708, + "loss": 2.4485, + "theoretical_loss": 3.4811071273283885, + "tokens_seen": 1682178048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986868686868687, + "loss": 2.7483, + "theoretical_loss": 3.481095431372383, + "tokens_seen": 1682243584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986666666666667, + "loss": 2.472, + "theoretical_loss": 3.4810837359995883, + "tokens_seen": 1682309120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986464646464647, + "loss": 2.6229, + "theoretical_loss": 3.481072041209954, + "tokens_seen": 1682374656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986262626262627, + "loss": 2.679, + "theoretical_loss": 3.4810603470034263, + "tokens_seen": 1682440192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986060606060606, + "loss": 2.5162, + "theoretical_loss": 3.481048653379955, + "tokens_seen": 1682505728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985858585858586, + "loss": 2.6945, + "theoretical_loss": 3.4810369603394884, + "tokens_seen": 1682571264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985656565656565, + "loss": 2.6558, + "theoretical_loss": 3.4810252678819733, + "tokens_seen": 1682636800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985454545454545, + "loss": 2.6452, + "theoretical_loss": 3.481013576007359, + "tokens_seen": 1682702336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985252525252525, + "loss": 2.6641, + "theoretical_loss": 3.481001884715594, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985050505050506, + "loss": 2.5645, + "theoretical_loss": 3.4809901940066257, + "tokens_seen": 1682833408 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 929249, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0171878337860107, + "objective/train/theoretical_loss": 3.4809785038804026, + "objective/train/tokens_used": 41758176, + "theoretical_loss": 3.4809785038804026, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984848484848485, + "loss": 2.6663, + "theoretical_loss": 3.4809785038804026, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984646464646465, + "loss": 2.733, + "theoretical_loss": 3.480966814336873, + "tokens_seen": 1682964480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984444444444445, + "loss": 2.5084, + "theoretical_loss": 3.480955125375986, + "tokens_seen": 1683030016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984242424242425, + "loss": 2.7494, + "theoretical_loss": 3.4809434369976886, + "tokens_seen": 1683095552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984040404040404, + "loss": 2.3954, + "theoretical_loss": 3.48093174920193, + "tokens_seen": 1683161088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983838383838384, + "loss": 2.5031, + "theoretical_loss": 3.480920061988657, + "tokens_seen": 1683226624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983636363636363, + "loss": 2.5551, + "theoretical_loss": 3.48090837535782, + "tokens_seen": 1683292160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983434343434344, + "loss": 2.5852, + "theoretical_loss": 3.480896689309366, + "tokens_seen": 1683357696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983232323232323, + "loss": 2.7372, + "theoretical_loss": 3.4808850038432437, + "tokens_seen": 1683423232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983030303030303, + "loss": 2.6248, + "theoretical_loss": 3.480873318959401, + "tokens_seen": 1683488768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982828282828282, + "loss": 2.3415, + "theoretical_loss": 3.4808616346577868, + "tokens_seen": 1683554304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982626262626263, + "loss": 2.6582, + "theoretical_loss": 3.480849950938349, + "tokens_seen": 1683619840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982424242424243, + "loss": 2.5619, + "theoretical_loss": 3.4808382678010363, + "tokens_seen": 1683685376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982222222222223, + "loss": 2.7625, + "theoretical_loss": 3.480826585245797, + "tokens_seen": 1683750912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982020202020202, + "loss": 2.5896, + "theoretical_loss": 3.480814903272579, + "tokens_seen": 1683816448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981818181818182, + "loss": 2.7364, + "theoretical_loss": 3.4808032218813305, + "tokens_seen": 1683881984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981616161616162, + "loss": 2.7586, + "theoretical_loss": 3.4807915410720005, + "tokens_seen": 1683947520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981414141414142, + "loss": 2.5097, + "theoretical_loss": 3.480779860844538, + "tokens_seen": 1684013056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981212121212121, + "loss": 2.7616, + "theoretical_loss": 3.4807681811988895, + "tokens_seen": 1684078592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981010101010101, + "loss": 2.5229, + "theoretical_loss": 3.480756502135005, + "tokens_seen": 1684144128 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498080808080808, + "loss": 2.5937, + "theoretical_loss": 3.4807448236528318, + "tokens_seen": 1684209664 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498060606060606, + "loss": 2.705, + "theoretical_loss": 3.480733145752319, + "tokens_seen": 1684275200 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498040404040404, + "loss": 2.6938, + "theoretical_loss": 3.4807214684334147, + "tokens_seen": 1684340736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980202020202021, + "loss": 2.6702, + "theoretical_loss": 3.480709791696068, + "tokens_seen": 1684406272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498, + "loss": 2.466, + "theoretical_loss": 3.4806981155402257, + "tokens_seen": 1684471808 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 929812, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.769881010055542, + "objective/train/theoretical_loss": 3.4806864399658375, + "objective/train/tokens_used": 43396576, + "theoretical_loss": 3.4806864399658375, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497979797979798, + "loss": 2.5807, + "theoretical_loss": 3.4806864399658375, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497959595959596, + "loss": 2.532, + "theoretical_loss": 3.4806747649728518, + "tokens_seen": 1684602880 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497939393939394, + "loss": 2.4939, + "theoretical_loss": 3.480663090561217, + "tokens_seen": 1684668416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979191919191919, + "loss": 2.6286, + "theoretical_loss": 3.480651416730881, + "tokens_seen": 1684733952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978989898989899, + "loss": 2.5486, + "theoretical_loss": 3.4806397434817926, + "tokens_seen": 1684799488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978787878787879, + "loss": 2.4135, + "theoretical_loss": 3.4806280708139, + "tokens_seen": 1684865024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978585858585858, + "loss": 2.487, + "theoretical_loss": 3.480616398727152, + "tokens_seen": 1684930560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978383838383838, + "loss": 2.5598, + "theoretical_loss": 3.480604727221497, + "tokens_seen": 1684996096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978181818181818, + "loss": 2.6736, + "theoretical_loss": 3.4805930562968834, + "tokens_seen": 1685061632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977979797979799, + "loss": 2.4844, + "theoretical_loss": 3.4805813859532595, + "tokens_seen": 1685127168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977777777777778, + "loss": 2.7091, + "theoretical_loss": 3.4805697161905744, + "tokens_seen": 1685192704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977575757575758, + "loss": 2.5024, + "theoretical_loss": 3.480558047008776, + "tokens_seen": 1685258240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977373737373738, + "loss": 2.5198, + "theoretical_loss": 3.480546378407812, + "tokens_seen": 1685323776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977171717171717, + "loss": 2.44, + "theoretical_loss": 3.4805347103876327, + "tokens_seen": 1685389312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976969696969697, + "loss": 2.524, + "theoretical_loss": 3.480523042948186, + "tokens_seen": 1685454848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976767676767677, + "loss": 2.6416, + "theoretical_loss": 3.48051137608942, + "tokens_seen": 1685520384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976565656565656, + "loss": 2.5126, + "theoretical_loss": 3.480499709811283, + "tokens_seen": 1685585920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976363636363636, + "loss": 2.6729, + "theoretical_loss": 3.4804880441137245, + "tokens_seen": 1685651456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976161616161616, + "loss": 2.4778, + "theoretical_loss": 3.480476378996692, + "tokens_seen": 1685716992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975959595959596, + "loss": 2.5328, + "theoretical_loss": 3.4804647144601346, + "tokens_seen": 1685782528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975757575757575, + "loss": 2.6505, + "theoretical_loss": 3.480453050504001, + "tokens_seen": 1685848064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975555555555556, + "loss": 2.5126, + "theoretical_loss": 3.4804413871282396, + "tokens_seen": 1685913600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975353535353536, + "loss": 2.6655, + "theoretical_loss": 3.480429724332798, + "tokens_seen": 1685979136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975151515151516, + "loss": 2.5183, + "theoretical_loss": 3.480418062117627, + "tokens_seen": 1686044672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974949494949495, + "loss": 2.2442, + "theoretical_loss": 3.480406400482673, + "tokens_seen": 1686110208 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 931049, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4905221462249756, + "objective/train/theoretical_loss": 3.4803947394278856, + "objective/train/tokens_used": 45034976, + "theoretical_loss": 3.4803947394278856, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974747474747475, + "loss": 2.4943, + "theoretical_loss": 3.4803947394278856, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974545454545454, + "loss": 2.2533, + "theoretical_loss": 3.4803830789532126, + "tokens_seen": 1686241280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974343434343435, + "loss": 2.7416, + "theoretical_loss": 3.4803714190586037, + "tokens_seen": 1686306816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974141414141414, + "loss": 2.4504, + "theoretical_loss": 3.4803597597440072, + "tokens_seen": 1686372352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973939393939394, + "loss": 2.7452, + "theoretical_loss": 3.4803481010093718, + "tokens_seen": 1686437888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973737373737373, + "loss": 2.6507, + "theoretical_loss": 3.4803364428546453, + "tokens_seen": 1686503424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973535353535353, + "loss": 2.2988, + "theoretical_loss": 3.4803247852797767, + "tokens_seen": 1686568960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973333333333334, + "loss": 2.5663, + "theoretical_loss": 3.480313128284715, + "tokens_seen": 1686634496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973131313131314, + "loss": 2.518, + "theoretical_loss": 3.4803014718694087, + "tokens_seen": 1686700032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972929292929293, + "loss": 2.4251, + "theoretical_loss": 3.4802898160338063, + "tokens_seen": 1686765568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972727272727273, + "loss": 2.5484, + "theoretical_loss": 3.480278160777856, + "tokens_seen": 1686831104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972525252525253, + "loss": 2.3676, + "theoretical_loss": 3.4802665061015077, + "tokens_seen": 1686896640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972323232323233, + "loss": 2.3593, + "theoretical_loss": 3.480254852004709, + "tokens_seen": 1686962176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972121212121212, + "loss": 2.5334, + "theoretical_loss": 3.4802431984874094, + "tokens_seen": 1687027712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971919191919192, + "loss": 2.5443, + "theoretical_loss": 3.4802315455495565, + "tokens_seen": 1687093248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971717171717171, + "loss": 2.473, + "theoretical_loss": 3.4802198931910993, + "tokens_seen": 1687158784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971515151515152, + "loss": 2.5458, + "theoretical_loss": 3.480208241411987, + "tokens_seen": 1687224320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313131313131, + "loss": 2.5911, + "theoretical_loss": 3.4801965902121683, + "tokens_seen": 1687289856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971111111111111, + "loss": 2.5036, + "theoretical_loss": 3.480184939591591, + "tokens_seen": 1687355392 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497090909090909, + "loss": 2.452, + "theoretical_loss": 3.480173289550205, + "tokens_seen": 1687420928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970707070707071, + "loss": 2.3563, + "theoretical_loss": 3.4801616400879585, + "tokens_seen": 1687486464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970505050505051, + "loss": 2.6628, + "theoretical_loss": 3.4801499912048, + "tokens_seen": 1687552000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970303030303031, + "loss": 2.4553, + "theoretical_loss": 3.480138342900678, + "tokens_seen": 1687617536 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497010101010101, + "loss": 2.6518, + "theoretical_loss": 3.480126695175542, + "tokens_seen": 1687683072 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496989898989899, + "loss": 2.6693, + "theoretical_loss": 3.48011504802934, + "tokens_seen": 1687748608 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 931336, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.531233072280884, + "objective/train/theoretical_loss": 3.4801034014620216, + "objective/train/tokens_used": 46673376, + "theoretical_loss": 3.4801034014620216, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496969696969697, + "loss": 2.5694, + "theoretical_loss": 3.4801034014620216, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496949494949495, + "loss": 2.4312, + "theoretical_loss": 3.4800917554735347, + "tokens_seen": 1687879680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969292929292929, + "loss": 2.6901, + "theoretical_loss": 3.4800801100638283, + "tokens_seen": 1687945216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969090909090909, + "loss": 2.7315, + "theoretical_loss": 3.480068465232852, + "tokens_seen": 1688010752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968888888888889, + "loss": 2.5066, + "theoretical_loss": 3.480056820980553, + "tokens_seen": 1688076288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968686868686868, + "loss": 2.4754, + "theoretical_loss": 3.4800451773068817, + "tokens_seen": 1688141824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968484848484849, + "loss": 2.8056, + "theoretical_loss": 3.4800335342117856, + "tokens_seen": 1688207360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968282828282829, + "loss": 2.6707, + "theoretical_loss": 3.4800218916952144, + "tokens_seen": 1688272896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968080808080808, + "loss": 2.6028, + "theoretical_loss": 3.4800102497571164, + "tokens_seen": 1688338432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967878787878788, + "loss": 2.447, + "theoretical_loss": 3.47999860839744, + "tokens_seen": 1688403968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967676767676768, + "loss": 2.5797, + "theoretical_loss": 3.479986967616135, + "tokens_seen": 1688469504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967474747474748, + "loss": 2.6184, + "theoretical_loss": 3.4799753274131495, + "tokens_seen": 1688535040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967272727272727, + "loss": 2.651, + "theoretical_loss": 3.479963687788433, + "tokens_seen": 1688600576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967070707070707, + "loss": 2.6058, + "theoretical_loss": 3.4799520487419335, + "tokens_seen": 1688666112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966868686868687, + "loss": 2.5568, + "theoretical_loss": 3.4799404102736005, + "tokens_seen": 1688731648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966666666666666, + "loss": 2.6325, + "theoretical_loss": 3.4799287723833823, + "tokens_seen": 1688797184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966464646464646, + "loss": 2.4631, + "theoretical_loss": 3.4799171350712284, + "tokens_seen": 1688862720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966262626262627, + "loss": 2.8322, + "theoretical_loss": 3.4799054983370867, + "tokens_seen": 1688928256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966060606060607, + "loss": 2.5475, + "theoretical_loss": 3.4798938621809077, + "tokens_seen": 1688993792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965858585858586, + "loss": 2.3884, + "theoretical_loss": 3.4798822266026384, + "tokens_seen": 1689059328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965656565656566, + "loss": 2.6883, + "theoretical_loss": 3.4798705916022286, + "tokens_seen": 1689124864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965454545454546, + "loss": 2.4336, + "theoretical_loss": 3.4798589571796272, + "tokens_seen": 1689190400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965252525252525, + "loss": 2.6315, + "theoretical_loss": 3.4798473233347833, + "tokens_seen": 1689255936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965050505050505, + "loss": 2.5791, + "theoretical_loss": 3.4798356900676444, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964848484848485, + "loss": 2.4727, + "theoretical_loss": 3.4798240573781616, + "tokens_seen": 1689387008 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 932508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.615687847137451, + "objective/train/theoretical_loss": 3.479812425266282, + "objective/train/tokens_used": 48311776, + "theoretical_loss": 3.479812425266282, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964646464646464, + "loss": 2.4249, + "theoretical_loss": 3.479812425266282, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964444444444444, + "loss": 2.4928, + "theoretical_loss": 3.4798007937319557, + "tokens_seen": 1689518080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964242424242424, + "loss": 2.7017, + "theoretical_loss": 3.479789162775131, + "tokens_seen": 1689583616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964040404040405, + "loss": 2.6564, + "theoretical_loss": 3.4797775323957567, + "tokens_seen": 1689649152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963838383838384, + "loss": 2.4665, + "theoretical_loss": 3.479765902593782, + "tokens_seen": 1689714688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963636363636364, + "loss": 2.7239, + "theoretical_loss": 3.479754273369156, + "tokens_seen": 1689780224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963434343434344, + "loss": 2.3918, + "theoretical_loss": 3.4797426447218274, + "tokens_seen": 1689845760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963232323232324, + "loss": 2.6125, + "theoretical_loss": 3.479731016651745, + "tokens_seen": 1689911296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963030303030303, + "loss": 2.9569, + "theoretical_loss": 3.4797193891588583, + "tokens_seen": 1689976832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962828282828283, + "loss": 2.43, + "theoretical_loss": 3.479707762243116, + "tokens_seen": 1690042368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962626262626262, + "loss": 2.442, + "theoretical_loss": 3.479696135904467, + "tokens_seen": 1690107904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962424242424243, + "loss": 2.9013, + "theoretical_loss": 3.47968451014286, + "tokens_seen": 1690173440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962222222222222, + "loss": 2.6167, + "theoretical_loss": 3.479672884958245, + "tokens_seen": 1690238976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962020202020202, + "loss": 2.7755, + "theoretical_loss": 3.4796612603505697, + "tokens_seen": 1690304512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961818181818181, + "loss": 2.6197, + "theoretical_loss": 3.4796496363197837, + "tokens_seen": 1690370048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961616161616161, + "loss": 2.6195, + "theoretical_loss": 3.479638012865836, + "tokens_seen": 1690435584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961414141414142, + "loss": 2.691, + "theoretical_loss": 3.4796263899886757, + "tokens_seen": 1690501120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961212121212122, + "loss": 2.7274, + "theoretical_loss": 3.479614767688252, + "tokens_seen": 1690566656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961010101010101, + "loss": 2.7084, + "theoretical_loss": 3.4796031459645134, + "tokens_seen": 1690632192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960808080808081, + "loss": 2.6796, + "theoretical_loss": 3.479591524817409, + "tokens_seen": 1690697728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960606060606061, + "loss": 2.3627, + "theoretical_loss": 3.4795799042468882, + "tokens_seen": 1690763264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960404040404041, + "loss": 2.5495, + "theoretical_loss": 3.4795682842529, + "tokens_seen": 1690828800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496020202020202, + "loss": 2.6268, + "theoretical_loss": 3.4795566648353935, + "tokens_seen": 1690894336 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496, + "loss": 2.614, + "theoretical_loss": 3.4795450459943167, + "tokens_seen": 1690959872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959797979797979, + "loss": 2.4627, + "theoretical_loss": 3.4795334277296206, + "tokens_seen": 1691025408 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 933011, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.424670696258545, + "objective/train/theoretical_loss": 3.479521810041253, + "objective/train/tokens_used": 49950176, + "theoretical_loss": 3.479521810041253, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959595959595959, + "loss": 2.6558, + "theoretical_loss": 3.479521810041253, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495939393939394, + "loss": 2.5806, + "theoretical_loss": 3.4795101929291627, + "tokens_seen": 1691156480 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495919191919192, + "loss": 2.6571, + "theoretical_loss": 3.4794985763932997, + "tokens_seen": 1691222016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958989898989899, + "loss": 2.6959, + "theoretical_loss": 3.4794869604336123, + "tokens_seen": 1691287552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958787878787879, + "loss": 2.4706, + "theoretical_loss": 3.47947534505005, + "tokens_seen": 1691353088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958585858585859, + "loss": 2.572, + "theoretical_loss": 3.4794637302425624, + "tokens_seen": 1691418624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958383838383839, + "loss": 2.7139, + "theoretical_loss": 3.479452116011098, + "tokens_seen": 1691484160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958181818181818, + "loss": 2.6099, + "theoretical_loss": 3.4794405023556054, + "tokens_seen": 1691549696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957979797979798, + "loss": 2.4053, + "theoretical_loss": 3.4794288892760346, + "tokens_seen": 1691615232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957777777777778, + "loss": 2.8233, + "theoretical_loss": 3.4794172767723346, + "tokens_seen": 1691680768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957575757575757, + "loss": 2.7725, + "theoretical_loss": 3.479405664844454, + "tokens_seen": 1691746304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957373737373737, + "loss": 2.6491, + "theoretical_loss": 3.479394053492343, + "tokens_seen": 1691811840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957171717171717, + "loss": 2.516, + "theoretical_loss": 3.4793824427159494, + "tokens_seen": 1691877376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956969696969698, + "loss": 2.474, + "theoretical_loss": 3.4793708325152237, + "tokens_seen": 1691942912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956767676767677, + "loss": 2.3708, + "theoretical_loss": 3.479359222890114, + "tokens_seen": 1692008448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956565656565657, + "loss": 2.6737, + "theoretical_loss": 3.47934761384057, + "tokens_seen": 1692073984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956363636363637, + "loss": 2.5308, + "theoretical_loss": 3.479336005366541, + "tokens_seen": 1692139520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956161616161616, + "loss": 2.4276, + "theoretical_loss": 3.4793243974679755, + "tokens_seen": 1692205056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955959595959596, + "loss": 2.6885, + "theoretical_loss": 3.479312790144823, + "tokens_seen": 1692270592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955757575757576, + "loss": 2.2714, + "theoretical_loss": 3.479301183397033, + "tokens_seen": 1692336128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955555555555556, + "loss": 2.4882, + "theoretical_loss": 3.479289577224555, + "tokens_seen": 1692401664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955353535353535, + "loss": 2.4553, + "theoretical_loss": 3.4792779716273365, + "tokens_seen": 1692467200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955151515151515, + "loss": 2.6182, + "theoretical_loss": 3.479266366605329, + "tokens_seen": 1692532736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954949494949495, + "loss": 2.5212, + "theoretical_loss": 3.4792547621584804, + "tokens_seen": 1692598272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954747474747474, + "loss": 2.4225, + "theoretical_loss": 3.47924315828674, + "tokens_seen": 1692663808 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 934010, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.852196455001831, + "objective/train/theoretical_loss": 3.479231554990057, + "objective/train/tokens_used": 51588576, + "theoretical_loss": 3.479231554990057, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954545454545455, + "loss": 2.6898, + "theoretical_loss": 3.479231554990057, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954343434343435, + "loss": 2.6057, + "theoretical_loss": 3.479219952268381, + "tokens_seen": 1692794880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954141414141415, + "loss": 2.5534, + "theoretical_loss": 3.4792083501216613, + "tokens_seen": 1692860416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953939393939394, + "loss": 2.602, + "theoretical_loss": 3.479196748549847, + "tokens_seen": 1692925952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953737373737374, + "loss": 2.6322, + "theoretical_loss": 3.4791851475528874, + "tokens_seen": 1692991488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953535353535354, + "loss": 2.5945, + "theoretical_loss": 3.4791735471307312, + "tokens_seen": 1693057024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953333333333334, + "loss": 2.6971, + "theoretical_loss": 3.4791619472833286, + "tokens_seen": 1693122560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953131313131313, + "loss": 2.6478, + "theoretical_loss": 3.479150348010628, + "tokens_seen": 1693188096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952929292929293, + "loss": 2.5666, + "theoretical_loss": 3.479138749312579, + "tokens_seen": 1693253632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952727272727272, + "loss": 2.5181, + "theoretical_loss": 3.4791271511891315, + "tokens_seen": 1693319168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952525252525252, + "loss": 2.566, + "theoretical_loss": 3.4791155536402343, + "tokens_seen": 1693384704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952323232323232, + "loss": 2.7651, + "theoretical_loss": 3.479103956665836, + "tokens_seen": 1693450240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952121212121213, + "loss": 2.5736, + "theoretical_loss": 3.479092360265887, + "tokens_seen": 1693515776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951919191919192, + "loss": 2.4578, + "theoretical_loss": 3.4790807644403365, + "tokens_seen": 1693581312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951717171717172, + "loss": 2.6557, + "theoretical_loss": 3.479069169189133, + "tokens_seen": 1693646848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951515151515152, + "loss": 2.6888, + "theoretical_loss": 3.479057574512227, + "tokens_seen": 1693712384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951313131313132, + "loss": 2.5443, + "theoretical_loss": 3.4790459804095666, + "tokens_seen": 1693777920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951111111111111, + "loss": 2.7016, + "theoretical_loss": 3.479034386881102, + "tokens_seen": 1693843456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950909090909091, + "loss": 2.4926, + "theoretical_loss": 3.4790227939267826, + "tokens_seen": 1693908992 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495070707070707, + "loss": 2.7287, + "theoretical_loss": 3.479011201546557, + "tokens_seen": 1693974528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950505050505051, + "loss": 2.71, + "theoretical_loss": 3.478999609740375, + "tokens_seen": 1694040064 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495030303030303, + "loss": 2.4208, + "theoretical_loss": 3.4789880185081863, + "tokens_seen": 1694105600 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495010101010101, + "loss": 2.4963, + "theoretical_loss": 3.4789764278499398, + "tokens_seen": 1694171136 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494989898989899, + "loss": 2.7918, + "theoretical_loss": 3.478964837765585, + "tokens_seen": 1694236672 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494969696969697, + "loss": 2.3404, + "theoretical_loss": 3.4789532482550714, + "tokens_seen": 1694302208 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 934404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7420923709869385, + "objective/train/theoretical_loss": 3.478941659318348, + "objective/train/tokens_used": 53226976, + "theoretical_loss": 3.478941659318348, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494949494949495, + "loss": 2.6966, + "theoretical_loss": 3.478941659318348, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494929292929293, + "loss": 2.652, + "theoretical_loss": 3.478930070955365, + "tokens_seen": 1694433280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949090909090909, + "loss": 2.5731, + "theoretical_loss": 3.478918483166071, + "tokens_seen": 1694498816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948888888888889, + "loss": 2.5032, + "theoretical_loss": 3.478906895950416, + "tokens_seen": 1694564352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948686868686869, + "loss": 2.7563, + "theoretical_loss": 3.4788953093083492, + "tokens_seen": 1694629888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948484848484849, + "loss": 2.5857, + "theoretical_loss": 3.4788837232398198, + "tokens_seen": 1694695424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948282828282828, + "loss": 2.5759, + "theoretical_loss": 3.4788721377447773, + "tokens_seen": 1694760960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948080808080808, + "loss": 2.4183, + "theoretical_loss": 3.4788605528231713, + "tokens_seen": 1694826496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947878787878787, + "loss": 2.4682, + "theoretical_loss": 3.478848968474951, + "tokens_seen": 1694892032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947676767676767, + "loss": 2.7972, + "theoretical_loss": 3.4788373847000664, + "tokens_seen": 1694957568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947474747474748, + "loss": 2.5319, + "theoretical_loss": 3.4788258014984663, + "tokens_seen": 1695023104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947272727272728, + "loss": 2.4632, + "theoretical_loss": 3.478814218870101, + "tokens_seen": 1695088640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947070707070707, + "loss": 2.5513, + "theoretical_loss": 3.478802636814919, + "tokens_seen": 1695154176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946868686868687, + "loss": 2.5416, + "theoretical_loss": 3.4787910553328705, + "tokens_seen": 1695219712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946666666666667, + "loss": 2.6542, + "theoretical_loss": 3.4787794744239044, + "tokens_seen": 1695285248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946464646464647, + "loss": 2.5943, + "theoretical_loss": 3.4787678940879707, + "tokens_seen": 1695350784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946262626262626, + "loss": 2.5594, + "theoretical_loss": 3.4787563143250186, + "tokens_seen": 1695416320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946060606060606, + "loss": 2.4973, + "theoretical_loss": 3.478744735134998, + "tokens_seen": 1695481856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945858585858586, + "loss": 2.5056, + "theoretical_loss": 3.4787331565178574, + "tokens_seen": 1695547392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945656565656565, + "loss": 2.5805, + "theoretical_loss": 3.4787215784735475, + "tokens_seen": 1695612928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945454545454545, + "loss": 2.5839, + "theoretical_loss": 3.4787100010020175, + "tokens_seen": 1695678464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945252525252526, + "loss": 2.551, + "theoretical_loss": 3.478698424103216, + "tokens_seen": 1695744000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945050505050506, + "loss": 2.7043, + "theoretical_loss": 3.4786868477770945, + "tokens_seen": 1695809536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944848484848485, + "loss": 2.6309, + "theoretical_loss": 3.4786752720236005, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944646464646465, + "loss": 2.7457, + "theoretical_loss": 3.4786636968426845, + "tokens_seen": 1695940608 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 935560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.167574882507324, + "objective/train/theoretical_loss": 3.478652122234296, + "objective/train/tokens_used": 54865376, + "theoretical_loss": 3.478652122234296, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944444444444445, + "loss": 2.4537, + "theoretical_loss": 3.478652122234296, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944242424242424, + "loss": 2.5082, + "theoretical_loss": 3.4786405481983844, + "tokens_seen": 1696071680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944040404040404, + "loss": 2.716, + "theoretical_loss": 3.4786289747348995, + "tokens_seen": 1696137216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943838383838384, + "loss": 2.5441, + "theoretical_loss": 3.478617401843791, + "tokens_seen": 1696202752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943636363636363, + "loss": 2.5488, + "theoretical_loss": 3.4786058295250077, + "tokens_seen": 1696268288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943434343434343, + "loss": 2.5682, + "theoretical_loss": 3.4785942577785, + "tokens_seen": 1696333824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943232323232323, + "loss": 2.4028, + "theoretical_loss": 3.4785826866042173, + "tokens_seen": 1696399360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943030303030303, + "loss": 2.5064, + "theoretical_loss": 3.478571116002109, + "tokens_seen": 1696464896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828282828283, + "loss": 2.4192, + "theoretical_loss": 3.4785595459721246, + "tokens_seen": 1696530432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942626262626263, + "loss": 2.6105, + "theoretical_loss": 3.478547976514214, + "tokens_seen": 1696595968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942424242424243, + "loss": 2.6893, + "theoretical_loss": 3.4785364076283267, + "tokens_seen": 1696661504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942222222222223, + "loss": 2.4376, + "theoretical_loss": 3.478524839314413, + "tokens_seen": 1696727040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942020202020202, + "loss": 2.7309, + "theoretical_loss": 3.478513271572421, + "tokens_seen": 1696792576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941818181818182, + "loss": 2.8496, + "theoretical_loss": 3.4785017044023014, + "tokens_seen": 1696858112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941616161616161, + "loss": 2.423, + "theoretical_loss": 3.4784901378040036, + "tokens_seen": 1696923648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941414141414142, + "loss": 2.494, + "theoretical_loss": 3.4784785717774778, + "tokens_seen": 1696989184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941212121212121, + "loss": 2.53, + "theoretical_loss": 3.4784670063226732, + "tokens_seen": 1697054720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941010101010101, + "loss": 2.749, + "theoretical_loss": 3.478455441439539, + "tokens_seen": 1697120256 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494080808080808, + "loss": 2.3697, + "theoretical_loss": 3.4784438771280257, + "tokens_seen": 1697185792 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494060606060606, + "loss": 2.6826, + "theoretical_loss": 3.4784323133880823, + "tokens_seen": 1697251328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940404040404041, + "loss": 2.5308, + "theoretical_loss": 3.4784207502196587, + "tokens_seen": 1697316864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940202020202021, + "loss": 2.4568, + "theoretical_loss": 3.4784091876227046, + "tokens_seen": 1697382400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494, + "loss": 2.7751, + "theoretical_loss": 3.47839762559717, + "tokens_seen": 1697447936 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493979797979798, + "loss": 2.5151, + "theoretical_loss": 3.478386064143004, + "tokens_seen": 1697513472 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493959595959596, + "loss": 2.8479, + "theoretical_loss": 3.4783745032601567, + "tokens_seen": 1697579008 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 936682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.305478572845459, + "objective/train/theoretical_loss": 3.4783629429485785, + "objective/train/tokens_used": 56503776, + "theoretical_loss": 3.4783629429485785, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493939393939394, + "loss": 2.5086, + "theoretical_loss": 3.4783629429485785, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939191919191919, + "loss": 2.7043, + "theoretical_loss": 3.4783513832082176, + "tokens_seen": 1697710080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938989898989899, + "loss": 2.3848, + "theoretical_loss": 3.4783398240390246, + "tokens_seen": 1697775616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938787878787878, + "loss": 2.7298, + "theoretical_loss": 3.4783282654409495, + "tokens_seen": 1697841152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938585858585859, + "loss": 2.5754, + "theoretical_loss": 3.478316707413941, + "tokens_seen": 1697906688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938383838383838, + "loss": 2.7063, + "theoretical_loss": 3.4783051499579507, + "tokens_seen": 1697972224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938181818181819, + "loss": 2.6402, + "theoretical_loss": 3.478293593072926, + "tokens_seen": 1698037760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937979797979798, + "loss": 2.4452, + "theoretical_loss": 3.4782820367588183, + "tokens_seen": 1698103296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937777777777778, + "loss": 2.6044, + "theoretical_loss": 3.478270481015577, + "tokens_seen": 1698168832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937575757575758, + "loss": 2.6482, + "theoretical_loss": 3.4782589258431518, + "tokens_seen": 1698234368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937373737373738, + "loss": 2.7206, + "theoretical_loss": 3.4782473712414923, + "tokens_seen": 1698299904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937171717171717, + "loss": 2.4687, + "theoretical_loss": 3.4782358172105483, + "tokens_seen": 1698365440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936969696969697, + "loss": 2.7044, + "theoretical_loss": 3.47822426375027, + "tokens_seen": 1698430976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936767676767677, + "loss": 2.5647, + "theoretical_loss": 3.478212710860607, + "tokens_seen": 1698496512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936565656565657, + "loss": 2.6277, + "theoretical_loss": 3.4782011585415087, + "tokens_seen": 1698562048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936363636363636, + "loss": 2.7133, + "theoretical_loss": 3.4781896067929257, + "tokens_seen": 1698627584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936161616161616, + "loss": 2.5221, + "theoretical_loss": 3.4781780556148068, + "tokens_seen": 1698693120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935959595959597, + "loss": 2.6434, + "theoretical_loss": 3.4781665050071027, + "tokens_seen": 1698758656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935757575757576, + "loss": 2.7506, + "theoretical_loss": 3.478154954969763, + "tokens_seen": 1698824192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935555555555556, + "loss": 2.578, + "theoretical_loss": 3.4781434055027365, + "tokens_seen": 1698889728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935353535353536, + "loss": 2.5255, + "theoretical_loss": 3.478131856605975, + "tokens_seen": 1698955264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935151515151515, + "loss": 2.4671, + "theoretical_loss": 3.4781203082794265, + "tokens_seen": 1699020800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934949494949495, + "loss": 2.5736, + "theoretical_loss": 3.4781087605230425, + "tokens_seen": 1699086336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934747474747475, + "loss": 2.5738, + "theoretical_loss": 3.4780972133367714, + "tokens_seen": 1699151872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934545454545455, + "loss": 2.6036, + "theoretical_loss": 3.478085666720564, + "tokens_seen": 1699217408 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 937279, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8765950202941895, + "objective/train/theoretical_loss": 3.47807412067437, + "objective/train/tokens_used": 58142176, + "theoretical_loss": 3.47807412067437, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934343434343434, + "loss": 2.6055, + "theoretical_loss": 3.47807412067437, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934141414141414, + "loss": 2.5349, + "theoretical_loss": 3.4780625751981384, + "tokens_seen": 1699348480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933939393939394, + "loss": 2.7046, + "theoretical_loss": 3.478051030291821, + "tokens_seen": 1699414016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933737373737373, + "loss": 2.6394, + "theoretical_loss": 3.4780394859553656, + "tokens_seen": 1699479552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933535353535354, + "loss": 2.4959, + "theoretical_loss": 3.4780279421887235, + "tokens_seen": 1699545088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933333333333334, + "loss": 2.7816, + "theoretical_loss": 3.478016398991844, + "tokens_seen": 1699610624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933131313131314, + "loss": 2.622, + "theoretical_loss": 3.478004856364677, + "tokens_seen": 1699676160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932929292929293, + "loss": 2.6067, + "theoretical_loss": 3.477993314307173, + "tokens_seen": 1699741696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932727272727273, + "loss": 2.6708, + "theoretical_loss": 3.4779817728192803, + "tokens_seen": 1699807232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932525252525253, + "loss": 2.7044, + "theoretical_loss": 3.477970231900951, + "tokens_seen": 1699872768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932323232323232, + "loss": 2.5886, + "theoretical_loss": 3.477958691552134, + "tokens_seen": 1699938304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932121212121212, + "loss": 2.4632, + "theoretical_loss": 3.4779471517727787, + "tokens_seen": 1700003840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931919191919192, + "loss": 2.6896, + "theoretical_loss": 3.4779356125628365, + "tokens_seen": 1700069376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931717171717171, + "loss": 2.655, + "theoretical_loss": 3.4779240739222557, + "tokens_seen": 1700134912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931515151515151, + "loss": 2.6488, + "theoretical_loss": 3.4779125358509875, + "tokens_seen": 1700200448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931313131313131, + "loss": 2.6737, + "theoretical_loss": 3.4779009983489813, + "tokens_seen": 1700265984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931111111111112, + "loss": 2.5862, + "theoretical_loss": 3.4778894614161873, + "tokens_seen": 1700331520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930909090909091, + "loss": 2.5534, + "theoretical_loss": 3.477877925052555, + "tokens_seen": 1700397056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930707070707071, + "loss": 2.6173, + "theoretical_loss": 3.4778663892580353, + "tokens_seen": 1700462592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930505050505051, + "loss": 2.7138, + "theoretical_loss": 3.477854854032577, + "tokens_seen": 1700528128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930303030303031, + "loss": 2.5778, + "theoretical_loss": 3.477843319376131, + "tokens_seen": 1700593664 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493010101010101, + "loss": 2.5768, + "theoretical_loss": 3.4778317852886476, + "tokens_seen": 1700659200 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492989898989899, + "loss": 2.8706, + "theoretical_loss": 3.4778202517700754, + "tokens_seen": 1700724736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929696969696969, + "loss": 2.4533, + "theoretical_loss": 3.4778087188203655, + "tokens_seen": 1700790272 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492949494949495, + "loss": 2.6694, + "theoretical_loss": 3.477797186439468, + "tokens_seen": 1700855808 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 938365, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9837400913238525, + "objective/train/theoretical_loss": 3.4777856546273327, + "objective/train/tokens_used": 59780576, + "theoretical_loss": 3.4777856546273327, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929292929292929, + "loss": 2.7473, + "theoretical_loss": 3.4777856546273327, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929090909090909, + "loss": 2.4495, + "theoretical_loss": 3.477774123383909, + "tokens_seen": 1700986880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928888888888888, + "loss": 2.609, + "theoretical_loss": 3.477762592709148, + "tokens_seen": 1701052416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928686868686869, + "loss": 2.5167, + "theoretical_loss": 3.477751062602999, + "tokens_seen": 1701117952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928484848484849, + "loss": 2.4444, + "theoretical_loss": 3.477739533065412, + "tokens_seen": 1701183488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928282828282829, + "loss": 2.5933, + "theoretical_loss": 3.4777280040963383, + "tokens_seen": 1701249024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928080808080808, + "loss": 2.707, + "theoretical_loss": 3.4777164756957264, + "tokens_seen": 1701314560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927878787878788, + "loss": 2.5917, + "theoretical_loss": 3.477704947863527, + "tokens_seen": 1701380096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927676767676768, + "loss": 2.5673, + "theoretical_loss": 3.4776934205996906, + "tokens_seen": 1701445632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927474747474748, + "loss": 2.8966, + "theoretical_loss": 3.4776818939041663, + "tokens_seen": 1701511168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927272727272727, + "loss": 2.6156, + "theoretical_loss": 3.477670367776905, + "tokens_seen": 1701576704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927070707070707, + "loss": 2.6147, + "theoretical_loss": 3.477658842217857, + "tokens_seen": 1701642240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926868686868686, + "loss": 2.613, + "theoretical_loss": 3.4776473172269715, + "tokens_seen": 1701707776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926666666666666, + "loss": 2.7808, + "theoretical_loss": 3.4776357928041994, + "tokens_seen": 1701773312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926464646464647, + "loss": 2.6945, + "theoretical_loss": 3.4776242689494903, + "tokens_seen": 1701838848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926262626262627, + "loss": 2.7847, + "theoretical_loss": 3.4776127456627948, + "tokens_seen": 1701904384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926060606060606, + "loss": 2.7205, + "theoretical_loss": 3.4776012229440623, + "tokens_seen": 1701969920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925858585858586, + "loss": 2.6992, + "theoretical_loss": 3.477589700793244, + "tokens_seen": 1702035456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925656565656566, + "loss": 2.6906, + "theoretical_loss": 3.4775781792102887, + "tokens_seen": 1702100992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925454545454546, + "loss": 2.659, + "theoretical_loss": 3.477566658195148, + "tokens_seen": 1702166528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925252525252525, + "loss": 2.727, + "theoretical_loss": 3.477555137747771, + "tokens_seen": 1702232064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925050505050505, + "loss": 2.7682, + "theoretical_loss": 3.4775436178681085, + "tokens_seen": 1702297600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924848484848485, + "loss": 2.5353, + "theoretical_loss": 3.4775320985561105, + "tokens_seen": 1702363136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924646464646464, + "loss": 2.6144, + "theoretical_loss": 3.477520579811727, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924444444444444, + "loss": 2.7138, + "theoretical_loss": 3.477509061634908, + "tokens_seen": 1702494208 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 938945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2912395000457764, + "objective/train/theoretical_loss": 3.4774975440256037, + "objective/train/tokens_used": 61418976, + "theoretical_loss": 3.4774975440256037, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924242424242425, + "loss": 2.7635, + "theoretical_loss": 3.4774975440256037, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924040404040405, + "loss": 2.761, + "theoretical_loss": 3.4774860269837653, + "tokens_seen": 1702625280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923838383838384, + "loss": 2.6293, + "theoretical_loss": 3.4774745105093414, + "tokens_seen": 1702690816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923636363636364, + "loss": 2.6983, + "theoretical_loss": 3.4774629946022837, + "tokens_seen": 1702756352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923434343434344, + "loss": 2.6871, + "theoretical_loss": 3.477451479262541, + "tokens_seen": 1702821888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923232323232323, + "loss": 2.7305, + "theoretical_loss": 3.477439964490065, + "tokens_seen": 1702887424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923030303030303, + "loss": 2.6126, + "theoretical_loss": 3.477428450284805, + "tokens_seen": 1702952960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922828282828283, + "loss": 2.365, + "theoretical_loss": 3.4774169366467116, + "tokens_seen": 1703018496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922626262626262, + "loss": 2.6455, + "theoretical_loss": 3.4774054235757346, + "tokens_seen": 1703084032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922424242424242, + "loss": 2.5675, + "theoretical_loss": 3.4773939110718244, + "tokens_seen": 1703149568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922222222222222, + "loss": 2.5027, + "theoretical_loss": 3.4773823991349317, + "tokens_seen": 1703215104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922020202020202, + "loss": 2.7077, + "theoretical_loss": 3.4773708877650065, + "tokens_seen": 1703280640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921818181818182, + "loss": 2.5908, + "theoretical_loss": 3.477359376961999, + "tokens_seen": 1703346176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921616161616162, + "loss": 2.6174, + "theoretical_loss": 3.477347866725859, + "tokens_seen": 1703411712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921414141414142, + "loss": 2.6228, + "theoretical_loss": 3.4773363570565374, + "tokens_seen": 1703477248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921212121212122, + "loss": 2.8639, + "theoretical_loss": 3.4773248479539847, + "tokens_seen": 1703542784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921010101010101, + "loss": 2.75, + "theoretical_loss": 3.47731333941815, + "tokens_seen": 1703608320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920808080808081, + "loss": 2.4292, + "theoretical_loss": 3.477301831448985, + "tokens_seen": 1703673856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920606060606061, + "loss": 2.6518, + "theoretical_loss": 3.4772903240464395, + "tokens_seen": 1703739392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920404040404041, + "loss": 2.7381, + "theoretical_loss": 3.477278817210463, + "tokens_seen": 1703804928 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492020202020202, + "loss": 2.7286, + "theoretical_loss": 3.4772673109410075, + "tokens_seen": 1703870464 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492, + "loss": 2.7131, + "theoretical_loss": 3.4772558052380216, + "tokens_seen": 1703936000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919797979797979, + "loss": 2.8491, + "theoretical_loss": 3.4772443001014564, + "tokens_seen": 1704001536 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491959595959596, + "loss": 2.6865, + "theoretical_loss": 3.4772327955312625, + "tokens_seen": 1704067072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491939393939394, + "loss": 2.4784, + "theoretical_loss": 3.47722129152739, + "tokens_seen": 1704132608 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 940087, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.313035726547241, + "objective/train/theoretical_loss": 3.477209788089789, + "objective/train/tokens_used": 63057376, + "theoretical_loss": 3.477209788089789, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491919191919192, + "loss": 2.5144, + "theoretical_loss": 3.477209788089789, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918989898989899, + "loss": 2.4958, + "theoretical_loss": 3.4771982852184102, + "tokens_seen": 1704263680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918787878787879, + "loss": 2.6901, + "theoretical_loss": 3.4771867829132033, + "tokens_seen": 1704329216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918585858585859, + "loss": 2.7283, + "theoretical_loss": 3.4771752811741194, + "tokens_seen": 1704394752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918383838383839, + "loss": 2.5867, + "theoretical_loss": 3.477163780001109, + "tokens_seen": 1704460288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918181818181818, + "loss": 2.6477, + "theoretical_loss": 3.477152279394122, + "tokens_seen": 1704525824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917979797979798, + "loss": 2.6709, + "theoretical_loss": 3.4771407793531086, + "tokens_seen": 1704591360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917777777777777, + "loss": 2.6038, + "theoretical_loss": 3.4771292798780196, + "tokens_seen": 1704656896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917575757575758, + "loss": 2.9226, + "theoretical_loss": 3.477117780968805, + "tokens_seen": 1704722432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917373737373737, + "loss": 2.5123, + "theoretical_loss": 3.4771062826254155, + "tokens_seen": 1704787968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917171717171718, + "loss": 2.5392, + "theoretical_loss": 3.477094784847802, + "tokens_seen": 1704853504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916969696969697, + "loss": 2.8546, + "theoretical_loss": 3.477083287635914, + "tokens_seen": 1704919040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916767676767677, + "loss": 2.5932, + "theoretical_loss": 3.4770717909897026, + "tokens_seen": 1704984576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916565656565657, + "loss": 2.6186, + "theoretical_loss": 3.4770602949091174, + "tokens_seen": 1705050112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916363636363637, + "loss": 2.6519, + "theoretical_loss": 3.47704879939411, + "tokens_seen": 1705115648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916161616161616, + "loss": 2.6116, + "theoretical_loss": 3.4770373044446297, + "tokens_seen": 1705181184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915959595959596, + "loss": 2.7875, + "theoretical_loss": 3.4770258100606277, + "tokens_seen": 1705246720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915757575757576, + "loss": 2.621, + "theoretical_loss": 3.4770143162420544, + "tokens_seen": 1705312256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915555555555556, + "loss": 2.7483, + "theoretical_loss": 3.47700282298886, + "tokens_seen": 1705377792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915353535353535, + "loss": 2.6616, + "theoretical_loss": 3.4769913303009945, + "tokens_seen": 1705443328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915151515151515, + "loss": 2.8028, + "theoretical_loss": 3.4769798381784094, + "tokens_seen": 1705508864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914949494949494, + "loss": 2.3576, + "theoretical_loss": 3.4769683466210544, + "tokens_seen": 1705574400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914747474747475, + "loss": 2.6636, + "theoretical_loss": 3.47695685562888, + "tokens_seen": 1705639936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914545454545455, + "loss": 2.7262, + "theoretical_loss": 3.4769453652018374, + "tokens_seen": 1705705472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343434343435, + "loss": 2.6114, + "theoretical_loss": 3.476933875339877, + "tokens_seen": 1705771008 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 941279, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1284117698669434, + "objective/train/theoretical_loss": 3.4769223860429483, + "objective/train/tokens_used": 64695776, + "theoretical_loss": 3.4769223860429483, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914141414141414, + "loss": 2.6038, + "theoretical_loss": 3.4769223860429483, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913939393939394, + "loss": 2.5564, + "theoretical_loss": 3.4769108973110026, + "tokens_seen": 1705902080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913737373737374, + "loss": 2.6878, + "theoretical_loss": 3.47689940914399, + "tokens_seen": 1705967616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913535353535354, + "loss": 2.7768, + "theoretical_loss": 3.476887921541862, + "tokens_seen": 1706033152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913333333333333, + "loss": 2.5874, + "theoretical_loss": 3.4768764345045677, + "tokens_seen": 1706098688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913131313131313, + "loss": 2.4851, + "theoretical_loss": 3.476864948032059, + "tokens_seen": 1706164224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912929292929293, + "loss": 2.3652, + "theoretical_loss": 3.4768534621242853, + "tokens_seen": 1706229760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912727272727272, + "loss": 2.3602, + "theoretical_loss": 3.4768419767811976, + "tokens_seen": 1706295296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912525252525252, + "loss": 2.6307, + "theoretical_loss": 3.4768304920027466, + "tokens_seen": 1706360832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912323232323233, + "loss": 2.6657, + "theoretical_loss": 3.4768190077888823, + "tokens_seen": 1706426368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912121212121213, + "loss": 2.5756, + "theoretical_loss": 3.4768075241395566, + "tokens_seen": 1706491904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911919191919192, + "loss": 2.7722, + "theoretical_loss": 3.4767960410547185, + "tokens_seen": 1706557440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911717171717172, + "loss": 2.288, + "theoretical_loss": 3.4767845585343196, + "tokens_seen": 1706622976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911515151515152, + "loss": 2.4128, + "theoretical_loss": 3.4767730765783096, + "tokens_seen": 1706688512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911313131313131, + "loss": 2.4081, + "theoretical_loss": 3.47676159518664, + "tokens_seen": 1706754048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911111111111111, + "loss": 2.6869, + "theoretical_loss": 3.4767501143592607, + "tokens_seen": 1706819584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910909090909091, + "loss": 2.5965, + "theoretical_loss": 3.4767386340961233, + "tokens_seen": 1706885120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491070707070707, + "loss": 2.5827, + "theoretical_loss": 3.476727154397177, + "tokens_seen": 1706950656 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491050505050505, + "loss": 2.7387, + "theoretical_loss": 3.4767156752623736, + "tokens_seen": 1707016192 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491030303030303, + "loss": 2.7383, + "theoretical_loss": 3.4767041966916628, + "tokens_seen": 1707081728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910101010101011, + "loss": 2.7158, + "theoretical_loss": 3.476692718684996, + "tokens_seen": 1707147264 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490989898989899, + "loss": 2.6993, + "theoretical_loss": 3.476681241242323, + "tokens_seen": 1707212800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490969696969697, + "loss": 2.6435, + "theoretical_loss": 3.4766697643635958, + "tokens_seen": 1707278336 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490949494949495, + "loss": 2.6064, + "theoretical_loss": 3.4766582880487635, + "tokens_seen": 1707343872 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490929292929293, + "loss": 2.5374, + "theoretical_loss": 3.4766468122977776, + "tokens_seen": 1707409408 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 941668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3641209602355957, + "objective/train/theoretical_loss": 3.4766353371105887, + "objective/train/tokens_used": 66334176, + "theoretical_loss": 3.4766353371105887, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909090909090909, + "loss": 2.6239, + "theoretical_loss": 3.4766353371105887, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908888888888889, + "loss": 2.5148, + "theoretical_loss": 3.4766238624871475, + "tokens_seen": 1707540480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908686868686868, + "loss": 2.5122, + "theoretical_loss": 3.4766123884274043, + "tokens_seen": 1707606016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908484848484849, + "loss": 2.9696, + "theoretical_loss": 3.4766009149313097, + "tokens_seen": 1707671552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908282828282828, + "loss": 2.4315, + "theoretical_loss": 3.476589441998815, + "tokens_seen": 1707737088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908080808080808, + "loss": 2.6819, + "theoretical_loss": 3.4765779696298704, + "tokens_seen": 1707802624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907878787878787, + "loss": 2.6707, + "theoretical_loss": 3.476566497824427, + "tokens_seen": 1707868160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907676767676768, + "loss": 2.6873, + "theoretical_loss": 3.476555026582435, + "tokens_seen": 1707933696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907474747474748, + "loss": 2.6117, + "theoretical_loss": 3.476543555903846, + "tokens_seen": 1707999232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907272727272728, + "loss": 2.7386, + "theoretical_loss": 3.4765320857886097, + "tokens_seen": 1708064768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907070707070707, + "loss": 2.7186, + "theoretical_loss": 3.476520616236677, + "tokens_seen": 1708130304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906868686868687, + "loss": 2.6979, + "theoretical_loss": 3.476509147247999, + "tokens_seen": 1708195840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906666666666667, + "loss": 2.8907, + "theoretical_loss": 3.4764976788225264, + "tokens_seen": 1708261376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906464646464647, + "loss": 2.6496, + "theoretical_loss": 3.4764862109602097, + "tokens_seen": 1708326912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906262626262626, + "loss": 2.7096, + "theoretical_loss": 3.4764747436609995, + "tokens_seen": 1708392448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906060606060606, + "loss": 2.7149, + "theoretical_loss": 3.476463276924847, + "tokens_seen": 1708457984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905858585858585, + "loss": 2.8988, + "theoretical_loss": 3.476451810751703, + "tokens_seen": 1708523520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905656565656565, + "loss": 2.5287, + "theoretical_loss": 3.4764403451415173, + "tokens_seen": 1708589056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905454545454546, + "loss": 2.4729, + "theoretical_loss": 3.476428880094242, + "tokens_seen": 1708654592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905252525252526, + "loss": 2.7736, + "theoretical_loss": 3.476417415609827, + "tokens_seen": 1708720128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905050505050505, + "loss": 2.732, + "theoretical_loss": 3.476405951688223, + "tokens_seen": 1708785664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904848484848485, + "loss": 2.6687, + "theoretical_loss": 3.476394488329382, + "tokens_seen": 1708851200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904646464646465, + "loss": 2.7529, + "theoretical_loss": 3.476383025533253, + "tokens_seen": 1708916736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904444444444445, + "loss": 2.5937, + "theoretical_loss": 3.4763715632997885, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904242424242424, + "loss": 2.7642, + "theoretical_loss": 3.476360101628938, + "tokens_seen": 1709047808 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 943105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5795464515686035, + "objective/train/theoretical_loss": 3.476348640520653, + "objective/train/tokens_used": 67972576, + "theoretical_loss": 3.476348640520653, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904040404040404, + "loss": 2.6005, + "theoretical_loss": 3.476348640520653, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903838383838384, + "loss": 2.4409, + "theoretical_loss": 3.476337179974884, + "tokens_seen": 1709178880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903636363636364, + "loss": 2.6039, + "theoretical_loss": 3.476325719991582, + "tokens_seen": 1709244416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903434343434343, + "loss": 2.5914, + "theoretical_loss": 3.476314260570698, + "tokens_seen": 1709309952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903232323232323, + "loss": 2.4368, + "theoretical_loss": 3.4763028017121824, + "tokens_seen": 1709375488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903030303030304, + "loss": 2.7762, + "theoretical_loss": 3.4762913434159866, + "tokens_seen": 1709441024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902828282828283, + "loss": 2.6016, + "theoretical_loss": 3.476279885682061, + "tokens_seen": 1709506560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902626262626263, + "loss": 2.6904, + "theoretical_loss": 3.4762684285103562, + "tokens_seen": 1709572096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902424242424243, + "loss": 2.6499, + "theoretical_loss": 3.476256971900824, + "tokens_seen": 1709637632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902222222222222, + "loss": 2.5542, + "theoretical_loss": 3.4762455158534147, + "tokens_seen": 1709703168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902020202020202, + "loss": 2.8664, + "theoretical_loss": 3.476234060368079, + "tokens_seen": 1709768704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901818181818182, + "loss": 2.7259, + "theoretical_loss": 3.4762226054447676, + "tokens_seen": 1709834240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901616161616162, + "loss": 2.7595, + "theoretical_loss": 3.476211151083432, + "tokens_seen": 1709899776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901414141414141, + "loss": 2.7021, + "theoretical_loss": 3.4761996972840232, + "tokens_seen": 1709965312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901212121212121, + "loss": 2.5646, + "theoretical_loss": 3.476188244046491, + "tokens_seen": 1710030848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901010101010101, + "loss": 2.7032, + "theoretical_loss": 3.4761767913707877, + "tokens_seen": 1710096384 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490080808080808, + "loss": 2.5891, + "theoretical_loss": 3.4761653392568634, + "tokens_seen": 1710161920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900606060606061, + "loss": 2.6411, + "theoretical_loss": 3.4761538877046694, + "tokens_seen": 1710227456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900404040404041, + "loss": 2.5804, + "theoretical_loss": 3.476142436714156, + "tokens_seen": 1710292992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900202020202021, + "loss": 2.6421, + "theoretical_loss": 3.4761309862852747, + "tokens_seen": 1710358528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049, + "loss": 2.891, + "theoretical_loss": 3.4761195364179764, + "tokens_seen": 1710424064 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489979797979798, + "loss": 2.6098, + "theoretical_loss": 3.4761080871122116, + "tokens_seen": 1710489600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489959595959596, + "loss": 2.8071, + "theoretical_loss": 3.476096638367932, + "tokens_seen": 1710555136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899393939393939, + "loss": 2.5849, + "theoretical_loss": 3.4760851901850875, + "tokens_seen": 1710620672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899191919191919, + "loss": 2.8232, + "theoretical_loss": 3.4760737425636297, + "tokens_seen": 1710686208 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 943708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.789628744125366, + "objective/train/theoretical_loss": 3.4760622955035103, + "objective/train/tokens_used": 69610976, + "theoretical_loss": 3.4760622955035103, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898989898989899, + "loss": 2.7948, + "theoretical_loss": 3.4760622955035103, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898787878787878, + "loss": 2.7579, + "theoretical_loss": 3.476050849004679, + "tokens_seen": 1710817280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898585858585858, + "loss": 2.8273, + "theoretical_loss": 3.4760394030670874, + "tokens_seen": 1710882816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898383838383839, + "loss": 2.5987, + "theoretical_loss": 3.476027957690686, + "tokens_seen": 1710948352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898181818181819, + "loss": 2.6575, + "theoretical_loss": 3.476016512875426, + "tokens_seen": 1711013888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897979797979798, + "loss": 2.792, + "theoretical_loss": 3.4760050686212596, + "tokens_seen": 1711079424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897777777777778, + "loss": 2.703, + "theoretical_loss": 3.475993624928136, + "tokens_seen": 1711144960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897575757575758, + "loss": 2.6028, + "theoretical_loss": 3.475982181796007, + "tokens_seen": 1711210496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897373737373738, + "loss": 2.8526, + "theoretical_loss": 3.475970739224824, + "tokens_seen": 1711276032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897171717171717, + "loss": 2.5888, + "theoretical_loss": 3.475959297214537, + "tokens_seen": 1711341568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896969696969697, + "loss": 2.7614, + "theoretical_loss": 3.475947855765098, + "tokens_seen": 1711407104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896767676767676, + "loss": 2.723, + "theoretical_loss": 3.4759364148764575, + "tokens_seen": 1711472640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896565656565657, + "loss": 2.5174, + "theoretical_loss": 3.4759249745485667, + "tokens_seen": 1711538176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896363636363636, + "loss": 2.6374, + "theoretical_loss": 3.4759135347813768, + "tokens_seen": 1711603712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896161616161617, + "loss": 2.7387, + "theoretical_loss": 3.4759020955748388, + "tokens_seen": 1711669248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895959595959596, + "loss": 2.6615, + "theoretical_loss": 3.4758906569289034, + "tokens_seen": 1711734784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895757575757576, + "loss": 2.6527, + "theoretical_loss": 3.475879218843522, + "tokens_seen": 1711800320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895555555555556, + "loss": 2.6785, + "theoretical_loss": 3.475867781318646, + "tokens_seen": 1711865856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895353535353536, + "loss": 2.4673, + "theoretical_loss": 3.4758563443542254, + "tokens_seen": 1711931392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895151515151515, + "loss": 2.6067, + "theoretical_loss": 3.4758449079502123, + "tokens_seen": 1711996928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894949494949495, + "loss": 2.6138, + "theoretical_loss": 3.475833472106557, + "tokens_seen": 1712062464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894747474747475, + "loss": 2.6808, + "theoretical_loss": 3.475822036823212, + "tokens_seen": 1712128000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894545454545455, + "loss": 2.6603, + "theoretical_loss": 3.4758106021001267, + "tokens_seen": 1712193536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894343434343434, + "loss": 2.4896, + "theoretical_loss": 3.4757991679372533, + "tokens_seen": 1712259072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894141414141414, + "loss": 2.5848, + "theoretical_loss": 3.4757877343345425, + "tokens_seen": 1712324608 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 944906, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.562922239303589, + "objective/train/theoretical_loss": 3.4757763012919454, + "objective/train/tokens_used": 71249376, + "theoretical_loss": 3.4757763012919454, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893939393939393, + "loss": 2.6692, + "theoretical_loss": 3.4757763012919454, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893737373737374, + "loss": 2.848, + "theoretical_loss": 3.475764868809413, + "tokens_seen": 1712455680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893535353535354, + "loss": 2.437, + "theoretical_loss": 3.4757534368868974, + "tokens_seen": 1712521216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893333333333334, + "loss": 2.7263, + "theoretical_loss": 3.4757420055243484, + "tokens_seen": 1712586752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893131313131313, + "loss": 2.6201, + "theoretical_loss": 3.4757305747217178, + "tokens_seen": 1712652288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892929292929293, + "loss": 2.774, + "theoretical_loss": 3.4757191444789566, + "tokens_seen": 1712717824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892727272727273, + "loss": 2.8183, + "theoretical_loss": 3.475707714796016, + "tokens_seen": 1712783360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892525252525253, + "loss": 2.6004, + "theoretical_loss": 3.4756962856728473, + "tokens_seen": 1712848896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892323232323232, + "loss": 2.8133, + "theoretical_loss": 3.4756848571094014, + "tokens_seen": 1712914432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892121212121212, + "loss": 2.8219, + "theoretical_loss": 3.4756734291056297, + "tokens_seen": 1712979968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891919191919192, + "loss": 2.826, + "theoretical_loss": 3.475662001661483, + "tokens_seen": 1713045504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891717171717171, + "loss": 2.6803, + "theoretical_loss": 3.475650574776914, + "tokens_seen": 1713111040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891515151515151, + "loss": 2.7393, + "theoretical_loss": 3.4756391484518714, + "tokens_seen": 1713176576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891313131313132, + "loss": 2.5757, + "theoretical_loss": 3.4756277226863084, + "tokens_seen": 1713242112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891111111111112, + "loss": 2.7433, + "theoretical_loss": 3.475616297480175, + "tokens_seen": 1713307648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890909090909091, + "loss": 2.5726, + "theoretical_loss": 3.475604872833424, + "tokens_seen": 1713373184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890707070707071, + "loss": 2.7851, + "theoretical_loss": 3.4755934487460047, + "tokens_seen": 1713438720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890505050505051, + "loss": 2.6237, + "theoretical_loss": 3.475582025217869, + "tokens_seen": 1713504256 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489030303030303, + "loss": 2.8517, + "theoretical_loss": 3.4755706022489687, + "tokens_seen": 1713569792 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489010101010101, + "loss": 2.4496, + "theoretical_loss": 3.475559179839254, + "tokens_seen": 1713635328 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488989898989899, + "loss": 2.4707, + "theoretical_loss": 3.4755477579886778, + "tokens_seen": 1713700864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889696969696969, + "loss": 2.7956, + "theoretical_loss": 3.4755363366971896, + "tokens_seen": 1713766400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889494949494949, + "loss": 2.6349, + "theoretical_loss": 3.4755249159647414, + "tokens_seen": 1713831936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889292929292929, + "loss": 2.6285, + "theoretical_loss": 3.4755134957912848, + "tokens_seen": 1713897472 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488909090909091, + "loss": 2.7714, + "theoretical_loss": 3.4755020761767703, + "tokens_seen": 1713963008 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 945396, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1092090606689453, + "objective/train/theoretical_loss": 3.47549065712115, + "objective/train/tokens_used": 72887776, + "theoretical_loss": 3.47549065712115, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888888888888889, + "loss": 2.4456, + "theoretical_loss": 3.47549065712115, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888686868686869, + "loss": 2.5141, + "theoretical_loss": 3.4754792386243745, + "tokens_seen": 1714094080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888484848484849, + "loss": 2.7276, + "theoretical_loss": 3.4754678206863954, + "tokens_seen": 1714159616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888282828282829, + "loss": 2.5177, + "theoretical_loss": 3.475456403307164, + "tokens_seen": 1714225152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888080808080808, + "loss": 2.7962, + "theoretical_loss": 3.4754449864866315, + "tokens_seen": 1714290688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887878787878788, + "loss": 2.5457, + "theoretical_loss": 3.475433570224749, + "tokens_seen": 1714356224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887676767676768, + "loss": 2.6354, + "theoretical_loss": 3.475422154521468, + "tokens_seen": 1714421760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887474747474748, + "loss": 2.6071, + "theoretical_loss": 3.47541073937674, + "tokens_seen": 1714487296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887272727272727, + "loss": 2.5188, + "theoretical_loss": 3.475399324790516, + "tokens_seen": 1714552832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887070707070707, + "loss": 2.874, + "theoretical_loss": 3.4753879107627483, + "tokens_seen": 1714618368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886868686868686, + "loss": 2.7343, + "theoretical_loss": 3.4753764972933867, + "tokens_seen": 1714683904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886666666666667, + "loss": 2.8105, + "theoretical_loss": 3.4753650843823833, + "tokens_seen": 1714749440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886464646464647, + "loss": 2.8456, + "theoretical_loss": 3.47535367202969, + "tokens_seen": 1714814976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886262626262627, + "loss": 2.5544, + "theoretical_loss": 3.475342260235257, + "tokens_seen": 1714880512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886060606060606, + "loss": 2.7038, + "theoretical_loss": 3.475330848999036, + "tokens_seen": 1714946048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885858585858586, + "loss": 2.4342, + "theoretical_loss": 3.4753194383209793, + "tokens_seen": 1715011584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656565656566, + "loss": 2.8637, + "theoretical_loss": 3.475308028201037, + "tokens_seen": 1715077120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885454545454546, + "loss": 2.8579, + "theoretical_loss": 3.4752966186391614, + "tokens_seen": 1715142656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885252525252525, + "loss": 2.7374, + "theoretical_loss": 3.4752852096353033, + "tokens_seen": 1715208192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885050505050505, + "loss": 2.6466, + "theoretical_loss": 3.4752738011894144, + "tokens_seen": 1715273728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884848484848484, + "loss": 2.7546, + "theoretical_loss": 3.4752623933014464, + "tokens_seen": 1715339264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884646464646465, + "loss": 2.5621, + "theoretical_loss": 3.47525098597135, + "tokens_seen": 1715404800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884444444444445, + "loss": 2.5583, + "theoretical_loss": 3.475239579199077, + "tokens_seen": 1715470336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884242424242425, + "loss": 2.6457, + "theoretical_loss": 3.4752281729845786, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884040404040404, + "loss": 2.7425, + "theoretical_loss": 3.4752167673278063, + "tokens_seen": 1715601408 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 946450, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.578782796859741, + "objective/train/theoretical_loss": 3.475205362228712, + "objective/train/tokens_used": 74526176, + "theoretical_loss": 3.475205362228712, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883838383838384, + "loss": 2.544, + "theoretical_loss": 3.475205362228712, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883636363636364, + "loss": 2.5762, + "theoretical_loss": 3.475193957687246, + "tokens_seen": 1715732480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883434343434344, + "loss": 2.6404, + "theoretical_loss": 3.4751825537033607, + "tokens_seen": 1715798016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883232323232323, + "loss": 2.5441, + "theoretical_loss": 3.4751711502770077, + "tokens_seen": 1715863552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883030303030303, + "loss": 2.6666, + "theoretical_loss": 3.4751597474081377, + "tokens_seen": 1715929088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882828282828283, + "loss": 2.4691, + "theoretical_loss": 3.4751483450967027, + "tokens_seen": 1715994624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882626262626263, + "loss": 2.8491, + "theoretical_loss": 3.4751369433426538, + "tokens_seen": 1716060160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882424242424243, + "loss": 2.6214, + "theoretical_loss": 3.4751255421459426, + "tokens_seen": 1716125696 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822222222222225, + "loss": 2.7773, + "theoretical_loss": 3.4751141415065208, + "tokens_seen": 1716191232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882020202020202, + "loss": 2.4918, + "theoretical_loss": 3.4751027414243394, + "tokens_seen": 1716256768 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818181818181816, + "loss": 2.5795, + "theoretical_loss": 3.4750913418993505, + "tokens_seen": 1716322304 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048816161616161617, + "loss": 2.5246, + "theoretical_loss": 3.4750799429315054, + "tokens_seen": 1716387840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881414141414142, + "loss": 2.514, + "theoretical_loss": 3.475068544520755, + "tokens_seen": 1716453376 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048812121212121214, + "loss": 2.639, + "theoretical_loss": 3.4750571466670515, + "tokens_seen": 1716518912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881010101010101, + "loss": 2.6768, + "theoretical_loss": 3.475045749370346, + "tokens_seen": 1716584448 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048808080808080805, + "loss": 2.7541, + "theoretical_loss": 3.4750343526305905, + "tokens_seen": 1716649984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880606060606061, + "loss": 2.5554, + "theoretical_loss": 3.475022956447736, + "tokens_seen": 1716715520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880404040404041, + "loss": 2.7061, + "theoretical_loss": 3.4750115608217342, + "tokens_seen": 1716781056 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802020202020203, + "loss": 2.5479, + "theoretical_loss": 3.4750001657525367, + "tokens_seen": 1716846592 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488, + "loss": 2.5462, + "theoretical_loss": 3.4749887712400955, + "tokens_seen": 1716912128 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048797979797979795, + "loss": 2.4832, + "theoretical_loss": 3.474977377284361, + "tokens_seen": 1716977664 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487959595959596, + "loss": 2.7024, + "theoretical_loss": 3.4749659838852858, + "tokens_seen": 1717043200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048793939393939397, + "loss": 2.841, + "theoretical_loss": 3.4749545910428212, + "tokens_seen": 1717108736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879191919191919, + "loss": 2.589, + "theoretical_loss": 3.474943198756918, + "tokens_seen": 1717174272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878989898989899, + "loss": 2.5626, + "theoretical_loss": 3.474931807027529, + "tokens_seen": 1717239808 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 947071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.492351531982422, + "objective/train/theoretical_loss": 3.474920415854605, + "objective/train/tokens_used": 76164576, + "theoretical_loss": 3.474920415854605, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048787878787878784, + "loss": 2.7781, + "theoretical_loss": 3.474920415854605, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878585858585859, + "loss": 2.6613, + "theoretical_loss": 3.474909025238098, + "tokens_seen": 1717370880 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048783838383838386, + "loss": 2.6513, + "theoretical_loss": 3.474897635177959, + "tokens_seen": 1717436416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878181818181818, + "loss": 2.5682, + "theoretical_loss": 3.4748862456741403, + "tokens_seen": 1717501952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877979797979798, + "loss": 2.7969, + "theoretical_loss": 3.4748748567265926, + "tokens_seen": 1717567488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877777777777778, + "loss": 2.7713, + "theoretical_loss": 3.474863468335269, + "tokens_seen": 1717633024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877575757575758, + "loss": 2.7294, + "theoretical_loss": 3.4748520805001193, + "tokens_seen": 1717698560 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048773737373737375, + "loss": 2.4451, + "theoretical_loss": 3.474840693221096, + "tokens_seen": 1717764096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877171717171717, + "loss": 2.5473, + "theoretical_loss": 3.4748293064981515, + "tokens_seen": 1717829632 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048769696969696967, + "loss": 2.6743, + "theoretical_loss": 3.474817920331236, + "tokens_seen": 1717895168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876767676767677, + "loss": 2.4018, + "theoretical_loss": 3.4748065347203023, + "tokens_seen": 1717960704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876565656565657, + "loss": 2.5568, + "theoretical_loss": 3.474795149665301, + "tokens_seen": 1718026240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048763636363636365, + "loss": 2.4997, + "theoretical_loss": 3.474783765166185, + "tokens_seen": 1718091776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876161616161616, + "loss": 2.5956, + "theoretical_loss": 3.4747723812229045, + "tokens_seen": 1718157312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875959595959596, + "loss": 2.4705, + "theoretical_loss": 3.4747609978354124, + "tokens_seen": 1718222848 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048757575757575757, + "loss": 2.8133, + "theoretical_loss": 3.4747496150036596, + "tokens_seen": 1718288384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875555555555556, + "loss": 2.6255, + "theoretical_loss": 3.4747382327275984, + "tokens_seen": 1718353920 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048753535353535354, + "loss": 2.3969, + "theoretical_loss": 3.47472685100718, + "tokens_seen": 1718419456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875151515151515, + "loss": 2.8356, + "theoretical_loss": 3.474715469842356, + "tokens_seen": 1718484992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874949494949495, + "loss": 2.4337, + "theoretical_loss": 3.4747040892330787, + "tokens_seen": 1718550528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048747474747474746, + "loss": 2.5269, + "theoretical_loss": 3.474692709179299, + "tokens_seen": 1718616064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874545454545455, + "loss": 2.7374, + "theoretical_loss": 3.474681329680969, + "tokens_seen": 1718681600 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048743434343434343, + "loss": 2.7276, + "theoretical_loss": 3.4746699507380407, + "tokens_seen": 1718747136 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048741414141414144, + "loss": 2.5347, + "theoretical_loss": 3.4746585723504655, + "tokens_seen": 1718812672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004873939393939394, + "loss": 2.6824, + "theoretical_loss": 3.4746471945181954, + "tokens_seen": 1718878208 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 948336, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.850125789642334, + "objective/train/theoretical_loss": 3.4746358172411815, + "objective/train/tokens_used": 77802976, + "theoretical_loss": 3.4746358172411815, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048737373737373736, + "loss": 2.6919, + "theoretical_loss": 3.4746358172411815, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048735353535353537, + "loss": 2.537, + "theoretical_loss": 3.474624440519376, + "tokens_seen": 1719009280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873333333333333, + "loss": 2.7071, + "theoretical_loss": 3.4746130643527304, + "tokens_seen": 1719074816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048731313131313134, + "loss": 2.3608, + "theoretical_loss": 3.474601688741197, + "tokens_seen": 1719140352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872929292929293, + "loss": 2.6354, + "theoretical_loss": 3.474590313684727, + "tokens_seen": 1719205888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048727272727272725, + "loss": 2.7612, + "theoretical_loss": 3.474578939183272, + "tokens_seen": 1719271424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048725252525252526, + "loss": 2.561, + "theoretical_loss": 3.474567565236785, + "tokens_seen": 1719336960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048723232323232327, + "loss": 2.7526, + "theoretical_loss": 3.474556191845216, + "tokens_seen": 1719402496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048721212121212123, + "loss": 2.8614, + "theoretical_loss": 3.4745448190085177, + "tokens_seen": 1719468032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871919191919192, + "loss": 2.3397, + "theoretical_loss": 3.4745334467266424, + "tokens_seen": 1719533568 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048717171717171714, + "loss": 2.4128, + "theoretical_loss": 3.4745220749995407, + "tokens_seen": 1719599104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048715151515151515, + "loss": 2.5001, + "theoretical_loss": 3.474510703827165, + "tokens_seen": 1719664640 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048713131313131316, + "loss": 2.8698, + "theoretical_loss": 3.474499333209468, + "tokens_seen": 1719730176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871111111111111, + "loss": 2.8356, + "theoretical_loss": 3.4744879631464, + "tokens_seen": 1719795712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870909090909091, + "loss": 2.5725, + "theoretical_loss": 3.474476593637913, + "tokens_seen": 1719861248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048707070707070704, + "loss": 2.4696, + "theoretical_loss": 3.47446522468396, + "tokens_seen": 1719926784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870505050505051, + "loss": 2.5063, + "theoretical_loss": 3.4744538562844913, + "tokens_seen": 1719992320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048703030303030306, + "loss": 2.478, + "theoretical_loss": 3.4744424884394602, + "tokens_seen": 1720057856 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487010101010101, + "loss": 2.609, + "theoretical_loss": 3.4744311211488172, + "tokens_seen": 1720123392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048698989898989897, + "loss": 2.8236, + "theoretical_loss": 3.4744197544125153, + "tokens_seen": 1720188928 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486969696969697, + "loss": 2.7664, + "theoretical_loss": 3.4744083882305055, + "tokens_seen": 1720254464 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486949494949495, + "loss": 2.6229, + "theoretical_loss": 3.4743970226027403, + "tokens_seen": 1720320000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048692929292929295, + "loss": 2.8784, + "theoretical_loss": 3.474385657529171, + "tokens_seen": 1720385536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869090909090909, + "loss": 2.6972, + "theoretical_loss": 3.47437429300975, + "tokens_seen": 1720451072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688888888888886, + "loss": 2.9794, + "theoretical_loss": 3.4743629290444282, + "tokens_seen": 1720516608 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 949353, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0498993396759033, + "objective/train/theoretical_loss": 3.4743515656331594, + "objective/train/tokens_used": 79441376, + "theoretical_loss": 3.4743515656331594, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686868686868693, + "loss": 2.8742, + "theoretical_loss": 3.4743515656331594, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868484848484849, + "loss": 2.6368, + "theoretical_loss": 3.474340202775893, + "tokens_seen": 1720647680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682828282828284, + "loss": 2.6623, + "theoretical_loss": 3.474328840472583, + "tokens_seen": 1720713216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868080808080808, + "loss": 2.5302, + "theoretical_loss": 3.47431747872318, + "tokens_seen": 1720778752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678787878787876, + "loss": 2.827, + "theoretical_loss": 3.4743061175276364, + "tokens_seen": 1720844288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867676767676768, + "loss": 2.6887, + "theoretical_loss": 3.474294756885904, + "tokens_seen": 1720909824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867474747474748, + "loss": 2.6388, + "theoretical_loss": 3.4742833967979347, + "tokens_seen": 1720975360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048672727272727274, + "loss": 2.5768, + "theoretical_loss": 3.474272037263681, + "tokens_seen": 1721040896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867070707070707, + "loss": 2.5206, + "theoretical_loss": 3.474260678283094, + "tokens_seen": 1721106432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866868686868687, + "loss": 2.7724, + "theoretical_loss": 3.474249319856126, + "tokens_seen": 1721171968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866666666666667, + "loss": 2.8046, + "theoretical_loss": 3.4742379619827286, + "tokens_seen": 1721237504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048664646464646467, + "loss": 2.6154, + "theoretical_loss": 3.4742266046628543, + "tokens_seen": 1721303040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048662626262626263, + "loss": 2.7199, + "theoretical_loss": 3.4742152478964545, + "tokens_seen": 1721368576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866060606060606, + "loss": 2.5633, + "theoretical_loss": 3.474203891683482, + "tokens_seen": 1721434112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865858585858586, + "loss": 2.7631, + "theoretical_loss": 3.474192536023888, + "tokens_seen": 1721499648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865656565656566, + "loss": 2.5908, + "theoretical_loss": 3.474181180917624, + "tokens_seen": 1721565184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048654545454545456, + "loss": 2.601, + "theoretical_loss": 3.474169826364643, + "tokens_seen": 1721630720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865252525252525, + "loss": 2.8114, + "theoretical_loss": 3.4741584723648975, + "tokens_seen": 1721696256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048650505050505053, + "loss": 2.7601, + "theoretical_loss": 3.4741471189183377, + "tokens_seen": 1721761792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864848484848485, + "loss": 2.546, + "theoretical_loss": 3.474135766024917, + "tokens_seen": 1721827328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864646464646465, + "loss": 2.4717, + "theoretical_loss": 3.4741244136845864, + "tokens_seen": 1721892864 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048644444444444446, + "loss": 2.68, + "theoretical_loss": 3.474113061897299, + "tokens_seen": 1721958400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864242424242424, + "loss": 2.5694, + "theoretical_loss": 3.474101710663006, + "tokens_seen": 1722023936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864040404040404, + "loss": 2.6604, + "theoretical_loss": 3.474090359981659, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863838383838384, + "loss": 2.6332, + "theoretical_loss": 3.4740790098532113, + "tokens_seen": 1722155008 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 949989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.493234634399414, + "objective/train/theoretical_loss": 3.4740676602776146, + "objective/train/tokens_used": 81079776, + "theoretical_loss": 3.4740676602776146, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863636363636364, + "loss": 2.5725, + "theoretical_loss": 3.4740676602776146, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048634343434343435, + "loss": 2.5757, + "theoretical_loss": 3.47405631125482, + "tokens_seen": 1722286080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048632323232323236, + "loss": 2.6726, + "theoretical_loss": 3.4740449627847805, + "tokens_seen": 1722351616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863030303030303, + "loss": 2.5449, + "theoretical_loss": 3.4740336148674476, + "tokens_seen": 1722417152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862828282828283, + "loss": 2.7329, + "theoretical_loss": 3.4740222675027734, + "tokens_seen": 1722482688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862626262626263, + "loss": 2.5414, + "theoretical_loss": 3.474010920690711, + "tokens_seen": 1722548224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048624242424242424, + "loss": 2.5322, + "theoretical_loss": 3.4739995744312107, + "tokens_seen": 1722613760 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048622222222222225, + "loss": 2.6438, + "theoretical_loss": 3.4739882287242256, + "tokens_seen": 1722679296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862020202020202, + "loss": 2.7798, + "theoretical_loss": 3.473976883569708, + "tokens_seen": 1722744832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048618181818181817, + "loss": 2.6148, + "theoretical_loss": 3.47396553896761, + "tokens_seen": 1722810368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861616161616162, + "loss": 2.7383, + "theoretical_loss": 3.4739541949178827, + "tokens_seen": 1722875904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861414141414142, + "loss": 2.5262, + "theoretical_loss": 3.4739428514204787, + "tokens_seen": 1722941440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048612121212121215, + "loss": 2.6558, + "theoretical_loss": 3.47393150847535, + "tokens_seen": 1723006976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861010101010101, + "loss": 2.7145, + "theoretical_loss": 3.4739201660824497, + "tokens_seen": 1723072512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048608080808080806, + "loss": 2.8424, + "theoretical_loss": 3.473908824241729, + "tokens_seen": 1723138048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048606060606060607, + "loss": 2.4815, + "theoretical_loss": 3.4738974829531397, + "tokens_seen": 1723203584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860404040404041, + "loss": 2.5685, + "theoretical_loss": 3.4738861422166343, + "tokens_seen": 1723269120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048602020202020204, + "loss": 2.5099, + "theoretical_loss": 3.4738748020321655, + "tokens_seen": 1723334656 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486, + "loss": 2.5624, + "theoretical_loss": 3.4738634623996845, + "tokens_seen": 1723400192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048597979797979795, + "loss": 2.8233, + "theoretical_loss": 3.473852123319144, + "tokens_seen": 1723465728 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048595959595959596, + "loss": 2.6669, + "theoretical_loss": 3.4738407847904966, + "tokens_seen": 1723531264 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485939393939394, + "loss": 2.6042, + "theoretical_loss": 3.4738294468136934, + "tokens_seen": 1723596800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048591919191919193, + "loss": 2.7076, + "theoretical_loss": 3.473818109388687, + "tokens_seen": 1723662336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858989898989899, + "loss": 2.6665, + "theoretical_loss": 3.47380677251543, + "tokens_seen": 1723727872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048587878787878784, + "loss": 2.7535, + "theoretical_loss": 3.473795436193874, + "tokens_seen": 1723793408 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 951217, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.653082847595215, + "objective/train/theoretical_loss": 3.473784100423971, + "objective/train/tokens_used": 82718176, + "theoretical_loss": 3.473784100423971, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858585858585859, + "loss": 2.8862, + "theoretical_loss": 3.473784100423971, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048583838383838387, + "loss": 2.6551, + "theoretical_loss": 3.4737727652056742, + "tokens_seen": 1723924480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858181818181818, + "loss": 2.5733, + "theoretical_loss": 3.4737614305389344, + "tokens_seen": 1723990016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857979797979798, + "loss": 2.7681, + "theoretical_loss": 3.473750096423705, + "tokens_seen": 1724055552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577777777777774, + "loss": 2.5725, + "theoretical_loss": 3.473738762859938, + "tokens_seen": 1724121088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857575757575758, + "loss": 2.7263, + "theoretical_loss": 3.4737274298475844, + "tokens_seen": 1724186624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573737373737376, + "loss": 2.6403, + "theoretical_loss": 3.4737160973865984, + "tokens_seen": 1724252160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857171717171717, + "loss": 2.5917, + "theoretical_loss": 3.4737047654769304, + "tokens_seen": 1724317696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856969696969697, + "loss": 2.7386, + "theoretical_loss": 3.473693434118534, + "tokens_seen": 1724383232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856767676767677, + "loss": 2.7532, + "theoretical_loss": 3.4736821033113605, + "tokens_seen": 1724448768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856565656565657, + "loss": 2.6684, + "theoretical_loss": 3.4736707730553626, + "tokens_seen": 1724514304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563636363636365, + "loss": 2.4919, + "theoretical_loss": 3.473659443350493, + "tokens_seen": 1724579840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856161616161616, + "loss": 2.739, + "theoretical_loss": 3.4736481141967026, + "tokens_seen": 1724645376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559595959595957, + "loss": 2.8205, + "theoretical_loss": 3.4736367855939445, + "tokens_seen": 1724710912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855757575757576, + "loss": 2.8129, + "theoretical_loss": 3.4736254575421706, + "tokens_seen": 1724776448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855555555555556, + "loss": 2.5104, + "theoretical_loss": 3.4736141300413337, + "tokens_seen": 1724841984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048553535353535355, + "loss": 2.5611, + "theoretical_loss": 3.4736028030913864, + "tokens_seen": 1724907520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855151515151515, + "loss": 2.564, + "theoretical_loss": 3.47359147669228, + "tokens_seen": 1724973056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854949494949495, + "loss": 2.6007, + "theoretical_loss": 3.473580150843967, + "tokens_seen": 1725038592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048547474747474747, + "loss": 2.7037, + "theoretical_loss": 3.4735688255464003, + "tokens_seen": 1725104128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854545454545455, + "loss": 2.6524, + "theoretical_loss": 3.4735575007995316, + "tokens_seen": 1725169664 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048543434343434344, + "loss": 2.6171, + "theoretical_loss": 3.4735461766033136, + "tokens_seen": 1725235200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854141414141414, + "loss": 2.682, + "theoretical_loss": 3.473534852957698, + "tokens_seen": 1725300736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853939393939394, + "loss": 2.7294, + "theoretical_loss": 3.4735235298626375, + "tokens_seen": 1725366272 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048537373737373736, + "loss": 2.5625, + "theoretical_loss": 3.473512207318085, + "tokens_seen": 1725431808 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 951846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9040403366088867, + "objective/train/theoretical_loss": 3.473500885323992, + "objective/train/tokens_used": 84356576, + "theoretical_loss": 3.473500885323992, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853535353535354, + "loss": 2.6504, + "theoretical_loss": 3.473500885323992, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533333333333333, + "loss": 2.8612, + "theoretical_loss": 3.473489563880311, + "tokens_seen": 1725562880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048531313131313134, + "loss": 2.747, + "theoretical_loss": 3.473478242986994, + "tokens_seen": 1725628416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852929292929293, + "loss": 2.6916, + "theoretical_loss": 3.4734669226439943, + "tokens_seen": 1725693952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852727272727273, + "loss": 2.5991, + "theoretical_loss": 3.473455602851264, + "tokens_seen": 1725759488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525252525252527, + "loss": 2.5984, + "theoretical_loss": 3.473444283608755, + "tokens_seen": 1725825024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852323232323232, + "loss": 2.772, + "theoretical_loss": 3.4734329649164195, + "tokens_seen": 1725890560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521212121212123, + "loss": 2.6357, + "theoretical_loss": 3.4734216467742103, + "tokens_seen": 1725956096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851919191919192, + "loss": 2.6213, + "theoretical_loss": 3.47341032918208, + "tokens_seen": 1726021632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851717171717172, + "loss": 2.8269, + "theoretical_loss": 3.4733990121399807, + "tokens_seen": 1726087168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048515151515151516, + "loss": 2.6231, + "theoretical_loss": 3.4733876956478644, + "tokens_seen": 1726152704 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513131313131317, + "loss": 2.6762, + "theoretical_loss": 3.4733763797056842, + "tokens_seen": 1726218240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851111111111111, + "loss": 2.7135, + "theoretical_loss": 3.473365064313392, + "tokens_seen": 1726283776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850909090909091, + "loss": 2.7963, + "theoretical_loss": 3.4733537494709408, + "tokens_seen": 1726349312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850707070707071, + "loss": 2.7273, + "theoretical_loss": 3.4733424351782816, + "tokens_seen": 1726414848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048505050505050505, + "loss": 2.9032, + "theoretical_loss": 3.4733311214353684, + "tokens_seen": 1726480384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503030303030306, + "loss": 2.7651, + "theoretical_loss": 3.4733198082421533, + "tokens_seen": 1726545920 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485010101010101, + "loss": 2.5911, + "theoretical_loss": 3.473308495598588, + "tokens_seen": 1726611456 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484989898989899, + "loss": 2.6404, + "theoretical_loss": 3.4732971835046254, + "tokens_seen": 1726676992 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484969696969697, + "loss": 2.6003, + "theoretical_loss": 3.4732858719602175, + "tokens_seen": 1726742528 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484949494949495, + "loss": 2.6377, + "theoretical_loss": 3.4732745609653177, + "tokens_seen": 1726808064 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048492929292929296, + "loss": 2.6006, + "theoretical_loss": 3.473263250519878, + "tokens_seen": 1726873600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849090909090909, + "loss": 2.6061, + "theoretical_loss": 3.4732519406238507, + "tokens_seen": 1726939136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048488888888888887, + "loss": 2.6086, + "theoretical_loss": 3.4732406312771884, + "tokens_seen": 1727004672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848686868686869, + "loss": 2.8007, + "theoretical_loss": 3.473229322479843, + "tokens_seen": 1727070208 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 953107, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.85353422164917, + "objective/train/theoretical_loss": 3.473218014231768, + "objective/train/tokens_used": 85994976, + "theoretical_loss": 3.473218014231768, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848484848484849, + "loss": 2.6263, + "theoretical_loss": 3.473218014231768, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048482828282828285, + "loss": 2.5669, + "theoretical_loss": 3.473206706532915, + "tokens_seen": 1727201280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848080808080808, + "loss": 2.4935, + "theoretical_loss": 3.473195399383237, + "tokens_seen": 1727266816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048478787878787876, + "loss": 2.5743, + "theoretical_loss": 3.4731840927826867, + "tokens_seen": 1727332352 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048476767676767683, + "loss": 2.5745, + "theoretical_loss": 3.473172786731216, + "tokens_seen": 1727397888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847474747474748, + "loss": 2.8035, + "theoretical_loss": 3.4731614812287774, + "tokens_seen": 1727463424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048472727272727274, + "loss": 2.7708, + "theoretical_loss": 3.4731501762753236, + "tokens_seen": 1727528960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847070707070707, + "loss": 2.5373, + "theoretical_loss": 3.4731388718708076, + "tokens_seen": 1727594496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048468686868686865, + "loss": 2.4428, + "theoretical_loss": 3.4731275680151814, + "tokens_seen": 1727660032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846666666666667, + "loss": 2.7221, + "theoretical_loss": 3.4731162647083975, + "tokens_seen": 1727725568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846464646464647, + "loss": 2.8533, + "theoretical_loss": 3.4731049619504084, + "tokens_seen": 1727791104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048462626262626263, + "loss": 2.4632, + "theoretical_loss": 3.473093659741167, + "tokens_seen": 1727856640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846060606060606, + "loss": 2.6879, + "theoretical_loss": 3.4730823580806254, + "tokens_seen": 1727922176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048458585858585855, + "loss": 2.7259, + "theoretical_loss": 3.473071056968737, + "tokens_seen": 1727987712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845656565656566, + "loss": 2.8179, + "theoretical_loss": 3.473059756405453, + "tokens_seen": 1728053248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048454545454545457, + "loss": 2.7101, + "theoretical_loss": 3.4730484563907273, + "tokens_seen": 1728118784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845252525252525, + "loss": 2.7504, + "theoretical_loss": 3.4730371569245113, + "tokens_seen": 1728184320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845050505050505, + "loss": 2.3934, + "theoretical_loss": 3.473025858006759, + "tokens_seen": 1728249856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844848484848485, + "loss": 2.4794, + "theoretical_loss": 3.4730145596374213, + "tokens_seen": 1728315392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844646464646465, + "loss": 2.4167, + "theoretical_loss": 3.4730032618164524, + "tokens_seen": 1728380928 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048444444444444446, + "loss": 2.7904, + "theoretical_loss": 3.4729919645438034, + "tokens_seen": 1728446464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844242424242424, + "loss": 2.5223, + "theoretical_loss": 3.472980667819428, + "tokens_seen": 1728512000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844040404040404, + "loss": 2.7771, + "theoretical_loss": 3.4729693716432783, + "tokens_seen": 1728577536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843838383838384, + "loss": 2.6938, + "theoretical_loss": 3.472958076015307, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843636363636364, + "loss": 2.6798, + "theoretical_loss": 3.472946780935467, + "tokens_seen": 1728708608 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 954102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.302818536758423, + "objective/train/theoretical_loss": 3.4729354864037107, + "objective/train/tokens_used": 87633376, + "theoretical_loss": 3.4729354864037107, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048434343434343435, + "loss": 2.6855, + "theoretical_loss": 3.4729354864037107, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843232323232323, + "loss": 2.6738, + "theoretical_loss": 3.4729241924199905, + "tokens_seen": 1728839680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843030303030303, + "loss": 2.552, + "theoretical_loss": 3.4729128989842595, + "tokens_seen": 1728905216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842828282828283, + "loss": 2.682, + "theoretical_loss": 3.4729016060964697, + "tokens_seen": 1728970752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842626262626263, + "loss": 2.4425, + "theoretical_loss": 3.4728903137565745, + "tokens_seen": 1729036288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048424242424242425, + "loss": 2.7817, + "theoretical_loss": 3.472879021964526, + "tokens_seen": 1729101824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842222222222222, + "loss": 2.825, + "theoretical_loss": 3.472867730720277, + "tokens_seen": 1729167360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842020202020202, + "loss": 2.5866, + "theoretical_loss": 3.4728564400237802, + "tokens_seen": 1729232896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048418181818181817, + "loss": 2.9395, + "theoretical_loss": 3.4728451498749884, + "tokens_seen": 1729298432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841616161616162, + "loss": 2.7942, + "theoretical_loss": 3.4728338602738544, + "tokens_seen": 1729363968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048414141414141414, + "loss": 2.5925, + "theoretical_loss": 3.4728225712203304, + "tokens_seen": 1729429504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048412121212121215, + "loss": 2.5568, + "theoretical_loss": 3.4728112827143693, + "tokens_seen": 1729495040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841010101010101, + "loss": 2.7105, + "theoretical_loss": 3.472799994755924, + "tokens_seen": 1729560576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048408080808080807, + "loss": 2.7117, + "theoretical_loss": 3.4727887073449466, + "tokens_seen": 1729626112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840606060606061, + "loss": 2.687, + "theoretical_loss": 3.4727774204813904, + "tokens_seen": 1729691648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048404040404040403, + "loss": 2.7663, + "theoretical_loss": 3.472766134165208, + "tokens_seen": 1729757184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048402020202020204, + "loss": 2.5287, + "theoretical_loss": 3.472754848396352, + "tokens_seen": 1729822720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484, + "loss": 2.6449, + "theoretical_loss": 3.4727435631747756, + "tokens_seen": 1729888256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397979797979796, + "loss": 2.7483, + "theoretical_loss": 3.4727322785004304, + "tokens_seen": 1729953792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048395959595959597, + "loss": 2.5295, + "theoretical_loss": 3.4727209943732706, + "tokens_seen": 1730019328 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483939393939394, + "loss": 2.597, + "theoretical_loss": 3.4727097107932474, + "tokens_seen": 1730084864 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048391919191919194, + "loss": 2.5403, + "theoretical_loss": 3.4726984277603146, + "tokens_seen": 1730150400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838989898989899, + "loss": 2.7501, + "theoretical_loss": 3.472687145274425, + "tokens_seen": 1730215936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048387878787878785, + "loss": 2.6343, + "theoretical_loss": 3.472675863335531, + "tokens_seen": 1730281472 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048385858585858586, + "loss": 2.5461, + "theoretical_loss": 3.472664581943585, + "tokens_seen": 1730347008 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 954813, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2854347229003906, + "objective/train/theoretical_loss": 3.47265330109854, + "objective/train/tokens_used": 89271776, + "theoretical_loss": 3.47265330109854, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048383838383838387, + "loss": 2.5293, + "theoretical_loss": 3.47265330109854, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381818181818183, + "loss": 2.7872, + "theoretical_loss": 3.4726420208003494, + "tokens_seen": 1730478080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837979797979798, + "loss": 2.7759, + "theoretical_loss": 3.472630741048966, + "tokens_seen": 1730543616 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377777777777774, + "loss": 2.4633, + "theoretical_loss": 3.4726194618443413, + "tokens_seen": 1730609152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837575757575758, + "loss": 2.6828, + "theoretical_loss": 3.4726081831864293, + "tokens_seen": 1730674688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373737373737377, + "loss": 2.6382, + "theoretical_loss": 3.472596905075182, + "tokens_seen": 1730740224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837171717171717, + "loss": 2.6293, + "theoretical_loss": 3.4725856275105533, + "tokens_seen": 1730805760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836969696969697, + "loss": 2.421, + "theoretical_loss": 3.472574350492495, + "tokens_seen": 1730871296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048367676767676764, + "loss": 2.5533, + "theoretical_loss": 3.47256307402096, + "tokens_seen": 1730936832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836565656565657, + "loss": 2.8264, + "theoretical_loss": 3.4725517980959015, + "tokens_seen": 1731002368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048363636363636366, + "loss": 2.4918, + "theoretical_loss": 3.4725405227172725, + "tokens_seen": 1731067904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836161616161616, + "loss": 2.5235, + "theoretical_loss": 3.472529247885025, + "tokens_seen": 1731133440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359595959595957, + "loss": 2.5957, + "theoretical_loss": 3.472517973599113, + "tokens_seen": 1731198976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357575757575764, + "loss": 2.8658, + "theoretical_loss": 3.4725066998594882, + "tokens_seen": 1731264512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835555555555556, + "loss": 2.5194, + "theoretical_loss": 3.4724954266661046, + "tokens_seen": 1731330048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048353535353535355, + "loss": 2.8079, + "theoretical_loss": 3.4724841540189137, + "tokens_seen": 1731395584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835151515151515, + "loss": 2.7591, + "theoretical_loss": 3.472472881917869, + "tokens_seen": 1731461120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349494949494946, + "loss": 2.6462, + "theoretical_loss": 3.4724616103629242, + "tokens_seen": 1731526656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048347474747474753, + "loss": 2.4711, + "theoretical_loss": 3.4724503393540314, + "tokens_seen": 1731592192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834545454545455, + "loss": 2.5492, + "theoretical_loss": 3.472439068891143, + "tokens_seen": 1731657728 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343434343434344, + "loss": 2.604, + "theoretical_loss": 3.4724277989742127, + "tokens_seen": 1731723264 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834141414141414, + "loss": 2.6603, + "theoretical_loss": 3.4724165296031924, + "tokens_seen": 1731788800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833939393939394, + "loss": 2.5478, + "theoretical_loss": 3.4724052607780367, + "tokens_seen": 1731854336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833737373737374, + "loss": 2.4989, + "theoretical_loss": 3.472393992498697, + "tokens_seen": 1731919872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833535353535354, + "loss": 2.5639, + "theoretical_loss": 3.4723827247651267, + "tokens_seen": 1731985408 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 956095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3975791931152344, + "objective/train/theoretical_loss": 3.472371457577279, + "objective/train/tokens_used": 90910176, + "theoretical_loss": 3.472371457577279, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048333333333333334, + "loss": 2.5653, + "theoretical_loss": 3.472371457577279, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833131313131313, + "loss": 2.6249, + "theoretical_loss": 3.472360190935106, + "tokens_seen": 1732116480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832929292929293, + "loss": 2.7115, + "theoretical_loss": 3.472348924838562, + "tokens_seen": 1732182016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832727272727273, + "loss": 2.7388, + "theoretical_loss": 3.4723376592875983, + "tokens_seen": 1732247552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048325252525252527, + "loss": 2.572, + "theoretical_loss": 3.4723263942821694, + "tokens_seen": 1732313088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048323232323232323, + "loss": 2.7735, + "theoretical_loss": 3.472315129822227, + "tokens_seen": 1732378624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048321212121212124, + "loss": 2.4005, + "theoretical_loss": 3.4723038659077248, + "tokens_seen": 1732444160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831919191919192, + "loss": 2.7573, + "theoretical_loss": 3.4722926025386154, + "tokens_seen": 1732509696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831717171717172, + "loss": 2.6348, + "theoretical_loss": 3.4722813397148515, + "tokens_seen": 1732575232 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048315151515151516, + "loss": 2.5441, + "theoretical_loss": 3.472270077436387, + "tokens_seen": 1732640768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831313131313131, + "loss": 2.6699, + "theoretical_loss": 3.4722588157031744, + "tokens_seen": 1732706304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048311111111111113, + "loss": 2.6538, + "theoretical_loss": 3.472247554515166, + "tokens_seen": 1732771840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830909090909091, + "loss": 2.4593, + "theoretical_loss": 3.4722362938723155, + "tokens_seen": 1732837376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830707070707071, + "loss": 2.5461, + "theoretical_loss": 3.4722250337745764, + "tokens_seen": 1732902912 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048305050505050506, + "loss": 2.7509, + "theoretical_loss": 3.472213774221901, + "tokens_seen": 1732968448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048303030303030307, + "loss": 2.5207, + "theoretical_loss": 3.4722025152142413, + "tokens_seen": 1733033984 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483010101010101, + "loss": 2.7144, + "theoretical_loss": 3.472191256751552, + "tokens_seen": 1733099520 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482989898989899, + "loss": 2.578, + "theoretical_loss": 3.472179998833786, + "tokens_seen": 1733165056 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482969696969697, + "loss": 2.6586, + "theoretical_loss": 3.4721687414608953, + "tokens_seen": 1733230592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294949494949495, + "loss": 2.826, + "theoretical_loss": 3.4721574846328336, + "tokens_seen": 1733296128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292929292929296, + "loss": 2.6236, + "theoretical_loss": 3.472146228349554, + "tokens_seen": 1733361664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829090909090909, + "loss": 2.5089, + "theoretical_loss": 3.472134972611009, + "tokens_seen": 1733427200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828888888888889, + "loss": 2.6058, + "theoretical_loss": 3.472123717417152, + "tokens_seen": 1733492736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828686868686869, + "loss": 2.4047, + "theoretical_loss": 3.472112462767936, + "tokens_seen": 1733558272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828484848484849, + "loss": 2.6876, + "theoretical_loss": 3.4721012086633145, + "tokens_seen": 1733623808 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 956643, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.931276798248291, + "objective/train/theoretical_loss": 3.4720899551032396, + "objective/train/tokens_used": 92548576, + "theoretical_loss": 3.4720899551032396, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048282828282828285, + "loss": 2.5012, + "theoretical_loss": 3.4720899551032396, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828080808080808, + "loss": 2.5515, + "theoretical_loss": 3.472078702087665, + "tokens_seen": 1733754880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048278787878787877, + "loss": 2.6655, + "theoretical_loss": 3.472067449616544, + "tokens_seen": 1733820416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827676767676768, + "loss": 2.7347, + "theoretical_loss": 3.4720561976898288, + "tokens_seen": 1733885952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827474747474748, + "loss": 2.7386, + "theoretical_loss": 3.4720449463074736, + "tokens_seen": 1733951488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048272727272727275, + "loss": 2.4823, + "theoretical_loss": 3.47203369546943, + "tokens_seen": 1734017024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827070707070707, + "loss": 2.6493, + "theoretical_loss": 3.472022445175653, + "tokens_seen": 1734082560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048268686868686866, + "loss": 2.746, + "theoretical_loss": 3.472011195426095, + "tokens_seen": 1734148096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266666666666667, + "loss": 2.4995, + "theoretical_loss": 3.4719999462207083, + "tokens_seen": 1734213632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826464646464647, + "loss": 2.6906, + "theoretical_loss": 3.4719886975594463, + "tokens_seen": 1734279168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048262626262626264, + "loss": 2.3716, + "theoretical_loss": 3.471977449442263, + "tokens_seen": 1734344704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826060606060606, + "loss": 2.4103, + "theoretical_loss": 3.47196620186911, + "tokens_seen": 1734410240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258585858585855, + "loss": 2.7933, + "theoretical_loss": 3.471954954839942, + "tokens_seen": 1734475776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825656565656566, + "loss": 2.6078, + "theoretical_loss": 3.4719437083547113, + "tokens_seen": 1734541312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825454545454546, + "loss": 2.5246, + "theoretical_loss": 3.4719324624133714, + "tokens_seen": 1734606848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252525252525253, + "loss": 2.755, + "theoretical_loss": 3.4719212170158755, + "tokens_seen": 1734672384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825050505050505, + "loss": 2.2628, + "theoretical_loss": 3.471909972162176, + "tokens_seen": 1734737920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048248484848484845, + "loss": 2.6222, + "theoretical_loss": 3.471898727852227, + "tokens_seen": 1734803456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824646464646465, + "loss": 2.6469, + "theoretical_loss": 3.471887484085981, + "tokens_seen": 1734868992 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244444444444447, + "loss": 2.6871, + "theoretical_loss": 3.4718762408633914, + "tokens_seen": 1734934528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824242424242424, + "loss": 2.5698, + "theoretical_loss": 3.471864998184411, + "tokens_seen": 1735000064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824040404040404, + "loss": 2.8557, + "theoretical_loss": 3.471853756048994, + "tokens_seen": 1735065600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823838383838384, + "loss": 2.6444, + "theoretical_loss": 3.4718425144570926, + "tokens_seen": 1735131136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823636363636364, + "loss": 2.6187, + "theoretical_loss": 3.471831273408661, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048234343434343436, + "loss": 2.5576, + "theoretical_loss": 3.471820032903651, + "tokens_seen": 1735262208 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 957795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0068306922912598, + "objective/train/theoretical_loss": 3.471808792942017, + "objective/train/tokens_used": 94186976, + "theoretical_loss": 3.471808792942017, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823232323232323, + "loss": 2.839, + "theoretical_loss": 3.471808792942017, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823030303030303, + "loss": 2.612, + "theoretical_loss": 3.4717975535237118, + "tokens_seen": 1735393280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822828282828283, + "loss": 2.5873, + "theoretical_loss": 3.4717863146486887, + "tokens_seen": 1735458816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822626262626263, + "loss": 2.7101, + "theoretical_loss": 3.4717750763169004, + "tokens_seen": 1735524352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224242424242425, + "loss": 2.5618, + "theoretical_loss": 3.471763838528301, + "tokens_seen": 1735589888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822222222222222, + "loss": 2.7326, + "theoretical_loss": 3.471752601282843, + "tokens_seen": 1735655424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822020202020202, + "loss": 2.3858, + "theoretical_loss": 3.47174136458048, + "tokens_seen": 1735720960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821818181818182, + "loss": 2.6821, + "theoretical_loss": 3.4717301284211657, + "tokens_seen": 1735786496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821616161616162, + "loss": 2.6055, + "theoretical_loss": 3.4717188928048524, + "tokens_seen": 1735852032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048214141414141415, + "loss": 2.8098, + "theoretical_loss": 3.4717076577314936, + "tokens_seen": 1735917568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821212121212121, + "loss": 2.555, + "theoretical_loss": 3.471696423201043, + "tokens_seen": 1735983104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821010101010101, + "loss": 2.7927, + "theoretical_loss": 3.4716851892134537, + "tokens_seen": 1736048640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048208080808080807, + "loss": 2.7408, + "theoretical_loss": 3.4716739557686793, + "tokens_seen": 1736114176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820606060606061, + "loss": 2.9396, + "theoretical_loss": 3.471662722866672, + "tokens_seen": 1736179712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204040404040404, + "loss": 2.5873, + "theoretical_loss": 3.4716514905073863, + "tokens_seen": 1736245248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048202020202020205, + "loss": 2.5494, + "theoretical_loss": 3.471640258690775, + "tokens_seen": 1736310784 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482, + "loss": 2.6813, + "theoretical_loss": 3.4716290274167907, + "tokens_seen": 1736376320 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048197979797979796, + "loss": 2.6974, + "theoretical_loss": 3.471617796685388, + "tokens_seen": 1736441856 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481959595959596, + "loss": 2.6864, + "theoretical_loss": 3.4716065664965194, + "tokens_seen": 1736507392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048193939393939393, + "loss": 2.7289, + "theoretical_loss": 3.4715953368501387, + "tokens_seen": 1736572928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048191919191919194, + "loss": 2.7123, + "theoretical_loss": 3.4715841077461986, + "tokens_seen": 1736638464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818989898989899, + "loss": 2.6387, + "theoretical_loss": 3.471572879184653, + "tokens_seen": 1736704000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818787878787879, + "loss": 2.763, + "theoretical_loss": 3.4715616511654552, + "tokens_seen": 1736769536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048185858585858587, + "loss": 2.7991, + "theoretical_loss": 3.4715504236885577, + "tokens_seen": 1736835072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818383838383839, + "loss": 2.7678, + "theoretical_loss": 3.4715391967539153, + "tokens_seen": 1736900608 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 958846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5784518718719482, + "objective/train/theoretical_loss": 3.47152797036148, + "objective/train/tokens_used": 95825376, + "theoretical_loss": 3.47152797036148, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048181818181818184, + "loss": 2.7097, + "theoretical_loss": 3.47152797036148, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817979797979798, + "loss": 2.537, + "theoretical_loss": 3.471516744511206, + "tokens_seen": 1737031680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817777777777778, + "loss": 2.6579, + "theoretical_loss": 3.471505519203046, + "tokens_seen": 1737097216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048175757575757576, + "loss": 2.7483, + "theoretical_loss": 3.471494294436954, + "tokens_seen": 1737162752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048173737373737377, + "loss": 2.5049, + "theoretical_loss": 3.471483070212883, + "tokens_seen": 1737228288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048171717171717173, + "loss": 2.7261, + "theoretical_loss": 3.471471846530787, + "tokens_seen": 1737293824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816969696969697, + "loss": 2.6989, + "theoretical_loss": 3.471460623390618, + "tokens_seen": 1737359360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816767676767677, + "loss": 2.6896, + "theoretical_loss": 3.4714494007923307, + "tokens_seen": 1737424896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816565656565657, + "loss": 2.8564, + "theoretical_loss": 3.471438178735878, + "tokens_seen": 1737490432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048163636363636366, + "loss": 2.6221, + "theoretical_loss": 3.4714269572212135, + "tokens_seen": 1737555968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816161616161616, + "loss": 2.733, + "theoretical_loss": 3.4714157362482907, + "tokens_seen": 1737621504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815959595959596, + "loss": 2.8006, + "theoretical_loss": 3.4714045158170626, + "tokens_seen": 1737687040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815757575757576, + "loss": 2.64, + "theoretical_loss": 3.4713932959274825, + "tokens_seen": 1737752576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815555555555556, + "loss": 2.7127, + "theoretical_loss": 3.4713820765795047, + "tokens_seen": 1737818112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048153535353535356, + "loss": 2.929, + "theoretical_loss": 3.4713708577730817, + "tokens_seen": 1737883648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815151515151515, + "loss": 2.7397, + "theoretical_loss": 3.471359639508168, + "tokens_seen": 1737949184 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048149494949494947, + "loss": 2.7401, + "theoretical_loss": 3.4713484217847155, + "tokens_seen": 1738014720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048147474747474754, + "loss": 2.6971, + "theoretical_loss": 3.471337204602679, + "tokens_seen": 1738080256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814545454545455, + "loss": 2.7005, + "theoretical_loss": 3.471325987962011, + "tokens_seen": 1738145792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048143434343434345, + "loss": 2.5562, + "theoretical_loss": 3.471314771862666, + "tokens_seen": 1738211328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814141414141414, + "loss": 2.667, + "theoretical_loss": 3.4713035563045964, + "tokens_seen": 1738276864 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048139393939393936, + "loss": 2.7304, + "theoretical_loss": 3.4712923412877563, + "tokens_seen": 1738342400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048137373737373743, + "loss": 2.521, + "theoretical_loss": 3.4712811268120993, + "tokens_seen": 1738407936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813535353535354, + "loss": 2.5026, + "theoretical_loss": 3.4712699128775784, + "tokens_seen": 1738473472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048133333333333334, + "loss": 2.799, + "theoretical_loss": 3.4712586994841477, + "tokens_seen": 1738539008 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 959288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.807804584503174, + "objective/train/theoretical_loss": 3.4712474866317597, + "objective/train/tokens_used": 97463776, + "theoretical_loss": 3.4712474866317597, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813131313131313, + "loss": 2.7561, + "theoretical_loss": 3.4712474866317597, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048129292929292926, + "loss": 2.7609, + "theoretical_loss": 3.4712362743203693, + "tokens_seen": 1738670080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812727272727273, + "loss": 2.797, + "theoretical_loss": 3.4712250625499284, + "tokens_seen": 1738735616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812525252525253, + "loss": 3.041, + "theoretical_loss": 3.471213851320392, + "tokens_seen": 1738801152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048123232323232323, + "loss": 2.8472, + "theoretical_loss": 3.471202640631713, + "tokens_seen": 1738866688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812121212121212, + "loss": 2.5309, + "theoretical_loss": 3.4711914304838443, + "tokens_seen": 1738932224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811919191919192, + "loss": 2.6006, + "theoretical_loss": 3.4711802208767404, + "tokens_seen": 1738997760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811717171717172, + "loss": 2.6453, + "theoretical_loss": 3.471169011810354, + "tokens_seen": 1739063296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048115151515151517, + "loss": 2.7669, + "theoretical_loss": 3.4711578032846404, + "tokens_seen": 1739128832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048113131313131313, + "loss": 2.3778, + "theoretical_loss": 3.4711465952995506, + "tokens_seen": 1739194368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811111111111111, + "loss": 2.7263, + "theoretical_loss": 3.47113538785504, + "tokens_seen": 1739259904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810909090909091, + "loss": 2.5269, + "theoretical_loss": 3.471124180951061, + "tokens_seen": 1739325440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810707070707071, + "loss": 2.8646, + "theoretical_loss": 3.471112974587568, + "tokens_seen": 1739390976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048105050505050506, + "loss": 2.6309, + "theoretical_loss": 3.4711017687645147, + "tokens_seen": 1739456512 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481030303030303, + "loss": 2.4525, + "theoretical_loss": 3.471090563481854, + "tokens_seen": 1739522048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048101010101010103, + "loss": 2.6909, + "theoretical_loss": 3.4710793587395394, + "tokens_seen": 1739587584 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480989898989899, + "loss": 2.8344, + "theoretical_loss": 3.4710681545375253, + "tokens_seen": 1739653120 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480969696969697, + "loss": 2.8379, + "theoretical_loss": 3.4710569508757647, + "tokens_seen": 1739718656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048094949494949496, + "loss": 2.708, + "theoretical_loss": 3.471045747754211, + "tokens_seen": 1739784192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809292929292929, + "loss": 2.6413, + "theoretical_loss": 3.4710345451728184, + "tokens_seen": 1739849728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809090909090909, + "loss": 2.6477, + "theoretical_loss": 3.4710233431315407, + "tokens_seen": 1739915264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808888888888889, + "loss": 2.4457, + "theoretical_loss": 3.4710121416303306, + "tokens_seen": 1739980800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808686868686869, + "loss": 2.6397, + "theoretical_loss": 3.471000940669142, + "tokens_seen": 1740046336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084848484848485, + "loss": 2.6021, + "theoretical_loss": 3.470989740247929, + "tokens_seen": 1740111872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048082828282828286, + "loss": 2.6974, + "theoretical_loss": 3.4709785403666453, + "tokens_seen": 1740177408 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 960545, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9208710193634033, + "objective/train/theoretical_loss": 3.4709673410252435, + "objective/train/tokens_used": 99102176, + "theoretical_loss": 3.4709673410252435, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808080808080808, + "loss": 2.7373, + "theoretical_loss": 3.4709673410252435, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807878787878788, + "loss": 2.8115, + "theoretical_loss": 3.4709561422236783, + "tokens_seen": 1740308480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807676767676768, + "loss": 2.7494, + "theoretical_loss": 3.470944943961903, + "tokens_seen": 1740374016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048074747474747474, + "loss": 2.6765, + "theoretical_loss": 3.4709337462398713, + "tokens_seen": 1740439552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072727272727275, + "loss": 2.9857, + "theoretical_loss": 3.470922549057537, + "tokens_seen": 1740505088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807070707070707, + "loss": 2.443, + "theoretical_loss": 3.470911352414853, + "tokens_seen": 1740570624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068686868686867, + "loss": 2.6088, + "theoretical_loss": 3.4709001563117736, + "tokens_seen": 1740636160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806666666666667, + "loss": 2.8324, + "theoretical_loss": 3.470888960748253, + "tokens_seen": 1740701696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806464646464647, + "loss": 2.7392, + "theoretical_loss": 3.470877765724244, + "tokens_seen": 1740767232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048062626262626264, + "loss": 2.5289, + "theoretical_loss": 3.4708665712397004, + "tokens_seen": 1740832768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806060606060606, + "loss": 2.7885, + "theoretical_loss": 3.4708553772945763, + "tokens_seen": 1740898304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058585858585856, + "loss": 2.5816, + "theoretical_loss": 3.4708441838888255, + "tokens_seen": 1740963840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048056565656565657, + "loss": 2.7401, + "theoretical_loss": 3.4708329910224007, + "tokens_seen": 1741029376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805454545454546, + "loss": 2.6518, + "theoretical_loss": 3.4708217986952574, + "tokens_seen": 1741094912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052525252525254, + "loss": 2.6621, + "theoretical_loss": 3.470810606907347, + "tokens_seen": 1741160448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805050505050505, + "loss": 2.468, + "theoretical_loss": 3.4707994156586253, + "tokens_seen": 1741225984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048048484848484845, + "loss": 2.56, + "theoretical_loss": 3.470788224949045, + "tokens_seen": 1741291520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804646464646465, + "loss": 2.6133, + "theoretical_loss": 3.47077703477856, + "tokens_seen": 1741357056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804444444444445, + "loss": 2.4266, + "theoretical_loss": 3.470765845147124, + "tokens_seen": 1741422592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042424242424243, + "loss": 2.5242, + "theoretical_loss": 3.470754656054691, + "tokens_seen": 1741488128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804040404040404, + "loss": 2.733, + "theoretical_loss": 3.4707434675012148, + "tokens_seen": 1741553664 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038383838383834, + "loss": 2.8107, + "theoretical_loss": 3.470732279486649, + "tokens_seen": 1741619200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803636363636364, + "loss": 2.6628, + "theoretical_loss": 3.4707210920109466, + "tokens_seen": 1741684736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034343434343437, + "loss": 2.7073, + "theoretical_loss": 3.470709905074062, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803232323232323, + "loss": 2.6392, + "theoretical_loss": 3.47069871867595, + "tokens_seen": 1741815808 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 961825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8929381370544434, + "objective/train/theoretical_loss": 3.470687532816563, + "objective/train/tokens_used": 100740576, + "theoretical_loss": 3.470687532816563, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803030303030303, + "loss": 2.7658, + "theoretical_loss": 3.470687532816563, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802828282828283, + "loss": 2.7095, + "theoretical_loss": 3.4706763474958553, + "tokens_seen": 1741946880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802626262626263, + "loss": 2.6271, + "theoretical_loss": 3.4706651627137806, + "tokens_seen": 1742012416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048024242424242426, + "loss": 2.6202, + "theoretical_loss": 3.4706539784702928, + "tokens_seen": 1742077952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802222222222222, + "loss": 2.6904, + "theoretical_loss": 3.470642794765345, + "tokens_seen": 1742143488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020202020202017, + "loss": 2.2801, + "theoretical_loss": 3.4706316115988924, + "tokens_seen": 1742209024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801818181818182, + "loss": 2.7515, + "theoretical_loss": 3.4706204289708875, + "tokens_seen": 1742274560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801616161616162, + "loss": 2.8955, + "theoretical_loss": 3.4706092468812852, + "tokens_seen": 1742340096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014141414141415, + "loss": 2.5899, + "theoretical_loss": 3.4705980653300386, + "tokens_seen": 1742405632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801212121212121, + "loss": 2.5155, + "theoretical_loss": 3.4705868843171017, + "tokens_seen": 1742471168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801010101010101, + "loss": 2.469, + "theoretical_loss": 3.4705757038424285, + "tokens_seen": 1742536704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048008080808080813, + "loss": 2.6819, + "theoretical_loss": 3.4705645239059724, + "tokens_seen": 1742602240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800606060606061, + "loss": 2.9573, + "theoretical_loss": 3.470553344507688, + "tokens_seen": 1742667776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004040404040404, + "loss": 2.6073, + "theoretical_loss": 3.4705421656475286, + "tokens_seen": 1742733312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480020202020202, + "loss": 2.5603, + "theoretical_loss": 3.470530987325448, + "tokens_seen": 1742798848 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 2.5829, + "theoretical_loss": 3.4705198095414005, + "tokens_seen": 1742864384 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479979797979798, + "loss": 2.6065, + "theoretical_loss": 3.4705086322953393, + "tokens_seen": 1742929920 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479959595959596, + "loss": 2.7619, + "theoretical_loss": 3.4704974555872194, + "tokens_seen": 1742995456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047993939393939394, + "loss": 2.5569, + "theoretical_loss": 3.4704862794169937, + "tokens_seen": 1743060992 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047991919191919195, + "loss": 2.7131, + "theoretical_loss": 3.470475103784616, + "tokens_seen": 1743126528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798989898989899, + "loss": 2.5933, + "theoretical_loss": 3.470463928690041, + "tokens_seen": 1743192064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798787878787879, + "loss": 2.505, + "theoretical_loss": 3.4704527541332224, + "tokens_seen": 1743257600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047985858585858587, + "loss": 2.608, + "theoretical_loss": 3.4704415801141133, + "tokens_seen": 1743323136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983838383838383, + "loss": 2.523, + "theoretical_loss": 3.4704304066326683, + "tokens_seen": 1743388672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047981818181818184, + "loss": 2.5742, + "theoretical_loss": 3.4704192336888413, + "tokens_seen": 1743454208 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 962337, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5374813079833984, + "objective/train/theoretical_loss": 3.470408061282587, + "objective/train/tokens_used": 102378976, + "theoretical_loss": 3.470408061282587, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797979797979798, + "loss": 2.3709, + "theoretical_loss": 3.470408061282587, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797777777777778, + "loss": 2.8128, + "theoretical_loss": 3.4703968894138573, + "tokens_seen": 1743585280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975757575757577, + "loss": 2.4218, + "theoretical_loss": 3.470385718082608, + "tokens_seen": 1743650816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797373737373738, + "loss": 2.7803, + "theoretical_loss": 3.470374547288792, + "tokens_seen": 1743716352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047971717171717173, + "loss": 2.8138, + "theoretical_loss": 3.470363377032364, + "tokens_seen": 1743781888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796969696969697, + "loss": 2.3758, + "theoretical_loss": 3.4703522073132773, + "tokens_seen": 1743847424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796767676767677, + "loss": 2.7284, + "theoretical_loss": 3.470341038131486, + "tokens_seen": 1743912960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047965656565656566, + "loss": 2.6017, + "theoretical_loss": 3.470329869486944, + "tokens_seen": 1743978496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047963636363636367, + "loss": 3.0104, + "theoretical_loss": 3.470318701379606, + "tokens_seen": 1744044032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796161616161616, + "loss": 2.6543, + "theoretical_loss": 3.4703075338094256, + "tokens_seen": 1744109568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795959595959596, + "loss": 2.7775, + "theoretical_loss": 3.4702963667763562, + "tokens_seen": 1744175104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795757575757576, + "loss": 2.5585, + "theoretical_loss": 3.470285200280352, + "tokens_seen": 1744240640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795555555555556, + "loss": 2.6517, + "theoretical_loss": 3.470274034321368, + "tokens_seen": 1744306176 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953535353535356, + "loss": 2.394, + "theoretical_loss": 3.470262868899357, + "tokens_seen": 1744371712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795151515151515, + "loss": 2.7431, + "theoretical_loss": 3.470251704014273, + "tokens_seen": 1744437248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794949494949495, + "loss": 2.6337, + "theoretical_loss": 3.4702405396660705, + "tokens_seen": 1744502784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794747474747475, + "loss": 2.5813, + "theoretical_loss": 3.4702293758547036, + "tokens_seen": 1744568320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794545454545455, + "loss": 2.7258, + "theoretical_loss": 3.470218212580126, + "tokens_seen": 1744633856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943434343434345, + "loss": 2.6717, + "theoretical_loss": 3.4702070498422923, + "tokens_seen": 1744699392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794141414141414, + "loss": 2.3807, + "theoretical_loss": 3.4701958876411556, + "tokens_seen": 1744764928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939393939393937, + "loss": 2.7787, + "theoretical_loss": 3.4701847259766705, + "tokens_seen": 1744830464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793737373737374, + "loss": 2.4776, + "theoretical_loss": 3.4701735648487912, + "tokens_seen": 1744896000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793535353535354, + "loss": 2.5712, + "theoretical_loss": 3.4701624042574717, + "tokens_seen": 1744961536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047933333333333335, + "loss": 2.6089, + "theoretical_loss": 3.4701512442026656, + "tokens_seen": 1745027072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793131313131313, + "loss": 2.6412, + "theoretical_loss": 3.470140084684327, + "tokens_seen": 1745092608 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 963376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.749994993209839, + "objective/train/theoretical_loss": 3.4701289257024106, + "objective/train/tokens_used": 104017376, + "theoretical_loss": 3.4701289257024106, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047929292929292926, + "loss": 2.6664, + "theoretical_loss": 3.4701289257024106, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792727272727273, + "loss": 2.6718, + "theoretical_loss": 3.47011776725687, + "tokens_seen": 1745223680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792525252525253, + "loss": 2.7504, + "theoretical_loss": 3.47010660934766, + "tokens_seen": 1745289216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923232323232324, + "loss": 2.894, + "theoretical_loss": 3.470095451974733, + "tokens_seen": 1745354752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792121212121212, + "loss": 2.6854, + "theoretical_loss": 3.4700842951380446, + "tokens_seen": 1745420288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919191919191915, + "loss": 2.602, + "theoretical_loss": 3.4700731388375483, + "tokens_seen": 1745485824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791717171717172, + "loss": 2.5376, + "theoretical_loss": 3.4700619830731982, + "tokens_seen": 1745551360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791515151515152, + "loss": 2.6504, + "theoretical_loss": 3.470050827844949, + "tokens_seen": 1745616896 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913131313131313, + "loss": 2.6948, + "theoretical_loss": 3.4700396731527543, + "tokens_seen": 1745682432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791111111111111, + "loss": 2.7571, + "theoretical_loss": 3.470028518996568, + "tokens_seen": 1745747968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790909090909091, + "loss": 2.4343, + "theoretical_loss": 3.4700173653763446, + "tokens_seen": 1745813504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790707070707071, + "loss": 2.7223, + "theoretical_loss": 3.470006212292038, + "tokens_seen": 1745879040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905050505050507, + "loss": 2.7366, + "theoretical_loss": 3.469995059743603, + "tokens_seen": 1745944576 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479030303030303, + "loss": 2.7484, + "theoretical_loss": 3.4699839077309926, + "tokens_seen": 1746010112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479010101010101, + "loss": 2.5738, + "theoretical_loss": 3.469972756254162, + "tokens_seen": 1746075648 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478989898989899, + "loss": 2.7647, + "theoretical_loss": 3.469961605313064, + "tokens_seen": 1746141184 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478969696969697, + "loss": 2.8169, + "theoretical_loss": 3.4699504549076545, + "tokens_seen": 1746206720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047894949494949496, + "loss": 2.7003, + "theoretical_loss": 3.4699393050378866, + "tokens_seen": 1746272256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789292929292929, + "loss": 2.7409, + "theoretical_loss": 3.4699281557037147, + "tokens_seen": 1746337792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047890909090909093, + "loss": 2.7155, + "theoretical_loss": 3.469917006905093, + "tokens_seen": 1746403328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788888888888889, + "loss": 2.8076, + "theoretical_loss": 3.4699058586419755, + "tokens_seen": 1746468864 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788686868686869, + "loss": 2.7611, + "theoretical_loss": 3.469894710914317, + "tokens_seen": 1746534400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047884848484848485, + "loss": 2.6351, + "theoretical_loss": 3.46988356372207, + "tokens_seen": 1746599936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788282828282828, + "loss": 2.8063, + "theoretical_loss": 3.469872417065191, + "tokens_seen": 1746665472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788080808080808, + "loss": 2.7563, + "theoretical_loss": 3.4698612709436327, + "tokens_seen": 1746731008 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 963934, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.45017409324646, + "objective/train/theoretical_loss": 3.46985012535735, + "objective/train/tokens_used": 105655776, + "theoretical_loss": 3.46985012535735, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787878787878788, + "loss": 2.6769, + "theoretical_loss": 3.46985012535735, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787676767676768, + "loss": 2.6498, + "theoretical_loss": 3.469838980306297, + "tokens_seen": 1746862080 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047874747474747475, + "loss": 2.7676, + "theoretical_loss": 3.469827835790427, + "tokens_seen": 1746927616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047872727272727276, + "loss": 2.7206, + "theoretical_loss": 3.4698166918096955, + "tokens_seen": 1746993152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787070707070707, + "loss": 2.6543, + "theoretical_loss": 3.469805548364056, + "tokens_seen": 1747058688 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047868686868686867, + "loss": 2.7194, + "theoretical_loss": 3.469794405453463, + "tokens_seen": 1747124224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786666666666667, + "loss": 2.7218, + "theoretical_loss": 3.4697832630778707, + "tokens_seen": 1747189760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047864646464646464, + "loss": 2.6617, + "theoretical_loss": 3.4697721212372334, + "tokens_seen": 1747255296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047862626262626265, + "loss": 2.4966, + "theoretical_loss": 3.4697609799315052, + "tokens_seen": 1747320832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786060606060606, + "loss": 2.5697, + "theoretical_loss": 3.4697498391606403, + "tokens_seen": 1747386368 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047858585858585856, + "loss": 2.792, + "theoretical_loss": 3.4697386989245933, + "tokens_seen": 1747451904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785656565656566, + "loss": 2.5246, + "theoretical_loss": 3.469727559223318, + "tokens_seen": 1747517440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785454545454546, + "loss": 2.7315, + "theoretical_loss": 3.469716420056769, + "tokens_seen": 1747582976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047852525252525254, + "loss": 2.7625, + "theoretical_loss": 3.469705281424901, + "tokens_seen": 1747648512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785050505050505, + "loss": 2.4715, + "theoretical_loss": 3.4696941433276676, + "tokens_seen": 1747714048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047848484848484846, + "loss": 2.7081, + "theoretical_loss": 3.469683005765023, + "tokens_seen": 1747779584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047846464646464647, + "loss": 2.5652, + "theoretical_loss": 3.4696718687369223, + "tokens_seen": 1747845120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784444444444445, + "loss": 2.6408, + "theoretical_loss": 3.469660732243319, + "tokens_seen": 1747910656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047842424242424244, + "loss": 2.6097, + "theoretical_loss": 3.4696495962841674, + "tokens_seen": 1747976192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784040404040404, + "loss": 2.6966, + "theoretical_loss": 3.469638460859423, + "tokens_seen": 1748041728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783838383838384, + "loss": 2.6111, + "theoretical_loss": 3.4696273259690384, + "tokens_seen": 1748107264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783636363636364, + "loss": 2.7655, + "theoretical_loss": 3.4696161916129693, + "tokens_seen": 1748172800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047834343434343437, + "loss": 2.5875, + "theoretical_loss": 3.4696050577911692, + "tokens_seen": 1748238336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047832323232323233, + "loss": 2.698, + "theoretical_loss": 3.469593924503593, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783030303030303, + "loss": 2.7526, + "theoretical_loss": 3.4695827917501947, + "tokens_seen": 1748369408 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 965147, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.617971897125244, + "objective/train/theoretical_loss": 3.4695716595309287, + "objective/train/tokens_used": 107294176, + "theoretical_loss": 3.4695716595309287, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782828282828283, + "loss": 2.5613, + "theoretical_loss": 3.4695716595309287, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782626262626263, + "loss": 2.9621, + "theoretical_loss": 3.4695605278457493, + "tokens_seen": 1748500480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047824242424242426, + "loss": 2.642, + "theoretical_loss": 3.469549396694611, + "tokens_seen": 1748566016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782222222222222, + "loss": 2.7141, + "theoretical_loss": 3.4695382660774685, + "tokens_seen": 1748631552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782020202020202, + "loss": 2.5499, + "theoretical_loss": 3.469527135994275, + "tokens_seen": 1748697088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047818181818181824, + "loss": 2.4784, + "theoretical_loss": 3.4695160064449864, + "tokens_seen": 1748762624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781616161616162, + "loss": 2.7111, + "theoretical_loss": 3.469504877429556, + "tokens_seen": 1748828160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047814141414141416, + "loss": 2.5869, + "theoretical_loss": 3.4694937489479383, + "tokens_seen": 1748893696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781212121212121, + "loss": 2.4923, + "theoretical_loss": 3.4694826210000884, + "tokens_seen": 1748959232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047810101010101007, + "loss": 2.6399, + "theoretical_loss": 3.46947149358596, + "tokens_seen": 1749024768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047808080808080814, + "loss": 2.807, + "theoretical_loss": 3.4694603667055075, + "tokens_seen": 1749090304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780606060606061, + "loss": 2.3863, + "theoretical_loss": 3.469449240358686, + "tokens_seen": 1749155840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047804040404040405, + "loss": 2.8256, + "theoretical_loss": 3.4694381145454494, + "tokens_seen": 1749221376 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478020202020202, + "loss": 2.7606, + "theoretical_loss": 3.469426989265752, + "tokens_seen": 1749286912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799999999999996, + "loss": 2.6758, + "theoretical_loss": 3.469415864519548, + "tokens_seen": 1749352448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047797979797979803, + "loss": 2.6428, + "theoretical_loss": 3.469404740306793, + "tokens_seen": 1749417984 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477959595959596, + "loss": 2.7873, + "theoretical_loss": 3.469393616627441, + "tokens_seen": 1749483520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047793939393939394, + "loss": 2.7165, + "theoretical_loss": 3.469382493481445, + "tokens_seen": 1749549056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779191919191919, + "loss": 2.7791, + "theoretical_loss": 3.4693713708687612, + "tokens_seen": 1749614592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778989898989899, + "loss": 2.8045, + "theoretical_loss": 3.469360248789343, + "tokens_seen": 1749680128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778787878787879, + "loss": 2.5269, + "theoretical_loss": 3.4693491272431456, + "tokens_seen": 1749745664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778585858585859, + "loss": 2.742, + "theoretical_loss": 3.469338006230123, + "tokens_seen": 1749811200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783838383838384, + "loss": 2.7999, + "theoretical_loss": 3.46932688575023, + "tokens_seen": 1749876736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778181818181818, + "loss": 2.782, + "theoretical_loss": 3.4693157658034206, + "tokens_seen": 1749942272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777979797979798, + "loss": 2.8005, + "theoretical_loss": 3.46930464638965, + "tokens_seen": 1750007808 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 966110, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6748225688934326, + "objective/train/theoretical_loss": 3.469293527508872, + "objective/train/tokens_used": 108932576, + "theoretical_loss": 3.469293527508872, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777777777777778, + "loss": 2.6659, + "theoretical_loss": 3.469293527508872, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775757575757577, + "loss": 2.7751, + "theoretical_loss": 3.4692824091610412, + "tokens_seen": 1750138880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047773737373737373, + "loss": 2.6498, + "theoretical_loss": 3.4692712913461126, + "tokens_seen": 1750204416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771717171717174, + "loss": 2.6655, + "theoretical_loss": 3.4692601740640403, + "tokens_seen": 1750269952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776969696969697, + "loss": 2.6462, + "theoretical_loss": 3.4692490573147787, + "tokens_seen": 1750335488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776767676767677, + "loss": 2.8046, + "theoretical_loss": 3.4692379410982825, + "tokens_seen": 1750401024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047765656565656566, + "loss": 2.5931, + "theoretical_loss": 3.4692268254145064, + "tokens_seen": 1750466560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776363636363636, + "loss": 2.7954, + "theoretical_loss": 3.4692157102634047, + "tokens_seen": 1750532096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761616161616163, + "loss": 2.6183, + "theoretical_loss": 3.469204595644932, + "tokens_seen": 1750597632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775959595959596, + "loss": 2.779, + "theoretical_loss": 3.4691934815590426, + "tokens_seen": 1750663168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775757575757576, + "loss": 2.8523, + "theoretical_loss": 3.4691823680056917, + "tokens_seen": 1750728704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047755555555555556, + "loss": 2.5914, + "theoretical_loss": 3.4691712549848326, + "tokens_seen": 1750794240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753535353535357, + "loss": 2.6619, + "theoretical_loss": 3.4691601424964214, + "tokens_seen": 1750859776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775151515151515, + "loss": 2.845, + "theoretical_loss": 3.469149030540412, + "tokens_seen": 1750925312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774949494949495, + "loss": 2.6006, + "theoretical_loss": 3.469137919116758, + "tokens_seen": 1750990848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774747474747475, + "loss": 2.6609, + "theoretical_loss": 3.4691268082254156, + "tokens_seen": 1751056384 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745454545454545, + "loss": 2.7653, + "theoretical_loss": 3.4691156978663384, + "tokens_seen": 1751121920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047743434343434346, + "loss": 2.7009, + "theoretical_loss": 3.4691045880394817, + "tokens_seen": 1751187456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774141414141414, + "loss": 2.8441, + "theoretical_loss": 3.469093478744799, + "tokens_seen": 1751252992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773939393939394, + "loss": 2.6236, + "theoretical_loss": 3.4690823699822455, + "tokens_seen": 1751318528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773737373737374, + "loss": 2.6359, + "theoretical_loss": 3.4690712617517763, + "tokens_seen": 1751384064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773535353535354, + "loss": 2.6701, + "theoretical_loss": 3.469060154053345, + "tokens_seen": 1751449600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733333333333335, + "loss": 2.5259, + "theoretical_loss": 3.469049046886907, + "tokens_seen": 1751515136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773131313131313, + "loss": 2.7385, + "theoretical_loss": 3.4690379402524165, + "tokens_seen": 1751580672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729292929292927, + "loss": 2.4743, + "theoretical_loss": 3.4690268341498287, + "tokens_seen": 1751646208 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 966555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6258327960968018, + "objective/train/theoretical_loss": 3.4690157285790972, + "objective/train/tokens_used": 110570976, + "theoretical_loss": 3.4690157285790972, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772727272727273, + "loss": 2.6509, + "theoretical_loss": 3.4690157285790972, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772525252525253, + "loss": 2.809, + "theoretical_loss": 3.4690046235401777, + "tokens_seen": 1751777280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047723232323232325, + "loss": 2.6795, + "theoretical_loss": 3.468993519033024, + "tokens_seen": 1751842816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772121212121212, + "loss": 2.7073, + "theoretical_loss": 3.4689824150575914, + "tokens_seen": 1751908352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047719191919191916, + "loss": 2.6694, + "theoretical_loss": 3.468971311613834, + "tokens_seen": 1751973888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771717171717172, + "loss": 2.6556, + "theoretical_loss": 3.4689602087017066, + "tokens_seen": 1752039424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771515151515152, + "loss": 2.7105, + "theoretical_loss": 3.468949106321164, + "tokens_seen": 1752104960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713131313131314, + "loss": 2.5831, + "theoretical_loss": 3.468938004472161, + "tokens_seen": 1752170496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771111111111111, + "loss": 2.644, + "theoretical_loss": 3.468926903154652, + "tokens_seen": 1752236032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047709090909090905, + "loss": 2.5805, + "theoretical_loss": 3.468915802368592, + "tokens_seen": 1752301568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770707070707071, + "loss": 2.8613, + "theoretical_loss": 3.4689047021139356, + "tokens_seen": 1752367104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770505050505051, + "loss": 2.6725, + "theoretical_loss": 3.4688936023906374, + "tokens_seen": 1752432640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047703030303030303, + "loss": 2.838, + "theoretical_loss": 3.4688825031986514, + "tokens_seen": 1752498176 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477010101010101, + "loss": 2.4936, + "theoretical_loss": 3.468871404537934, + "tokens_seen": 1752563712 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476989898989899, + "loss": 2.7542, + "theoretical_loss": 3.4688603064084376, + "tokens_seen": 1752629248 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476969696969697, + "loss": 2.6908, + "theoretical_loss": 3.468849208810119, + "tokens_seen": 1752694784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047694949494949497, + "loss": 2.6054, + "theoretical_loss": 3.4688381117429317, + "tokens_seen": 1752760320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769292929292929, + "loss": 2.5833, + "theoretical_loss": 3.468827015206831, + "tokens_seen": 1752825856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769090909090909, + "loss": 3.0322, + "theoretical_loss": 3.4688159192017713, + "tokens_seen": 1752891392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768888888888889, + "loss": 2.5106, + "theoretical_loss": 3.468804823727708, + "tokens_seen": 1752956928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768686868686869, + "loss": 2.5625, + "theoretical_loss": 3.4687937287845947, + "tokens_seen": 1753022464 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047684848484848486, + "loss": 2.8008, + "theoretical_loss": 3.468782634372387, + "tokens_seen": 1753088000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768282828282828, + "loss": 2.5919, + "theoretical_loss": 3.4687715404910398, + "tokens_seen": 1753153536 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047680808080808083, + "loss": 2.6055, + "theoretical_loss": 3.4687604471405065, + "tokens_seen": 1753219072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767878787878788, + "loss": 2.7941, + "theoretical_loss": 3.4687493543207433, + "tokens_seen": 1753284608 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 967544, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.789055347442627, + "objective/train/theoretical_loss": 3.468738262031705, + "objective/train/tokens_used": 112209376, + "theoretical_loss": 3.468738262031705, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767676767676768, + "loss": 2.6272, + "theoretical_loss": 3.468738262031705, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047674747474747475, + "loss": 2.7313, + "theoretical_loss": 3.468727170273345, + "tokens_seen": 1753415680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767272727272727, + "loss": 2.6265, + "theoretical_loss": 3.4687160790456195, + "tokens_seen": 1753481216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767070707070707, + "loss": 2.5517, + "theoretical_loss": 3.468704988348483, + "tokens_seen": 1753546752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047668686868686873, + "loss": 2.6153, + "theoretical_loss": 3.4686938981818893, + "tokens_seen": 1753612288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766666666666667, + "loss": 2.698, + "theoretical_loss": 3.468682808545794, + "tokens_seen": 1753677824 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047664646464646465, + "loss": 2.5592, + "theoretical_loss": 3.4686717194401524, + "tokens_seen": 1753743360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662626262626266, + "loss": 2.6959, + "theoretical_loss": 3.4686606308649184, + "tokens_seen": 1753808896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766060606060606, + "loss": 2.7546, + "theoretical_loss": 3.468649542820047, + "tokens_seen": 1753874432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765858585858586, + "loss": 2.7334, + "theoretical_loss": 3.468638455305493, + "tokens_seen": 1753939968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765656565656566, + "loss": 2.7488, + "theoretical_loss": 3.4686273683212114, + "tokens_seen": 1754005504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654545454545454, + "loss": 2.5704, + "theoretical_loss": 3.468616281867157, + "tokens_seen": 1754071040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047652525252525255, + "loss": 2.7951, + "theoretical_loss": 3.468605195943285, + "tokens_seen": 1754136576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765050505050505, + "loss": 2.7617, + "theoretical_loss": 3.46859411054955, + "tokens_seen": 1754202112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764848484848485, + "loss": 2.7249, + "theoretical_loss": 3.468583025685906, + "tokens_seen": 1754267648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764646464646465, + "loss": 2.8578, + "theoretical_loss": 3.468571941352309, + "tokens_seen": 1754333184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764444444444445, + "loss": 2.7968, + "theoretical_loss": 3.4685608575487135, + "tokens_seen": 1754398720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047642424242424244, + "loss": 2.5969, + "theoretical_loss": 3.4685497742750737, + "tokens_seen": 1754464256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764040404040404, + "loss": 2.6532, + "theoretical_loss": 3.4685386915313456, + "tokens_seen": 1754529792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763838383838384, + "loss": 2.5712, + "theoretical_loss": 3.4685276093174835, + "tokens_seen": 1754595328 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047636363636363637, + "loss": 2.7184, + "theoretical_loss": 3.468516527633442, + "tokens_seen": 1754660864 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763434343434344, + "loss": 2.666, + "theoretical_loss": 3.4685054464791767, + "tokens_seen": 1754726400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047632323232323233, + "loss": 2.704, + "theoretical_loss": 3.4684943658546414, + "tokens_seen": 1754791936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763030303030303, + "loss": 2.5184, + "theoretical_loss": 3.4684832857597923, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762828282828283, + "loss": 2.5821, + "theoretical_loss": 3.4684722061945834, + "tokens_seen": 1754923008 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 968097, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9240682125091553, + "objective/train/theoretical_loss": 3.46846112715897, + "objective/train/tokens_used": 113847776, + "theoretical_loss": 3.46846112715897, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762626262626263, + "loss": 2.6846, + "theoretical_loss": 3.46846112715897, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047624242424242427, + "loss": 2.686, + "theoretical_loss": 3.4684500486529064, + "tokens_seen": 1755054080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762222222222222, + "loss": 2.6209, + "theoretical_loss": 3.468438970676348, + "tokens_seen": 1755119616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762020202020202, + "loss": 2.5909, + "theoretical_loss": 3.4684278932292503, + "tokens_seen": 1755185152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761818181818182, + "loss": 2.7275, + "theoretical_loss": 3.468416816311567, + "tokens_seen": 1755250688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761616161616162, + "loss": 2.7313, + "theoretical_loss": 3.4684057399232544, + "tokens_seen": 1755316224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614141414141416, + "loss": 2.5685, + "theoretical_loss": 3.468394664064266, + "tokens_seen": 1755381760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761212121212121, + "loss": 2.6424, + "theoretical_loss": 3.468383588734558, + "tokens_seen": 1755447296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761010101010101, + "loss": 2.3658, + "theoretical_loss": 3.468372513934084, + "tokens_seen": 1755512832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760808080808081, + "loss": 2.8822, + "theoretical_loss": 3.468361439662801, + "tokens_seen": 1755578368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760606060606061, + "loss": 2.768, + "theoretical_loss": 3.468350365920662, + "tokens_seen": 1755643904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047604040404040406, + "loss": 2.4781, + "theoretical_loss": 3.468339292707623, + "tokens_seen": 1755709440 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476020202020202, + "loss": 2.509, + "theoretical_loss": 3.468328220023638, + "tokens_seen": 1755774976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047599999999999997, + "loss": 2.556, + "theoretical_loss": 3.468317147868663, + "tokens_seen": 1755840512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047597979797979803, + "loss": 2.6309, + "theoretical_loss": 3.4683060762426523, + "tokens_seen": 1755906048 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475959595959596, + "loss": 2.6173, + "theoretical_loss": 3.4682950051455617, + "tokens_seen": 1755971584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047593939393939395, + "loss": 2.502, + "theoretical_loss": 3.4682839345773457, + "tokens_seen": 1756037120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759191919191919, + "loss": 2.7037, + "theoretical_loss": 3.468272864537959, + "tokens_seen": 1756102656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047589898989898986, + "loss": 2.9021, + "theoretical_loss": 3.468261795027357, + "tokens_seen": 1756168192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047587878787878793, + "loss": 2.6436, + "theoretical_loss": 3.4682507260454942, + "tokens_seen": 1756233728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758585858585859, + "loss": 2.5381, + "theoretical_loss": 3.4682396575923264, + "tokens_seen": 1756299264 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047583838383838384, + "loss": 2.6176, + "theoretical_loss": 3.4682285896678087, + "tokens_seen": 1756364800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758181818181818, + "loss": 2.5754, + "theoretical_loss": 3.468217522271895, + "tokens_seen": 1756430336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757979797979798, + "loss": 2.6936, + "theoretical_loss": 3.468206455404541, + "tokens_seen": 1756495872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757777777777778, + "loss": 2.4587, + "theoretical_loss": 3.468195389065702, + "tokens_seen": 1756561408 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 969105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5639123916625977, + "objective/train/theoretical_loss": 3.4681843232553327, + "objective/train/tokens_used": 115486176, + "theoretical_loss": 3.4681843232553327, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757575757575758, + "loss": 2.5915, + "theoretical_loss": 3.4681843232553327, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047573737373737373, + "loss": 2.5335, + "theoretical_loss": 3.4681732579733886, + "tokens_seen": 1756692480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757171717171717, + "loss": 2.4869, + "theoretical_loss": 3.4681621932198237, + "tokens_seen": 1756758016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756969696969697, + "loss": 2.8309, + "theoretical_loss": 3.4681511289945943, + "tokens_seen": 1756823552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756767676767677, + "loss": 2.832, + "theoretical_loss": 3.4681400652976544, + "tokens_seen": 1756889088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047565656565656567, + "loss": 2.6746, + "theoretical_loss": 3.46812900212896, + "tokens_seen": 1756954624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756363636363636, + "loss": 2.6248, + "theoretical_loss": 3.4681179394884656, + "tokens_seen": 1757020160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047561616161616164, + "loss": 2.5682, + "theoretical_loss": 3.468106877376126, + "tokens_seen": 1757085696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755959595959596, + "loss": 2.6822, + "theoretical_loss": 3.4680958157918975, + "tokens_seen": 1757151232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755757575757576, + "loss": 2.8153, + "theoretical_loss": 3.468084754735734, + "tokens_seen": 1757216768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047555555555555556, + "loss": 2.7997, + "theoretical_loss": 3.4680736942075914, + "tokens_seen": 1757282304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755353535353535, + "loss": 2.6609, + "theoretical_loss": 3.468062634207424, + "tokens_seen": 1757347840 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047551515151515153, + "loss": 2.8563, + "theoretical_loss": 3.4680515747351874, + "tokens_seen": 1757413376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754949494949495, + "loss": 2.6059, + "theoretical_loss": 3.4680405157908365, + "tokens_seen": 1757478912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754747474747475, + "loss": 2.8125, + "theoretical_loss": 3.4680294573743264, + "tokens_seen": 1757544448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047545454545454545, + "loss": 2.7202, + "theoretical_loss": 3.4680183994856133, + "tokens_seen": 1757609984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047543434343434347, + "loss": 2.6924, + "theoretical_loss": 3.4680073421246504, + "tokens_seen": 1757675520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754141414141414, + "loss": 2.6198, + "theoretical_loss": 3.4679962852913944, + "tokens_seen": 1757741056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753939393939394, + "loss": 2.6276, + "theoretical_loss": 3.4679852289857998, + "tokens_seen": 1757806592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753737373737374, + "loss": 2.7586, + "theoretical_loss": 3.4679741732078213, + "tokens_seen": 1757872128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047535353535353535, + "loss": 2.4646, + "theoretical_loss": 3.4679631179574155, + "tokens_seen": 1757937664 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047533333333333336, + "loss": 2.4745, + "theoretical_loss": 3.467952063234536, + "tokens_seen": 1758003200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753131313131313, + "loss": 2.7784, + "theoretical_loss": 3.467941009039139, + "tokens_seen": 1758068736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047529292929292927, + "loss": 2.7118, + "theoretical_loss": 3.4679299553711793, + "tokens_seen": 1758134272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752727272727273, + "loss": 2.8808, + "theoretical_loss": 3.4679189022306116, + "tokens_seen": 1758199808 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 970436, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.550333023071289, + "objective/train/theoretical_loss": 3.467907849617392, + "objective/train/tokens_used": 117124576, + "theoretical_loss": 3.467907849617392, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752525252525253, + "loss": 2.6393, + "theoretical_loss": 3.467907849617392, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047523232323232325, + "loss": 2.488, + "theoretical_loss": 3.467896797531475, + "tokens_seen": 1758330880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752121212121212, + "loss": 2.7108, + "theoretical_loss": 3.467885745972816, + "tokens_seen": 1758396416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047519191919191916, + "loss": 2.7059, + "theoretical_loss": 3.4678746949413704, + "tokens_seen": 1758461952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751717171717172, + "loss": 2.5608, + "theoretical_loss": 3.467863644437093, + "tokens_seen": 1758527488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751515151515152, + "loss": 2.7159, + "theoretical_loss": 3.467852594459939, + "tokens_seen": 1758593024 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047513131313131314, + "loss": 2.6068, + "theoretical_loss": 3.4678415450098643, + "tokens_seen": 1758658560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751111111111111, + "loss": 2.5219, + "theoretical_loss": 3.4678304960868234, + "tokens_seen": 1758724096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047509090909090906, + "loss": 2.5771, + "theoretical_loss": 3.467819447690772, + "tokens_seen": 1758789632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750707070707071, + "loss": 2.6642, + "theoretical_loss": 3.4678083998216653, + "tokens_seen": 1758855168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750505050505051, + "loss": 2.6359, + "theoretical_loss": 3.467797352479458, + "tokens_seen": 1758920704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047503030303030304, + "loss": 2.6904, + "theoretical_loss": 3.4677863056641054, + "tokens_seen": 1758986240 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475010101010101, + "loss": 2.5726, + "theoretical_loss": 3.4677752593755633, + "tokens_seen": 1759051776 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474989898989899, + "loss": 2.689, + "theoretical_loss": 3.467764213613787, + "tokens_seen": 1759117312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474969696969697, + "loss": 2.5023, + "theoretical_loss": 3.4677531683787306, + "tokens_seen": 1759182848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494949494949497, + "loss": 2.5113, + "theoretical_loss": 3.4677421236703507, + "tokens_seen": 1759248384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047492929292929293, + "loss": 2.7619, + "theoretical_loss": 3.4677310794886025, + "tokens_seen": 1759313920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749090909090909, + "loss": 2.5726, + "theoretical_loss": 3.46772003583344, + "tokens_seen": 1759379456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047488888888888895, + "loss": 2.424, + "theoretical_loss": 3.46770899270482, + "tokens_seen": 1759444992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748686868686869, + "loss": 2.705, + "theoretical_loss": 3.467697950102697, + "tokens_seen": 1759510528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047484848484848487, + "loss": 2.8012, + "theoretical_loss": 3.467686908027026, + "tokens_seen": 1759576064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748282828282828, + "loss": 2.6163, + "theoretical_loss": 3.467675866477763, + "tokens_seen": 1759641600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748080808080808, + "loss": 2.7308, + "theoretical_loss": 3.4676648254548628, + "tokens_seen": 1759707136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478787878787884, + "loss": 2.6315, + "theoretical_loss": 3.467653784958281, + "tokens_seen": 1759772672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747676767676768, + "loss": 2.8835, + "theoretical_loss": 3.467642744987973, + "tokens_seen": 1759838208 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 971034, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.658268690109253, + "objective/train/theoretical_loss": 3.467631705543894, + "objective/train/tokens_used": 118762976, + "theoretical_loss": 3.467631705543894, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047474747474747476, + "loss": 2.441, + "theoretical_loss": 3.467631705543894, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747272727272727, + "loss": 2.4756, + "theoretical_loss": 3.467620666625999, + "tokens_seen": 1759969280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470707070707067, + "loss": 2.5877, + "theoretical_loss": 3.4676096282342432, + "tokens_seen": 1760034816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047468686868686874, + "loss": 2.4157, + "theoretical_loss": 3.4675985903685826, + "tokens_seen": 1760100352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746666666666667, + "loss": 2.6185, + "theoretical_loss": 3.4675875530289724, + "tokens_seen": 1760165888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047464646464646465, + "loss": 2.626, + "theoretical_loss": 3.4675765162153676, + "tokens_seen": 1760231424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746262626262626, + "loss": 2.6303, + "theoretical_loss": 3.467565479927724, + "tokens_seen": 1760296960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746060606060606, + "loss": 2.4665, + "theoretical_loss": 3.4675544441659962, + "tokens_seen": 1760362496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047458585858585863, + "loss": 2.4859, + "theoretical_loss": 3.4675434089301405, + "tokens_seen": 1760428032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745656565656566, + "loss": 2.7329, + "theoretical_loss": 3.467532374220112, + "tokens_seen": 1760493568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454545454545454, + "loss": 2.7522, + "theoretical_loss": 3.4675213400358653, + "tokens_seen": 1760559104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745252525252525, + "loss": 2.6709, + "theoretical_loss": 3.4675103063773567, + "tokens_seen": 1760624640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745050505050505, + "loss": 2.3443, + "theoretical_loss": 3.4674992732445413, + "tokens_seen": 1760690176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744848484848485, + "loss": 2.5856, + "theoretical_loss": 3.4674882406373744, + "tokens_seen": 1760755712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744646464646465, + "loss": 2.8132, + "theoretical_loss": 3.4674772085558114, + "tokens_seen": 1760821248 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444444444444444, + "loss": 2.691, + "theoretical_loss": 3.4674661769998076, + "tokens_seen": 1760886784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047442424242424245, + "loss": 2.8995, + "theoretical_loss": 3.4674551459693186, + "tokens_seen": 1760952320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744040404040404, + "loss": 2.5649, + "theoretical_loss": 3.4674441154643, + "tokens_seen": 1761017856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743838383838384, + "loss": 2.7209, + "theoretical_loss": 3.467433085484706, + "tokens_seen": 1761083392 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436363636363637, + "loss": 2.8425, + "theoretical_loss": 3.467422056030494, + "tokens_seen": 1761148928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047434343434343433, + "loss": 2.7739, + "theoretical_loss": 3.467411027101618, + "tokens_seen": 1761214464 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432323232323234, + "loss": 2.6837, + "theoretical_loss": 3.467399998698034, + "tokens_seen": 1761280000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743030303030303, + "loss": 2.7335, + "theoretical_loss": 3.467388970819697, + "tokens_seen": 1761345536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742828282828283, + "loss": 2.6222, + "theoretical_loss": 3.4673779434665635, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047426262626262626, + "loss": 2.6726, + "theoretical_loss": 3.4673669166385874, + "tokens_seen": 1761476608 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 972196, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6714515686035156, + "objective/train/theoretical_loss": 3.4673558903357247, + "objective/train/tokens_used": 120401376, + "theoretical_loss": 3.4673558903357247, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742424242424243, + "loss": 2.6306, + "theoretical_loss": 3.4673558903357247, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047422222222222223, + "loss": 2.8951, + "theoretical_loss": 3.4673448645579317, + "tokens_seen": 1761607680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742020202020202, + "loss": 2.7546, + "theoretical_loss": 3.467333839305163, + "tokens_seen": 1761673216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741818181818182, + "loss": 2.5618, + "theoretical_loss": 3.4673228145773747, + "tokens_seen": 1761738752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416161616161616, + "loss": 2.5558, + "theoretical_loss": 3.467311790374521, + "tokens_seen": 1761804288 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047414141414141417, + "loss": 2.3849, + "theoretical_loss": 3.467300766696559, + "tokens_seen": 1761869824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741212121212121, + "loss": 2.6992, + "theoretical_loss": 3.467289743543443, + "tokens_seen": 1761935360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741010101010101, + "loss": 2.6239, + "theoretical_loss": 3.4672787209151292, + "tokens_seen": 1762000896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740808080808081, + "loss": 2.61, + "theoretical_loss": 3.4672676988115727, + "tokens_seen": 1762066432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740606060606061, + "loss": 2.7707, + "theoretical_loss": 3.4672566772327293, + "tokens_seen": 1762131968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047404040404040406, + "loss": 2.5288, + "theoretical_loss": 3.4672456561785543, + "tokens_seen": 1762197504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474020202020202, + "loss": 2.5156, + "theoretical_loss": 3.467234635649003, + "tokens_seen": 1762263040 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474, + "loss": 2.3634, + "theoretical_loss": 3.4672236156440315, + "tokens_seen": 1762328576 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473979797979798, + "loss": 2.3909, + "theoretical_loss": 3.467212596163595, + "tokens_seen": 1762394112 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473959595959596, + "loss": 2.6485, + "theoretical_loss": 3.467201577207649, + "tokens_seen": 1762459648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047393939393939395, + "loss": 2.9205, + "theoretical_loss": 3.467190558776149, + "tokens_seen": 1762525184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739191919191919, + "loss": 2.4281, + "theoretical_loss": 3.4671795408690507, + "tokens_seen": 1762590720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047389898989898987, + "loss": 2.6767, + "theoretical_loss": 3.4671685234863094, + "tokens_seen": 1762656256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047387878787878793, + "loss": 2.6607, + "theoretical_loss": 3.467157506627881, + "tokens_seen": 1762721792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738585858585859, + "loss": 2.4989, + "theoretical_loss": 3.4671464902937204, + "tokens_seen": 1762787328 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047383838383838385, + "loss": 2.7566, + "theoretical_loss": 3.4671354744837837, + "tokens_seen": 1762852864 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738181818181818, + "loss": 2.4784, + "theoretical_loss": 3.4671244591980264, + "tokens_seen": 1762918400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047379797979797976, + "loss": 2.6094, + "theoretical_loss": 3.4671134444364045, + "tokens_seen": 1762983936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737777777777778, + "loss": 2.3573, + "theoretical_loss": 3.467102430198873, + "tokens_seen": 1763049472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737575757575758, + "loss": 2.7262, + "theoretical_loss": 3.4670914164853874, + "tokens_seen": 1763115008 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 972718, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.458674430847168, + "objective/train/theoretical_loss": 3.4670804032959035, + "objective/train/tokens_used": 122039776, + "theoretical_loss": 3.4670804032959035, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047373737373737374, + "loss": 2.5665, + "theoretical_loss": 3.4670804032959035, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737171717171717, + "loss": 2.4811, + "theoretical_loss": 3.4670693906303773, + "tokens_seen": 1763246080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736969696969697, + "loss": 2.4546, + "theoretical_loss": 3.4670583784887636, + "tokens_seen": 1763311616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736767676767677, + "loss": 2.5793, + "theoretical_loss": 3.4670473668710184, + "tokens_seen": 1763377152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736565656565657, + "loss": 2.5163, + "theoretical_loss": 3.467036355777098, + "tokens_seen": 1763442688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047363636363636363, + "loss": 2.5888, + "theoretical_loss": 3.467025345206956, + "tokens_seen": 1763508224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736161616161616, + "loss": 2.7923, + "theoretical_loss": 3.4670143351605502, + "tokens_seen": 1763573760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735959595959596, + "loss": 2.7194, + "theoretical_loss": 3.4670033256378354, + "tokens_seen": 1763639296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735757575757576, + "loss": 2.4734, + "theoretical_loss": 3.466992316638767, + "tokens_seen": 1763704832 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047355555555555557, + "loss": 2.7535, + "theoretical_loss": 3.4669813081633007, + "tokens_seen": 1763770368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735353535353535, + "loss": 2.7308, + "theoretical_loss": 3.466970300211393, + "tokens_seen": 1763835904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047351515151515154, + "loss": 2.579, + "theoretical_loss": 3.466959292782998, + "tokens_seen": 1763901440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734949494949495, + "loss": 2.5358, + "theoretical_loss": 3.4669482858780727, + "tokens_seen": 1763966976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734747474747475, + "loss": 2.7295, + "theoretical_loss": 3.4669372794965723, + "tokens_seen": 1764032512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047345454545454546, + "loss": 2.474, + "theoretical_loss": 3.4669262736384523, + "tokens_seen": 1764098048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734343434343434, + "loss": 2.7381, + "theoretical_loss": 3.4669152683036684, + "tokens_seen": 1764163584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047341414141414143, + "loss": 2.7042, + "theoretical_loss": 3.466904263492176, + "tokens_seen": 1764229120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733939393939394, + "loss": 2.8351, + "theoretical_loss": 3.466893259203932, + "tokens_seen": 1764294656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733737373737374, + "loss": 2.7247, + "theoretical_loss": 3.4668822554388905, + "tokens_seen": 1764360192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047335353535353535, + "loss": 2.5802, + "theoretical_loss": 3.4668712521970084, + "tokens_seen": 1764425728 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047333333333333336, + "loss": 2.6002, + "theoretical_loss": 3.4668602494782403, + "tokens_seen": 1764491264 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733131313131313, + "loss": 2.6269, + "theoretical_loss": 3.466849247282543, + "tokens_seen": 1764556800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732929292929293, + "loss": 2.6215, + "theoretical_loss": 3.466838245609872, + "tokens_seen": 1764622336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732727272727273, + "loss": 2.6113, + "theoretical_loss": 3.466827244460182, + "tokens_seen": 1764687872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325252525252525, + "loss": 2.534, + "theoretical_loss": 3.46681624383343, + "tokens_seen": 1764753408 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 973844, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.547422170639038, + "objective/train/theoretical_loss": 3.466805243729571, + "objective/train/tokens_used": 123678176, + "theoretical_loss": 3.466805243729571, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047323232323232326, + "loss": 2.5461, + "theoretical_loss": 3.466805243729571, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732121212121212, + "loss": 2.4588, + "theoretical_loss": 3.466794244148561, + "tokens_seen": 1764884480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731919191919192, + "loss": 2.4346, + "theoretical_loss": 3.4667832450903555, + "tokens_seen": 1764950016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731717171717172, + "loss": 2.6804, + "theoretical_loss": 3.4667722465549105, + "tokens_seen": 1765015552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731515151515152, + "loss": 2.5802, + "theoretical_loss": 3.4667612485421815, + "tokens_seen": 1765081088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047313131313131315, + "loss": 2.6148, + "theoretical_loss": 3.4667502510521246, + "tokens_seen": 1765146624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731111111111111, + "loss": 2.625, + "theoretical_loss": 3.466739254084695, + "tokens_seen": 1765212160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730909090909091, + "loss": 2.6565, + "theoretical_loss": 3.466728257639849, + "tokens_seen": 1765277696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730707070707071, + "loss": 2.6265, + "theoretical_loss": 3.4667172617175424, + "tokens_seen": 1765343232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730505050505051, + "loss": 2.5729, + "theoretical_loss": 3.4667062663177304, + "tokens_seen": 1765408768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303030303030304, + "loss": 2.706, + "theoretical_loss": 3.4666952714403694, + "tokens_seen": 1765474304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473010101010101, + "loss": 2.5028, + "theoretical_loss": 3.4666842770854145, + "tokens_seen": 1765539840 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472989898989899, + "loss": 2.7347, + "theoretical_loss": 3.466673283252822, + "tokens_seen": 1765605376 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472969696969697, + "loss": 2.2926, + "theoretical_loss": 3.4666622899425477, + "tokens_seen": 1765670912 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472949494949495, + "loss": 2.7329, + "theoretical_loss": 3.4666512971545473, + "tokens_seen": 1765736448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047292929292929293, + "loss": 2.7155, + "theoretical_loss": 3.4666403048887764, + "tokens_seen": 1765801984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729090909090909, + "loss": 2.676, + "theoretical_loss": 3.4666293131451917, + "tokens_seen": 1765867520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728888888888889, + "loss": 2.5448, + "theoretical_loss": 3.4666183219237476, + "tokens_seen": 1765933056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728686868686869, + "loss": 2.6503, + "theoretical_loss": 3.4666073312244006, + "tokens_seen": 1765998592 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047284848484848487, + "loss": 2.7058, + "theoretical_loss": 3.466596341047107, + "tokens_seen": 1766064128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047282828282828283, + "loss": 2.517, + "theoretical_loss": 3.4665853513918217, + "tokens_seen": 1766129664 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728080808080808, + "loss": 2.626, + "theoretical_loss": 3.466574362258501, + "tokens_seen": 1766195200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727878787878788, + "loss": 2.6928, + "theoretical_loss": 3.4665633736471015, + "tokens_seen": 1766260736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727676767676768, + "loss": 2.6162, + "theoretical_loss": 3.466552385557578, + "tokens_seen": 1766326272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047274747474747476, + "loss": 2.5867, + "theoretical_loss": 3.4665413979898863, + "tokens_seen": 1766391808 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 974264, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8634729385375977, + "objective/train/theoretical_loss": 3.466530410943983, + "objective/train/tokens_used": 125316576, + "theoretical_loss": 3.466530410943983, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727272727272727, + "loss": 2.755, + "theoretical_loss": 3.466530410943983, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727070707070707, + "loss": 2.6146, + "theoretical_loss": 3.466519424419823, + "tokens_seen": 1766522880 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047268686868686874, + "loss": 2.8209, + "theoretical_loss": 3.4665084384173634, + "tokens_seen": 1766588416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726666666666667, + "loss": 2.7291, + "theoretical_loss": 3.4664974529365593, + "tokens_seen": 1766653952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047264646464646466, + "loss": 2.7675, + "theoretical_loss": 3.466486467977367, + "tokens_seen": 1766719488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726262626262626, + "loss": 2.4818, + "theoretical_loss": 3.4664754835397416, + "tokens_seen": 1766785024 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047260606060606057, + "loss": 2.5268, + "theoretical_loss": 3.4664644996236396, + "tokens_seen": 1766850560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047258585858585864, + "loss": 2.5537, + "theoretical_loss": 3.4664535162290164, + "tokens_seen": 1766916096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725656565656566, + "loss": 2.643, + "theoretical_loss": 3.466442533355829, + "tokens_seen": 1766981632 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047254545454545455, + "loss": 2.6887, + "theoretical_loss": 3.466431551004032, + "tokens_seen": 1767047168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725252525252525, + "loss": 2.6244, + "theoretical_loss": 3.4664205691735823, + "tokens_seen": 1767112704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725050505050505, + "loss": 2.5747, + "theoretical_loss": 3.466409587864436, + "tokens_seen": 1767178240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047248484848484853, + "loss": 2.3783, + "theoretical_loss": 3.466398607076547, + "tokens_seen": 1767243776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724646464646465, + "loss": 2.6642, + "theoretical_loss": 3.4663876268098734, + "tokens_seen": 1767309312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047244444444444444, + "loss": 2.4038, + "theoretical_loss": 3.466376647064371, + "tokens_seen": 1767374848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724242424242424, + "loss": 2.5637, + "theoretical_loss": 3.4663656678399946, + "tokens_seen": 1767440384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724040404040404, + "loss": 2.6683, + "theoretical_loss": 3.4663546891367005, + "tokens_seen": 1767505920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723838383838384, + "loss": 2.6663, + "theoretical_loss": 3.4663437109544453, + "tokens_seen": 1767571456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723636363636364, + "loss": 2.7499, + "theoretical_loss": 3.466332733293184, + "tokens_seen": 1767636992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047234343434343433, + "loss": 2.6059, + "theoretical_loss": 3.4663217561528734, + "tokens_seen": 1767702528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047232323232323235, + "loss": 2.8539, + "theoretical_loss": 3.466310779533469, + "tokens_seen": 1767768064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723030303030303, + "loss": 2.6061, + "theoretical_loss": 3.466299803434927, + "tokens_seen": 1767833600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722828282828283, + "loss": 2.7606, + "theoretical_loss": 3.4662888278572033, + "tokens_seen": 1767899136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047226262626262627, + "loss": 2.602, + "theoretical_loss": 3.4662778528002534, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722424242424242, + "loss": 2.6908, + "theoretical_loss": 3.466266878264034, + "tokens_seen": 1768030208 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 975251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7586162090301514, + "objective/train/theoretical_loss": 3.4662559042485013, + "objective/train/tokens_used": 126954976, + "theoretical_loss": 3.4662559042485013, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047222222222222224, + "loss": 2.6865, + "theoretical_loss": 3.4662559042485013, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722020202020202, + "loss": 2.5986, + "theoretical_loss": 3.4662449307536103, + "tokens_seen": 1768161280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721818181818182, + "loss": 2.5451, + "theoretical_loss": 3.466233957779318, + "tokens_seen": 1768226816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047216161616161616, + "loss": 2.5286, + "theoretical_loss": 3.4662229853255795, + "tokens_seen": 1768292352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721414141414142, + "loss": 2.6367, + "theoretical_loss": 3.466212013392351, + "tokens_seen": 1768357888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047212121212121213, + "loss": 2.6325, + "theoretical_loss": 3.4662010419795894, + "tokens_seen": 1768423424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721010101010101, + "loss": 2.5137, + "theoretical_loss": 3.46619007108725, + "tokens_seen": 1768488960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720808080808081, + "loss": 2.4045, + "theoretical_loss": 3.4661791007152885, + "tokens_seen": 1768554496 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047206060606060606, + "loss": 2.6393, + "theoretical_loss": 3.466168130863662, + "tokens_seen": 1768620032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047204040404040407, + "loss": 2.4247, + "theoretical_loss": 3.4661571615323252, + "tokens_seen": 1768685568 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472020202020202, + "loss": 2.4163, + "theoretical_loss": 3.466146192721235, + "tokens_seen": 1768751104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472, + "loss": 2.6915, + "theoretical_loss": 3.4661352244303476, + "tokens_seen": 1768816640 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471979797979798, + "loss": 2.4705, + "theoretical_loss": 3.4661242566596187, + "tokens_seen": 1768882176 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471959595959596, + "loss": 2.6915, + "theoretical_loss": 3.4661132894090043, + "tokens_seen": 1768947712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193939393939396, + "loss": 2.5054, + "theoretical_loss": 3.46610232267846, + "tokens_seen": 1769013248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719191919191919, + "loss": 2.6938, + "theoretical_loss": 3.4660913564679436, + "tokens_seen": 1769078784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718989898989899, + "loss": 2.5973, + "theoretical_loss": 3.4660803907774094, + "tokens_seen": 1769144320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718787878787879, + "loss": 2.6252, + "theoretical_loss": 3.4660694256068143, + "tokens_seen": 1769209856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718585858585859, + "loss": 2.7995, + "theoretical_loss": 3.4660584609561145, + "tokens_seen": 1769275392 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047183838383838385, + "loss": 2.5113, + "theoretical_loss": 3.466047496825265, + "tokens_seen": 1769340928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718181818181818, + "loss": 2.58, + "theoretical_loss": 3.4660365332142233, + "tokens_seen": 1769406464 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047179797979797977, + "loss": 2.6229, + "theoretical_loss": 3.4660255701229445, + "tokens_seen": 1769472000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177777777777783, + "loss": 2.625, + "theoretical_loss": 3.4660146075513856, + "tokens_seen": 1769537536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717575757575758, + "loss": 2.4516, + "theoretical_loss": 3.466003645499502, + "tokens_seen": 1769603072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173737373737374, + "loss": 2.6413, + "theoretical_loss": 3.46599268396725, + "tokens_seen": 1769668608 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 976290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.645479679107666, + "objective/train/theoretical_loss": 3.465981722954586, + "objective/train/tokens_used": 128593376, + "theoretical_loss": 3.465981722954586, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717171717171717, + "loss": 2.7062, + "theoretical_loss": 3.465981722954586, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169696969696966, + "loss": 2.7589, + "theoretical_loss": 3.465970762461466, + "tokens_seen": 1769799680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716767676767677, + "loss": 2.8116, + "theoretical_loss": 3.4659598024878457, + "tokens_seen": 1769865216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716565656565657, + "loss": 2.8078, + "theoretical_loss": 3.4659488430336816, + "tokens_seen": 1769930752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163636363636364, + "loss": 2.2874, + "theoretical_loss": 3.46593788409893, + "tokens_seen": 1769996288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716161616161616, + "loss": 2.2027, + "theoretical_loss": 3.4659269256835468, + "tokens_seen": 1770061824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715959595959596, + "loss": 2.7392, + "theoretical_loss": 3.4659159677874882, + "tokens_seen": 1770127360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715757575757576, + "loss": 2.8195, + "theoretical_loss": 3.4659050104107103, + "tokens_seen": 1770192896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715555555555556, + "loss": 2.6863, + "theoretical_loss": 3.4658940535531695, + "tokens_seen": 1770258432 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047153535353535353, + "loss": 2.4416, + "theoretical_loss": 3.465883097214822, + "tokens_seen": 1770323968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715151515151515, + "loss": 2.5736, + "theoretical_loss": 3.4658721413956237, + "tokens_seen": 1770389504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047149494949494955, + "loss": 2.5487, + "theoretical_loss": 3.465861186095531, + "tokens_seen": 1770455040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714747474747475, + "loss": 2.9009, + "theoretical_loss": 3.4658502313144997, + "tokens_seen": 1770520576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047145454545454547, + "loss": 2.8561, + "theoretical_loss": 3.4658392770524866, + "tokens_seen": 1770586112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714343434343434, + "loss": 2.646, + "theoretical_loss": 3.4658283233094473, + "tokens_seen": 1770651648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714141414141414, + "loss": 2.6522, + "theoretical_loss": 3.4658173700853387, + "tokens_seen": 1770717184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139393939393944, + "loss": 2.7383, + "theoretical_loss": 3.4658064173801164, + "tokens_seen": 1770782720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713737373737374, + "loss": 2.4971, + "theoretical_loss": 3.465795465193737, + "tokens_seen": 1770848256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135353535353536, + "loss": 2.8238, + "theoretical_loss": 3.465784513526156, + "tokens_seen": 1770913792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713333333333333, + "loss": 2.5457, + "theoretical_loss": 3.465773562377331, + "tokens_seen": 1770979328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713131313131313, + "loss": 2.7584, + "theoretical_loss": 3.465762611747217, + "tokens_seen": 1771044864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047129292929292934, + "loss": 2.5373, + "theoretical_loss": 3.4657516616357706, + "tokens_seen": 1771110400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712727272727273, + "loss": 2.7518, + "theoretical_loss": 3.4657407120429484, + "tokens_seen": 1771175936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125252525252525, + "loss": 2.6246, + "theoretical_loss": 3.4657297629687056, + "tokens_seen": 1771241472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712323232323232, + "loss": 2.6083, + "theoretical_loss": 3.465718814413, + "tokens_seen": 1771307008 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 976909, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4419713020324707, + "objective/train/theoretical_loss": 3.4657078663757868, + "objective/train/tokens_used": 130231776, + "theoretical_loss": 3.4657078663757868, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712121212121212, + "loss": 2.683, + "theoretical_loss": 3.4657078663757868, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047119191919191923, + "loss": 2.7196, + "theoretical_loss": 3.4656969188570224, + "tokens_seen": 1771438080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711717171717172, + "loss": 2.5106, + "theoretical_loss": 3.4656859718566633, + "tokens_seen": 1771503616 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115151515151514, + "loss": 2.4445, + "theoretical_loss": 3.465675025374666, + "tokens_seen": 1771569152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047113131313131316, + "loss": 2.4538, + "theoretical_loss": 3.4656640794109856, + "tokens_seen": 1771634688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711111111111111, + "loss": 2.6856, + "theoretical_loss": 3.4656531339655796, + "tokens_seen": 1771700224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710909090909091, + "loss": 2.771, + "theoretical_loss": 3.4656421890384044, + "tokens_seen": 1771765760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710707070707071, + "loss": 2.5379, + "theoretical_loss": 3.4656312446294155, + "tokens_seen": 1771831296 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047105050505050504, + "loss": 2.587, + "theoretical_loss": 3.465620300738569, + "tokens_seen": 1771896832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047103030303030305, + "loss": 2.5692, + "theoretical_loss": 3.4656093573658224, + "tokens_seen": 1771962368 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471010101010101, + "loss": 2.7043, + "theoretical_loss": 3.465598414511131, + "tokens_seen": 1772027904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470989898989899, + "loss": 2.6211, + "theoretical_loss": 3.4655874721744517, + "tokens_seen": 1772093440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047096969696969697, + "loss": 2.6399, + "theoretical_loss": 3.46557653035574, + "tokens_seen": 1772158976 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470949494949495, + "loss": 2.5832, + "theoretical_loss": 3.465565589054954, + "tokens_seen": 1772224512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047092929292929294, + "loss": 2.6915, + "theoretical_loss": 3.4655546482720476, + "tokens_seen": 1772290048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709090909090909, + "loss": 2.5416, + "theoretical_loss": 3.4655437080069786, + "tokens_seen": 1772355584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708888888888889, + "loss": 2.6132, + "theoretical_loss": 3.4655327682597035, + "tokens_seen": 1772421120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047086868686868687, + "loss": 2.4983, + "theoretical_loss": 3.4655218290301786, + "tokens_seen": 1772486656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708484848484849, + "loss": 2.6079, + "theoretical_loss": 3.465510890318359, + "tokens_seen": 1772552192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047082828282828283, + "loss": 2.7404, + "theoretical_loss": 3.465499952124203, + "tokens_seen": 1772617728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708080808080808, + "loss": 2.7041, + "theoretical_loss": 3.465489014447665, + "tokens_seen": 1772683264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707878787878788, + "loss": 2.6753, + "theoretical_loss": 3.4654780772887026, + "tokens_seen": 1772748800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707676767676768, + "loss": 2.8614, + "theoretical_loss": 3.465467140647272, + "tokens_seen": 1772814336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047074747474747477, + "loss": 2.5925, + "theoretical_loss": 3.4654562045233295, + "tokens_seen": 1772879872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707272727272727, + "loss": 2.7646, + "theoretical_loss": 3.465445268916832, + "tokens_seen": 1772945408 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 977811, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4676389694213867, + "objective/train/theoretical_loss": 3.4654343338277345, + "objective/train/tokens_used": 131870176, + "theoretical_loss": 3.4654343338277345, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707070707070707, + "loss": 2.5732, + "theoretical_loss": 3.4654343338277345, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706868686868687, + "loss": 2.8103, + "theoretical_loss": 3.4654233992559944, + "tokens_seen": 1773076480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706666666666667, + "loss": 2.4505, + "theoretical_loss": 3.465412465201568, + "tokens_seen": 1773142016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047064646464646466, + "loss": 2.3781, + "theoretical_loss": 3.465401531664412, + "tokens_seen": 1773207552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706262626262626, + "loss": 2.8003, + "theoretical_loss": 3.4653905986444817, + "tokens_seen": 1773273088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706060606060606, + "loss": 2.6426, + "theoretical_loss": 3.465379666141735, + "tokens_seen": 1773338624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047058585858585864, + "loss": 2.5942, + "theoretical_loss": 3.4653687341561277, + "tokens_seen": 1773404160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705656565656566, + "loss": 2.628, + "theoretical_loss": 3.4653578026876155, + "tokens_seen": 1773469696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047054545454545455, + "loss": 2.6492, + "theoretical_loss": 3.4653468717361555, + "tokens_seen": 1773535232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705252525252525, + "loss": 2.6411, + "theoretical_loss": 3.4653359413017046, + "tokens_seen": 1773600768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047050505050505047, + "loss": 2.7541, + "theoretical_loss": 3.465325011384219, + "tokens_seen": 1773666304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047048484848484853, + "loss": 2.4253, + "theoretical_loss": 3.465314081983654, + "tokens_seen": 1773731840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704646464646465, + "loss": 2.7165, + "theoretical_loss": 3.4653031530999674, + "tokens_seen": 1773797376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047044444444444445, + "loss": 2.7943, + "theoretical_loss": 3.4652922247331155, + "tokens_seen": 1773862912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704242424242424, + "loss": 2.5949, + "theoretical_loss": 3.465281296883054, + "tokens_seen": 1773928448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704040404040404, + "loss": 2.5194, + "theoretical_loss": 3.4652703695497404, + "tokens_seen": 1773993984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703838383838384, + "loss": 2.8284, + "theoretical_loss": 3.46525944273313, + "tokens_seen": 1774059520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703636363636364, + "loss": 2.5426, + "theoretical_loss": 3.4652485164331805, + "tokens_seen": 1774125056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047034343434343434, + "loss": 2.6549, + "theoretical_loss": 3.4652375906498474, + "tokens_seen": 1774190592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703232323232323, + "loss": 2.5567, + "theoretical_loss": 3.4652266653830877, + "tokens_seen": 1774256128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703030303030303, + "loss": 2.6891, + "theoretical_loss": 3.465215740632858, + "tokens_seen": 1774321664 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702828282828283, + "loss": 2.6164, + "theoretical_loss": 3.465204816399114, + "tokens_seen": 1774387200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702626262626263, + "loss": 2.6243, + "theoretical_loss": 3.465193892681813, + "tokens_seen": 1774452736 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047024242424242423, + "loss": 2.6201, + "theoretical_loss": 3.4651829694809115, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047022222222222224, + "loss": 2.6683, + "theoretical_loss": 3.465172046796366, + "tokens_seen": 1774583808 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 978517, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.493600368499756, + "objective/train/theoretical_loss": 3.4651611246281324, + "objective/train/tokens_used": 133508576, + "theoretical_loss": 3.4651611246281324, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702020202020202, + "loss": 2.5756, + "theoretical_loss": 3.4651611246281324, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701818181818182, + "loss": 2.7666, + "theoretical_loss": 3.465150202976168, + "tokens_seen": 1774714880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047016161616161617, + "loss": 2.6612, + "theoretical_loss": 3.465139281840429, + "tokens_seen": 1774780416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701414141414141, + "loss": 2.5653, + "theoretical_loss": 3.465128361220872, + "tokens_seen": 1774845952 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047012121212121214, + "loss": 2.6465, + "theoretical_loss": 3.465117441117453, + "tokens_seen": 1774911488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701010101010101, + "loss": 2.5144, + "theoretical_loss": 3.4651065215301298, + "tokens_seen": 1774977024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700808080808081, + "loss": 2.7176, + "theoretical_loss": 3.4650956024588577, + "tokens_seen": 1775042560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047006060606060606, + "loss": 2.7295, + "theoretical_loss": 3.4650846839035943, + "tokens_seen": 1775108096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047004040404040407, + "loss": 2.5483, + "theoretical_loss": 3.465073765864295, + "tokens_seen": 1775173632 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047002020202020203, + "loss": 2.5907, + "theoretical_loss": 3.465062848340917, + "tokens_seen": 1775239168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047, + "loss": 2.6084, + "theoretical_loss": 3.4650519313334174, + "tokens_seen": 1775304704 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469979797979798, + "loss": 2.4807, + "theoretical_loss": 3.465041014841751, + "tokens_seen": 1775370240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046995959595959595, + "loss": 2.6201, + "theoretical_loss": 3.465030098865877, + "tokens_seen": 1775435776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046993939393939396, + "loss": 2.4878, + "theoretical_loss": 3.4650191834057504, + "tokens_seen": 1775501312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699191919191919, + "loss": 2.6224, + "theoretical_loss": 3.465008268461328, + "tokens_seen": 1775566848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698989898989899, + "loss": 2.653, + "theoretical_loss": 3.4649973540325663, + "tokens_seen": 1775632384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698787878787879, + "loss": 2.5147, + "theoretical_loss": 3.4649864401194215, + "tokens_seen": 1775697920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698585858585859, + "loss": 2.4981, + "theoretical_loss": 3.4649755267218514, + "tokens_seen": 1775763456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046983838383838386, + "loss": 2.6706, + "theoretical_loss": 3.464964613839812, + "tokens_seen": 1775828992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698181818181818, + "loss": 2.6469, + "theoretical_loss": 3.4649537014732594, + "tokens_seen": 1775894528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697979797979798, + "loss": 2.6877, + "theoretical_loss": 3.464942789622151, + "tokens_seen": 1775960064 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697777777777778, + "loss": 2.6401, + "theoretical_loss": 3.464931878286443, + "tokens_seen": 1776025600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697575757575758, + "loss": 2.7416, + "theoretical_loss": 3.4649209674660923, + "tokens_seen": 1776091136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046973737373737375, + "loss": 2.716, + "theoretical_loss": 3.464910057161056, + "tokens_seen": 1776156672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697171717171717, + "loss": 2.775, + "theoretical_loss": 3.4648991473712893, + "tokens_seen": 1776222208 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 979554, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.655245542526245, + "objective/train/theoretical_loss": 3.4648882380967505, + "objective/train/tokens_used": 135146976, + "theoretical_loss": 3.4648882380967505, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696969696969697, + "loss": 2.735, + "theoretical_loss": 3.4648882380967505, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046967676767676773, + "loss": 2.7324, + "theoretical_loss": 3.464877329337395, + "tokens_seen": 1776353280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696565656565657, + "loss": 2.6467, + "theoretical_loss": 3.4648664210931797, + "tokens_seen": 1776418816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046963636363636364, + "loss": 2.6666, + "theoretical_loss": 3.4648555133640624, + "tokens_seen": 1776484352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696161616161616, + "loss": 2.694, + "theoretical_loss": 3.464844606149998, + "tokens_seen": 1776549888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695959595959596, + "loss": 2.5421, + "theoretical_loss": 3.4648336994509448, + "tokens_seen": 1776615424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695757575757576, + "loss": 2.4765, + "theoretical_loss": 3.4648227932668583, + "tokens_seen": 1776680960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695555555555556, + "loss": 2.7016, + "theoretical_loss": 3.464811887597696, + "tokens_seen": 1776746496 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046953535353535354, + "loss": 2.3606, + "theoretical_loss": 3.464800982443414, + "tokens_seen": 1776812032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695151515151515, + "loss": 2.7383, + "theoretical_loss": 3.4647900778039693, + "tokens_seen": 1776877568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694949494949495, + "loss": 2.5567, + "theoretical_loss": 3.464779173679319, + "tokens_seen": 1776943104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694747474747475, + "loss": 2.6411, + "theoretical_loss": 3.4647682700694187, + "tokens_seen": 1777008640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046945454545454547, + "loss": 2.4005, + "theoretical_loss": 3.464757366974226, + "tokens_seen": 1777074176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046943434343434343, + "loss": 2.7334, + "theoretical_loss": 3.4647464643936976, + "tokens_seen": 1777139712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694141414141414, + "loss": 2.5828, + "theoretical_loss": 3.4647355623277902, + "tokens_seen": 1777205248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046939393939393945, + "loss": 2.3715, + "theoretical_loss": 3.4647246607764606, + "tokens_seen": 1777270784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693737373737374, + "loss": 2.6547, + "theoretical_loss": 3.4647137597396647, + "tokens_seen": 1777336320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046935353535353536, + "loss": 2.4004, + "theoretical_loss": 3.4647028592173603, + "tokens_seen": 1777401856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693333333333333, + "loss": 2.7303, + "theoretical_loss": 3.464691959209503, + "tokens_seen": 1777467392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693131313131313, + "loss": 2.5245, + "theoretical_loss": 3.4646810597160513, + "tokens_seen": 1777532928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046929292929292934, + "loss": 2.6391, + "theoretical_loss": 3.46467016073696, + "tokens_seen": 1777598464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692727272727273, + "loss": 2.6976, + "theoretical_loss": 3.4646592622721872, + "tokens_seen": 1777664000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046925252525252526, + "loss": 2.5603, + "theoretical_loss": 3.4646483643216897, + "tokens_seen": 1777729536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692323232323232, + "loss": 2.4618, + "theoretical_loss": 3.4646374668854234, + "tokens_seen": 1777795072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692121212121212, + "loss": 2.8756, + "theoretical_loss": 3.4646265699633454, + "tokens_seen": 1777860608 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 979848, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.122527599334717, + "objective/train/theoretical_loss": 3.464615673555413, + "objective/train/tokens_used": 136785376, + "theoretical_loss": 3.464615673555413, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046919191919191924, + "loss": 2.7605, + "theoretical_loss": 3.464615673555413, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691717171717172, + "loss": 2.6706, + "theoretical_loss": 3.4646047776615823, + "tokens_seen": 1777991680 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046915151515151515, + "loss": 2.5914, + "theoretical_loss": 3.4645938822818105, + "tokens_seen": 1778057216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691313131313131, + "loss": 2.8862, + "theoretical_loss": 3.4645829874160543, + "tokens_seen": 1778122752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691111111111111, + "loss": 2.6125, + "theoretical_loss": 3.4645720930642705, + "tokens_seen": 1778188288 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046909090909090913, + "loss": 2.6488, + "theoretical_loss": 3.4645611992264156, + "tokens_seen": 1778253824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690707070707071, + "loss": 2.5569, + "theoretical_loss": 3.4645503059024474, + "tokens_seen": 1778319360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046905050505050504, + "loss": 2.4418, + "theoretical_loss": 3.4645394130923215, + "tokens_seen": 1778384896 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046903030303030305, + "loss": 2.6375, + "theoretical_loss": 3.4645285207959953, + "tokens_seen": 1778450432 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469010101010101, + "loss": 2.5087, + "theoretical_loss": 3.464517629013426, + "tokens_seen": 1778515968 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468989898989899, + "loss": 2.4809, + "theoretical_loss": 3.4645067377445695, + "tokens_seen": 1778581504 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468969696969697, + "loss": 2.696, + "theoretical_loss": 3.464495846989384, + "tokens_seen": 1778647040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046894949494949494, + "loss": 2.7445, + "theoretical_loss": 3.4644849567478246, + "tokens_seen": 1778712576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892929292929295, + "loss": 2.7525, + "theoretical_loss": 3.4644740670198497, + "tokens_seen": 1778778112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689090909090909, + "loss": 2.6765, + "theoretical_loss": 3.464463177805415, + "tokens_seen": 1778843648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688888888888889, + "loss": 2.6821, + "theoretical_loss": 3.4644522891044787, + "tokens_seen": 1778909184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046886868686868687, + "loss": 2.8012, + "theoretical_loss": 3.4644414009169964, + "tokens_seen": 1778974720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688484848484849, + "loss": 2.573, + "theoretical_loss": 3.4644305132429256, + "tokens_seen": 1779040256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046882828282828284, + "loss": 2.8178, + "theoretical_loss": 3.464419626082223, + "tokens_seen": 1779105792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688080808080808, + "loss": 2.7468, + "theoretical_loss": 3.4644087394348455, + "tokens_seen": 1779171328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687878787878788, + "loss": 2.5975, + "theoretical_loss": 3.46439785330075, + "tokens_seen": 1779236864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046876767676767676, + "loss": 2.7253, + "theoretical_loss": 3.4643869676798937, + "tokens_seen": 1779302400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687474747474748, + "loss": 2.6003, + "theoretical_loss": 3.464376082572233, + "tokens_seen": 1779367936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872727272727273, + "loss": 2.6038, + "theoretical_loss": 3.464365197977725, + "tokens_seen": 1779433472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687070707070707, + "loss": 2.7135, + "theoretical_loss": 3.464354313896327, + "tokens_seen": 1779499008 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 981168, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4408981800079346, + "objective/train/theoretical_loss": 3.464343430327995, + "objective/train/tokens_used": 138423776, + "theoretical_loss": 3.464343430327995, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686868686868687, + "loss": 2.6659, + "theoretical_loss": 3.464343430327995, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686666666666667, + "loss": 2.545, + "theoretical_loss": 3.4643325472726865, + "tokens_seen": 1779630080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864646464646467, + "loss": 2.5786, + "theoretical_loss": 3.4643216647303583, + "tokens_seen": 1779695616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686262626262626, + "loss": 2.6326, + "theoretical_loss": 3.4643107827009674, + "tokens_seen": 1779761152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686060606060606, + "loss": 2.476, + "theoretical_loss": 3.4642999011844715, + "tokens_seen": 1779826688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685858585858586, + "loss": 2.5703, + "theoretical_loss": 3.464289020180826, + "tokens_seen": 1779892224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656565656566, + "loss": 2.5252, + "theoretical_loss": 3.464278139689989, + "tokens_seen": 1779957760 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046854545454545456, + "loss": 2.653, + "theoretical_loss": 3.4642672597119173, + "tokens_seen": 1780023296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685252525252525, + "loss": 2.5044, + "theoretical_loss": 3.4642563802465673, + "tokens_seen": 1780088832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685050505050505, + "loss": 2.5998, + "theoretical_loss": 3.464245501293896, + "tokens_seen": 1780154368 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046848484848484854, + "loss": 2.5963, + "theoretical_loss": 3.464234622853861, + "tokens_seen": 1780219904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684646464646465, + "loss": 2.6996, + "theoretical_loss": 3.4642237449264193, + "tokens_seen": 1780285440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046844444444444445, + "loss": 2.702, + "theoretical_loss": 3.4642128675115273, + "tokens_seen": 1780350976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684242424242424, + "loss": 2.7233, + "theoretical_loss": 3.4642019906091424, + "tokens_seen": 1780416512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046840404040404037, + "loss": 2.7893, + "theoretical_loss": 3.464191114219221, + "tokens_seen": 1780482048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838383838383843, + "loss": 2.4928, + "theoretical_loss": 3.464180238341721, + "tokens_seen": 1780547584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683636363636364, + "loss": 2.7807, + "theoretical_loss": 3.4641693629765986, + "tokens_seen": 1780613120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834343434343435, + "loss": 2.4966, + "theoretical_loss": 3.464158488123811, + "tokens_seen": 1780678656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683232323232323, + "loss": 2.6617, + "theoretical_loss": 3.4641476137833154, + "tokens_seen": 1780744192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046830303030303026, + "loss": 2.8356, + "theoretical_loss": 3.464136739955069, + "tokens_seen": 1780809728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682828282828283, + "loss": 2.577, + "theoretical_loss": 3.464125866639028, + "tokens_seen": 1780875264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682626262626263, + "loss": 2.6046, + "theoretical_loss": 3.4641149938351505, + "tokens_seen": 1780940800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046824242424242424, + "loss": 2.7569, + "theoretical_loss": 3.464104121543393, + "tokens_seen": 1781006336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682222222222222, + "loss": 2.6343, + "theoretical_loss": 3.4640932497637125, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682020202020202, + "loss": 2.9327, + "theoretical_loss": 3.464082378496066, + "tokens_seen": 1781137408 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 982366, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5046448707580566, + "objective/train/theoretical_loss": 3.4640715077404103, + "objective/train/tokens_used": 140062176, + "theoretical_loss": 3.4640715077404103, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681818181818182, + "loss": 2.4701, + "theoretical_loss": 3.4640715077404103, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681616161616162, + "loss": 2.5085, + "theoretical_loss": 3.4640606374967033, + "tokens_seen": 1781268480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046814141414141413, + "loss": 2.7434, + "theoretical_loss": 3.464049767764901, + "tokens_seen": 1781334016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681212121212121, + "loss": 2.681, + "theoretical_loss": 3.4640388985449615, + "tokens_seen": 1781399552 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810101010101015, + "loss": 2.8738, + "theoretical_loss": 3.4640280298368413, + "tokens_seen": 1781465088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680808080808081, + "loss": 2.5953, + "theoretical_loss": 3.4640171616404976, + "tokens_seen": 1781530624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046806060606060607, + "loss": 2.8082, + "theoretical_loss": 3.4640062939558867, + "tokens_seen": 1781596160 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468040404040404, + "loss": 2.6266, + "theoretical_loss": 3.4639954267829673, + "tokens_seen": 1781661696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802020202020203, + "loss": 2.7488, + "theoretical_loss": 3.4639845601216948, + "tokens_seen": 1781727232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800000000000005, + "loss": 2.6267, + "theoretical_loss": 3.4639736939720276, + "tokens_seen": 1781792768 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467979797979798, + "loss": 2.8528, + "theoretical_loss": 3.4639628283339223, + "tokens_seen": 1781858304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046795959595959596, + "loss": 2.4593, + "theoretical_loss": 3.463951963207336, + "tokens_seen": 1781923840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679393939393939, + "loss": 2.7433, + "theoretical_loss": 3.463941098592225, + "tokens_seen": 1781989376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046791919191919193, + "loss": 2.7575, + "theoretical_loss": 3.463930234488548, + "tokens_seen": 1782054912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046789898989898994, + "loss": 2.8066, + "theoretical_loss": 3.4639193708962615, + "tokens_seen": 1782120448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678787878787879, + "loss": 2.6911, + "theoretical_loss": 3.4639085078153222, + "tokens_seen": 1782185984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046785858585858585, + "loss": 2.5327, + "theoretical_loss": 3.4638976452456873, + "tokens_seen": 1782251520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046783838383838386, + "loss": 2.713, + "theoretical_loss": 3.4638867831873137, + "tokens_seen": 1782317056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678181818181818, + "loss": 2.7197, + "theoretical_loss": 3.4638759216401596, + "tokens_seen": 1782382592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046779797979797983, + "loss": 2.7073, + "theoretical_loss": 3.463865060604181, + "tokens_seen": 1782448128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677777777777778, + "loss": 2.5768, + "theoretical_loss": 3.4638542000793358, + "tokens_seen": 1782513664 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046775757575757574, + "loss": 2.7761, + "theoretical_loss": 3.463843340065581, + "tokens_seen": 1782579200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046773737373737376, + "loss": 2.67, + "theoretical_loss": 3.463832480562873, + "tokens_seen": 1782644736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677171717171717, + "loss": 2.6575, + "theoretical_loss": 3.463821621571171, + "tokens_seen": 1782710272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676969696969697, + "loss": 2.5326, + "theoretical_loss": 3.4638107630904296, + "tokens_seen": 1782775808 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 982914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4014761447906494, + "objective/train/theoretical_loss": 3.4637999051206076, + "objective/train/tokens_used": 141700576, + "theoretical_loss": 3.4637999051206076, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676767676767677, + "loss": 2.5757, + "theoretical_loss": 3.4637999051206076, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676565656565657, + "loss": 2.7185, + "theoretical_loss": 3.4637890476616615, + "tokens_seen": 1782906880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046763636363636365, + "loss": 2.4657, + "theoretical_loss": 3.4637781907135485, + "tokens_seen": 1782972416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676161616161616, + "loss": 2.753, + "theoretical_loss": 3.4637673342762265, + "tokens_seen": 1783037952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675959595959596, + "loss": 2.6718, + "theoretical_loss": 3.463756478349652, + "tokens_seen": 1783103488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675757575757576, + "loss": 2.8216, + "theoretical_loss": 3.463745622933782, + "tokens_seen": 1783169024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675555555555556, + "loss": 2.5559, + "theoretical_loss": 3.4637347680285746, + "tokens_seen": 1783234560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046753535353535354, + "loss": 2.7446, + "theoretical_loss": 3.463723913633987, + "tokens_seen": 1783300096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675151515151515, + "loss": 2.7602, + "theoretical_loss": 3.463713059749975, + "tokens_seen": 1783365632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674949494949495, + "loss": 2.5877, + "theoretical_loss": 3.4637022063764977, + "tokens_seen": 1783431168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674747474747475, + "loss": 2.7978, + "theoretical_loss": 3.4636913535135108, + "tokens_seen": 1783496704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674545454545455, + "loss": 2.547, + "theoretical_loss": 3.4636805011609724, + "tokens_seen": 1783562240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046743434343434343, + "loss": 2.8632, + "theoretical_loss": 3.463669649318839, + "tokens_seen": 1783627776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674141414141414, + "loss": 2.57, + "theoretical_loss": 3.4636587979870685, + "tokens_seen": 1783693312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673939393939394, + "loss": 2.5764, + "theoretical_loss": 3.463647947165618, + "tokens_seen": 1783758848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673737373737374, + "loss": 2.9032, + "theoretical_loss": 3.4636370968544448, + "tokens_seen": 1783824384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046735353535353537, + "loss": 2.7317, + "theoretical_loss": 3.463626247053506, + "tokens_seen": 1783889920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673333333333333, + "loss": 2.6338, + "theoretical_loss": 3.4636153977627586, + "tokens_seen": 1783955456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673131313131313, + "loss": 2.7822, + "theoretical_loss": 3.4636045489821607, + "tokens_seen": 1784020992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046729292929292935, + "loss": 2.497, + "theoretical_loss": 3.4635937007116686, + "tokens_seen": 1784086528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672727272727273, + "loss": 2.509, + "theoretical_loss": 3.4635828529512405, + "tokens_seen": 1784152064 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046725252525252526, + "loss": 2.6947, + "theoretical_loss": 3.4635720057008332, + "tokens_seen": 1784217600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672323232323232, + "loss": 2.5064, + "theoretical_loss": 3.463561158960404, + "tokens_seen": 1784283136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672121212121212, + "loss": 2.3541, + "theoretical_loss": 3.4635503127299097, + "tokens_seen": 1784348672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046719191919191924, + "loss": 2.5577, + "theoretical_loss": 3.4635394670093085, + "tokens_seen": 1784414208 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 984019, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.804720401763916, + "objective/train/theoretical_loss": 3.4635286217985572, + "objective/train/tokens_used": 143338976, + "theoretical_loss": 3.4635286217985572, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671717171717172, + "loss": 2.8734, + "theoretical_loss": 3.4635286217985572, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046715151515151516, + "loss": 2.841, + "theoretical_loss": 3.4635177770976133, + "tokens_seen": 1784545280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671313131313131, + "loss": 2.4586, + "theoretical_loss": 3.463506932906434, + "tokens_seen": 1784610816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671111111111111, + "loss": 2.3573, + "theoretical_loss": 3.463496089224977, + "tokens_seen": 1784676352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046709090909090913, + "loss": 2.7561, + "theoretical_loss": 3.4634852460531986, + "tokens_seen": 1784741888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670707070707071, + "loss": 2.8304, + "theoretical_loss": 3.463474403391057, + "tokens_seen": 1784807424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046705050505050505, + "loss": 2.566, + "theoretical_loss": 3.46346356123851, + "tokens_seen": 1784872960 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467030303030303, + "loss": 2.7447, + "theoretical_loss": 3.4634527195955136, + "tokens_seen": 1784938496 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467010101010101, + "loss": 2.6607, + "theoretical_loss": 3.463441878462026, + "tokens_seen": 1785004032 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466989898989899, + "loss": 2.4519, + "theoretical_loss": 3.463431037838004, + "tokens_seen": 1785069568 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466969696969697, + "loss": 2.7172, + "theoretical_loss": 3.463420197723406, + "tokens_seen": 1785135104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694949494949494, + "loss": 2.4123, + "theoretical_loss": 3.463409358118188, + "tokens_seen": 1785200640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046692929292929295, + "loss": 2.7201, + "theoretical_loss": 3.4633985190223084, + "tokens_seen": 1785266176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669090909090909, + "loss": 2.7307, + "theoretical_loss": 3.4633876804357246, + "tokens_seen": 1785331712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668888888888889, + "loss": 2.5424, + "theoretical_loss": 3.463376842358393, + "tokens_seen": 1785397248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668686868686869, + "loss": 2.8041, + "theoretical_loss": 3.463366004790272, + "tokens_seen": 1785462784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046684848484848483, + "loss": 2.7058, + "theoretical_loss": 3.4633551677313186, + "tokens_seen": 1785528320 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682828282828284, + "loss": 2.6283, + "theoretical_loss": 3.46334433118149, + "tokens_seen": 1785593856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668080808080808, + "loss": 2.4156, + "theoretical_loss": 3.463333495140744, + "tokens_seen": 1785659392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667878787878788, + "loss": 2.4195, + "theoretical_loss": 3.4633226596090374, + "tokens_seen": 1785724928 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046676767676767677, + "loss": 2.4871, + "theoretical_loss": 3.463311824586328, + "tokens_seen": 1785790464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667474747474748, + "loss": 2.4098, + "theoretical_loss": 3.4633009900725735, + "tokens_seen": 1785856000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046672727272727274, + "loss": 2.8379, + "theoretical_loss": 3.463290156067731, + "tokens_seen": 1785921536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667070707070707, + "loss": 2.5884, + "theoretical_loss": 3.4632793225717573, + "tokens_seen": 1785987072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666868686868687, + "loss": 2.7167, + "theoretical_loss": 3.463268489584611, + "tokens_seen": 1786052608 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 984652, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4918596744537354, + "objective/train/theoretical_loss": 3.463257657106249, + "objective/train/tokens_used": 144977376, + "theoretical_loss": 3.463257657106249, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046666666666666666, + "loss": 2.6791, + "theoretical_loss": 3.463257657106249, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666464646464647, + "loss": 2.5787, + "theoretical_loss": 3.463246825136628, + "tokens_seen": 1786183680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046662626262626263, + "loss": 2.4831, + "theoretical_loss": 3.463235993675707, + "tokens_seen": 1786249216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666060606060606, + "loss": 2.6653, + "theoretical_loss": 3.4632251627234423, + "tokens_seen": 1786314752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665858585858586, + "loss": 2.604, + "theoretical_loss": 3.4632143322797915, + "tokens_seen": 1786380288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665656565656566, + "loss": 2.458, + "theoretical_loss": 3.4632035023447125, + "tokens_seen": 1786445824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046654545454545457, + "loss": 2.6402, + "theoretical_loss": 3.4631926729181624, + "tokens_seen": 1786511360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665252525252525, + "loss": 2.5807, + "theoretical_loss": 3.4631818440000988, + "tokens_seen": 1786576896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665050505050505, + "loss": 2.7028, + "theoretical_loss": 3.4631710155904787, + "tokens_seen": 1786642432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664848484848485, + "loss": 2.6881, + "theoretical_loss": 3.4631601876892604, + "tokens_seen": 1786707968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664646464646465, + "loss": 2.6738, + "theoretical_loss": 3.4631493602964007, + "tokens_seen": 1786773504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046644444444444446, + "loss": 2.5745, + "theoretical_loss": 3.4631385334118576, + "tokens_seen": 1786839040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664242424242424, + "loss": 2.6133, + "theoretical_loss": 3.4631277070355884, + "tokens_seen": 1786904576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046640404040404037, + "loss": 2.6516, + "theoretical_loss": 3.463116881167551, + "tokens_seen": 1786970112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663838383838384, + "loss": 2.6627, + "theoretical_loss": 3.4631060558077014, + "tokens_seen": 1787035648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663636363636364, + "loss": 2.6789, + "theoretical_loss": 3.463095230955999, + "tokens_seen": 1787101184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046634343434343435, + "loss": 2.7396, + "theoretical_loss": 3.4630844066124, + "tokens_seen": 1787166720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663232323232323, + "loss": 2.5693, + "theoretical_loss": 3.463073582776863, + "tokens_seen": 1787232256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663030303030303, + "loss": 2.8965, + "theoretical_loss": 3.463062759449344, + "tokens_seen": 1787297792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046628282828282833, + "loss": 2.7113, + "theoretical_loss": 3.4630519366298023, + "tokens_seen": 1787363328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662626262626263, + "loss": 2.7535, + "theoretical_loss": 3.4630411143181945, + "tokens_seen": 1787428864 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046624242424242424, + "loss": 2.6914, + "theoretical_loss": 3.4630302925144782, + "tokens_seen": 1787494400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662222222222222, + "loss": 2.4684, + "theoretical_loss": 3.463019471218611, + "tokens_seen": 1787559936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662020202020202, + "loss": 2.4305, + "theoretical_loss": 3.4630086504305497, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661818181818182, + "loss": 2.5705, + "theoretical_loss": 3.462997830150253, + "tokens_seen": 1787691008 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 985931, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.233638286590576, + "objective/train/theoretical_loss": 3.4629870103776783, + "objective/train/tokens_used": 146615776, + "theoretical_loss": 3.4629870103776783, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661616161616162, + "loss": 2.5084, + "theoretical_loss": 3.4629870103776783, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046614141414141414, + "loss": 2.4923, + "theoretical_loss": 3.462976191112783, + "tokens_seen": 1787822080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661212121212121, + "loss": 2.4434, + "theoretical_loss": 3.4629653723555247, + "tokens_seen": 1787887616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046610101010101016, + "loss": 2.669, + "theoretical_loss": 3.4629545541058606, + "tokens_seen": 1787953152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660808080808081, + "loss": 2.6602, + "theoretical_loss": 3.4629437363637483, + "tokens_seen": 1788018688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046606060606060607, + "loss": 2.6326, + "theoretical_loss": 3.462932919129146, + "tokens_seen": 1788084224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046604040404040403, + "loss": 2.5248, + "theoretical_loss": 3.4629221024020107, + "tokens_seen": 1788149760 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466020202020202, + "loss": 2.4801, + "theoretical_loss": 3.4629112861823, + "tokens_seen": 1788215296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046600000000000005, + "loss": 2.6761, + "theoretical_loss": 3.4629004704699726, + "tokens_seen": 1788280832 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465979797979798, + "loss": 2.6667, + "theoretical_loss": 3.462889655264984, + "tokens_seen": 1788346368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595959595959597, + "loss": 2.8018, + "theoretical_loss": 3.462878840567294, + "tokens_seen": 1788411904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659393939393939, + "loss": 2.7818, + "theoretical_loss": 3.4628680263768588, + "tokens_seen": 1788477440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591919191919193, + "loss": 2.6761, + "theoretical_loss": 3.4628572126936366, + "tokens_seen": 1788542976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046589898989898994, + "loss": 2.5054, + "theoretical_loss": 3.4628463995175847, + "tokens_seen": 1788608512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658787878787879, + "loss": 2.6396, + "theoretical_loss": 3.4628355868486613, + "tokens_seen": 1788674048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046585858585858586, + "loss": 2.595, + "theoretical_loss": 3.4628247746868235, + "tokens_seen": 1788739584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658383838383838, + "loss": 2.6262, + "theoretical_loss": 3.462813963032029, + "tokens_seen": 1788805120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658181818181818, + "loss": 2.6693, + "theoretical_loss": 3.4628031518842355, + "tokens_seen": 1788870656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046579797979797984, + "loss": 2.5535, + "theoretical_loss": 3.462792341243401, + "tokens_seen": 1788936192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657777777777778, + "loss": 2.4625, + "theoretical_loss": 3.4627815311094823, + "tokens_seen": 1789001728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575757575757575, + "loss": 2.4874, + "theoretical_loss": 3.462770721482438, + "tokens_seen": 1789067264 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046573737373737376, + "loss": 2.3284, + "theoretical_loss": 3.4627599123622255, + "tokens_seen": 1789132800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171717171717, + "loss": 2.6288, + "theoretical_loss": 3.462749103748802, + "tokens_seen": 1789198336 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046569696969696973, + "loss": 2.4654, + "theoretical_loss": 3.4627382956421258, + "tokens_seen": 1789263872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656767676767677, + "loss": 2.5071, + "theoretical_loss": 3.4627274880421544, + "tokens_seen": 1789329408 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 986490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.360241413116455, + "objective/train/theoretical_loss": 3.462716680948845, + "objective/train/tokens_used": 148254176, + "theoretical_loss": 3.462716680948845, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565656565656564, + "loss": 2.8045, + "theoretical_loss": 3.462716680948845, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046563636363636365, + "loss": 2.7073, + "theoretical_loss": 3.462705874362156, + "tokens_seen": 1789460480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656161616161616, + "loss": 2.4757, + "theoretical_loss": 3.462695068282045, + "tokens_seen": 1789526016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655959595959596, + "loss": 2.5969, + "theoretical_loss": 3.4626842627084695, + "tokens_seen": 1789591552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655757575757576, + "loss": 2.7488, + "theoretical_loss": 3.4626734576413867, + "tokens_seen": 1789657088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655555555555556, + "loss": 2.6564, + "theoretical_loss": 3.4626626530807547, + "tokens_seen": 1789722624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553535353535355, + "loss": 2.5831, + "theoretical_loss": 3.462651849026532, + "tokens_seen": 1789788160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655151515151515, + "loss": 2.6404, + "theoretical_loss": 3.4626410454786756, + "tokens_seen": 1789853696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654949494949495, + "loss": 2.4886, + "theoretical_loss": 3.462630242437143, + "tokens_seen": 1789919232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046547474747474747, + "loss": 2.7851, + "theoretical_loss": 3.462619439901892, + "tokens_seen": 1789984768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654545454545455, + "loss": 2.4352, + "theoretical_loss": 3.462608637872881, + "tokens_seen": 1790050304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046543434343434344, + "loss": 2.5899, + "theoretical_loss": 3.462597836350067, + "tokens_seen": 1790115840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654141414141414, + "loss": 2.4442, + "theoretical_loss": 3.4625870353334083, + "tokens_seen": 1790181376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653939393939394, + "loss": 2.4023, + "theoretical_loss": 3.462576234822863, + "tokens_seen": 1790246912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653737373737374, + "loss": 2.5182, + "theoretical_loss": 3.462565434818387, + "tokens_seen": 1790312448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653535353535354, + "loss": 2.6444, + "theoretical_loss": 3.46255463531994, + "tokens_seen": 1790377984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533333333333333, + "loss": 2.5226, + "theoretical_loss": 3.4625438363274794, + "tokens_seen": 1790443520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653131313131313, + "loss": 2.7895, + "theoretical_loss": 3.462533037840962, + "tokens_seen": 1790509056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652929292929293, + "loss": 2.5776, + "theoretical_loss": 3.4625222398603466, + "tokens_seen": 1790574592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652727272727273, + "loss": 2.5917, + "theoretical_loss": 3.4625114423855905, + "tokens_seen": 1790640128 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525252525252527, + "loss": 2.7743, + "theoretical_loss": 3.4625006454166516, + "tokens_seen": 1790705664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652323232323232, + "loss": 2.7714, + "theoretical_loss": 3.4624898489534877, + "tokens_seen": 1790771200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652121212121212, + "loss": 2.5768, + "theoretical_loss": 3.4624790529960574, + "tokens_seen": 1790836736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519191919191925, + "loss": 2.751, + "theoretical_loss": 3.462468257544317, + "tokens_seen": 1790902272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651717171717172, + "loss": 2.5792, + "theoretical_loss": 3.4624574625982247, + "tokens_seen": 1790967808 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 987679, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5805740356445312, + "objective/train/theoretical_loss": 3.462446668157739, + "objective/train/tokens_used": 149892576, + "theoretical_loss": 3.462446668157739, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046515151515151516, + "loss": 2.4389, + "theoretical_loss": 3.462446668157739, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651313131313131, + "loss": 2.5047, + "theoretical_loss": 3.4624358742228174, + "tokens_seen": 1791098880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651111111111111, + "loss": 2.6406, + "theoretical_loss": 3.462425080793418, + "tokens_seen": 1791164416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046509090909090914, + "loss": 2.5138, + "theoretical_loss": 3.462414287869498, + "tokens_seen": 1791229952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650707070707071, + "loss": 2.7199, + "theoretical_loss": 3.4624034954510154, + "tokens_seen": 1791295488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046505050505050505, + "loss": 2.5726, + "theoretical_loss": 3.4623927035379283, + "tokens_seen": 1791361024 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465030303030303, + "loss": 2.5712, + "theoretical_loss": 3.462381912130194, + "tokens_seen": 1791426560 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046501010101010097, + "loss": 2.4468, + "theoretical_loss": 3.462371121227772, + "tokens_seen": 1791492096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046498989898989903, + "loss": 2.628, + "theoretical_loss": 3.462360330830618, + "tokens_seen": 1791557632 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464969696969697, + "loss": 2.8937, + "theoretical_loss": 3.462349540938691, + "tokens_seen": 1791623168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046494949494949495, + "loss": 2.5733, + "theoretical_loss": 3.4623387515519486, + "tokens_seen": 1791688704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649292929292929, + "loss": 2.7154, + "theoretical_loss": 3.462327962670349, + "tokens_seen": 1791754240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649090909090909, + "loss": 2.3878, + "theoretical_loss": 3.462317174293849, + "tokens_seen": 1791819776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648888888888889, + "loss": 2.773, + "theoretical_loss": 3.462306386422408, + "tokens_seen": 1791885312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648686868686869, + "loss": 2.6715, + "theoretical_loss": 3.462295599055983, + "tokens_seen": 1791950848 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046484848484848484, + "loss": 2.6902, + "theoretical_loss": 3.4622848121945324, + "tokens_seen": 1792016384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648282828282828, + "loss": 2.551, + "theoretical_loss": 3.4622740258380134, + "tokens_seen": 1792081920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648080808080808, + "loss": 2.4135, + "theoretical_loss": 3.4622632399863846, + "tokens_seen": 1792147456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647878787878788, + "loss": 2.5175, + "theoretical_loss": 3.4622524546396027, + "tokens_seen": 1792212992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647676767676768, + "loss": 2.7417, + "theoretical_loss": 3.4622416697976277, + "tokens_seen": 1792278528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046474747474747473, + "loss": 2.4755, + "theoretical_loss": 3.4622308854604156, + "tokens_seen": 1792344064 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046472727272727274, + "loss": 2.5535, + "theoretical_loss": 3.462220101627925, + "tokens_seen": 1792409600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647070707070707, + "loss": 2.6122, + "theoretical_loss": 3.462209318300114, + "tokens_seen": 1792475136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646868686868687, + "loss": 2.7892, + "theoretical_loss": 3.46219853547694, + "tokens_seen": 1792540672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046466666666666667, + "loss": 2.6861, + "theoretical_loss": 3.4621877531583616, + "tokens_seen": 1792606208 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 988330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5130808353424072, + "objective/train/theoretical_loss": 3.462176971344337, + "objective/train/tokens_used": 151530976, + "theoretical_loss": 3.462176971344337, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646464646464646, + "loss": 2.5728, + "theoretical_loss": 3.462176971344337, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046462626262626264, + "loss": 2.4717, + "theoretical_loss": 3.4621661900348224, + "tokens_seen": 1792737280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046460606060606065, + "loss": 2.6032, + "theoretical_loss": 3.4621554092297777, + "tokens_seen": 1792802816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645858585858586, + "loss": 2.6182, + "theoretical_loss": 3.46214462892916, + "tokens_seen": 1792868352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046456565656565656, + "loss": 2.7285, + "theoretical_loss": 3.462133849132927, + "tokens_seen": 1792933888 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046454545454545457, + "loss": 2.4959, + "theoretical_loss": 3.462123069841038, + "tokens_seen": 1792999424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046452525252525253, + "loss": 2.5876, + "theoretical_loss": 3.462112291053449, + "tokens_seen": 1793064960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046450505050505054, + "loss": 2.52, + "theoretical_loss": 3.4621015127701193, + "tokens_seen": 1793130496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644848484848485, + "loss": 2.5387, + "theoretical_loss": 3.4620907349910066, + "tokens_seen": 1793196032 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046446464646464645, + "loss": 2.4564, + "theoretical_loss": 3.462079957716069, + "tokens_seen": 1793261568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046444444444444446, + "loss": 2.7585, + "theoretical_loss": 3.4620691809452637, + "tokens_seen": 1793327104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644242424242424, + "loss": 2.5675, + "theoretical_loss": 3.46205840467855, + "tokens_seen": 1793392640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046440404040404043, + "loss": 2.6589, + "theoretical_loss": 3.462047628915885, + "tokens_seen": 1793458176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643838383838384, + "loss": 2.5341, + "theoretical_loss": 3.462036853657227, + "tokens_seen": 1793523712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643636363636364, + "loss": 2.81, + "theoretical_loss": 3.4620260789025337, + "tokens_seen": 1793589248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046434343434343436, + "loss": 2.6829, + "theoretical_loss": 3.4620153046517634, + "tokens_seen": 1793654784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643232323232323, + "loss": 2.7504, + "theoretical_loss": 3.462004530904874, + "tokens_seen": 1793720320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643030303030303, + "loss": 2.6757, + "theoretical_loss": 3.461993757661824, + "tokens_seen": 1793785856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642828282828283, + "loss": 2.5854, + "theoretical_loss": 3.461982984922571, + "tokens_seen": 1793851392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642626262626263, + "loss": 2.7475, + "theoretical_loss": 3.4619722126870727, + "tokens_seen": 1793916928 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046424242424242425, + "loss": 2.6829, + "theoretical_loss": 3.461961440955288, + "tokens_seen": 1793982464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642222222222222, + "loss": 2.5497, + "theoretical_loss": 3.461950669727174, + "tokens_seen": 1794048000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642020202020202, + "loss": 2.5674, + "theoretical_loss": 3.4619398990026893, + "tokens_seen": 1794113536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046418181818181823, + "loss": 2.6553, + "theoretical_loss": 3.461929128781792, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641616161616162, + "loss": 2.7199, + "theoretical_loss": 3.4619183590644402, + "tokens_seen": 1794244608 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 989671, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.687277317047119, + "objective/train/theoretical_loss": 3.4619075898505915, + "objective/train/tokens_used": 153169376, + "theoretical_loss": 3.4619075898505915, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046414141414141414, + "loss": 2.7491, + "theoretical_loss": 3.4619075898505915, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641212121212121, + "loss": 2.6619, + "theoretical_loss": 3.461896821140204, + "tokens_seen": 1794375680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641010101010101, + "loss": 2.6628, + "theoretical_loss": 3.4618860529332363, + "tokens_seen": 1794441216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640808080808081, + "loss": 2.4893, + "theoretical_loss": 3.461875285229646, + "tokens_seen": 1794506752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640606060606061, + "loss": 2.6323, + "theoretical_loss": 3.461864518029392, + "tokens_seen": 1794572288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046404040404040403, + "loss": 2.7328, + "theoretical_loss": 3.4618537513324315, + "tokens_seen": 1794637824 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464020202020202, + "loss": 2.6513, + "theoretical_loss": 3.4618429851387233, + "tokens_seen": 1794703360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046400000000000006, + "loss": 2.5925, + "theoretical_loss": 3.4618322194482243, + "tokens_seen": 1794768896 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463979797979798, + "loss": 2.4511, + "theoretical_loss": 3.4618214542608943, + "tokens_seen": 1794834432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046395959595959597, + "loss": 2.7825, + "theoretical_loss": 3.46181068957669, + "tokens_seen": 1794899968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393939393939393, + "loss": 2.6791, + "theoretical_loss": 3.46179992539557, + "tokens_seen": 1794965504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639191919191919, + "loss": 2.6882, + "theoretical_loss": 3.4617891617174923, + "tokens_seen": 1795031040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389898989898995, + "loss": 2.7291, + "theoretical_loss": 3.4617783985424158, + "tokens_seen": 1795096576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638787878787879, + "loss": 2.7468, + "theoretical_loss": 3.4617676358702973, + "tokens_seen": 1795162112 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385858585858586, + "loss": 2.6849, + "theoretical_loss": 3.461756873701096, + "tokens_seen": 1795227648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638383838383838, + "loss": 2.5844, + "theoretical_loss": 3.4617461120347697, + "tokens_seen": 1795293184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381818181818183, + "loss": 2.5013, + "theoretical_loss": 3.461735350871277, + "tokens_seen": 1795358720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046379797979797984, + "loss": 2.564, + "theoretical_loss": 3.461724590210575, + "tokens_seen": 1795424256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637777777777778, + "loss": 2.6563, + "theoretical_loss": 3.4617138300526222, + "tokens_seen": 1795489792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375757575757576, + "loss": 2.4432, + "theoretical_loss": 3.4617030703973777, + "tokens_seen": 1795555328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637373737373737, + "loss": 2.7619, + "theoretical_loss": 3.461692311244798, + "tokens_seen": 1795620864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637171717171717, + "loss": 2.8329, + "theoretical_loss": 3.4616815525948432, + "tokens_seen": 1795686400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046369696969696974, + "loss": 2.434, + "theoretical_loss": 3.4616707944474703, + "tokens_seen": 1795751936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636767676767677, + "loss": 2.5695, + "theoretical_loss": 3.4616600368026376, + "tokens_seen": 1795817472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046365656565656565, + "loss": 2.3648, + "theoretical_loss": 3.4616492796603033, + "tokens_seen": 1795883008 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 990185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.496757984161377, + "objective/train/theoretical_loss": 3.461638523020426, + "objective/train/tokens_used": 154807776, + "theoretical_loss": 3.461638523020426, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046363636363636366, + "loss": 2.5464, + "theoretical_loss": 3.461638523020426, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636161616161616, + "loss": 2.6964, + "theoretical_loss": 3.4616277668829634, + "tokens_seen": 1796014080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359595959595963, + "loss": 2.5699, + "theoretical_loss": 3.461617011247874, + "tokens_seen": 1796079616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635757575757576, + "loss": 2.6995, + "theoretical_loss": 3.461606256115116, + "tokens_seen": 1796145152 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046355555555555554, + "loss": 2.5355, + "theoretical_loss": 3.461595501484647, + "tokens_seen": 1796210688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046353535353535355, + "loss": 2.4066, + "theoretical_loss": 3.4615847473564263, + "tokens_seen": 1796276224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635151515151515, + "loss": 2.573, + "theoretical_loss": 3.4615739937304113, + "tokens_seen": 1796341760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634949494949495, + "loss": 2.5721, + "theoretical_loss": 3.4615632406065604, + "tokens_seen": 1796407296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634747474747475, + "loss": 2.782, + "theoretical_loss": 3.461552487984832, + "tokens_seen": 1796472832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634545454545455, + "loss": 2.3226, + "theoretical_loss": 3.4615417358651843, + "tokens_seen": 1796538368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343434343434345, + "loss": 2.6606, + "theoretical_loss": 3.461530984247575, + "tokens_seen": 1796603904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634141414141414, + "loss": 2.6738, + "theoretical_loss": 3.4615202331319637, + "tokens_seen": 1796669440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633939393939394, + "loss": 2.6571, + "theoretical_loss": 3.4615094825183066, + "tokens_seen": 1796734976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046337373737373737, + "loss": 2.5698, + "theoretical_loss": 3.4614987324065645, + "tokens_seen": 1796800512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633535353535354, + "loss": 2.6218, + "theoretical_loss": 3.4614879827966933, + "tokens_seen": 1796866048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046333333333333334, + "loss": 2.6216, + "theoretical_loss": 3.461477233688653, + "tokens_seen": 1796931584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633131313131313, + "loss": 2.6791, + "theoretical_loss": 3.4614664850824, + "tokens_seen": 1796997120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632929292929293, + "loss": 2.6627, + "theoretical_loss": 3.4614557369778947, + "tokens_seen": 1797062656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632727272727273, + "loss": 2.7609, + "theoretical_loss": 3.461444989375094, + "tokens_seen": 1797128192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632525252525253, + "loss": 2.689, + "theoretical_loss": 3.461434242273957, + "tokens_seen": 1797193728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046323232323232323, + "loss": 2.352, + "theoretical_loss": 3.461423495674441, + "tokens_seen": 1797259264 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632121212121212, + "loss": 2.7198, + "theoretical_loss": 3.461412749576505, + "tokens_seen": 1797324800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631919191919192, + "loss": 2.5821, + "theoretical_loss": 3.4614020039801074, + "tokens_seen": 1797390336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631717171717172, + "loss": 2.5262, + "theoretical_loss": 3.4613912588852065, + "tokens_seen": 1797455872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046315151515151517, + "loss": 2.5769, + "theoretical_loss": 3.46138051429176, + "tokens_seen": 1797521408 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 991489, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6060218811035156, + "objective/train/theoretical_loss": 3.4613697701997266, + "objective/train/tokens_used": 156446176, + "theoretical_loss": 3.4613697701997266, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631313131313131, + "loss": 2.5123, + "theoretical_loss": 3.4613697701997266, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631111111111111, + "loss": 2.7132, + "theoretical_loss": 3.461359026609065, + "tokens_seen": 1797652480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630909090909091, + "loss": 2.5873, + "theoretical_loss": 3.4613482835197322, + "tokens_seen": 1797718016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630707070707071, + "loss": 2.737, + "theoretical_loss": 3.461337540931688, + "tokens_seen": 1797783552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046305050505050506, + "loss": 2.5273, + "theoretical_loss": 3.4613267988448913, + "tokens_seen": 1797849088 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463030303030303, + "loss": 2.561, + "theoretical_loss": 3.4613160572592983, + "tokens_seen": 1797914624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046301010101010097, + "loss": 2.7165, + "theoretical_loss": 3.4613053161748684, + "tokens_seen": 1797980160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298989898989904, + "loss": 2.7665, + "theoretical_loss": 3.46129457559156, + "tokens_seen": 1798045696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462969696969697, + "loss": 2.846, + "theoretical_loss": 3.461283835509332, + "tokens_seen": 1798111232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294949494949495, + "loss": 2.7031, + "theoretical_loss": 3.4612730959281413, + "tokens_seen": 1798176768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629292929292929, + "loss": 2.7652, + "theoretical_loss": 3.4612623568479477, + "tokens_seen": 1798242304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629090909090909, + "loss": 2.5421, + "theoretical_loss": 3.461251618268709, + "tokens_seen": 1798307840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046288888888888893, + "loss": 2.7416, + "theoretical_loss": 3.4612408801903833, + "tokens_seen": 1798373376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628686868686869, + "loss": 2.6124, + "theoretical_loss": 3.4612301426129295, + "tokens_seen": 1798438912 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284848484848484, + "loss": 2.8337, + "theoretical_loss": 3.4612194055363057, + "tokens_seen": 1798504448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628282828282828, + "loss": 2.5391, + "theoretical_loss": 3.4612086689604706, + "tokens_seen": 1798569984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046280808080808087, + "loss": 2.6909, + "theoretical_loss": 3.461197932885382, + "tokens_seen": 1798635520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627878787878788, + "loss": 2.4735, + "theoretical_loss": 3.461187197310999, + "tokens_seen": 1798701056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627676767676768, + "loss": 2.4016, + "theoretical_loss": 3.4611764622372796, + "tokens_seen": 1798766592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274747474747474, + "loss": 2.7274, + "theoretical_loss": 3.4611657276641816, + "tokens_seen": 1798832128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627272727272727, + "loss": 2.4681, + "theoretical_loss": 3.4611549935916646, + "tokens_seen": 1798897664 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046270707070707076, + "loss": 2.6583, + "theoretical_loss": 3.4611442600196867, + "tokens_seen": 1798963200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626868686868687, + "loss": 2.5414, + "theoretical_loss": 3.461133526948206, + "tokens_seen": 1799028736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626666666666667, + "loss": 2.7138, + "theoretical_loss": 3.4611227943771814, + "tokens_seen": 1799094272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046264646464646463, + "loss": 2.6, + "theoretical_loss": 3.4611120623065705, + "tokens_seen": 1799159808 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 992118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7015321254730225, + "objective/train/theoretical_loss": 3.4611013307363323, + "objective/train/tokens_used": 158084576, + "theoretical_loss": 3.4611013307363323, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046262626262626264, + "loss": 2.5245, + "theoretical_loss": 3.4611013307363323, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260606060606065, + "loss": 2.5428, + "theoretical_loss": 3.4610905996664254, + "tokens_seen": 1799290880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625858585858586, + "loss": 2.9118, + "theoretical_loss": 3.4610798690968076, + "tokens_seen": 1799356416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046256565656565657, + "loss": 2.7124, + "theoretical_loss": 3.461069139027438, + "tokens_seen": 1799421952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625454545454545, + "loss": 2.6946, + "theoretical_loss": 3.461058409458275, + "tokens_seen": 1799487488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252525252525253, + "loss": 2.7679, + "theoretical_loss": 3.461047680389277, + "tokens_seen": 1799553024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046250505050505054, + "loss": 2.8121, + "theoretical_loss": 3.461036951820402, + "tokens_seen": 1799618560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624848484848485, + "loss": 2.5779, + "theoretical_loss": 3.4610262237516096, + "tokens_seen": 1799684096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046246464646464646, + "loss": 2.6062, + "theoretical_loss": 3.4610154961828568, + "tokens_seen": 1799749632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244444444444447, + "loss": 2.6283, + "theoretical_loss": 3.461004769114103, + "tokens_seen": 1799815168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624242424242424, + "loss": 2.6631, + "theoretical_loss": 3.4609940425453067, + "tokens_seen": 1799880704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046240404040404044, + "loss": 2.6806, + "theoretical_loss": 3.460983316476426, + "tokens_seen": 1799946240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623838383838384, + "loss": 2.7087, + "theoretical_loss": 3.46097259090742, + "tokens_seen": 1800011776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236363636363635, + "loss": 2.4683, + "theoretical_loss": 3.4609618658382466, + "tokens_seen": 1800077312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046234343434343436, + "loss": 2.8184, + "theoretical_loss": 3.4609511412688647, + "tokens_seen": 1800142848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623232323232323, + "loss": 2.5615, + "theoretical_loss": 3.4609404171992324, + "tokens_seen": 1800208384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230303030303033, + "loss": 2.5703, + "theoretical_loss": 3.460929693629309, + "tokens_seen": 1800273920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622828282828283, + "loss": 2.5853, + "theoretical_loss": 3.4609189705590513, + "tokens_seen": 1800339456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622626262626263, + "loss": 2.8325, + "theoretical_loss": 3.46090824798842, + "tokens_seen": 1800404992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046224242424242425, + "loss": 2.4077, + "theoretical_loss": 3.460897525917373, + "tokens_seen": 1800470528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622222222222222, + "loss": 2.45, + "theoretical_loss": 3.4608868043458676, + "tokens_seen": 1800536064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622020202020202, + "loss": 2.7897, + "theoretical_loss": 3.4608760832738636, + "tokens_seen": 1800601600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621818181818182, + "loss": 2.7197, + "theoretical_loss": 3.4608653627013193, + "tokens_seen": 1800667136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621616161616162, + "loss": 2.5448, + "theoretical_loss": 3.4608546426281928, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214141414141415, + "loss": 2.6301, + "theoretical_loss": 3.460843923054443, + "tokens_seen": 1800798208 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 993246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.732495069503784, + "objective/train/theoretical_loss": 3.4608332039800294, + "objective/train/tokens_used": 159722976, + "theoretical_loss": 3.4608332039800294, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621212121212121, + "loss": 2.7341, + "theoretical_loss": 3.4608332039800294, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621010101010101, + "loss": 2.6709, + "theoretical_loss": 3.460822485404909, + "tokens_seen": 1800929280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620808080808081, + "loss": 2.6878, + "theoretical_loss": 3.460811767329041, + "tokens_seen": 1800994816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620606060606061, + "loss": 2.8197, + "theoretical_loss": 3.460801049752384, + "tokens_seen": 1801060352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046204040404040404, + "loss": 2.4737, + "theoretical_loss": 3.4607903326748968, + "tokens_seen": 1801125888 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462020202020202, + "loss": 2.6015, + "theoretical_loss": 3.4607796160965374, + "tokens_seen": 1801191424 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462, + "loss": 2.7105, + "theoretical_loss": 3.460768900017265, + "tokens_seen": 1801256960 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461979797979798, + "loss": 2.7153, + "theoretical_loss": 3.4607581844370383, + "tokens_seen": 1801322496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461959595959596, + "loss": 2.7215, + "theoretical_loss": 3.460747469355815, + "tokens_seen": 1801388032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046193939393939393, + "loss": 2.7523, + "theoretical_loss": 3.4607367547735546, + "tokens_seen": 1801453568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619191919191919, + "loss": 2.6356, + "theoretical_loss": 3.460726040690215, + "tokens_seen": 1801519104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046189898989898996, + "loss": 2.8216, + "theoretical_loss": 3.4607153271057554, + "tokens_seen": 1801584640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618787878787879, + "loss": 2.6657, + "theoretical_loss": 3.460704614020135, + "tokens_seen": 1801650176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046185858585858587, + "loss": 2.5136, + "theoretical_loss": 3.460693901433311, + "tokens_seen": 1801715712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618383838383838, + "loss": 2.7718, + "theoretical_loss": 3.4606831893452425, + "tokens_seen": 1801781248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618181818181818, + "loss": 2.645, + "theoretical_loss": 3.460672477755889, + "tokens_seen": 1801846784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046179797979797985, + "loss": 2.637, + "theoretical_loss": 3.4606617666652078, + "tokens_seen": 1801912320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617777777777778, + "loss": 2.6257, + "theoretical_loss": 3.4606510560731585, + "tokens_seen": 1801977856 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046175757575757576, + "loss": 2.5434, + "theoretical_loss": 3.4606403459796997, + "tokens_seen": 1802043392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617373737373737, + "loss": 2.5597, + "theoretical_loss": 3.4606296363847897, + "tokens_seen": 1802108928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617171717171717, + "loss": 2.4782, + "theoretical_loss": 3.4606189272883876, + "tokens_seen": 1802174464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046169696969696974, + "loss": 2.5543, + "theoretical_loss": 3.460608218690451, + "tokens_seen": 1802240000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616767676767677, + "loss": 2.6303, + "theoretical_loss": 3.46059751059094, + "tokens_seen": 1802305536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046165656565656565, + "loss": 2.3844, + "theoretical_loss": 3.4605868029898126, + "tokens_seen": 1802371072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616363636363636, + "loss": 2.6324, + "theoretical_loss": 3.4605760958870277, + "tokens_seen": 1802436608 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 994266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.712080717086792, + "objective/train/theoretical_loss": 3.460565389282543, + "objective/train/tokens_used": 161361376, + "theoretical_loss": 3.460565389282543, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616161616161616, + "loss": 2.6, + "theoretical_loss": 3.460565389282543, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046159595959595963, + "loss": 2.8756, + "theoretical_loss": 3.4605546831763188, + "tokens_seen": 1802567680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615757575757576, + "loss": 2.7354, + "theoretical_loss": 3.460543977568313, + "tokens_seen": 1802633216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046155555555555555, + "loss": 2.6013, + "theoretical_loss": 3.460533272458484, + "tokens_seen": 1802698752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615353535353535, + "loss": 2.8307, + "theoretical_loss": 3.4605225678467906, + "tokens_seen": 1802764288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615151515151515, + "loss": 2.7824, + "theoretical_loss": 3.460511863733192, + "tokens_seen": 1802829824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614949494949495, + "loss": 2.4709, + "theoretical_loss": 3.460501160117647, + "tokens_seen": 1802895360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614747474747475, + "loss": 2.6541, + "theoretical_loss": 3.4604904570001134, + "tokens_seen": 1802960896 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046145454545454544, + "loss": 2.684, + "theoretical_loss": 3.4604797543805503, + "tokens_seen": 1803026432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046143434343434345, + "loss": 2.6394, + "theoretical_loss": 3.460469052258917, + "tokens_seen": 1803091968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614141414141414, + "loss": 2.5222, + "theoretical_loss": 3.460458350635172, + "tokens_seen": 1803157504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613939393939394, + "loss": 2.4386, + "theoretical_loss": 3.460447649509274, + "tokens_seen": 1803223040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613737373737374, + "loss": 2.5735, + "theoretical_loss": 3.4604369488811813, + "tokens_seen": 1803288576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046135353535353533, + "loss": 2.5963, + "theoretical_loss": 3.4604262487508533, + "tokens_seen": 1803354112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046133333333333334, + "loss": 2.6435, + "theoretical_loss": 3.460415549118248, + "tokens_seen": 1803419648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613131313131313, + "loss": 2.5554, + "theoretical_loss": 3.460404849983325, + "tokens_seen": 1803485184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612929292929293, + "loss": 2.5415, + "theoretical_loss": 3.4603941513460423, + "tokens_seen": 1803550720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046127272727272727, + "loss": 2.6456, + "theoretical_loss": 3.4603834532063598, + "tokens_seen": 1803616256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612525252525253, + "loss": 2.6177, + "theoretical_loss": 3.460372755564235, + "tokens_seen": 1803681792 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046123232323232324, + "loss": 2.3797, + "theoretical_loss": 3.4603620584196273, + "tokens_seen": 1803747328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612121212121212, + "loss": 2.5797, + "theoretical_loss": 3.4603513617724957, + "tokens_seen": 1803812864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611919191919192, + "loss": 2.6529, + "theoretical_loss": 3.460340665622798, + "tokens_seen": 1803878400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046117171717171716, + "loss": 2.5033, + "theoretical_loss": 3.460329969970495, + "tokens_seen": 1803943936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046115151515151517, + "loss": 2.7475, + "theoretical_loss": 3.460319274815543, + "tokens_seen": 1804009472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046113131313131313, + "loss": 2.4978, + "theoretical_loss": 3.460308580157902, + "tokens_seen": 1804075008 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 994874, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3973050117492676, + "objective/train/theoretical_loss": 3.460297885997531, + "objective/train/tokens_used": 162999776, + "theoretical_loss": 3.460297885997531, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046111111111111114, + "loss": 2.5142, + "theoretical_loss": 3.460297885997531, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610909090909091, + "loss": 2.5792, + "theoretical_loss": 3.4602871923343894, + "tokens_seen": 1804206080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610707070707071, + "loss": 2.6572, + "theoretical_loss": 3.4602764991684345, + "tokens_seen": 1804271616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046105050505050506, + "loss": 2.7006, + "theoretical_loss": 3.460265806499626, + "tokens_seen": 1804337152 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461030303030303, + "loss": 2.5644, + "theoretical_loss": 3.4602551143279228, + "tokens_seen": 1804402688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046101010101010103, + "loss": 2.5553, + "theoretical_loss": 3.4602444226532834, + "tokens_seen": 1804468224 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460989898989899, + "loss": 2.4872, + "theoretical_loss": 3.4602337314756664, + "tokens_seen": 1804533760 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460969696969697, + "loss": 2.5919, + "theoretical_loss": 3.460223040795032, + "tokens_seen": 1804599296 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046094949494949496, + "loss": 2.5868, + "theoretical_loss": 3.4602123506113376, + "tokens_seen": 1804664832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609292929292929, + "loss": 2.557, + "theoretical_loss": 3.4602016609245423, + "tokens_seen": 1804730368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609090909090909, + "loss": 2.4903, + "theoretical_loss": 3.460190971734605, + "tokens_seen": 1804795904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046088888888888894, + "loss": 2.6423, + "theoretical_loss": 3.4601802830414856, + "tokens_seen": 1804861440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608686868686869, + "loss": 2.607, + "theoretical_loss": 3.4601695948451416, + "tokens_seen": 1804926976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084848484848485, + "loss": 2.4623, + "theoretical_loss": 3.4601589071455323, + "tokens_seen": 1804992512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608282828282828, + "loss": 2.5671, + "theoretical_loss": 3.460148219942617, + "tokens_seen": 1805058048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608080808080808, + "loss": 2.7849, + "theoretical_loss": 3.4601375332363538, + "tokens_seen": 1805123584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046078787878787883, + "loss": 2.4045, + "theoretical_loss": 3.4601268470267024, + "tokens_seen": 1805189120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607676767676768, + "loss": 2.6255, + "theoretical_loss": 3.4601161613136213, + "tokens_seen": 1805254656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046074747474747474, + "loss": 2.3319, + "theoretical_loss": 3.4601054760970698, + "tokens_seen": 1805320192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607272727272727, + "loss": 2.6209, + "theoretical_loss": 3.460094791377006, + "tokens_seen": 1805385728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070707070707077, + "loss": 2.5799, + "theoretical_loss": 3.4600841071533894, + "tokens_seen": 1805451264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606868686868687, + "loss": 2.6091, + "theoretical_loss": 3.460073423426179, + "tokens_seen": 1805516800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606666666666667, + "loss": 2.8417, + "theoretical_loss": 3.460062740195333, + "tokens_seen": 1805582336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046064646464646464, + "loss": 2.5454, + "theoretical_loss": 3.4600520574608113, + "tokens_seen": 1805647872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606262626262626, + "loss": 2.7244, + "theoretical_loss": 3.4600413752225725, + "tokens_seen": 1805713408 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 995886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.640719413757324, + "objective/train/theoretical_loss": 3.460030693480575, + "objective/train/tokens_used": 164638176, + "theoretical_loss": 3.460030693480575, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060606060606066, + "loss": 2.7838, + "theoretical_loss": 3.460030693480575, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605858585858586, + "loss": 2.5454, + "theoretical_loss": 3.460020012234778, + "tokens_seen": 1805844480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046056565656565657, + "loss": 2.737, + "theoretical_loss": 3.460009331485141, + "tokens_seen": 1805910016 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054545454545453, + "loss": 2.6839, + "theoretical_loss": 3.459998651231622, + "tokens_seen": 1805975552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046052525252525254, + "loss": 2.5493, + "theoretical_loss": 3.459987971474181, + "tokens_seen": 1806041088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050505050505055, + "loss": 2.4706, + "theoretical_loss": 3.4599772922127765, + "tokens_seen": 1806106624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604848484848485, + "loss": 2.7013, + "theoretical_loss": 3.4599666134473672, + "tokens_seen": 1806172160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046464646464646, + "loss": 2.7035, + "theoretical_loss": 3.459955935177912, + "tokens_seen": 1806237696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604444444444444, + "loss": 2.6302, + "theoretical_loss": 3.45994525740437, + "tokens_seen": 1806303232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042424242424243, + "loss": 2.5569, + "theoretical_loss": 3.459934580126701, + "tokens_seen": 1806368768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046040404040404044, + "loss": 2.6291, + "theoretical_loss": 3.459923903344863, + "tokens_seen": 1806434304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603838383838384, + "loss": 2.6722, + "theoretical_loss": 3.4599132270588155, + "tokens_seen": 1806499840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046036363636363636, + "loss": 2.8284, + "theoretical_loss": 3.4599025512685166, + "tokens_seen": 1806565376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046034343434343437, + "loss": 2.836, + "theoretical_loss": 3.459891875973927, + "tokens_seen": 1806630912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603232323232323, + "loss": 2.5876, + "theoretical_loss": 3.459881201175004, + "tokens_seen": 1806696448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030303030303034, + "loss": 2.5903, + "theoretical_loss": 3.4598705268717076, + "tokens_seen": 1806761984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602828282828283, + "loss": 2.5872, + "theoretical_loss": 3.459859853063996, + "tokens_seen": 1806827520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046026262626262625, + "loss": 2.7085, + "theoretical_loss": 3.4598491797518287, + "tokens_seen": 1806893056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046024242424242426, + "loss": 2.6058, + "theoretical_loss": 3.4598385069351654, + "tokens_seen": 1806958592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602222222222222, + "loss": 2.431, + "theoretical_loss": 3.459827834613964, + "tokens_seen": 1807024128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046020202020202023, + "loss": 2.6627, + "theoretical_loss": 3.459817162788184, + "tokens_seen": 1807089664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601818181818182, + "loss": 2.6105, + "theoretical_loss": 3.4598064914577846, + "tokens_seen": 1807155200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601616161616162, + "loss": 2.6375, + "theoretical_loss": 3.4597958206227246, + "tokens_seen": 1807220736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046014141414141415, + "loss": 2.6245, + "theoretical_loss": 3.459785150282963, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601212121212121, + "loss": 2.7088, + "theoretical_loss": 3.459774480438459, + "tokens_seen": 1807351808 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 996582, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.828779935836792, + "objective/train/theoretical_loss": 3.4597638110891715, + "objective/train/tokens_used": 166276576, + "theoretical_loss": 3.4597638110891715, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601010101010101, + "loss": 2.6118, + "theoretical_loss": 3.4597638110891715, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600808080808081, + "loss": 2.55, + "theoretical_loss": 3.4597531422350603, + "tokens_seen": 1807482880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600606060606061, + "loss": 2.9119, + "theoretical_loss": 3.459742473876083, + "tokens_seen": 1807548416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046004040404040405, + "loss": 2.4969, + "theoretical_loss": 3.4597318060122, + "tokens_seen": 1807613952 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460020202020202, + "loss": 2.3569, + "theoretical_loss": 3.4597211386433697, + "tokens_seen": 1807679488 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 2.8525, + "theoretical_loss": 3.4597104717695517, + "tokens_seen": 1807745024 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459979797979798, + "loss": 2.6927, + "theoretical_loss": 3.459699805390705, + "tokens_seen": 1807810560 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459959595959596, + "loss": 2.6665, + "theoretical_loss": 3.4596891395067875, + "tokens_seen": 1807876096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993939393939394, + "loss": 2.6101, + "theoretical_loss": 3.45967847411776, + "tokens_seen": 1807941632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599191919191919, + "loss": 2.6743, + "theoretical_loss": 3.4596678092235806, + "tokens_seen": 1808007168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598989898989899, + "loss": 2.7123, + "theoretical_loss": 3.4596571448242086, + "tokens_seen": 1808072704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598787878787879, + "loss": 2.589, + "theoretical_loss": 3.459646480919603, + "tokens_seen": 1808138240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598585858585859, + "loss": 2.4773, + "theoretical_loss": 3.4596358175097235, + "tokens_seen": 1808203776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045983838383838383, + "loss": 2.4748, + "theoretical_loss": 3.4596251545945282, + "tokens_seen": 1808269312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598181818181818, + "loss": 2.5673, + "theoretical_loss": 3.4596144921739773, + "tokens_seen": 1808334848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597979797979798, + "loss": 2.4766, + "theoretical_loss": 3.45960383024803, + "tokens_seen": 1808400384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597777777777778, + "loss": 2.7642, + "theoretical_loss": 3.459593168816644, + "tokens_seen": 1808465920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045975757575757577, + "loss": 2.7496, + "theoretical_loss": 3.4595825078797793, + "tokens_seen": 1808531456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597373737373737, + "loss": 2.8011, + "theoretical_loss": 3.4595718474373953, + "tokens_seen": 1808596992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597171717171717, + "loss": 2.8709, + "theoretical_loss": 3.459561187489451, + "tokens_seen": 1808662528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969696969696975, + "loss": 2.6683, + "theoretical_loss": 3.459550528035905, + "tokens_seen": 1808728064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596767676767677, + "loss": 2.6207, + "theoretical_loss": 3.4595398690767176, + "tokens_seen": 1808793600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045965656565656566, + "loss": 2.4637, + "theoretical_loss": 3.459529210611847, + "tokens_seen": 1808859136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596363636363636, + "loss": 2.7937, + "theoretical_loss": 3.4595185526412524, + "tokens_seen": 1808924672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596161616161616, + "loss": 2.6215, + "theoretical_loss": 3.4595078951648937, + "tokens_seen": 1808990208 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 997711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4244730472564697, + "objective/train/theoretical_loss": 3.4594972381827294, + "objective/train/tokens_used": 167914976, + "theoretical_loss": 3.4594972381827294, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045959595959595964, + "loss": 2.64, + "theoretical_loss": 3.4594972381827294, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595757575757576, + "loss": 2.6338, + "theoretical_loss": 3.4594865816947182, + "tokens_seen": 1809121280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045955555555555555, + "loss": 2.5629, + "theoretical_loss": 3.4594759257008207, + "tokens_seen": 1809186816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595353535353535, + "loss": 2.5733, + "theoretical_loss": 3.4594652702009956, + "tokens_seen": 1809252352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595151515151515, + "loss": 2.6077, + "theoretical_loss": 3.4594546151952015, + "tokens_seen": 1809317888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045949494949494953, + "loss": 2.4091, + "theoretical_loss": 3.4594439606833975, + "tokens_seen": 1809383424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594747474747475, + "loss": 2.6934, + "theoretical_loss": 3.459433306665544, + "tokens_seen": 1809448960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045945454545454545, + "loss": 2.6156, + "theoretical_loss": 3.4594226531415986, + "tokens_seen": 1809514496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594343434343434, + "loss": 2.8258, + "theoretical_loss": 3.4594120001115223, + "tokens_seen": 1809580032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045941414141414147, + "loss": 2.7045, + "theoretical_loss": 3.459401347575273, + "tokens_seen": 1809645568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593939393939394, + "loss": 2.6075, + "theoretical_loss": 3.4593906955328104, + "tokens_seen": 1809711104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593737373737374, + "loss": 2.6631, + "theoretical_loss": 3.4593800439840936, + "tokens_seen": 1809776640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045935353535353534, + "loss": 2.8016, + "theoretical_loss": 3.459369392929082, + "tokens_seen": 1809842176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045933333333333335, + "loss": 2.7707, + "theoretical_loss": 3.4593587423677348, + "tokens_seen": 1809907712 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931313131313136, + "loss": 2.5944, + "theoretical_loss": 3.459348092300011, + "tokens_seen": 1809973248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592929292929293, + "loss": 2.536, + "theoretical_loss": 3.45933744272587, + "tokens_seen": 1810038784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592727272727273, + "loss": 2.6021, + "theoretical_loss": 3.459326793645271, + "tokens_seen": 1810104320 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045925252525252523, + "loss": 2.6998, + "theoretical_loss": 3.4593161450581738, + "tokens_seen": 1810169856 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923232323232324, + "loss": 2.8461, + "theoretical_loss": 3.4593054969645367, + "tokens_seen": 1810235392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045921212121212125, + "loss": 2.7473, + "theoretical_loss": 3.45929484936432, + "tokens_seen": 1810300928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591919191919192, + "loss": 2.6926, + "theoretical_loss": 3.459284202257482, + "tokens_seen": 1810366464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045917171717171717, + "loss": 2.6265, + "theoretical_loss": 3.459273555643983, + "tokens_seen": 1810432000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591515151515152, + "loss": 2.5741, + "theoretical_loss": 3.4592629095237815, + "tokens_seen": 1810497536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045913131313131313, + "loss": 2.6005, + "theoretical_loss": 3.459252263896837, + "tokens_seen": 1810563072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045911111111111115, + "loss": 2.6592, + "theoretical_loss": 3.4592416187631088, + "tokens_seen": 1810628608 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 998243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6315853595733643, + "objective/train/theoretical_loss": 3.4592309741225558, + "objective/train/tokens_used": 169553376, + "theoretical_loss": 3.4592309741225558, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590909090909091, + "loss": 2.5426, + "theoretical_loss": 3.4592309741225558, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045907070707070706, + "loss": 2.5075, + "theoretical_loss": 3.459220329975138, + "tokens_seen": 1810759680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045905050505050507, + "loss": 2.7955, + "theoretical_loss": 3.4592096863208144, + "tokens_seen": 1810825216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045903030303030303, + "loss": 2.5196, + "theoretical_loss": 3.4591990431595443, + "tokens_seen": 1810890752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045901010101010104, + "loss": 2.6317, + "theoretical_loss": 3.4591884004912874, + "tokens_seen": 1810956288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458989898989899, + "loss": 2.3922, + "theoretical_loss": 3.4591777583160024, + "tokens_seen": 1811021824 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458969696969697, + "loss": 2.9019, + "theoretical_loss": 3.4591671166336484, + "tokens_seen": 1811087360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045894949494949496, + "loss": 2.7095, + "theoretical_loss": 3.459156475444186, + "tokens_seen": 1811152896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589292929292929, + "loss": 2.7239, + "theoretical_loss": 3.4591458347475736, + "tokens_seen": 1811218432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045890909090909093, + "loss": 2.7473, + "theoretical_loss": 3.4591351945437703, + "tokens_seen": 1811283968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588888888888889, + "loss": 2.6645, + "theoretical_loss": 3.4591245548327363, + "tokens_seen": 1811349504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588686868686869, + "loss": 2.8029, + "theoretical_loss": 3.45911391561443, + "tokens_seen": 1811415040 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045884848484848486, + "loss": 2.5806, + "theoretical_loss": 3.4591032768888117, + "tokens_seen": 1811480576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588282828282828, + "loss": 2.6284, + "theoretical_loss": 3.4590926386558403, + "tokens_seen": 1811546112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588080808080808, + "loss": 2.767, + "theoretical_loss": 3.459082000915475, + "tokens_seen": 1811611648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045878787878787883, + "loss": 2.792, + "theoretical_loss": 3.4590713636676753, + "tokens_seen": 1811677184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587676767676768, + "loss": 2.7077, + "theoretical_loss": 3.4590607269124005, + "tokens_seen": 1811742720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045874747474747475, + "loss": 2.6499, + "theoretical_loss": 3.459050090649611, + "tokens_seen": 1811808256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587272727272727, + "loss": 2.8369, + "theoretical_loss": 3.4590394548792642, + "tokens_seen": 1811873792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587070707070707, + "loss": 2.3474, + "theoretical_loss": 3.459028819601321, + "tokens_seen": 1811939328 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045868686868686873, + "loss": 2.7419, + "theoretical_loss": 3.459018184815741, + "tokens_seen": 1812004864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586666666666667, + "loss": 2.7241, + "theoretical_loss": 3.459007550522482, + "tokens_seen": 1812070400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045864646464646464, + "loss": 2.5917, + "theoretical_loss": 3.458996916721505, + "tokens_seen": 1812135936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586262626262626, + "loss": 2.5461, + "theoretical_loss": 3.4589862834127683, + "tokens_seen": 1812201472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045860606060606066, + "loss": 2.429, + "theoretical_loss": 3.458975650596232, + "tokens_seen": 1812267008 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 999229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6778602600097656, + "objective/train/theoretical_loss": 3.458965018271855, + "objective/train/tokens_used": 171191776, + "theoretical_loss": 3.458965018271855, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585858585858586, + "loss": 2.6921, + "theoretical_loss": 3.458965018271855, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585656565656566, + "loss": 2.6781, + "theoretical_loss": 3.458954386439598, + "tokens_seen": 1812398080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045854545454545453, + "loss": 2.8004, + "theoretical_loss": 3.458943755099419, + "tokens_seen": 1812463616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585252525252525, + "loss": 2.5061, + "theoretical_loss": 3.4589331242512777, + "tokens_seen": 1812529152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045850505050505056, + "loss": 2.6991, + "theoretical_loss": 3.458922493895134, + "tokens_seen": 1812594688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584848484848485, + "loss": 2.59, + "theoretical_loss": 3.458911864030947, + "tokens_seen": 1812660224 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045846464646464647, + "loss": 2.7665, + "theoretical_loss": 3.4589012346586765, + "tokens_seen": 1812725760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584444444444444, + "loss": 2.5644, + "theoretical_loss": 3.458890605778281, + "tokens_seen": 1812791296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584242424242424, + "loss": 2.6733, + "theoretical_loss": 3.4588799773897216, + "tokens_seen": 1812856832 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045840404040404045, + "loss": 2.495, + "theoretical_loss": 3.4588693494929563, + "tokens_seen": 1812922368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583838383838384, + "loss": 2.4922, + "theoretical_loss": 3.458858722087945, + "tokens_seen": 1812987904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045836363636363636, + "loss": 2.4791, + "theoretical_loss": 3.4588480951746474, + "tokens_seen": 1813053440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583434343434343, + "loss": 2.6036, + "theoretical_loss": 3.4588374687530226, + "tokens_seen": 1813118976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045832323232323233, + "loss": 2.4228, + "theoretical_loss": 3.4588268428230307, + "tokens_seen": 1813184512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045830303030303034, + "loss": 2.747, + "theoretical_loss": 3.4588162173846304, + "tokens_seen": 1813250048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582828282828283, + "loss": 2.4541, + "theoretical_loss": 3.458805592437782, + "tokens_seen": 1813315584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045826262626262626, + "loss": 2.4754, + "theoretical_loss": 3.458794967982444, + "tokens_seen": 1813381120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582424242424242, + "loss": 2.3365, + "theoretical_loss": 3.458784344018577, + "tokens_seen": 1813446656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582222222222222, + "loss": 2.8325, + "theoretical_loss": 3.45877372054614, + "tokens_seen": 1813512192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045820202020202023, + "loss": 2.7678, + "theoretical_loss": 3.4587630975650923, + "tokens_seen": 1813577728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581818181818182, + "loss": 2.5188, + "theoretical_loss": 3.4587524750753937, + "tokens_seen": 1813643264 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045816161616161615, + "loss": 2.6459, + "theoretical_loss": 3.4587418530770035, + "tokens_seen": 1813708800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045814141414141416, + "loss": 2.477, + "theoretical_loss": 3.4587312315698813, + "tokens_seen": 1813774336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581212121212121, + "loss": 2.5432, + "theoretical_loss": 3.458720610553987, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581010101010101, + "loss": 2.7704, + "theoretical_loss": 3.4587099900292797, + "tokens_seen": 1813905408 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 999785, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5763378143310547, + "objective/train/theoretical_loss": 3.4586993699957187, + "objective/train/tokens_used": 172830176, + "theoretical_loss": 3.4586993699957187, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580808080808081, + "loss": 2.5818, + "theoretical_loss": 3.4586993699957187, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045806060606060604, + "loss": 2.3931, + "theoretical_loss": 3.4586887504532644, + "tokens_seen": 1814036480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045804040404040405, + "loss": 2.455, + "theoretical_loss": 3.458678131401876, + "tokens_seen": 1814102016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458020202020202, + "loss": 2.5927, + "theoretical_loss": 3.4586675128415125, + "tokens_seen": 1814167552 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458, + "loss": 2.9255, + "theoretical_loss": 3.4586568947721337, + "tokens_seen": 1814233088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457979797979798, + "loss": 2.5883, + "theoretical_loss": 3.4586462771936994, + "tokens_seen": 1814298624 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457959595959596, + "loss": 2.618, + "theoretical_loss": 3.4586356601061694, + "tokens_seen": 1814364160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045793939393939394, + "loss": 2.4486, + "theoretical_loss": 3.4586250435095027, + "tokens_seen": 1814429696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579191919191919, + "loss": 2.7142, + "theoretical_loss": 3.4586144274036594, + "tokens_seen": 1814495232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578989898989899, + "loss": 2.4711, + "theoretical_loss": 3.4586038117885987, + "tokens_seen": 1814560768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045787878787878787, + "loss": 2.5087, + "theoretical_loss": 3.4585931966642804, + "tokens_seen": 1814626304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578585858585859, + "loss": 2.622, + "theoretical_loss": 3.458582582030664, + "tokens_seen": 1814691840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783838383838384, + "loss": 2.472, + "theoretical_loss": 3.4585719678877087, + "tokens_seen": 1814757376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578181818181818, + "loss": 2.8238, + "theoretical_loss": 3.458561354235375, + "tokens_seen": 1814822912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577979797979798, + "loss": 2.4024, + "theoretical_loss": 3.458550741073622, + "tokens_seen": 1814888448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577777777777778, + "loss": 2.6186, + "theoretical_loss": 3.458540128402409, + "tokens_seen": 1814953984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045775757575757577, + "loss": 2.609, + "theoretical_loss": 3.458529516221696, + "tokens_seen": 1815019520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045773737373737373, + "loss": 2.565, + "theoretical_loss": 3.4585189045314424, + "tokens_seen": 1815085056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045771717171717174, + "loss": 2.7391, + "theoretical_loss": 3.458508293331609, + "tokens_seen": 1815150592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576969696969697, + "loss": 2.5499, + "theoretical_loss": 3.458497682622153, + "tokens_seen": 1815216128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576767676767677, + "loss": 2.7469, + "theoretical_loss": 3.4584870724030363, + "tokens_seen": 1815281664 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045765656565656567, + "loss": 2.6686, + "theoretical_loss": 3.4584764626742173, + "tokens_seen": 1815347200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576363636363636, + "loss": 2.6245, + "theoretical_loss": 3.4584658534356567, + "tokens_seen": 1815412736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045761616161616163, + "loss": 2.6962, + "theoretical_loss": 3.458455244687313, + "tokens_seen": 1815478272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045759595959595964, + "loss": 2.6647, + "theoretical_loss": 3.458444636429146, + "tokens_seen": 1815543808 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1000504, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.835871458053589, + "objective/train/theoretical_loss": 3.458434028661116, + "objective/train/tokens_used": 174468576, + "theoretical_loss": 3.458434028661116, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575757575757576, + "loss": 2.7497, + "theoretical_loss": 3.458434028661116, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045755555555555556, + "loss": 2.6998, + "theoretical_loss": 3.4584234213831824, + "tokens_seen": 1815674880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575353535353535, + "loss": 2.5959, + "theoretical_loss": 3.458412814595305, + "tokens_seen": 1815740416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575151515151515, + "loss": 2.6133, + "theoretical_loss": 3.458402208297443, + "tokens_seen": 1815805952 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045749494949494954, + "loss": 2.6405, + "theoretical_loss": 3.4583916024895567, + "tokens_seen": 1815871488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574747474747475, + "loss": 2.7544, + "theoretical_loss": 3.458380997171605, + "tokens_seen": 1815937024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045745454545454545, + "loss": 2.6652, + "theoretical_loss": 3.4583703923435483, + "tokens_seen": 1816002560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574343434343434, + "loss": 2.7187, + "theoretical_loss": 3.458359788005346, + "tokens_seen": 1816068096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574141414141415, + "loss": 2.6787, + "theoretical_loss": 3.458349184156958, + "tokens_seen": 1816133632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739393939393943, + "loss": 2.6822, + "theoretical_loss": 3.458338580798343, + "tokens_seen": 1816199168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573737373737374, + "loss": 2.6664, + "theoretical_loss": 3.4583279779294624, + "tokens_seen": 1816264704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045735353535353534, + "loss": 2.6565, + "theoretical_loss": 3.458317375550275, + "tokens_seen": 1816330240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573333333333333, + "loss": 2.6583, + "theoretical_loss": 3.4583067736607402, + "tokens_seen": 1816395776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045731313131313137, + "loss": 2.9378, + "theoretical_loss": 3.4582961722608188, + "tokens_seen": 1816461312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572929292929293, + "loss": 2.6101, + "theoretical_loss": 3.458285571350469, + "tokens_seen": 1816526848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572727272727273, + "loss": 2.8678, + "theoretical_loss": 3.4582749709296516, + "tokens_seen": 1816592384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045725252525252524, + "loss": 2.5882, + "theoretical_loss": 3.458264370998326, + "tokens_seen": 1816657920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045723232323232325, + "loss": 2.5968, + "theoretical_loss": 3.4582537715564525, + "tokens_seen": 1816723456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045721212121212126, + "loss": 2.7255, + "theoretical_loss": 3.45824317260399, + "tokens_seen": 1816788992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571919191919192, + "loss": 2.7617, + "theoretical_loss": 3.458232574140899, + "tokens_seen": 1816854528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045717171717171717, + "loss": 2.4575, + "theoretical_loss": 3.4582219761671382, + "tokens_seen": 1816920064 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045715151515151513, + "loss": 2.66, + "theoretical_loss": 3.4582113786826683, + "tokens_seen": 1816985600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045713131313131314, + "loss": 2.527, + "theoretical_loss": 3.4582007816874487, + "tokens_seen": 1817051136 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045711111111111115, + "loss": 2.4134, + "theoretical_loss": 3.4581901851814396, + "tokens_seen": 1817116672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004570909090909091, + "loss": 2.8572, + "theoretical_loss": 3.4581795891646, + "tokens_seen": 1817182208 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1001884, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.327115058898926, + "objective/train/theoretical_loss": 3.4581689936368907, + "objective/train/tokens_used": 176106976, + "theoretical_loss": 3.4581689936368907, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045707070707070706, + "loss": 2.5736, + "theoretical_loss": 3.4581689936368907, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570505050505051, + "loss": 2.8025, + "theoretical_loss": 3.45815839859827, + "tokens_seen": 1817313280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045703030303030303, + "loss": 2.6386, + "theoretical_loss": 3.458147804048699, + "tokens_seen": 1817378816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045701010101010104, + "loss": 2.719, + "theoretical_loss": 3.458137209988138, + "tokens_seen": 1817444352 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456989898989899, + "loss": 2.5823, + "theoretical_loss": 3.4581266164165445, + "tokens_seen": 1817509888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045696969696969696, + "loss": 2.7439, + "theoretical_loss": 3.45811602333388, + "tokens_seen": 1817575424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045694949494949497, + "loss": 2.4384, + "theoretical_loss": 3.4581054307401047, + "tokens_seen": 1817640960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569292929292929, + "loss": 2.8483, + "theoretical_loss": 3.458094838635177, + "tokens_seen": 1817706496 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045690909090909094, + "loss": 2.6419, + "theoretical_loss": 3.458084247019057, + "tokens_seen": 1817772032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568888888888889, + "loss": 2.7798, + "theoretical_loss": 3.458073655891706, + "tokens_seen": 1817837568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568686868686869, + "loss": 2.5721, + "theoretical_loss": 3.458063065253082, + "tokens_seen": 1817903104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045684848484848486, + "loss": 2.8557, + "theoretical_loss": 3.4580524751031456, + "tokens_seen": 1817968640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568282828282828, + "loss": 2.6035, + "theoretical_loss": 3.4580418854418573, + "tokens_seen": 1818034176 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045680808080808083, + "loss": 2.5388, + "theoretical_loss": 3.4580312962691755, + "tokens_seen": 1818099712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567878787878788, + "loss": 2.5895, + "theoretical_loss": 3.458020707585061, + "tokens_seen": 1818165248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567676767676768, + "loss": 2.4687, + "theoretical_loss": 3.4580101193894732, + "tokens_seen": 1818230784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045674747474747475, + "loss": 2.461, + "theoretical_loss": 3.457999531682373, + "tokens_seen": 1818296320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567272727272727, + "loss": 2.5961, + "theoretical_loss": 3.4579889444637186, + "tokens_seen": 1818361856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567070707070707, + "loss": 2.5384, + "theoretical_loss": 3.457978357733471, + "tokens_seen": 1818427392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045668686868686873, + "loss": 2.729, + "theoretical_loss": 3.4579677714915897, + "tokens_seen": 1818492928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566666666666667, + "loss": 2.8327, + "theoretical_loss": 3.457957185738035, + "tokens_seen": 1818558464 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045664646464646465, + "loss": 2.4631, + "theoretical_loss": 3.4579466004727664, + "tokens_seen": 1818624000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566262626262626, + "loss": 2.7424, + "theoretical_loss": 3.4579360156957435, + "tokens_seen": 1818689536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566060606060606, + "loss": 2.7302, + "theoretical_loss": 3.4579254314069265, + "tokens_seen": 1818755072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565858585858586, + "loss": 2.8242, + "theoretical_loss": 3.4579148476062755, + "tokens_seen": 1818820608 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1002311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1334288120269775, + "objective/train/theoretical_loss": 3.4579042642937505, + "objective/train/tokens_used": 177745376, + "theoretical_loss": 3.4579042642937505, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565656565656566, + "loss": 2.9767, + "theoretical_loss": 3.4579042642937505, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045654545454545454, + "loss": 2.3271, + "theoretical_loss": 3.4578936814693106, + "tokens_seen": 1818951680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565252525252525, + "loss": 2.7679, + "theoretical_loss": 3.4578830991329164, + "tokens_seen": 1819017216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565050505050505, + "loss": 2.4984, + "theoretical_loss": 3.4578725172845273, + "tokens_seen": 1819082752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564848484848485, + "loss": 2.6605, + "theoretical_loss": 3.457861935924104, + "tokens_seen": 1819148288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564646464646465, + "loss": 2.582, + "theoretical_loss": 3.4578513550516057, + "tokens_seen": 1819213824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045644444444444443, + "loss": 2.6528, + "theoretical_loss": 3.457840774666993, + "tokens_seen": 1819279360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564242424242424, + "loss": 2.6102, + "theoretical_loss": 3.457830194770225, + "tokens_seen": 1819344896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045640404040404045, + "loss": 2.647, + "theoretical_loss": 3.4578196153612617, + "tokens_seen": 1819410432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563838383838384, + "loss": 2.8564, + "theoretical_loss": 3.457809036440064, + "tokens_seen": 1819475968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045636363636363637, + "loss": 2.581, + "theoretical_loss": 3.4577984580065912, + "tokens_seen": 1819541504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563434343434343, + "loss": 2.8138, + "theoretical_loss": 3.4577878800608026, + "tokens_seen": 1819607040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563232323232323, + "loss": 2.4587, + "theoretical_loss": 3.4577773026026595, + "tokens_seen": 1819672576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045630303030303035, + "loss": 2.6125, + "theoretical_loss": 3.457766725632121, + "tokens_seen": 1819738112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562828282828283, + "loss": 2.7351, + "theoretical_loss": 3.457756149149147, + "tokens_seen": 1819803648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045626262626262626, + "loss": 2.8924, + "theoretical_loss": 3.457745573153698, + "tokens_seen": 1819869184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562424242424242, + "loss": 2.6131, + "theoretical_loss": 3.4577349976457334, + "tokens_seen": 1819934720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045622222222222223, + "loss": 2.7737, + "theoretical_loss": 3.4577244226252137, + "tokens_seen": 1820000256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045620202020202024, + "loss": 2.6232, + "theoretical_loss": 3.457713848092099, + "tokens_seen": 1820065792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561818181818182, + "loss": 2.7592, + "theoretical_loss": 3.4577032740463483, + "tokens_seen": 1820131328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045616161616161615, + "loss": 2.7347, + "theoretical_loss": 3.4576927004879225, + "tokens_seen": 1820196864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561414141414141, + "loss": 2.6675, + "theoretical_loss": 3.4576821274167813, + "tokens_seen": 1820262400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561212121212121, + "loss": 2.815, + "theoretical_loss": 3.4576715548328845, + "tokens_seen": 1820327936 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045610101010101013, + "loss": 2.8066, + "theoretical_loss": 3.4576609827361926, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560808080808081, + "loss": 2.6566, + "theoretical_loss": 3.4576504111266653, + "tokens_seen": 1820459008 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1003206, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.652801990509033, + "objective/train/theoretical_loss": 3.457639840004262, + "objective/train/tokens_used": 179383776, + "theoretical_loss": 3.457639840004262, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045606060606060605, + "loss": 2.7231, + "theoretical_loss": 3.457639840004262, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045604040404040406, + "loss": 2.5342, + "theoretical_loss": 3.4576292693689443, + "tokens_seen": 1820590080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045602020202020207, + "loss": 2.6447, + "theoretical_loss": 3.4576186992206708, + "tokens_seen": 1820655616 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456, + "loss": 2.5253, + "theoretical_loss": 3.4576081295594023, + "tokens_seen": 1820721152 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455979797979798, + "loss": 2.5932, + "theoretical_loss": 3.457597560385098, + "tokens_seen": 1820786688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045595959595959594, + "loss": 2.4873, + "theoretical_loss": 3.4575869916977195, + "tokens_seen": 1820852224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045593939393939395, + "loss": 2.7011, + "theoretical_loss": 3.4575764234972253, + "tokens_seen": 1820917760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045591919191919196, + "loss": 2.6617, + "theoretical_loss": 3.4575658557835753, + "tokens_seen": 1820983296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558989898989899, + "loss": 2.7763, + "theoretical_loss": 3.457555288556731, + "tokens_seen": 1821048832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558787878787879, + "loss": 2.813, + "theoretical_loss": 3.4575447218166517, + "tokens_seen": 1821114368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558585858585859, + "loss": 2.7815, + "theoretical_loss": 3.4575341555632972, + "tokens_seen": 1821179904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045583838383838384, + "loss": 2.755, + "theoretical_loss": 3.4575235897966277, + "tokens_seen": 1821245440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045581818181818185, + "loss": 2.4572, + "theoretical_loss": 3.457513024516604, + "tokens_seen": 1821310976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557979797979798, + "loss": 2.9435, + "theoretical_loss": 3.457502459723185, + "tokens_seen": 1821376512 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045577777777777777, + "loss": 2.6106, + "theoretical_loss": 3.4574918954163314, + "tokens_seen": 1821442048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557575757575758, + "loss": 2.4757, + "theoretical_loss": 3.4574813315960036, + "tokens_seen": 1821507584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045573737373737374, + "loss": 2.5581, + "theoretical_loss": 3.4574707682621613, + "tokens_seen": 1821573120 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045571717171717175, + "loss": 2.8554, + "theoretical_loss": 3.4574602054147645, + "tokens_seen": 1821638656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556969696969697, + "loss": 2.6493, + "theoretical_loss": 3.4574496430537733, + "tokens_seen": 1821704192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556767676767677, + "loss": 2.9531, + "theoretical_loss": 3.4574390811791478, + "tokens_seen": 1821769728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045565656565656567, + "loss": 2.811, + "theoretical_loss": 3.4574285197908483, + "tokens_seen": 1821835264 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045563636363636363, + "loss": 2.8732, + "theoretical_loss": 3.4574179588888345, + "tokens_seen": 1821900800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045561616161616164, + "loss": 2.7586, + "theoretical_loss": 3.4574073984730678, + "tokens_seen": 1821966336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555959595959596, + "loss": 2.6515, + "theoretical_loss": 3.4573968385435068, + "tokens_seen": 1822031872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555757575757576, + "loss": 3.0081, + "theoretical_loss": 3.4573862791001124, + "tokens_seen": 1822097408 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1004309, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1638598442077637, + "objective/train/theoretical_loss": 3.4573757201428443, + "objective/train/tokens_used": 181022176, + "theoretical_loss": 3.4573757201428443, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045555555555555556, + "loss": 2.8326, + "theoretical_loss": 3.4573757201428443, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555353535353535, + "loss": 2.7456, + "theoretical_loss": 3.457365161671663, + "tokens_seen": 1822228480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045551515151515153, + "loss": 2.6156, + "theoretical_loss": 3.457354603686529, + "tokens_seen": 1822294016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045549494949494954, + "loss": 2.7395, + "theoretical_loss": 3.457344046187401, + "tokens_seen": 1822359552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554747474747475, + "loss": 2.901, + "theoretical_loss": 3.457333489174241, + "tokens_seen": 1822425088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045545454545454546, + "loss": 2.5662, + "theoretical_loss": 3.4573229326470076, + "tokens_seen": 1822490624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554343434343434, + "loss": 2.5802, + "theoretical_loss": 3.457312376605662, + "tokens_seen": 1822556160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554141414141414, + "loss": 2.6943, + "theoretical_loss": 3.457301821050164, + "tokens_seen": 1822621696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045539393939393944, + "loss": 2.5732, + "theoretical_loss": 3.4572912659804738, + "tokens_seen": 1822687232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553737373737374, + "loss": 2.783, + "theoretical_loss": 3.4572807113965514, + "tokens_seen": 1822752768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045535353535353535, + "loss": 2.7854, + "theoretical_loss": 3.4572701572983573, + "tokens_seen": 1822818304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553333333333333, + "loss": 2.3922, + "theoretical_loss": 3.457259603685851, + "tokens_seen": 1822883840 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045531313131313137, + "loss": 2.6926, + "theoretical_loss": 3.4572490505589943, + "tokens_seen": 1822949376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045529292929292933, + "loss": 2.8008, + "theoretical_loss": 3.4572384979177455, + "tokens_seen": 1823014912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552727272727273, + "loss": 2.3679, + "theoretical_loss": 3.4572279457620656, + "tokens_seen": 1823080448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045525252525252524, + "loss": 2.6327, + "theoretical_loss": 3.457217394091915, + "tokens_seen": 1823145984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552323232323232, + "loss": 2.7428, + "theoretical_loss": 3.4572068429072536, + "tokens_seen": 1823211520 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045521212121212126, + "loss": 2.7718, + "theoretical_loss": 3.457196292208042, + "tokens_seen": 1823277056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551919191919192, + "loss": 2.8718, + "theoretical_loss": 3.4571857419942402, + "tokens_seen": 1823342592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551717171717172, + "loss": 2.5766, + "theoretical_loss": 3.457175192265808, + "tokens_seen": 1823408128 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045515151515151513, + "loss": 2.8171, + "theoretical_loss": 3.457164643022706, + "tokens_seen": 1823473664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551313131313131, + "loss": 2.6179, + "theoretical_loss": 3.4571540942648946, + "tokens_seen": 1823539200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045511111111111116, + "loss": 2.6814, + "theoretical_loss": 3.4571435459923334, + "tokens_seen": 1823604736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550909090909091, + "loss": 2.4929, + "theoretical_loss": 3.457132998204984, + "tokens_seen": 1823670272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045507070707070707, + "loss": 2.7795, + "theoretical_loss": 3.457122450902805, + "tokens_seen": 1823735808 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1004868, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2328946590423584, + "objective/train/theoretical_loss": 3.4571119040857576, + "objective/train/tokens_used": 182660576, + "theoretical_loss": 3.4571119040857576, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045505050505050503, + "loss": 2.337, + "theoretical_loss": 3.4571119040857576, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045503030303030304, + "loss": 2.9013, + "theoretical_loss": 3.4571013577538015, + "tokens_seen": 1823866880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045501010101010105, + "loss": 2.7025, + "theoretical_loss": 3.457090811906898, + "tokens_seen": 1823932416 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454989898989899, + "loss": 2.9696, + "theoretical_loss": 3.457080266545006, + "tokens_seen": 1823997952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045496969696969696, + "loss": 2.6379, + "theoretical_loss": 3.457069721668087, + "tokens_seen": 1824063488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549494949494949, + "loss": 2.7066, + "theoretical_loss": 3.4570591772761006, + "tokens_seen": 1824129024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045492929292929293, + "loss": 2.756, + "theoretical_loss": 3.4570486333690065, + "tokens_seen": 1824194560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045490909090909094, + "loss": 2.7721, + "theoretical_loss": 3.4570380899467663, + "tokens_seen": 1824260096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548888888888889, + "loss": 2.7186, + "theoretical_loss": 3.4570275470093397, + "tokens_seen": 1824325632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045486868686868686, + "loss": 2.7226, + "theoretical_loss": 3.457017004556687, + "tokens_seen": 1824391168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045484848484848487, + "loss": 2.6073, + "theoretical_loss": 3.457006462588768, + "tokens_seen": 1824456704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548282828282828, + "loss": 2.5764, + "theoretical_loss": 3.456995921105544, + "tokens_seen": 1824522240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045480808080808083, + "loss": 2.7873, + "theoretical_loss": 3.4569853801069743, + "tokens_seen": 1824587776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547878787878788, + "loss": 2.6969, + "theoretical_loss": 3.4569748395930198, + "tokens_seen": 1824653312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045476767676767675, + "loss": 2.7108, + "theoretical_loss": 3.4569642995636407, + "tokens_seen": 1824718848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045474747474747476, + "loss": 2.8133, + "theoretical_loss": 3.4569537600187976, + "tokens_seen": 1824784384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547272727272727, + "loss": 2.5824, + "theoretical_loss": 3.4569432209584496, + "tokens_seen": 1824849920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045470707070707073, + "loss": 2.7374, + "theoretical_loss": 3.4569326823825586, + "tokens_seen": 1824915456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546868686868687, + "loss": 2.4365, + "theoretical_loss": 3.4569221442910845, + "tokens_seen": 1824980992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546666666666667, + "loss": 2.9807, + "theoretical_loss": 3.4569116066839873, + "tokens_seen": 1825046528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045464646464646465, + "loss": 2.7103, + "theoretical_loss": 3.4569010695612272, + "tokens_seen": 1825112064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546262626262626, + "loss": 2.7703, + "theoretical_loss": 3.456890532922765, + "tokens_seen": 1825177600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546060606060606, + "loss": 2.5735, + "theoretical_loss": 3.456879996768561, + "tokens_seen": 1825243136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545858585858586, + "loss": 2.5735, + "theoretical_loss": 3.4568694610985755, + "tokens_seen": 1825308672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545656565656566, + "loss": 2.5422, + "theoretical_loss": 3.456858925912768, + "tokens_seen": 1825374208 + }, + { + "debugging/Self-BLEU-5": 0.5101952161435601, + "debugging/distinct-1-grams": 0.7564583571701602, + "debugging/distinct-2-grams": 0.9365913780414189, + "debugging/entropy-1-grams": 5.753082263870947, + "debugging/entropy-2-grams": 6.573169152318852, + "debugging/length": 571.4, + "debugging/num_segments": 10, + "epoch": 0.1, + "objective/train/docs_used": 1006237, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5774552822113037, + "objective/train/theoretical_loss": 3.4568483912111008, + "objective/train/tokens_used": 184298976, + "theoretical_loss": 3.4568483912111008, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045454545454545455, + "loss": 2.8626, + "theoretical_loss": 3.4568483912111008, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545252525252525, + "loss": 2.7059, + "theoretical_loss": 3.4568378569935323, + "tokens_seen": 1825505280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545050505050505, + "loss": 2.4361, + "theoretical_loss": 3.4568273232600237, + "tokens_seen": 1825570816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544848484848485, + "loss": 2.6962, + "theoretical_loss": 3.456816790010536, + "tokens_seen": 1825636352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544646464646465, + "loss": 2.8448, + "theoretical_loss": 3.4568062572450287, + "tokens_seen": 1825701888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045444444444444444, + "loss": 2.7327, + "theoretical_loss": 3.456795724963462, + "tokens_seen": 1825767424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544242424242424, + "loss": 2.7395, + "theoretical_loss": 3.456785193165797, + "tokens_seen": 1825832960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544040404040404, + "loss": 2.7404, + "theoretical_loss": 3.456774661851995, + "tokens_seen": 1825898496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543838383838384, + "loss": 2.6062, + "theoretical_loss": 3.4567641310220134, + "tokens_seen": 1825964032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543636363636364, + "loss": 2.7898, + "theoretical_loss": 3.4567536006758157, + "tokens_seen": 1826029568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045434343434343433, + "loss": 2.6872, + "theoretical_loss": 3.456743070813361, + "tokens_seen": 1826095104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543232323232323, + "loss": 2.5771, + "theoretical_loss": 3.4567325414346093, + "tokens_seen": 1826160640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045430303030303035, + "loss": 2.8704, + "theoretical_loss": 3.4567220125395215, + "tokens_seen": 1826226176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542828282828283, + "loss": 2.7221, + "theoretical_loss": 3.456711484128059, + "tokens_seen": 1826291712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045426262626262627, + "loss": 2.7092, + "theoretical_loss": 3.45670095620018, + "tokens_seen": 1826357248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542424242424242, + "loss": 2.6278, + "theoretical_loss": 3.456690428755847, + "tokens_seen": 1826422784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045422222222222223, + "loss": 2.6251, + "theoretical_loss": 3.4566799017950194, + "tokens_seen": 1826488320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045420202020202025, + "loss": 2.5685, + "theoretical_loss": 3.4566693753176576, + "tokens_seen": 1826553856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541818181818182, + "loss": 2.5096, + "theoretical_loss": 3.4566588493237234, + "tokens_seen": 1826619392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045416161616161616, + "loss": 2.6342, + "theoretical_loss": 3.4566483238131753, + "tokens_seen": 1826684928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541414141414141, + "loss": 2.8359, + "theoretical_loss": 3.456637798785975, + "tokens_seen": 1826750464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541212121212122, + "loss": 2.5948, + "theoretical_loss": 3.4566272742420825, + "tokens_seen": 1826816000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045410101010101014, + "loss": 2.586, + "theoretical_loss": 3.4566167501814586, + "tokens_seen": 1826881536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540808080808081, + "loss": 2.5007, + "theoretical_loss": 3.4566062266040634, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045406060606060605, + "loss": 2.7515, + "theoretical_loss": 3.4565957035098576, + "tokens_seen": 1827012608 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1006887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7742507457733154, + "objective/train/theoretical_loss": 3.4565851808988013, + "objective/train/tokens_used": 185937376, + "theoretical_loss": 3.4565851808988013, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454040404040404, + "loss": 2.6239, + "theoretical_loss": 3.4565851808988013, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540202020202021, + "loss": 2.68, + "theoretical_loss": 3.4565746587708563, + "tokens_seen": 1827143680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045400000000000003, + "loss": 2.5783, + "theoretical_loss": 3.456564137125981, + "tokens_seen": 1827209216 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453979797979798, + "loss": 2.852, + "theoretical_loss": 3.456553615964138, + "tokens_seen": 1827274752 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045395959595959594, + "loss": 2.5331, + "theoretical_loss": 3.4565430952852862, + "tokens_seen": 1827340288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045393939393939396, + "loss": 2.4804, + "theoretical_loss": 3.456532575089387, + "tokens_seen": 1827405824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045391919191919197, + "loss": 2.7907, + "theoretical_loss": 3.4565220553764004, + "tokens_seen": 1827471360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538989898989899, + "loss": 2.5951, + "theoretical_loss": 3.4565115361462873, + "tokens_seen": 1827536896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538787878787879, + "loss": 2.5612, + "theoretical_loss": 3.456501017399008, + "tokens_seen": 1827602432 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045385858585858584, + "loss": 2.5128, + "theoretical_loss": 3.4564904991345236, + "tokens_seen": 1827667968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045383838383838385, + "loss": 2.7096, + "theoretical_loss": 3.4564799813527936, + "tokens_seen": 1827733504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045381818181818186, + "loss": 2.6215, + "theoretical_loss": 3.4564694640537788, + "tokens_seen": 1827799040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537979797979798, + "loss": 2.6144, + "theoretical_loss": 3.4564589472374405, + "tokens_seen": 1827864576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537777777777778, + "loss": 2.4467, + "theoretical_loss": 3.4564484309037384, + "tokens_seen": 1827930112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537575757575758, + "loss": 2.8435, + "theoretical_loss": 3.456437915052634, + "tokens_seen": 1827995648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045373737373737374, + "loss": 2.7307, + "theoretical_loss": 3.456427399684087, + "tokens_seen": 1828061184 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045371717171717175, + "loss": 2.4665, + "theoretical_loss": 3.456416884798058, + "tokens_seen": 1828126720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536969696969697, + "loss": 2.6755, + "theoretical_loss": 3.456406370394508, + "tokens_seen": 1828192256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045367676767676767, + "loss": 2.6087, + "theoretical_loss": 3.456395856473397, + "tokens_seen": 1828257792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536565656565657, + "loss": 2.4562, + "theoretical_loss": 3.456385343034686, + "tokens_seen": 1828323328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045363636363636363, + "loss": 2.5869, + "theoretical_loss": 3.456374830078336, + "tokens_seen": 1828388864 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045361616161616164, + "loss": 2.6877, + "theoretical_loss": 3.456364317604306, + "tokens_seen": 1828454400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535959595959596, + "loss": 2.7657, + "theoretical_loss": 3.456353805612559, + "tokens_seen": 1828519936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535757575757576, + "loss": 2.8022, + "theoretical_loss": 3.4563432941030534, + "tokens_seen": 1828585472 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045355555555555557, + "loss": 2.6839, + "theoretical_loss": 3.4563327830757506, + "tokens_seen": 1828651008 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1008261, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3774466514587402, + "objective/train/theoretical_loss": 3.4563222725306115, + "objective/train/tokens_used": 187575776, + "theoretical_loss": 3.4563222725306115, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535353535353535, + "loss": 2.6512, + "theoretical_loss": 3.4563222725306115, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045351515151515154, + "loss": 2.8056, + "theoretical_loss": 3.4563117624675965, + "tokens_seen": 1828782080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534949494949495, + "loss": 2.4355, + "theoretical_loss": 3.456301252886666, + "tokens_seen": 1828847616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534747474747475, + "loss": 2.7577, + "theoretical_loss": 3.456290743787781, + "tokens_seen": 1828913152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045345454545454546, + "loss": 2.6925, + "theoretical_loss": 3.456280235170902, + "tokens_seen": 1828978688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534343434343434, + "loss": 2.8243, + "theoretical_loss": 3.4562697270359886, + "tokens_seen": 1829044224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045341414141414143, + "loss": 2.7169, + "theoretical_loss": 3.456259219383003, + "tokens_seen": 1829109760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045339393939393944, + "loss": 2.5994, + "theoretical_loss": 3.456248712211905, + "tokens_seen": 1829175296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533737373737374, + "loss": 2.7856, + "theoretical_loss": 3.4562382055226557, + "tokens_seen": 1829240832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045335353535353535, + "loss": 2.584, + "theoretical_loss": 3.456227699315215, + "tokens_seen": 1829306368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533333333333333, + "loss": 2.7333, + "theoretical_loss": 3.456217193589544, + "tokens_seen": 1829371904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533131313131313, + "loss": 2.5758, + "theoretical_loss": 3.4562066883456035, + "tokens_seen": 1829437440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045329292929292933, + "loss": 2.6453, + "theoretical_loss": 3.456196183583354, + "tokens_seen": 1829502976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532727272727273, + "loss": 2.7702, + "theoretical_loss": 3.4561856793027568, + "tokens_seen": 1829568512 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045325252525252525, + "loss": 2.65, + "theoretical_loss": 3.4561751755037706, + "tokens_seen": 1829634048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532323232323232, + "loss": 2.7699, + "theoretical_loss": 3.4561646721863584, + "tokens_seen": 1829699584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532121212121212, + "loss": 2.6576, + "theoretical_loss": 3.4561541693504796, + "tokens_seen": 1829765120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531919191919192, + "loss": 2.5966, + "theoretical_loss": 3.4561436669960948, + "tokens_seen": 1829830656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531717171717172, + "loss": 2.5938, + "theoretical_loss": 3.4561331651231653, + "tokens_seen": 1829896192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045315151515151514, + "loss": 2.7886, + "theoretical_loss": 3.4561226637316516, + "tokens_seen": 1829961728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531313131313131, + "loss": 2.7663, + "theoretical_loss": 3.456112162821514, + "tokens_seen": 1830027264 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045311111111111116, + "loss": 2.7715, + "theoretical_loss": 3.456101662392714, + "tokens_seen": 1830092800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530909090909091, + "loss": 2.5694, + "theoretical_loss": 3.4560911624452113, + "tokens_seen": 1830158336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530707070707071, + "loss": 2.5395, + "theoretical_loss": 3.456080662978967, + "tokens_seen": 1830223872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045305050505050503, + "loss": 2.4467, + "theoretical_loss": 3.4560701639939424, + "tokens_seen": 1830289408 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1009482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.900181770324707, + "objective/train/theoretical_loss": 3.4560596654900975, + "objective/train/tokens_used": 189214176, + "theoretical_loss": 3.4560596654900975, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453030303030303, + "loss": 2.6643, + "theoretical_loss": 3.4560596654900975, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045301010101010106, + "loss": 2.6989, + "theoretical_loss": 3.456049167467393, + "tokens_seen": 1830420480 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452989898989899, + "loss": 2.6867, + "theoretical_loss": 3.45603866992579, + "tokens_seen": 1830486016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045296969696969697, + "loss": 2.452, + "theoretical_loss": 3.456028172865249, + "tokens_seen": 1830551552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529494949494949, + "loss": 2.9193, + "theoretical_loss": 3.4560176762857315, + "tokens_seen": 1830617088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045292929292929294, + "loss": 2.5696, + "theoretical_loss": 3.456007180187197, + "tokens_seen": 1830682624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045290909090909095, + "loss": 2.7878, + "theoretical_loss": 3.455996684569607, + "tokens_seen": 1830748160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528888888888889, + "loss": 2.4411, + "theoretical_loss": 3.4559861894329216, + "tokens_seen": 1830813696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045286868686868686, + "loss": 2.7437, + "theoretical_loss": 3.4559756947771025, + "tokens_seen": 1830879232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528484848484848, + "loss": 2.7311, + "theoretical_loss": 3.45596520060211, + "tokens_seen": 1830944768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045282828282828283, + "loss": 2.6225, + "theoretical_loss": 3.4559547069079044, + "tokens_seen": 1831010304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045280808080808084, + "loss": 2.6697, + "theoretical_loss": 3.455944213694447, + "tokens_seen": 1831075840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527878787878788, + "loss": 2.5926, + "theoretical_loss": 3.4559337209616987, + "tokens_seen": 1831141376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045276767676767675, + "loss": 2.7966, + "theoretical_loss": 3.45592322870962, + "tokens_seen": 1831206912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045274747474747477, + "loss": 2.5301, + "theoretical_loss": 3.4559127369381715, + "tokens_seen": 1831272448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527272727272727, + "loss": 2.6866, + "theoretical_loss": 3.4559022456473145, + "tokens_seen": 1831337984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045270707070707073, + "loss": 2.681, + "theoretical_loss": 3.455891754837009, + "tokens_seen": 1831403520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526868686868687, + "loss": 2.3233, + "theoretical_loss": 3.4558812645072168, + "tokens_seen": 1831469056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045266666666666665, + "loss": 2.6848, + "theoretical_loss": 3.455870774657898, + "tokens_seen": 1831534592 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045264646464646466, + "loss": 2.6255, + "theoretical_loss": 3.4558602852890132, + "tokens_seen": 1831600128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526262626262626, + "loss": 2.7053, + "theoretical_loss": 3.4558497964005244, + "tokens_seen": 1831665664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526060606060606, + "loss": 2.6355, + "theoretical_loss": 3.455839307992391, + "tokens_seen": 1831731200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525858585858586, + "loss": 2.7105, + "theoretical_loss": 3.455828820064575, + "tokens_seen": 1831796736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525656565656566, + "loss": 2.8606, + "theoretical_loss": 3.455818332617036, + "tokens_seen": 1831862272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045254545454545455, + "loss": 2.9341, + "theoretical_loss": 3.455807845649735, + "tokens_seen": 1831927808 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1009995, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7280309200286865, + "objective/train/theoretical_loss": 3.4557973591626343, + "objective/train/tokens_used": 190852576, + "theoretical_loss": 3.4557973591626343, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045252525252525256, + "loss": 2.636, + "theoretical_loss": 3.4557973591626343, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525050505050505, + "loss": 2.6817, + "theoretical_loss": 3.4557868731556933, + "tokens_seen": 1832058880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524848484848485, + "loss": 2.5711, + "theoretical_loss": 3.455776387628873, + "tokens_seen": 1832124416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524646464646465, + "loss": 2.5362, + "theoretical_loss": 3.4557659025821352, + "tokens_seen": 1832189952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045244444444444444, + "loss": 2.7667, + "theoretical_loss": 3.45575541801544, + "tokens_seen": 1832255488 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045242424242424245, + "loss": 2.78, + "theoretical_loss": 3.455744933928748, + "tokens_seen": 1832321024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524040404040404, + "loss": 2.6392, + "theoretical_loss": 3.45573445032202, + "tokens_seen": 1832386560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523838383838384, + "loss": 2.5771, + "theoretical_loss": 3.455723967195218, + "tokens_seen": 1832452096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523636363636364, + "loss": 2.3533, + "theoretical_loss": 3.4557134845483013, + "tokens_seen": 1832517632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045234343434343434, + "loss": 2.5183, + "theoretical_loss": 3.4557030023812323, + "tokens_seen": 1832583168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045232323232323235, + "loss": 2.7558, + "theoretical_loss": 3.4556925206939706, + "tokens_seen": 1832648704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523030303030303, + "loss": 2.6891, + "theoretical_loss": 3.455682039486478, + "tokens_seen": 1832714240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522828282828283, + "loss": 2.5169, + "theoretical_loss": 3.455671558758715, + "tokens_seen": 1832779776 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045226262626262627, + "loss": 2.6283, + "theoretical_loss": 3.4556610785106425, + "tokens_seen": 1832845312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045224242424242423, + "loss": 2.7893, + "theoretical_loss": 3.455650598742221, + "tokens_seen": 1832910848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045222222222222224, + "loss": 2.8487, + "theoretical_loss": 3.4556401194534123, + "tokens_seen": 1832976384 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045220202020202025, + "loss": 2.5464, + "theoretical_loss": 3.4556296406441764, + "tokens_seen": 1833041920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521818181818182, + "loss": 2.7207, + "theoretical_loss": 3.4556191623144747, + "tokens_seen": 1833107456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045216161616161616, + "loss": 2.5345, + "theoretical_loss": 3.455608684464268, + "tokens_seen": 1833172992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521414141414141, + "loss": 2.5418, + "theoretical_loss": 3.4555982070935176, + "tokens_seen": 1833238528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045212121212121213, + "loss": 2.877, + "theoretical_loss": 3.4555877302021836, + "tokens_seen": 1833304064 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045210101010101014, + "loss": 2.577, + "theoretical_loss": 3.455577253790228, + "tokens_seen": 1833369600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520808080808081, + "loss": 2.6027, + "theoretical_loss": 3.4555667778576105, + "tokens_seen": 1833435136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045206060606060606, + "loss": 2.8754, + "theoretical_loss": 3.455556302404293, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452040404040404, + "loss": 2.8005, + "theoretical_loss": 3.4555458274302353, + "tokens_seen": 1833566208 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1010832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6558239459991455, + "objective/train/theoretical_loss": 3.4555353529354003, + "objective/train/tokens_used": 192490976, + "theoretical_loss": 3.4555353529354003, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520202020202021, + "loss": 2.4985, + "theoretical_loss": 3.4555353529354003, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045200000000000004, + "loss": 2.7386, + "theoretical_loss": 3.4555248789197472, + "tokens_seen": 1833697280 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451979797979798, + "loss": 2.6204, + "theoretical_loss": 3.455514405383237, + "tokens_seen": 1833762816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045195959595959595, + "loss": 2.7539, + "theoretical_loss": 3.455503932325832, + "tokens_seen": 1833828352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519393939393939, + "loss": 2.7639, + "theoretical_loss": 3.4554934597474922, + "tokens_seen": 1833893888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045191919191919197, + "loss": 2.6223, + "theoretical_loss": 3.4554829876481787, + "tokens_seen": 1833959424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045189898989898993, + "loss": 2.7164, + "theoretical_loss": 3.4554725160278528, + "tokens_seen": 1834024960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518787878787879, + "loss": 2.3732, + "theoretical_loss": 3.4554620448864743, + "tokens_seen": 1834090496 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045185858585858584, + "loss": 2.6354, + "theoretical_loss": 3.4554515742240057, + "tokens_seen": 1834156032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518383838383838, + "loss": 2.6698, + "theoretical_loss": 3.4554411040404074, + "tokens_seen": 1834221568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045181818181818186, + "loss": 2.6226, + "theoretical_loss": 3.45543063433564, + "tokens_seen": 1834287104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517979797979798, + "loss": 2.6863, + "theoretical_loss": 3.4554201651096648, + "tokens_seen": 1834352640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517777777777778, + "loss": 2.6369, + "theoretical_loss": 3.455409696362443, + "tokens_seen": 1834418176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045175757575757574, + "loss": 2.5681, + "theoretical_loss": 3.4553992280939356, + "tokens_seen": 1834483712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173737373737375, + "loss": 2.6862, + "theoretical_loss": 3.4553887603041034, + "tokens_seen": 1834549248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045171717171717176, + "loss": 2.5591, + "theoretical_loss": 3.4553782929929073, + "tokens_seen": 1834614784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516969696969697, + "loss": 2.6327, + "theoretical_loss": 3.4553678261603085, + "tokens_seen": 1834680320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045167676767676767, + "loss": 2.6442, + "theoretical_loss": 3.455357359806268, + "tokens_seen": 1834745856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045165656565656563, + "loss": 2.6768, + "theoretical_loss": 3.455346893930747, + "tokens_seen": 1834811392 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045163636363636364, + "loss": 2.4703, + "theoretical_loss": 3.4553364285337063, + "tokens_seen": 1834876928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045161616161616165, + "loss": 2.7648, + "theoretical_loss": 3.4553259636151066, + "tokens_seen": 1834942464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515959595959596, + "loss": 2.6158, + "theoretical_loss": 3.45531549917491, + "tokens_seen": 1835008000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045157575757575756, + "loss": 2.8335, + "theoretical_loss": 3.455305035213076, + "tokens_seen": 1835073536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515555555555556, + "loss": 2.7982, + "theoretical_loss": 3.455294571729567, + "tokens_seen": 1835139072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045153535353535353, + "loss": 2.8067, + "theoretical_loss": 3.455284108724344, + "tokens_seen": 1835204608 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1011110, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.309302806854248, + "objective/train/theoretical_loss": 3.4552736461973668, + "objective/train/tokens_used": 194129376, + "theoretical_loss": 3.4552736461973668, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045151515151515154, + "loss": 2.5007, + "theoretical_loss": 3.4552736461973668, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514949494949495, + "loss": 2.5452, + "theoretical_loss": 3.455263184148598, + "tokens_seen": 1835335680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045147474747474746, + "loss": 2.6689, + "theoretical_loss": 3.455252722577997, + "tokens_seen": 1835401216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045145454545454547, + "loss": 2.6253, + "theoretical_loss": 3.455242261485527, + "tokens_seen": 1835466752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514343434343434, + "loss": 2.4733, + "theoretical_loss": 3.4552318008711476, + "tokens_seen": 1835532288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045141414141414144, + "loss": 2.4469, + "theoretical_loss": 3.45522134073482, + "tokens_seen": 1835597824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513939393939394, + "loss": 2.7587, + "theoretical_loss": 3.455210881076505, + "tokens_seen": 1835663360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513737373737374, + "loss": 2.5127, + "theoretical_loss": 3.455200421896165, + "tokens_seen": 1835728896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045135353535353536, + "loss": 2.8253, + "theoretical_loss": 3.45518996319376, + "tokens_seen": 1835794432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513333333333333, + "loss": 2.5292, + "theoretical_loss": 3.4551795049692515, + "tokens_seen": 1835859968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045131313131313133, + "loss": 2.7594, + "theoretical_loss": 3.4551690472226007, + "tokens_seen": 1835925504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512929292929293, + "loss": 2.6391, + "theoretical_loss": 3.455158589953768, + "tokens_seen": 1835991040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512727272727273, + "loss": 2.6793, + "theoretical_loss": 3.455148133162715, + "tokens_seen": 1836056576 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125252525252525, + "loss": 2.7743, + "theoretical_loss": 3.455137676849403, + "tokens_seen": 1836122112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512323232323232, + "loss": 2.7776, + "theoretical_loss": 3.455127221013793, + "tokens_seen": 1836187648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512121212121212, + "loss": 2.7312, + "theoretical_loss": 3.4551167656558457, + "tokens_seen": 1836253184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045119191919191923, + "loss": 2.8942, + "theoretical_loss": 3.4551063107755233, + "tokens_seen": 1836318720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511717171717172, + "loss": 2.6964, + "theoretical_loss": 3.4550958563727856, + "tokens_seen": 1836384256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045115151515151515, + "loss": 2.4926, + "theoretical_loss": 3.4550854024475948, + "tokens_seen": 1836449792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511313131313131, + "loss": 2.8051, + "theoretical_loss": 3.4550749489999113, + "tokens_seen": 1836515328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511111111111111, + "loss": 2.7998, + "theoretical_loss": 3.4550644960296966, + "tokens_seen": 1836580864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510909090909091, + "loss": 2.4606, + "theoretical_loss": 3.455054043536912, + "tokens_seen": 1836646400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510707070707071, + "loss": 2.5874, + "theoretical_loss": 3.455043591521518, + "tokens_seen": 1836711936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045105050505050504, + "loss": 2.5794, + "theoretical_loss": 3.4550331399834766, + "tokens_seen": 1836777472 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451030303030303, + "loss": 2.7082, + "theoretical_loss": 3.4550226889227487, + "tokens_seen": 1836843008 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1012365, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6850903034210205, + "objective/train/theoretical_loss": 3.455012238339296, + "objective/train/tokens_used": 195767776, + "theoretical_loss": 3.455012238339296, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045101010101010106, + "loss": 2.617, + "theoretical_loss": 3.455012238339296, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450989898989899, + "loss": 2.8187, + "theoretical_loss": 3.455001788233078, + "tokens_seen": 1836974080 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450969696969697, + "loss": 2.7156, + "theoretical_loss": 3.4549913386040574, + "tokens_seen": 1837039616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045094949494949493, + "loss": 2.5967, + "theoretical_loss": 3.454980889452195, + "tokens_seen": 1837105152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509292929292929, + "loss": 2.4446, + "theoretical_loss": 3.4549704407774517, + "tokens_seen": 1837170688 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045090909090909095, + "loss": 2.6417, + "theoretical_loss": 3.454959992579789, + "tokens_seen": 1837236224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508888888888889, + "loss": 2.592, + "theoretical_loss": 3.4549495448591685, + "tokens_seen": 1837301760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045086868686868687, + "loss": 2.8509, + "theoretical_loss": 3.4549390976155503, + "tokens_seen": 1837367296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508484848484848, + "loss": 2.6198, + "theoretical_loss": 3.4549286508488963, + "tokens_seen": 1837432832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508282828282829, + "loss": 2.4963, + "theoretical_loss": 3.454918204559168, + "tokens_seen": 1837498368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045080808080808085, + "loss": 2.5869, + "theoretical_loss": 3.4549077587463257, + "tokens_seen": 1837563904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507878787878788, + "loss": 2.6174, + "theoretical_loss": 3.4548973134103322, + "tokens_seen": 1837629440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045076767676767676, + "loss": 2.6871, + "theoretical_loss": 3.4548868685511467, + "tokens_seen": 1837694976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507474747474747, + "loss": 2.8478, + "theoretical_loss": 3.4548764241687318, + "tokens_seen": 1837760512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507272727272728, + "loss": 2.5321, + "theoretical_loss": 3.4548659802630484, + "tokens_seen": 1837826048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045070707070707074, + "loss": 2.7471, + "theoretical_loss": 3.454855536834058, + "tokens_seen": 1837891584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506868686868687, + "loss": 2.8312, + "theoretical_loss": 3.4548450938817212, + "tokens_seen": 1837957120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045066666666666665, + "loss": 2.5958, + "theoretical_loss": 3.454834651406, + "tokens_seen": 1838022656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045064646464646466, + "loss": 2.6896, + "theoretical_loss": 3.454824209406855, + "tokens_seen": 1838088192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506262626262627, + "loss": 2.7956, + "theoretical_loss": 3.4548137678842474, + "tokens_seen": 1838153728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045060606060606063, + "loss": 2.626, + "theoretical_loss": 3.4548033268381397, + "tokens_seen": 1838219264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505858585858586, + "loss": 2.7092, + "theoretical_loss": 3.454792886268492, + "tokens_seen": 1838284800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045056565656565655, + "loss": 2.8204, + "theoretical_loss": 3.4547824461752654, + "tokens_seen": 1838350336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045054545454545456, + "loss": 2.9183, + "theoretical_loss": 3.4547720065584215, + "tokens_seen": 1838415872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045052525252525257, + "loss": 2.7341, + "theoretical_loss": 3.454761567417922, + "tokens_seen": 1838481408 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1013519, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.807917356491089, + "objective/train/theoretical_loss": 3.454751128753728, + "objective/train/tokens_used": 197406176, + "theoretical_loss": 3.454751128753728, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505050505050505, + "loss": 2.8162, + "theoretical_loss": 3.454751128753728, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504848484848485, + "loss": 2.6302, + "theoretical_loss": 3.4547406905658002, + "tokens_seen": 1838612480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504646464646465, + "loss": 2.6506, + "theoretical_loss": 3.454730252854101, + "tokens_seen": 1838678016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045044444444444445, + "loss": 2.7298, + "theoretical_loss": 3.454719815618591, + "tokens_seen": 1838743552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045042424242424246, + "loss": 2.7674, + "theoretical_loss": 3.4547093788592314, + "tokens_seen": 1838809088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504040404040404, + "loss": 2.5375, + "theoretical_loss": 3.4546989425759835, + "tokens_seen": 1838874624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503838383838384, + "loss": 2.5675, + "theoretical_loss": 3.454688506768809, + "tokens_seen": 1838940160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503636363636364, + "loss": 2.8516, + "theoretical_loss": 3.4546780714376695, + "tokens_seen": 1839005696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045034343434343434, + "loss": 2.5366, + "theoretical_loss": 3.454667636582525, + "tokens_seen": 1839071232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045032323232323235, + "loss": 2.511, + "theoretical_loss": 3.4546572022033377, + "tokens_seen": 1839136768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503030303030303, + "loss": 2.8897, + "theoretical_loss": 3.4546467683000692, + "tokens_seen": 1839202304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502828282828283, + "loss": 2.7794, + "theoretical_loss": 3.4546363348726805, + "tokens_seen": 1839267840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502626262626263, + "loss": 2.832, + "theoretical_loss": 3.4546259019211334, + "tokens_seen": 1839333376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045024242424242423, + "loss": 2.6955, + "theoretical_loss": 3.4546154694453883, + "tokens_seen": 1839398912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045022222222222225, + "loss": 2.6539, + "theoretical_loss": 3.454605037445407, + "tokens_seen": 1839464448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502020202020202, + "loss": 2.4324, + "theoretical_loss": 3.454594605921151, + "tokens_seen": 1839529984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501818181818182, + "loss": 2.6089, + "theoretical_loss": 3.454584174872582, + "tokens_seen": 1839595520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045016161616161617, + "loss": 2.7772, + "theoretical_loss": 3.454573744299661, + "tokens_seen": 1839661056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501414141414141, + "loss": 2.7491, + "theoretical_loss": 3.454563314202349, + "tokens_seen": 1839726592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045012121212121214, + "loss": 2.7735, + "theoretical_loss": 3.454552884580607, + "tokens_seen": 1839792128 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045010101010101015, + "loss": 2.666, + "theoretical_loss": 3.4545424554343978, + "tokens_seen": 1839857664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500808080808081, + "loss": 2.7511, + "theoretical_loss": 3.454532026763682, + "tokens_seen": 1839923200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045006060606060606, + "loss": 2.4584, + "theoretical_loss": 3.4545215985684212, + "tokens_seen": 1839988736 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450040404040404, + "loss": 2.7286, + "theoretical_loss": 3.4545111708485763, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045002020202020203, + "loss": 2.8811, + "theoretical_loss": 3.454500743604109, + "tokens_seen": 1840119808 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1014020, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2581570148468018, + "objective/train/theoretical_loss": 3.4544903168349808, + "objective/train/tokens_used": 199044576, + "theoretical_loss": 3.4544903168349808, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045000000000000004, + "loss": 2.6365, + "theoretical_loss": 3.4544903168349808, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449979797979798, + "loss": 2.6191, + "theoretical_loss": 3.454479890541153, + "tokens_seen": 1840250880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044995959595959596, + "loss": 2.6941, + "theoretical_loss": 3.454469464722587, + "tokens_seen": 1840316416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499393939393939, + "loss": 2.8638, + "theoretical_loss": 3.4544590393792443, + "tokens_seen": 1840381952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499191919191919, + "loss": 2.5423, + "theoretical_loss": 3.4544486145110858, + "tokens_seen": 1840447488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044989898989898993, + "loss": 2.8338, + "theoretical_loss": 3.454438190118074, + "tokens_seen": 1840513024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498787878787879, + "loss": 2.854, + "theoretical_loss": 3.4544277662001694, + "tokens_seen": 1840578560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044985858585858585, + "loss": 2.5746, + "theoretical_loss": 3.454417342757334, + "tokens_seen": 1840644096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498383838383838, + "loss": 2.5932, + "theoretical_loss": 3.4544069197895286, + "tokens_seen": 1840709632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044981818181818187, + "loss": 2.7428, + "theoretical_loss": 3.454396497296715, + "tokens_seen": 1840775168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044979797979797983, + "loss": 2.5032, + "theoretical_loss": 3.454386075278855, + "tokens_seen": 1840840704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497777777777778, + "loss": 2.6914, + "theoretical_loss": 3.4543756537359096, + "tokens_seen": 1840906240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044975757575757574, + "loss": 2.4781, + "theoretical_loss": 3.4543652326678402, + "tokens_seen": 1840971776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497373737373737, + "loss": 2.8693, + "theoretical_loss": 3.454354812074609, + "tokens_seen": 1841037312 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044971717171717176, + "loss": 2.5425, + "theoretical_loss": 3.454344391956176, + "tokens_seen": 1841102848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496969696969697, + "loss": 2.6198, + "theoretical_loss": 3.4543339723125035, + "tokens_seen": 1841168384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496767676767677, + "loss": 2.6949, + "theoretical_loss": 3.4543235531435537, + "tokens_seen": 1841233920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044965656565656563, + "loss": 2.6307, + "theoretical_loss": 3.4543131344492872, + "tokens_seen": 1841299456 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044963636363636364, + "loss": 2.632, + "theoretical_loss": 3.454302716229665, + "tokens_seen": 1841364992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044961616161616166, + "loss": 2.7287, + "theoretical_loss": 3.45429229848465, + "tokens_seen": 1841430528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495959595959596, + "loss": 2.521, + "theoretical_loss": 3.454281881214203, + "tokens_seen": 1841496064 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044957575757575757, + "loss": 2.6339, + "theoretical_loss": 3.454271464418285, + "tokens_seen": 1841561600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495555555555555, + "loss": 2.7453, + "theoretical_loss": 3.454261048096858, + "tokens_seen": 1841627136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044953535353535354, + "loss": 2.5324, + "theoretical_loss": 3.4542506322498836, + "tokens_seen": 1841692672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044951515151515155, + "loss": 2.5193, + "theoretical_loss": 3.454240216877323, + "tokens_seen": 1841758208 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1015263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0066089630126953, + "objective/train/theoretical_loss": 3.454229801979138, + "objective/train/tokens_used": 200682976, + "theoretical_loss": 3.454229801979138, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494949494949495, + "loss": 2.759, + "theoretical_loss": 3.454229801979138, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044947474747474746, + "loss": 2.5906, + "theoretical_loss": 3.45421938755529, + "tokens_seen": 1841889280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494545454545455, + "loss": 2.5523, + "theoretical_loss": 3.4542089736057404, + "tokens_seen": 1841954816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044943434343434343, + "loss": 2.7433, + "theoretical_loss": 3.4541985601304503, + "tokens_seen": 1842020352 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044941414141414144, + "loss": 2.6943, + "theoretical_loss": 3.4541881471293827, + "tokens_seen": 1842085888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493939393939394, + "loss": 2.7566, + "theoretical_loss": 3.4541777346024976, + "tokens_seen": 1842151424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044937373737373735, + "loss": 2.6265, + "theoretical_loss": 3.454167322549757, + "tokens_seen": 1842216960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044935353535353537, + "loss": 2.7595, + "theoretical_loss": 3.4541569109711228, + "tokens_seen": 1842282496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493333333333333, + "loss": 2.3543, + "theoretical_loss": 3.454146499866556, + "tokens_seen": 1842348032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044931313131313133, + "loss": 2.6629, + "theoretical_loss": 3.4541360892360187, + "tokens_seen": 1842413568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492929292929293, + "loss": 2.6116, + "theoretical_loss": 3.4541256790794725, + "tokens_seen": 1842479104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492727272727273, + "loss": 2.6179, + "theoretical_loss": 3.4541152693968784, + "tokens_seen": 1842544640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044925252525252526, + "loss": 2.8316, + "theoretical_loss": 3.454104860188198, + "tokens_seen": 1842610176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492323232323232, + "loss": 2.542, + "theoretical_loss": 3.4540944514533933, + "tokens_seen": 1842675712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492121212121212, + "loss": 2.6138, + "theoretical_loss": 3.454084043192426, + "tokens_seen": 1842741248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491919191919192, + "loss": 2.7504, + "theoretical_loss": 3.4540736354052575, + "tokens_seen": 1842806784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491717171717172, + "loss": 2.663, + "theoretical_loss": 3.454063228091848, + "tokens_seen": 1842872320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044915151515151515, + "loss": 2.6304, + "theoretical_loss": 3.4540528212521617, + "tokens_seen": 1842937856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044913131313131316, + "loss": 2.5554, + "theoretical_loss": 3.4540424148861586, + "tokens_seen": 1843003392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491111111111111, + "loss": 2.6124, + "theoretical_loss": 3.4540320089938, + "tokens_seen": 1843068928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044909090909090913, + "loss": 2.8087, + "theoretical_loss": 3.454021603575048, + "tokens_seen": 1843134464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490707070707071, + "loss": 2.9358, + "theoretical_loss": 3.454011198629865, + "tokens_seen": 1843200000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044905050505050504, + "loss": 2.5626, + "theoretical_loss": 3.454000794158211, + "tokens_seen": 1843265536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044903030303030306, + "loss": 2.4122, + "theoretical_loss": 3.453990390160049, + "tokens_seen": 1843331072 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449010101010101, + "loss": 2.8471, + "theoretical_loss": 3.4539799866353396, + "tokens_seen": 1843396608 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1015947, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9678730964660645, + "objective/train/theoretical_loss": 3.453969583584045, + "objective/train/tokens_used": 202321376, + "theoretical_loss": 3.453969583584045, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448989898989899, + "loss": 2.7793, + "theoretical_loss": 3.453969583584045, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448969696969697, + "loss": 2.5612, + "theoretical_loss": 3.4539591810061268, + "tokens_seen": 1843527680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044894949494949494, + "loss": 2.6931, + "theoretical_loss": 3.453948778901547, + "tokens_seen": 1843593216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044892929292929295, + "loss": 2.5578, + "theoretical_loss": 3.4539383772702665, + "tokens_seen": 1843658752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044890909090909096, + "loss": 2.6837, + "theoretical_loss": 3.4539279761122472, + "tokens_seen": 1843724288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488888888888889, + "loss": 2.615, + "theoretical_loss": 3.4539175754274507, + "tokens_seen": 1843789824 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044886868686868687, + "loss": 2.6294, + "theoretical_loss": 3.453907175215839, + "tokens_seen": 1843855360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044884848484848483, + "loss": 2.5685, + "theoretical_loss": 3.4538967754773733, + "tokens_seen": 1843920896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044882828282828284, + "loss": 2.6948, + "theoretical_loss": 3.453886376212015, + "tokens_seen": 1843986432 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044880808080808085, + "loss": 2.6051, + "theoretical_loss": 3.4538759774197265, + "tokens_seen": 1844051968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487878787878788, + "loss": 2.7497, + "theoretical_loss": 3.4538655791004693, + "tokens_seen": 1844117504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044876767676767677, + "loss": 2.9335, + "theoretical_loss": 3.4538551812542053, + "tokens_seen": 1844183040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487474747474747, + "loss": 2.4906, + "theoretical_loss": 3.4538447838808954, + "tokens_seen": 1844248576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487272727272728, + "loss": 2.6569, + "theoretical_loss": 3.453834386980502, + "tokens_seen": 1844314112 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044870707070707074, + "loss": 2.7522, + "theoretical_loss": 3.453823990552986, + "tokens_seen": 1844379648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486868686868687, + "loss": 2.2554, + "theoretical_loss": 3.4538135945983104, + "tokens_seen": 1844445184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044866666666666666, + "loss": 2.5178, + "theoretical_loss": 3.453803199116435, + "tokens_seen": 1844510720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486464646464646, + "loss": 2.5563, + "theoretical_loss": 3.453792804107324, + "tokens_seen": 1844576256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486262626262627, + "loss": 2.7275, + "theoretical_loss": 3.4537824095709366, + "tokens_seen": 1844641792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044860606060606064, + "loss": 2.2752, + "theoretical_loss": 3.4537720155072362, + "tokens_seen": 1844707328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485858585858586, + "loss": 2.7008, + "theoretical_loss": 3.4537616219161835, + "tokens_seen": 1844772864 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044856565656565655, + "loss": 2.7969, + "theoretical_loss": 3.4537512287977403, + "tokens_seen": 1844838400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485454545454545, + "loss": 2.7409, + "theoretical_loss": 3.4537408361518693, + "tokens_seen": 1844903936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485252525252526, + "loss": 2.4061, + "theoretical_loss": 3.4537304439785315, + "tokens_seen": 1844969472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044850505050505053, + "loss": 2.663, + "theoretical_loss": 3.4537200522776885, + "tokens_seen": 1845035008 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1017203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1169590950012207, + "objective/train/theoretical_loss": 3.4537096610493023, + "objective/train/tokens_used": 203959776, + "theoretical_loss": 3.4537096610493023, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484848484848485, + "loss": 2.6011, + "theoretical_loss": 3.4537096610493023, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044846464646464644, + "loss": 2.5262, + "theoretical_loss": 3.4536992702933347, + "tokens_seen": 1845166080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044844444444444445, + "loss": 2.5678, + "theoretical_loss": 3.453688880009747, + "tokens_seen": 1845231616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044842424242424247, + "loss": 2.5909, + "theoretical_loss": 3.4536784901985014, + "tokens_seen": 1845297152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484040404040404, + "loss": 2.6497, + "theoretical_loss": 3.4536681008595593, + "tokens_seen": 1845362688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483838383838384, + "loss": 2.7736, + "theoretical_loss": 3.4536577119928835, + "tokens_seen": 1845428224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044836363636363634, + "loss": 2.6749, + "theoretical_loss": 3.4536473235984344, + "tokens_seen": 1845493760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044834343434343435, + "loss": 2.6772, + "theoretical_loss": 3.4536369356761742, + "tokens_seen": 1845559296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044832323232323236, + "loss": 2.6142, + "theoretical_loss": 3.4536265482260653, + "tokens_seen": 1845624832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483030303030303, + "loss": 2.6519, + "theoretical_loss": 3.4536161612480685, + "tokens_seen": 1845690368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828282828282827, + "loss": 2.735, + "theoretical_loss": 3.453605774742146, + "tokens_seen": 1845755904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482626262626263, + "loss": 2.825, + "theoretical_loss": 3.4535953887082598, + "tokens_seen": 1845821440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044824242424242424, + "loss": 2.8881, + "theoretical_loss": 3.4535850031463715, + "tokens_seen": 1845886976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044822222222222225, + "loss": 2.6401, + "theoretical_loss": 3.453574618056443, + "tokens_seen": 1845952512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482020202020202, + "loss": 2.6842, + "theoretical_loss": 3.4535642334384358, + "tokens_seen": 1846018048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044818181818181816, + "loss": 2.7116, + "theoretical_loss": 3.4535538492923123, + "tokens_seen": 1846083584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481616161616162, + "loss": 2.5188, + "theoretical_loss": 3.4535434656180337, + "tokens_seen": 1846149120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044814141414141413, + "loss": 2.5895, + "theoretical_loss": 3.4535330824155617, + "tokens_seen": 1846214656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044812121212121214, + "loss": 2.5826, + "theoretical_loss": 3.453522699684859, + "tokens_seen": 1846280192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481010101010101, + "loss": 2.6438, + "theoretical_loss": 3.4535123174258864, + "tokens_seen": 1846345728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480808080808081, + "loss": 2.6586, + "theoretical_loss": 3.4535019356386067, + "tokens_seen": 1846411264 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044806060606060607, + "loss": 2.6629, + "theoretical_loss": 3.4534915543229805, + "tokens_seen": 1846476800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448040404040404, + "loss": 2.7671, + "theoretical_loss": 3.4534811734789708, + "tokens_seen": 1846542336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044802020202020204, + "loss": 2.7187, + "theoretical_loss": 3.453470793106539, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448, + "loss": 2.6632, + "theoretical_loss": 3.453460413205647, + "tokens_seen": 1846673408 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1017618, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3263816833496094, + "objective/train/theoretical_loss": 3.453450033776256, + "objective/train/tokens_used": 205598176, + "theoretical_loss": 3.453450033776256, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447979797979798, + "loss": 2.5091, + "theoretical_loss": 3.453450033776256, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044795959595959596, + "loss": 2.5978, + "theoretical_loss": 3.453439654818329, + "tokens_seen": 1846804480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479393939393939, + "loss": 2.7581, + "theoretical_loss": 3.4534292763318275, + "tokens_seen": 1846870016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044791919191919193, + "loss": 2.7108, + "theoretical_loss": 3.4534188983167127, + "tokens_seen": 1846935552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044789898989898994, + "loss": 2.6206, + "theoretical_loss": 3.453408520772947, + "tokens_seen": 1847001088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478787878787879, + "loss": 2.4462, + "theoretical_loss": 3.453398143700492, + "tokens_seen": 1847066624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044785858585858585, + "loss": 2.6832, + "theoretical_loss": 3.4533877670993096, + "tokens_seen": 1847132160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478383838383838, + "loss": 2.5257, + "theoretical_loss": 3.4533773909693624, + "tokens_seen": 1847197696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478181818181818, + "loss": 2.721, + "theoretical_loss": 3.4533670153106115, + "tokens_seen": 1847263232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044779797979797983, + "loss": 2.7503, + "theoretical_loss": 3.453356640123019, + "tokens_seen": 1847328768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477777777777778, + "loss": 2.8506, + "theoretical_loss": 3.4533462654065463, + "tokens_seen": 1847394304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044775757575757575, + "loss": 2.7111, + "theoretical_loss": 3.4533358911611565, + "tokens_seen": 1847459840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477373737373737, + "loss": 2.7537, + "theoretical_loss": 3.4533255173868103, + "tokens_seen": 1847525376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044771717171717177, + "loss": 2.7598, + "theoretical_loss": 3.4533151440834704, + "tokens_seen": 1847590912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476969696969697, + "loss": 2.5996, + "theoretical_loss": 3.4533047712510982, + "tokens_seen": 1847656448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476767676767677, + "loss": 2.5975, + "theoretical_loss": 3.453294398889656, + "tokens_seen": 1847721984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044765656565656564, + "loss": 2.8062, + "theoretical_loss": 3.453284026999105, + "tokens_seen": 1847787520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476363636363636, + "loss": 2.7529, + "theoretical_loss": 3.4532736555794084, + "tokens_seen": 1847853056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044761616161616166, + "loss": 2.457, + "theoretical_loss": 3.453263284630527, + "tokens_seen": 1847918592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475959595959596, + "loss": 2.3476, + "theoretical_loss": 3.453252914152423, + "tokens_seen": 1847984128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475757575757576, + "loss": 2.5465, + "theoretical_loss": 3.4532425441450587, + "tokens_seen": 1848049664 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044755555555555553, + "loss": 2.7043, + "theoretical_loss": 3.4532321746083956, + "tokens_seen": 1848115200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044753535353535354, + "loss": 2.527, + "theoretical_loss": 3.4532218055423956, + "tokens_seen": 1848180736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044751515151515155, + "loss": 2.6198, + "theoretical_loss": 3.4532114369470213, + "tokens_seen": 1848246272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474949494949495, + "loss": 2.7901, + "theoretical_loss": 3.453201068822234, + "tokens_seen": 1848311808 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1018803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.287393093109131, + "objective/train/theoretical_loss": 3.453190701167996, + "objective/train/tokens_used": 207236576, + "theoretical_loss": 3.453190701167996, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044747474747474747, + "loss": 2.4596, + "theoretical_loss": 3.453190701167996, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474545454545454, + "loss": 2.6234, + "theoretical_loss": 3.4531803339842693, + "tokens_seen": 1848442880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044743434343434344, + "loss": 2.9293, + "theoretical_loss": 3.453169967271015, + "tokens_seen": 1848508416 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044741414141414145, + "loss": 2.6174, + "theoretical_loss": 3.4531596010281964, + "tokens_seen": 1848573952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473939393939394, + "loss": 2.5937, + "theoretical_loss": 3.4531492352557747, + "tokens_seen": 1848639488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044737373737373736, + "loss": 2.6619, + "theoretical_loss": 3.453138869953712, + "tokens_seen": 1848705024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044735353535353537, + "loss": 2.7539, + "theoretical_loss": 3.4531285051219704, + "tokens_seen": 1848770560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473333333333334, + "loss": 2.8116, + "theoretical_loss": 3.4531181407605116, + "tokens_seen": 1848836096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044731313131313134, + "loss": 2.7512, + "theoretical_loss": 3.453107776869298, + "tokens_seen": 1848901632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472929292929293, + "loss": 2.8565, + "theoretical_loss": 3.4530974134482917, + "tokens_seen": 1848967168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044727272727272725, + "loss": 2.6797, + "theoretical_loss": 3.4530870504974542, + "tokens_seen": 1849032704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044725252525252526, + "loss": 2.5755, + "theoretical_loss": 3.4530766880167474, + "tokens_seen": 1849098240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472323232323233, + "loss": 2.5315, + "theoretical_loss": 3.453066326006134, + "tokens_seen": 1849163776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044721212121212123, + "loss": 2.7439, + "theoretical_loss": 3.4530559644655754, + "tokens_seen": 1849229312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471919191919192, + "loss": 2.7021, + "theoretical_loss": 3.453045603395034, + "tokens_seen": 1849294848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471717171717172, + "loss": 2.4656, + "theoretical_loss": 3.4530352427944715, + "tokens_seen": 1849360384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044715151515151516, + "loss": 2.5479, + "theoretical_loss": 3.45302488266385, + "tokens_seen": 1849425920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044713131313131317, + "loss": 2.778, + "theoretical_loss": 3.453014523003132, + "tokens_seen": 1849491456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471111111111111, + "loss": 2.9284, + "theoretical_loss": 3.453004163812279, + "tokens_seen": 1849556992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470909090909091, + "loss": 2.6908, + "theoretical_loss": 3.452993805091253, + "tokens_seen": 1849622528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470707070707071, + "loss": 2.711, + "theoretical_loss": 3.4529834468400162, + "tokens_seen": 1849688064 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044705050505050505, + "loss": 2.7749, + "theoretical_loss": 3.4529730890585313, + "tokens_seen": 1849753600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044703030303030306, + "loss": 2.4908, + "theoretical_loss": 3.4529627317467595, + "tokens_seen": 1849819136 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447010101010101, + "loss": 2.5087, + "theoretical_loss": 3.4529523749046627, + "tokens_seen": 1849884672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044698989898989903, + "loss": 2.6053, + "theoretical_loss": 3.4529420185322035, + "tokens_seen": 1849950208 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1019318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6232709884643555, + "objective/train/theoretical_loss": 3.4529316626293443, + "objective/train/tokens_used": 208874976, + "theoretical_loss": 3.4529316626293443, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446969696969697, + "loss": 2.5128, + "theoretical_loss": 3.4529316626293443, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044694949494949494, + "loss": 2.6452, + "theoretical_loss": 3.4529213071960463, + "tokens_seen": 1850081280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044692929292929295, + "loss": 2.74, + "theoretical_loss": 3.4529109522322727, + "tokens_seen": 1850146816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469090909090909, + "loss": 2.6177, + "theoretical_loss": 3.452900597737984, + "tokens_seen": 1850212352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468888888888889, + "loss": 2.609, + "theoretical_loss": 3.452890243713144, + "tokens_seen": 1850277888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468686868686869, + "loss": 2.3597, + "theoretical_loss": 3.452879890157713, + "tokens_seen": 1850343424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044684848484848484, + "loss": 2.6272, + "theoretical_loss": 3.452869537071655, + "tokens_seen": 1850408960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044682828282828285, + "loss": 2.591, + "theoretical_loss": 3.4528591844549306, + "tokens_seen": 1850474496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468080808080808, + "loss": 2.7254, + "theoretical_loss": 3.4528488323075024, + "tokens_seen": 1850540032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467878787878788, + "loss": 2.5987, + "theoretical_loss": 3.4528384806293326, + "tokens_seen": 1850605568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044676767676767677, + "loss": 2.7431, + "theoretical_loss": 3.452828129420384, + "tokens_seen": 1850671104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044674747474747473, + "loss": 2.4767, + "theoretical_loss": 3.452817778680617, + "tokens_seen": 1850736640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044672727272727274, + "loss": 2.5841, + "theoretical_loss": 3.4528074284099954, + "tokens_seen": 1850802176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044670707070707075, + "loss": 2.5436, + "theoretical_loss": 3.45279707860848, + "tokens_seen": 1850867712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466868686868687, + "loss": 2.7338, + "theoretical_loss": 3.452786729276034, + "tokens_seen": 1850933248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044666666666666666, + "loss": 2.629, + "theoretical_loss": 3.452776380412619, + "tokens_seen": 1850998784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466464646464646, + "loss": 2.5704, + "theoretical_loss": 3.4527660320181974, + "tokens_seen": 1851064320 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044662626262626263, + "loss": 2.6984, + "theoretical_loss": 3.4527556840927307, + "tokens_seen": 1851129856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044660606060606064, + "loss": 2.5546, + "theoretical_loss": 3.452745336636182, + "tokens_seen": 1851195392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465858585858586, + "loss": 2.7072, + "theoretical_loss": 3.452734989648513, + "tokens_seen": 1851260928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044656565656565656, + "loss": 2.5535, + "theoretical_loss": 3.4527246431296854, + "tokens_seen": 1851326464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465454545454545, + "loss": 2.678, + "theoretical_loss": 3.4527142970796625, + "tokens_seen": 1851392000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465252525252526, + "loss": 2.683, + "theoretical_loss": 3.452703951498405, + "tokens_seen": 1851457536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044650505050505054, + "loss": 2.7782, + "theoretical_loss": 3.452693606385876, + "tokens_seen": 1851523072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464848484848485, + "loss": 2.79, + "theoretical_loss": 3.452683261742038, + "tokens_seen": 1851588608 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1019593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.985586643218994, + "objective/train/theoretical_loss": 3.4526729175668525, + "objective/train/tokens_used": 210513376, + "theoretical_loss": 3.4526729175668525, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044646464646464645, + "loss": 2.7467, + "theoretical_loss": 3.4526729175668525, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464444444444444, + "loss": 2.6811, + "theoretical_loss": 3.452662573860281, + "tokens_seen": 1851719680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044642424242424247, + "loss": 2.4987, + "theoretical_loss": 3.452652230622288, + "tokens_seen": 1851785216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044640404040404043, + "loss": 2.6987, + "theoretical_loss": 3.4526418878528333, + "tokens_seen": 1851850752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463838383838384, + "loss": 2.4518, + "theoretical_loss": 3.4526315455518803, + "tokens_seen": 1851916288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044636363636363634, + "loss": 2.7949, + "theoretical_loss": 3.452621203719391, + "tokens_seen": 1851981824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044634343434343435, + "loss": 2.6532, + "theoretical_loss": 3.452610862355327, + "tokens_seen": 1852047360 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044632323232323236, + "loss": 2.8367, + "theoretical_loss": 3.452600521459652, + "tokens_seen": 1852112896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463030303030303, + "loss": 2.6774, + "theoretical_loss": 3.4525901810323263, + "tokens_seen": 1852178432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462828282828283, + "loss": 2.6519, + "theoretical_loss": 3.4525798410733133, + "tokens_seen": 1852243968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044626262626262623, + "loss": 2.5936, + "theoretical_loss": 3.4525695015825755, + "tokens_seen": 1852309504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044624242424242425, + "loss": 2.6858, + "theoretical_loss": 3.4525591625600742, + "tokens_seen": 1852375040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044622222222222226, + "loss": 2.692, + "theoretical_loss": 3.4525488240057722, + "tokens_seen": 1852440576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462020202020202, + "loss": 2.6024, + "theoretical_loss": 3.4525384859196313, + "tokens_seen": 1852506112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044618181818181817, + "loss": 2.6869, + "theoretical_loss": 3.4525281483016146, + "tokens_seen": 1852571648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461616161616162, + "loss": 2.7454, + "theoretical_loss": 3.4525178111516834, + "tokens_seen": 1852637184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044614141414141414, + "loss": 2.6114, + "theoretical_loss": 3.4525074744698, + "tokens_seen": 1852702720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044612121212121215, + "loss": 2.6078, + "theoretical_loss": 3.4524971382559277, + "tokens_seen": 1852768256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461010101010101, + "loss": 2.7454, + "theoretical_loss": 3.4524868025100277, + "tokens_seen": 1852833792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044608080808080806, + "loss": 2.6441, + "theoretical_loss": 3.4524764672320627, + "tokens_seen": 1852899328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460606060606061, + "loss": 2.5881, + "theoretical_loss": 3.4524661324219945, + "tokens_seen": 1852964864 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044604040404040403, + "loss": 2.6752, + "theoretical_loss": 3.452455798079786, + "tokens_seen": 1853030400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044602020202020204, + "loss": 2.5053, + "theoretical_loss": 3.4524454642053994, + "tokens_seen": 1853095936 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446, + "loss": 2.603, + "theoretical_loss": 3.4524351307987966, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445979797979798, + "loss": 2.8048, + "theoretical_loss": 3.45242479785994, + "tokens_seen": 1853227008 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1020707, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.561333656311035, + "objective/train/theoretical_loss": 3.452414465388792, + "objective/train/tokens_used": 212151776, + "theoretical_loss": 3.452414465388792, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044595959595959597, + "loss": 2.6155, + "theoretical_loss": 3.452414465388792, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459393939393939, + "loss": 2.8532, + "theoretical_loss": 3.452404133385315, + "tokens_seen": 1853358080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591919191919193, + "loss": 2.5471, + "theoretical_loss": 3.4523938018494706, + "tokens_seen": 1853423616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458989898989899, + "loss": 2.6863, + "theoretical_loss": 3.452383470781222, + "tokens_seen": 1853489152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458787878787879, + "loss": 2.5466, + "theoretical_loss": 3.4523731401805313, + "tokens_seen": 1853554688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044585858585858586, + "loss": 2.7701, + "theoretical_loss": 3.4523628100473607, + "tokens_seen": 1853620224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458383838383838, + "loss": 2.6979, + "theoretical_loss": 3.4523524803816716, + "tokens_seen": 1853685760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044581818181818183, + "loss": 2.8536, + "theoretical_loss": 3.4523421511834282, + "tokens_seen": 1853751296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044579797979797984, + "loss": 2.5863, + "theoretical_loss": 3.4523318224525914, + "tokens_seen": 1853816832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457777777777778, + "loss": 2.8932, + "theoretical_loss": 3.452321494189124, + "tokens_seen": 1853882368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575757575757575, + "loss": 2.893, + "theoretical_loss": 3.452311166392988, + "tokens_seen": 1853947904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457373737373737, + "loss": 2.5877, + "theoretical_loss": 3.4523008390641463, + "tokens_seen": 1854013440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457171717171717, + "loss": 2.5738, + "theoretical_loss": 3.452290512202561, + "tokens_seen": 1854078976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044569696969696973, + "loss": 2.6439, + "theoretical_loss": 3.4522801858081946, + "tokens_seen": 1854144512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456767676767677, + "loss": 2.6565, + "theoretical_loss": 3.4522698598810084, + "tokens_seen": 1854210048 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044565656565656564, + "loss": 2.4392, + "theoretical_loss": 3.452259534420967, + "tokens_seen": 1854275584 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044563636363636366, + "loss": 2.4127, + "theoretical_loss": 3.4522492094280297, + "tokens_seen": 1854341120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044561616161616167, + "loss": 2.7188, + "theoretical_loss": 3.4522388849021617, + "tokens_seen": 1854406656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455959595959596, + "loss": 2.5797, + "theoretical_loss": 3.4522285608433236, + "tokens_seen": 1854472192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455757575757576, + "loss": 2.8985, + "theoretical_loss": 3.4522182372514783, + "tokens_seen": 1854537728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044555555555555554, + "loss": 2.5284, + "theoretical_loss": 3.4522079141265882, + "tokens_seen": 1854603264 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044553535353535355, + "loss": 2.6146, + "theoretical_loss": 3.4521975914686163, + "tokens_seen": 1854668800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044551515151515156, + "loss": 2.6305, + "theoretical_loss": 3.4521872692775237, + "tokens_seen": 1854734336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454949494949495, + "loss": 2.566, + "theoretical_loss": 3.4521769475532738, + "tokens_seen": 1854799872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454747474747475, + "loss": 2.7986, + "theoretical_loss": 3.4521666262958286, + "tokens_seen": 1854865408 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1021895, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6850180625915527, + "objective/train/theoretical_loss": 3.4521563055051505, + "objective/train/tokens_used": 213790176, + "theoretical_loss": 3.4521563055051505, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044545454545454543, + "loss": 2.5554, + "theoretical_loss": 3.4521563055051505, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454343434343435, + "loss": 2.6896, + "theoretical_loss": 3.452145985181202, + "tokens_seen": 1854996480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044541414141414145, + "loss": 2.8255, + "theoretical_loss": 3.4521356653239454, + "tokens_seen": 1855062016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453939393939394, + "loss": 2.5766, + "theoretical_loss": 3.452125345933343, + "tokens_seen": 1855127552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044537373737373737, + "loss": 2.8282, + "theoretical_loss": 3.452115027009358, + "tokens_seen": 1855193088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453535353535353, + "loss": 2.7015, + "theoretical_loss": 3.4521047085519516, + "tokens_seen": 1855258624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453333333333334, + "loss": 2.7837, + "theoretical_loss": 3.4520943905610872, + "tokens_seen": 1855324160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531313131313135, + "loss": 2.5069, + "theoretical_loss": 3.4520840730367266, + "tokens_seen": 1855389696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452929292929293, + "loss": 2.5115, + "theoretical_loss": 3.452073755978833, + "tokens_seen": 1855455232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527272727272726, + "loss": 2.906, + "theoretical_loss": 3.4520634393873673, + "tokens_seen": 1855520768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452525252525252, + "loss": 2.5688, + "theoretical_loss": 3.452053123262294, + "tokens_seen": 1855586304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452323232323233, + "loss": 2.5582, + "theoretical_loss": 3.452042807603574, + "tokens_seen": 1855651840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044521212121212124, + "loss": 2.632, + "theoretical_loss": 3.45203249241117, + "tokens_seen": 1855717376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451919191919192, + "loss": 2.7484, + "theoretical_loss": 3.452022177685045, + "tokens_seen": 1855782912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044517171717171715, + "loss": 2.7794, + "theoretical_loss": 3.4520118634251613, + "tokens_seen": 1855848448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044515151515151516, + "loss": 2.4757, + "theoretical_loss": 3.452001549631481, + "tokens_seen": 1855913984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451313131313132, + "loss": 2.7872, + "theoretical_loss": 3.451991236303967, + "tokens_seen": 1855979520 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044511111111111113, + "loss": 2.7724, + "theoretical_loss": 3.4519809234425813, + "tokens_seen": 1856045056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450909090909091, + "loss": 2.5745, + "theoretical_loss": 3.4519706110472868, + "tokens_seen": 1856110592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044507070707070704, + "loss": 2.6076, + "theoretical_loss": 3.451960299118046, + "tokens_seen": 1856176128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044505050505050506, + "loss": 2.7614, + "theoretical_loss": 3.451949987654821, + "tokens_seen": 1856241664 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044503030303030307, + "loss": 2.6492, + "theoretical_loss": 3.4519396766575747, + "tokens_seen": 1856307200 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445010101010101, + "loss": 2.5089, + "theoretical_loss": 3.451929366126269, + "tokens_seen": 1856372736 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444989898989899, + "loss": 2.8286, + "theoretical_loss": 3.4519190560608672, + "tokens_seen": 1856438272 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444969696969697, + "loss": 2.7507, + "theoretical_loss": 3.451908746461331, + "tokens_seen": 1856503808 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1022391, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0687434673309326, + "objective/train/theoretical_loss": 3.4518984373276234, + "objective/train/tokens_used": 215428576, + "theoretical_loss": 3.4518984373276234, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044494949494949495, + "loss": 2.7867, + "theoretical_loss": 3.4518984373276234, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044492929292929296, + "loss": 2.713, + "theoretical_loss": 3.4518881286597067, + "tokens_seen": 1856634880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449090909090909, + "loss": 2.8357, + "theoretical_loss": 3.4518778204575433, + "tokens_seen": 1856700416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044488888888888887, + "loss": 2.7542, + "theoretical_loss": 3.4518675127210967, + "tokens_seen": 1856765952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448686868686869, + "loss": 2.4508, + "theoretical_loss": 3.4518572054503283, + "tokens_seen": 1856831488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044484848484848484, + "loss": 2.7486, + "theoretical_loss": 3.4518468986452007, + "tokens_seen": 1856897024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044482828282828285, + "loss": 2.5418, + "theoretical_loss": 3.4518365923056766, + "tokens_seen": 1856962560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448080808080808, + "loss": 2.4934, + "theoretical_loss": 3.4518262864317193, + "tokens_seen": 1857028096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447878787878788, + "loss": 2.6511, + "theoretical_loss": 3.45181598102329, + "tokens_seen": 1857093632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447676767676768, + "loss": 2.6574, + "theoretical_loss": 3.4518056760803524, + "tokens_seen": 1857159168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044474747474747473, + "loss": 2.5949, + "theoretical_loss": 3.451795371602868, + "tokens_seen": 1857224704 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044472727272727274, + "loss": 2.5193, + "theoretical_loss": 3.4517850675908006, + "tokens_seen": 1857290240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447070707070707, + "loss": 2.7867, + "theoretical_loss": 3.451774764044112, + "tokens_seen": 1857355776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446868686868687, + "loss": 2.755, + "theoretical_loss": 3.4517644609627647, + "tokens_seen": 1857421312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044466666666666667, + "loss": 2.6344, + "theoretical_loss": 3.4517541583467213, + "tokens_seen": 1857486848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446464646464646, + "loss": 2.667, + "theoretical_loss": 3.451743856195945, + "tokens_seen": 1857552384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044462626262626264, + "loss": 2.8186, + "theoretical_loss": 3.4517335545103967, + "tokens_seen": 1857617920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044460606060606065, + "loss": 2.3931, + "theoretical_loss": 3.4517232532900413, + "tokens_seen": 1857683456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445858585858586, + "loss": 2.6651, + "theoretical_loss": 3.45171295253484, + "tokens_seen": 1857748992 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044456565656565656, + "loss": 2.6855, + "theoretical_loss": 3.451702652244755, + "tokens_seen": 1857814528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445454545454545, + "loss": 2.609, + "theoretical_loss": 3.45169235241975, + "tokens_seen": 1857880064 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044452525252525253, + "loss": 2.716, + "theoretical_loss": 3.4516820530597867, + "tokens_seen": 1857945600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044450505050505054, + "loss": 2.572, + "theoretical_loss": 3.4516717541648285, + "tokens_seen": 1858011136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444848484848485, + "loss": 2.4947, + "theoretical_loss": 3.4516614557348375, + "tokens_seen": 1858076672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044446464646464645, + "loss": 2.7446, + "theoretical_loss": 3.451651157769777, + "tokens_seen": 1858142208 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1023648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7927091121673584, + "objective/train/theoretical_loss": 3.451640860269608, + "objective/train/tokens_used": 217066976, + "theoretical_loss": 3.451640860269608, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444444444444444, + "loss": 2.4124, + "theoretical_loss": 3.451640860269608, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444242424242425, + "loss": 2.5427, + "theoretical_loss": 3.4516305632342945, + "tokens_seen": 1858273280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044440404040404043, + "loss": 2.5259, + "theoretical_loss": 3.4516202666637987, + "tokens_seen": 1858338816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443838383838384, + "loss": 2.6627, + "theoretical_loss": 3.451609970558083, + "tokens_seen": 1858404352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044436363636363635, + "loss": 2.7357, + "theoretical_loss": 3.4515996749171114, + "tokens_seen": 1858469888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443434343434343, + "loss": 2.7667, + "theoretical_loss": 3.4515893797408443, + "tokens_seen": 1858535424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044432323232323237, + "loss": 2.5378, + "theoretical_loss": 3.4515790850292465, + "tokens_seen": 1858600960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443030303030303, + "loss": 2.5758, + "theoretical_loss": 3.4515687907822787, + "tokens_seen": 1858666496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442828282828283, + "loss": 2.536, + "theoretical_loss": 3.451558496999905, + "tokens_seen": 1858732032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044426262626262624, + "loss": 2.6107, + "theoretical_loss": 3.451548203682088, + "tokens_seen": 1858797568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044424242424242425, + "loss": 2.7001, + "theoretical_loss": 3.451537910828789, + "tokens_seen": 1858863104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044422222222222226, + "loss": 2.6327, + "theoretical_loss": 3.4515276184399717, + "tokens_seen": 1858928640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442020202020202, + "loss": 2.5813, + "theoretical_loss": 3.451517326515599, + "tokens_seen": 1858994176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441818181818182, + "loss": 2.7582, + "theoretical_loss": 3.4515070350556334, + "tokens_seen": 1859059712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044416161616161613, + "loss": 2.8071, + "theoretical_loss": 3.4514967440600364, + "tokens_seen": 1859125248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044414141414141414, + "loss": 2.7393, + "theoretical_loss": 3.451486453528773, + "tokens_seen": 1859190784 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044412121212121215, + "loss": 2.5984, + "theoretical_loss": 3.4514761634618036, + "tokens_seen": 1859256320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441010101010101, + "loss": 2.734, + "theoretical_loss": 3.4514658738590915, + "tokens_seen": 1859321856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044408080808080807, + "loss": 2.7378, + "theoretical_loss": 3.4514555847206, + "tokens_seen": 1859387392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440606060606061, + "loss": 2.7765, + "theoretical_loss": 3.451445296046292, + "tokens_seen": 1859452928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044404040404040404, + "loss": 2.5779, + "theoretical_loss": 3.4514350078361296, + "tokens_seen": 1859518464 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044402020202020205, + "loss": 2.4927, + "theoretical_loss": 3.4514247200900754, + "tokens_seen": 1859584000 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444, + "loss": 2.6562, + "theoretical_loss": 3.4514144328080922, + "tokens_seen": 1859649536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044397979797979796, + "loss": 2.9958, + "theoretical_loss": 3.4514041459901432, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044395959595959597, + "loss": 2.7743, + "theoretical_loss": 3.4513938596361906, + "tokens_seen": 1859780608 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1024242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3011066913604736, + "objective/train/theoretical_loss": 3.451383573746197, + "objective/train/tokens_used": 218705376, + "theoretical_loss": 3.451383573746197, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443939393939394, + "loss": 2.4715, + "theoretical_loss": 3.451383573746197, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044391919191919194, + "loss": 2.6938, + "theoretical_loss": 3.4513732883201254, + "tokens_seen": 1859911680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438989898989899, + "loss": 2.6091, + "theoretical_loss": 3.451363003357939, + "tokens_seen": 1859977216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438787878787879, + "loss": 2.6714, + "theoretical_loss": 3.4513527188596, + "tokens_seen": 1860042752 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044385858585858587, + "loss": 2.6668, + "theoretical_loss": 3.4513424348250705, + "tokens_seen": 1860108288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438383838383839, + "loss": 2.8937, + "theoretical_loss": 3.4513321512543147, + "tokens_seen": 1860173824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044381818181818183, + "loss": 2.8199, + "theoretical_loss": 3.451321868147294, + "tokens_seen": 1860239360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437979797979798, + "loss": 2.6694, + "theoretical_loss": 3.451311585503972, + "tokens_seen": 1860304896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437777777777778, + "loss": 2.7428, + "theoretical_loss": 3.4513013033243114, + "tokens_seen": 1860370432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044375757575757576, + "loss": 2.5192, + "theoretical_loss": 3.4512910216082746, + "tokens_seen": 1860435968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044373737373737377, + "loss": 2.5591, + "theoretical_loss": 3.4512807403558243, + "tokens_seen": 1860501504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437171717171717, + "loss": 2.5545, + "theoretical_loss": 3.451270459566924, + "tokens_seen": 1860567040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044369696969696974, + "loss": 2.7749, + "theoretical_loss": 3.451260179241536, + "tokens_seen": 1860632576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436767676767677, + "loss": 2.2815, + "theoretical_loss": 3.451249899379622, + "tokens_seen": 1860698112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044365656565656565, + "loss": 2.8408, + "theoretical_loss": 3.4512396199811466, + "tokens_seen": 1860763648 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044363636363636366, + "loss": 2.5795, + "theoretical_loss": 3.451229341046072, + "tokens_seen": 1860829184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436161616161616, + "loss": 2.7172, + "theoretical_loss": 3.4512190625743604, + "tokens_seen": 1860894720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044359595959595963, + "loss": 2.7811, + "theoretical_loss": 3.451208784565975, + "tokens_seen": 1860960256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435757575757576, + "loss": 2.5311, + "theoretical_loss": 3.4511985070208793, + "tokens_seen": 1861025792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044355555555555554, + "loss": 2.4705, + "theoretical_loss": 3.4511882299390346, + "tokens_seen": 1861091328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044353535353535355, + "loss": 2.8382, + "theoretical_loss": 3.4511779533204043, + "tokens_seen": 1861156864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435151515151515, + "loss": 2.5743, + "theoretical_loss": 3.4511676771649524, + "tokens_seen": 1861222400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434949494949495, + "loss": 2.7183, + "theoretical_loss": 3.4511574014726403, + "tokens_seen": 1861287936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434747474747475, + "loss": 2.7241, + "theoretical_loss": 3.4511471262434306, + "tokens_seen": 1861353472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044345454545454544, + "loss": 2.7197, + "theoretical_loss": 3.4511368514772878, + "tokens_seen": 1861419008 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1025485, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8361258506774902, + "objective/train/theoretical_loss": 3.451126577174173, + "objective/train/tokens_used": 220343776, + "theoretical_loss": 3.451126577174173, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044343434343434345, + "loss": 2.6971, + "theoretical_loss": 3.451126577174173, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044341414141414146, + "loss": 2.5431, + "theoretical_loss": 3.4511163033340497, + "tokens_seen": 1861550080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433939393939394, + "loss": 2.3856, + "theoretical_loss": 3.451106029956881, + "tokens_seen": 1861615616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044337373737373737, + "loss": 2.4283, + "theoretical_loss": 3.45109575704263, + "tokens_seen": 1861681152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044335353535353533, + "loss": 2.6594, + "theoretical_loss": 3.451085484591258, + "tokens_seen": 1861746688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044333333333333334, + "loss": 2.663, + "theoretical_loss": 3.4510752126027295, + "tokens_seen": 1861812224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044331313131313135, + "loss": 2.6829, + "theoretical_loss": 3.451064941077007, + "tokens_seen": 1861877760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432929292929293, + "loss": 2.6757, + "theoretical_loss": 3.451054670014053, + "tokens_seen": 1861943296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044327272727272726, + "loss": 2.665, + "theoretical_loss": 3.4510443994138305, + "tokens_seen": 1862008832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432525252525252, + "loss": 2.6872, + "theoretical_loss": 3.451034129276302, + "tokens_seen": 1862074368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432323232323233, + "loss": 2.6956, + "theoretical_loss": 3.451023859601431, + "tokens_seen": 1862139904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044321212121212124, + "loss": 2.6309, + "theoretical_loss": 3.45101359038918, + "tokens_seen": 1862205440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431919191919192, + "loss": 2.7008, + "theoretical_loss": 3.4510033216395124, + "tokens_seen": 1862270976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044317171717171716, + "loss": 2.6083, + "theoretical_loss": 3.4509930533523905, + "tokens_seen": 1862336512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431515151515151, + "loss": 2.6467, + "theoretical_loss": 3.450982785527777, + "tokens_seen": 1862402048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431313131313132, + "loss": 2.9646, + "theoretical_loss": 3.4509725181656354, + "tokens_seen": 1862467584 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044311111111111114, + "loss": 2.6964, + "theoretical_loss": 3.4509622512659286, + "tokens_seen": 1862533120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430909090909091, + "loss": 2.5257, + "theoretical_loss": 3.4509519848286185, + "tokens_seen": 1862598656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044307070707070705, + "loss": 2.5954, + "theoretical_loss": 3.4509417188536693, + "tokens_seen": 1862664192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044305050505050506, + "loss": 2.6717, + "theoretical_loss": 3.4509314533410436, + "tokens_seen": 1862729728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044303030303030307, + "loss": 2.5898, + "theoretical_loss": 3.450921188290704, + "tokens_seen": 1862795264 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044301010101010103, + "loss": 2.6666, + "theoretical_loss": 3.450910923702613, + "tokens_seen": 1862860800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442989898989899, + "loss": 2.7131, + "theoretical_loss": 3.4509006595767344, + "tokens_seen": 1862926336 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044296969696969694, + "loss": 2.4907, + "theoretical_loss": 3.4508903959130306, + "tokens_seen": 1862991872 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294949494949495, + "loss": 2.5954, + "theoretical_loss": 3.4508801327114647, + "tokens_seen": 1863057408 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1026014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.185640573501587, + "objective/train/theoretical_loss": 3.450869869972, + "objective/train/tokens_used": 221982176, + "theoretical_loss": 3.450869869972, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044292929292929296, + "loss": 2.6933, + "theoretical_loss": 3.450869869972, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429090909090909, + "loss": 2.906, + "theoretical_loss": 3.450859607694599, + "tokens_seen": 1863188480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428888888888889, + "loss": 2.5886, + "theoretical_loss": 3.4508493458792246, + "tokens_seen": 1863254016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428686868686869, + "loss": 2.6838, + "theoretical_loss": 3.4508390845258394, + "tokens_seen": 1863319552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044284848484848485, + "loss": 2.7344, + "theoretical_loss": 3.4508288236344073, + "tokens_seen": 1863385088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044282828282828286, + "loss": 2.7642, + "theoretical_loss": 3.4508185632048907, + "tokens_seen": 1863450624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428080808080808, + "loss": 2.5856, + "theoretical_loss": 3.4508083032372525, + "tokens_seen": 1863516160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044278787878787877, + "loss": 2.4, + "theoretical_loss": 3.4507980437314556, + "tokens_seen": 1863581696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427676767676768, + "loss": 2.8011, + "theoretical_loss": 3.4507877846874635, + "tokens_seen": 1863647232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044274747474747474, + "loss": 2.6666, + "theoretical_loss": 3.450777526105239, + "tokens_seen": 1863712768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044272727272727275, + "loss": 2.6065, + "theoretical_loss": 3.4507672679847445, + "tokens_seen": 1863778304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427070707070707, + "loss": 2.651, + "theoretical_loss": 3.450757010325944, + "tokens_seen": 1863843840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426868686868687, + "loss": 2.8708, + "theoretical_loss": 3.450746753128799, + "tokens_seen": 1863909376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426666666666667, + "loss": 2.6971, + "theoretical_loss": 3.450736496393274, + "tokens_seen": 1863974912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044264646464646463, + "loss": 2.8966, + "theoretical_loss": 3.4507262401193315, + "tokens_seen": 1864040448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044262626262626264, + "loss": 2.7598, + "theoretical_loss": 3.4507159843069344, + "tokens_seen": 1864105984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426060606060606, + "loss": 2.6357, + "theoretical_loss": 3.450705728956045, + "tokens_seen": 1864171520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425858585858586, + "loss": 2.6469, + "theoretical_loss": 3.4506954740666274, + "tokens_seen": 1864237056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044256565656565657, + "loss": 2.7603, + "theoretical_loss": 3.450685219638644, + "tokens_seen": 1864302592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425454545454545, + "loss": 2.5525, + "theoretical_loss": 3.450674965672059, + "tokens_seen": 1864368128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044252525252525254, + "loss": 2.4534, + "theoretical_loss": 3.4506647121668332, + "tokens_seen": 1864433664 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044250505050505055, + "loss": 2.7848, + "theoretical_loss": 3.450654459122932, + "tokens_seen": 1864499200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424848484848485, + "loss": 2.7757, + "theoretical_loss": 3.450644206540316, + "tokens_seen": 1864564736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246464646464646, + "loss": 2.3675, + "theoretical_loss": 3.4506339544189504, + "tokens_seen": 1864630272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424444444444444, + "loss": 2.9488, + "theoretical_loss": 3.450623702758797, + "tokens_seen": 1864695808 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1027424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.689354419708252, + "objective/train/theoretical_loss": 3.4506134515598195, + "objective/train/tokens_used": 223620576, + "theoretical_loss": 3.4506134515598195, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044242424242424243, + "loss": 2.8144, + "theoretical_loss": 3.4506134515598195, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044240404040404044, + "loss": 2.7622, + "theoretical_loss": 3.450603200821981, + "tokens_seen": 1864826880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423838383838384, + "loss": 2.4991, + "theoretical_loss": 3.4505929505452437, + "tokens_seen": 1864892416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044236363636363635, + "loss": 2.7188, + "theoretical_loss": 3.450582700729571, + "tokens_seen": 1864957952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423434343434343, + "loss": 2.4162, + "theoretical_loss": 3.4505724513749265, + "tokens_seen": 1865023488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423232323232324, + "loss": 2.6481, + "theoretical_loss": 3.4505622024812728, + "tokens_seen": 1865089024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230303030303033, + "loss": 2.8747, + "theoretical_loss": 3.450551954048573, + "tokens_seen": 1865154560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422828282828283, + "loss": 2.652, + "theoretical_loss": 3.4505417060767902, + "tokens_seen": 1865220096 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044226262626262625, + "loss": 2.5749, + "theoretical_loss": 3.4505314585658877, + "tokens_seen": 1865285632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422424242424242, + "loss": 2.5441, + "theoretical_loss": 3.450521211515828, + "tokens_seen": 1865351168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044222222222222227, + "loss": 2.6244, + "theoretical_loss": 3.4505109649265755, + "tokens_seen": 1865416704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422020202020202, + "loss": 2.8392, + "theoretical_loss": 3.4505007187980916, + "tokens_seen": 1865482240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421818181818182, + "loss": 2.7035, + "theoretical_loss": 3.4504904731303405, + "tokens_seen": 1865547776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044216161616161614, + "loss": 2.767, + "theoretical_loss": 3.4504802279232845, + "tokens_seen": 1865613312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421414141414142, + "loss": 2.6047, + "theoretical_loss": 3.4504699831768875, + "tokens_seen": 1865678848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044212121212121216, + "loss": 2.3905, + "theoretical_loss": 3.4504597388911122, + "tokens_seen": 1865744384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421010101010101, + "loss": 2.7433, + "theoretical_loss": 3.4504494950659215, + "tokens_seen": 1865809920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420808080808081, + "loss": 2.7318, + "theoretical_loss": 3.4504392517012796, + "tokens_seen": 1865875456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044206060606060603, + "loss": 2.7193, + "theoretical_loss": 3.450429008797148, + "tokens_seen": 1865940992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420404040404041, + "loss": 2.6802, + "theoretical_loss": 3.450418766353491, + "tokens_seen": 1866006528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044202020202020205, + "loss": 2.5807, + "theoretical_loss": 3.450408524370271, + "tokens_seen": 1866072064 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442, + "loss": 2.5472, + "theoretical_loss": 3.450398282847452, + "tokens_seen": 1866137600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044197979797979797, + "loss": 2.7356, + "theoretical_loss": 3.4503880417849966, + "tokens_seen": 1866203136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419595959595959, + "loss": 2.7341, + "theoretical_loss": 3.450377801182868, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441939393939394, + "loss": 2.8592, + "theoretical_loss": 3.4503675610410287, + "tokens_seen": 1866334208 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1028516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.410780429840088, + "objective/train/theoretical_loss": 3.4503573213594434, + "objective/train/tokens_used": 225258976, + "theoretical_loss": 3.4503573213594434, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044191919191919195, + "loss": 2.5768, + "theoretical_loss": 3.4503573213594434, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418989898989899, + "loss": 2.6665, + "theoretical_loss": 3.450347082138074, + "tokens_seen": 1866465280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044187878787878786, + "loss": 2.7986, + "theoretical_loss": 3.4503368433768835, + "tokens_seen": 1866530816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044185858585858587, + "loss": 2.6747, + "theoretical_loss": 3.450326605075836, + "tokens_seen": 1866596352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418383838383839, + "loss": 2.6103, + "theoretical_loss": 3.450316367234894, + "tokens_seen": 1866661888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044181818181818184, + "loss": 2.3946, + "theoretical_loss": 3.450306129854021, + "tokens_seen": 1866727424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417979797979798, + "loss": 2.7839, + "theoretical_loss": 3.45029589293318, + "tokens_seen": 1866792960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044177777777777775, + "loss": 2.5326, + "theoretical_loss": 3.4502856564723343, + "tokens_seen": 1866858496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044175757575757576, + "loss": 2.6544, + "theoretical_loss": 3.450275420471447, + "tokens_seen": 1866924032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417373737373738, + "loss": 2.5638, + "theoretical_loss": 3.4502651849304815, + "tokens_seen": 1866989568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044171717171717173, + "loss": 2.8576, + "theoretical_loss": 3.4502549498494, + "tokens_seen": 1867055104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416969696969697, + "loss": 2.5272, + "theoretical_loss": 3.4502447152281674, + "tokens_seen": 1867120640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416767676767677, + "loss": 2.7956, + "theoretical_loss": 3.4502344810667456, + "tokens_seen": 1867186176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044165656565656566, + "loss": 2.7912, + "theoretical_loss": 3.4502242473650986, + "tokens_seen": 1867251712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044163636363636367, + "loss": 2.6048, + "theoretical_loss": 3.4502140141231887, + "tokens_seen": 1867317248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416161616161616, + "loss": 2.652, + "theoretical_loss": 3.4502037813409796, + "tokens_seen": 1867382784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415959595959596, + "loss": 3.0165, + "theoretical_loss": 3.450193549018435, + "tokens_seen": 1867448320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415757575757576, + "loss": 2.4882, + "theoretical_loss": 3.4501833171555174, + "tokens_seen": 1867513856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044155555555555555, + "loss": 2.6933, + "theoretical_loss": 3.4501730857521906, + "tokens_seen": 1867579392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044153535353535356, + "loss": 2.5284, + "theoretical_loss": 3.4501628548084167, + "tokens_seen": 1867644928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415151515151515, + "loss": 2.6121, + "theoretical_loss": 3.4501526243241605, + "tokens_seen": 1867710464 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044149494949494953, + "loss": 2.6236, + "theoretical_loss": 3.4501423942993843, + "tokens_seen": 1867776000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414747474747475, + "loss": 2.724, + "theoretical_loss": 3.4501321647340513, + "tokens_seen": 1867841536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044145454545454544, + "loss": 2.5005, + "theoretical_loss": 3.4501219356281254, + "tokens_seen": 1867907072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044143434343434345, + "loss": 2.7259, + "theoretical_loss": 3.450111706981569, + "tokens_seen": 1867972608 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1029117, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.526183605194092, + "objective/train/theoretical_loss": 3.4501014787943456, + "objective/train/tokens_used": 226897376, + "theoretical_loss": 3.4501014787943456, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414141414141414, + "loss": 2.4437, + "theoretical_loss": 3.4501014787943456, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413939393939394, + "loss": 2.4661, + "theoretical_loss": 3.4500912510664192, + "tokens_seen": 1868103680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413737373737374, + "loss": 2.5983, + "theoretical_loss": 3.450081023797752, + "tokens_seen": 1868169216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044135353535353533, + "loss": 2.4983, + "theoretical_loss": 3.4500707969883084, + "tokens_seen": 1868234752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044133333333333335, + "loss": 2.5441, + "theoretical_loss": 3.4500605706380503, + "tokens_seen": 1868300288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044131313131313136, + "loss": 2.7335, + "theoretical_loss": 3.450050344746942, + "tokens_seen": 1868365824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412929292929293, + "loss": 2.6975, + "theoretical_loss": 3.450040119314947, + "tokens_seen": 1868431360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044127272727272727, + "loss": 2.7856, + "theoretical_loss": 3.450029894342028, + "tokens_seen": 1868496896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412525252525252, + "loss": 2.7327, + "theoretical_loss": 3.450019669828148, + "tokens_seen": 1868562432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044123232323232324, + "loss": 2.394, + "theoretical_loss": 3.4500094457732704, + "tokens_seen": 1868627968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044121212121212125, + "loss": 2.8116, + "theoretical_loss": 3.449999222177359, + "tokens_seen": 1868693504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411919191919192, + "loss": 2.6899, + "theoretical_loss": 3.449988999040377, + "tokens_seen": 1868759040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044117171717171716, + "loss": 2.7514, + "theoretical_loss": 3.449978776362288, + "tokens_seen": 1868824576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411515151515151, + "loss": 2.7282, + "theoretical_loss": 3.4499685541430543, + "tokens_seen": 1868890112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411313131313132, + "loss": 2.7994, + "theoretical_loss": 3.4499583323826397, + "tokens_seen": 1868955648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044111111111111114, + "loss": 2.7855, + "theoretical_loss": 3.4499481110810075, + "tokens_seen": 1869021184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410909090909091, + "loss": 2.6568, + "theoretical_loss": 3.4499378902381217, + "tokens_seen": 1869086720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044107070707070706, + "loss": 2.6813, + "theoretical_loss": 3.449927669853945, + "tokens_seen": 1869152256 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441050505050505, + "loss": 2.5984, + "theoretical_loss": 3.4499174499284404, + "tokens_seen": 1869217792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410303030303031, + "loss": 2.7613, + "theoretical_loss": 3.449907230461572, + "tokens_seen": 1869283328 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044101010101010103, + "loss": 2.708, + "theoretical_loss": 3.4498970114533023, + "tokens_seen": 1869348864 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440989898989899, + "loss": 2.5817, + "theoretical_loss": 3.4498867929035955, + "tokens_seen": 1869414400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044096969696969695, + "loss": 2.7155, + "theoretical_loss": 3.4498765748124143, + "tokens_seen": 1869479936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094949494949496, + "loss": 2.931, + "theoretical_loss": 3.449866357179723, + "tokens_seen": 1869545472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044092929292929297, + "loss": 2.7079, + "theoretical_loss": 3.4498561400054832, + "tokens_seen": 1869611008 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1029470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.821355104446411, + "objective/train/theoretical_loss": 3.44984592328966, + "objective/train/tokens_used": 228535776, + "theoretical_loss": 3.44984592328966, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044090909090909093, + "loss": 2.7606, + "theoretical_loss": 3.44984592328966, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408888888888889, + "loss": 2.6028, + "theoretical_loss": 3.449835707032216, + "tokens_seen": 1869742080 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086868686868684, + "loss": 2.73, + "theoretical_loss": 3.4498254912331148, + "tokens_seen": 1869807616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044084848484848485, + "loss": 2.5199, + "theoretical_loss": 3.4498152758923197, + "tokens_seen": 1869873152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044082828282828286, + "loss": 2.6589, + "theoretical_loss": 3.4498050610097932, + "tokens_seen": 1869938688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408080808080808, + "loss": 2.7433, + "theoretical_loss": 3.4497948465855, + "tokens_seen": 1870004224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407878787878788, + "loss": 2.7544, + "theoretical_loss": 3.4497846326194033, + "tokens_seen": 1870069760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407676767676768, + "loss": 2.585, + "theoretical_loss": 3.449774419111466, + "tokens_seen": 1870135296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044074747474747474, + "loss": 2.6629, + "theoretical_loss": 3.449764206061652, + "tokens_seen": 1870200832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044072727272727276, + "loss": 2.7832, + "theoretical_loss": 3.4497539934699235, + "tokens_seen": 1870266368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407070707070707, + "loss": 2.7545, + "theoretical_loss": 3.4497437813362453, + "tokens_seen": 1870331904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044068686868686867, + "loss": 2.8311, + "theoretical_loss": 3.44973356966058, + "tokens_seen": 1870397440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406666666666667, + "loss": 3.011, + "theoretical_loss": 3.449723358442892, + "tokens_seen": 1870462976 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044064646464646464, + "loss": 2.5545, + "theoretical_loss": 3.449713147683143, + "tokens_seen": 1870528512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044062626262626265, + "loss": 2.8115, + "theoretical_loss": 3.449702937381298, + "tokens_seen": 1870594048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406060606060606, + "loss": 2.8322, + "theoretical_loss": 3.44969272753732, + "tokens_seen": 1870659584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405858585858586, + "loss": 2.8499, + "theoretical_loss": 3.449682518151172, + "tokens_seen": 1870725120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405656565656566, + "loss": 2.9033, + "theoretical_loss": 3.4496723092228176, + "tokens_seen": 1870790656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044054545454545453, + "loss": 2.5401, + "theoretical_loss": 3.449662100752221, + "tokens_seen": 1870856192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044052525252525254, + "loss": 2.7093, + "theoretical_loss": 3.449651892739344, + "tokens_seen": 1870921728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405050505050505, + "loss": 2.4943, + "theoretical_loss": 3.449641685184152, + "tokens_seen": 1870987264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404848484848485, + "loss": 2.7457, + "theoretical_loss": 3.449631478086607, + "tokens_seen": 1871052800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044046464646464647, + "loss": 2.6909, + "theoretical_loss": 3.4496212714466727, + "tokens_seen": 1871118336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404444444444445, + "loss": 2.8722, + "theoretical_loss": 3.449611065264313, + "tokens_seen": 1871183872 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044042424242424243, + "loss": 2.7971, + "theoretical_loss": 3.449600859539491, + "tokens_seen": 1871249408 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1030757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.707038402557373, + "objective/train/theoretical_loss": 3.4495906542721704, + "objective/train/tokens_used": 230174176, + "theoretical_loss": 3.4495906542721704, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044040404040404044, + "loss": 2.6459, + "theoretical_loss": 3.4495906542721704, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403838383838384, + "loss": 2.5098, + "theoretical_loss": 3.4495804494623146, + "tokens_seen": 1871380480 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044036363636363636, + "loss": 2.7132, + "theoretical_loss": 3.4495702451098875, + "tokens_seen": 1871446016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044034343434343437, + "loss": 2.6227, + "theoretical_loss": 3.449560041214851, + "tokens_seen": 1871511552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403232323232323, + "loss": 2.7804, + "theoretical_loss": 3.4495498377771705, + "tokens_seen": 1871577088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044030303030303034, + "loss": 2.7748, + "theoretical_loss": 3.4495396347968086, + "tokens_seen": 1871642624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402828282828283, + "loss": 2.6117, + "theoretical_loss": 3.4495294322737293, + "tokens_seen": 1871708160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044026262626262625, + "loss": 2.6279, + "theoretical_loss": 3.449519230207895, + "tokens_seen": 1871773696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044024242424242426, + "loss": 2.7321, + "theoretical_loss": 3.4495090285992704, + "tokens_seen": 1871839232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402222222222222, + "loss": 2.7325, + "theoretical_loss": 3.4494988274478184, + "tokens_seen": 1871904768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044020202020202023, + "loss": 2.7222, + "theoretical_loss": 3.4494886267535025, + "tokens_seen": 1871970304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401818181818182, + "loss": 2.5767, + "theoretical_loss": 3.4494784265162863, + "tokens_seen": 1872035840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044016161616161614, + "loss": 2.425, + "theoretical_loss": 3.4494682267361334, + "tokens_seen": 1872101376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044014141414141416, + "loss": 2.5256, + "theoretical_loss": 3.449458027413007, + "tokens_seen": 1872166912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044012121212121217, + "loss": 2.5125, + "theoretical_loss": 3.4494478285468713, + "tokens_seen": 1872232448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401010101010101, + "loss": 2.6882, + "theoretical_loss": 3.4494376301376892, + "tokens_seen": 1872297984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400808080808081, + "loss": 2.6736, + "theoretical_loss": 3.4494274321854244, + "tokens_seen": 1872363520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044006060606060604, + "loss": 2.6651, + "theoretical_loss": 3.4494172346900402, + "tokens_seen": 1872429056 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044004040404040405, + "loss": 2.7091, + "theoretical_loss": 3.449407037651501, + "tokens_seen": 1872494592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044002020202020206, + "loss": 2.6583, + "theoretical_loss": 3.4493968410697695, + "tokens_seen": 1872560128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044, + "loss": 2.7233, + "theoretical_loss": 3.449386644944809, + "tokens_seen": 1872625664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043997979797979797, + "loss": 2.6792, + "theoretical_loss": 3.449376449276584, + "tokens_seen": 1872691200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043995959595959593, + "loss": 2.6617, + "theoretical_loss": 3.449366254065058, + "tokens_seen": 1872756736 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439939393939394, + "loss": 2.8347, + "theoretical_loss": 3.449356059310194, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043991919191919195, + "loss": 2.6349, + "theoretical_loss": 3.4493458650119555, + "tokens_seen": 1872887808 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1031383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7423930168151855, + "objective/train/theoretical_loss": 3.4493356711703065, + "objective/train/tokens_used": 231812576, + "theoretical_loss": 3.4493356711703065, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398989898989899, + "loss": 2.6487, + "theoretical_loss": 3.4493356711703065, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043987878787878787, + "loss": 2.6297, + "theoretical_loss": 3.4493254777852096, + "tokens_seen": 1873018880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398585858585858, + "loss": 2.742, + "theoretical_loss": 3.44931528485663, + "tokens_seen": 1873084416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398383838383839, + "loss": 2.7249, + "theoretical_loss": 3.4493050923845305, + "tokens_seen": 1873149952 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043981818181818184, + "loss": 2.4449, + "theoretical_loss": 3.4492949003688747, + "tokens_seen": 1873215488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397979797979798, + "loss": 2.6473, + "theoretical_loss": 3.4492847088096257, + "tokens_seen": 1873281024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043977777777777776, + "loss": 2.7582, + "theoretical_loss": 3.4492745177067476, + "tokens_seen": 1873346560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043975757575757577, + "loss": 2.7019, + "theoretical_loss": 3.449264327060204, + "tokens_seen": 1873412096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397373737373738, + "loss": 2.5911, + "theoretical_loss": 3.4492541368699587, + "tokens_seen": 1873477632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043971717171717174, + "loss": 2.5838, + "theoretical_loss": 3.449243947135974, + "tokens_seen": 1873543168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396969696969697, + "loss": 2.7736, + "theoretical_loss": 3.4492337578582157, + "tokens_seen": 1873608704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043967676767676765, + "loss": 2.7215, + "theoretical_loss": 3.449223569036646, + "tokens_seen": 1873674240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043965656565656566, + "loss": 2.6328, + "theoretical_loss": 3.449213380671229, + "tokens_seen": 1873739776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043963636363636367, + "loss": 3.0429, + "theoretical_loss": 3.4492031927619275, + "tokens_seen": 1873805312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043961616161616163, + "loss": 2.6799, + "theoretical_loss": 3.4491930053087065, + "tokens_seen": 1873870848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395959595959596, + "loss": 2.6578, + "theoretical_loss": 3.4491828183115283, + "tokens_seen": 1873936384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395757575757576, + "loss": 2.5755, + "theoretical_loss": 3.449172631770357, + "tokens_seen": 1874001920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043955555555555555, + "loss": 2.5907, + "theoretical_loss": 3.4491624456851566, + "tokens_seen": 1874067456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043953535353535357, + "loss": 2.6247, + "theoretical_loss": 3.4491522600558904, + "tokens_seen": 1874132992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395151515151515, + "loss": 2.7296, + "theoretical_loss": 3.4491420748825226, + "tokens_seen": 1874198528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394949494949495, + "loss": 2.5939, + "theoretical_loss": 3.449131890165016, + "tokens_seen": 1874264064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394747474747475, + "loss": 2.6861, + "theoretical_loss": 3.449121705903335, + "tokens_seen": 1874329600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043945454545454545, + "loss": 2.6701, + "theoretical_loss": 3.4491115220974424, + "tokens_seen": 1874395136 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043943434343434346, + "loss": 2.4645, + "theoretical_loss": 3.449101338747303, + "tokens_seen": 1874460672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394141414141414, + "loss": 2.5077, + "theoretical_loss": 3.449091155852879, + "tokens_seen": 1874526208 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1032514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0022151470184326, + "objective/train/theoretical_loss": 3.4490809734141354, + "objective/train/tokens_used": 233450976, + "theoretical_loss": 3.4490809734141354, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393939393939394, + "loss": 2.7027, + "theoretical_loss": 3.4490809734141354, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393737373737374, + "loss": 2.6109, + "theoretical_loss": 3.4490707914310352, + "tokens_seen": 1874657280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043935353535353534, + "loss": 2.7608, + "theoretical_loss": 3.449060609903543, + "tokens_seen": 1874722816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933333333333335, + "loss": 2.5745, + "theoretical_loss": 3.4490504288316215, + "tokens_seen": 1874788352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393131313131313, + "loss": 2.6831, + "theoretical_loss": 3.4490402482152343, + "tokens_seen": 1874853888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392929292929293, + "loss": 2.8604, + "theoretical_loss": 3.4490300680543458, + "tokens_seen": 1874919424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392727272727273, + "loss": 2.5409, + "theoretical_loss": 3.449019888348919, + "tokens_seen": 1874984960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043925252525252523, + "loss": 2.5154, + "theoretical_loss": 3.4490097090989185, + "tokens_seen": 1875050496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043923232323232324, + "loss": 2.6237, + "theoretical_loss": 3.4489995303043073, + "tokens_seen": 1875116032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921212121212125, + "loss": 2.4967, + "theoretical_loss": 3.448989351965049, + "tokens_seen": 1875181568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391919191919192, + "loss": 2.6085, + "theoretical_loss": 3.448979174081108, + "tokens_seen": 1875247104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043917171717171717, + "loss": 2.7374, + "theoretical_loss": 3.4489689966524475, + "tokens_seen": 1875312640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391515151515151, + "loss": 2.3794, + "theoretical_loss": 3.4489588196790315, + "tokens_seen": 1875378176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043913131313131314, + "loss": 2.6384, + "theoretical_loss": 3.448948643160823, + "tokens_seen": 1875443712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043911111111111115, + "loss": 2.7276, + "theoretical_loss": 3.448938467097787, + "tokens_seen": 1875509248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390909090909091, + "loss": 2.5831, + "theoretical_loss": 3.448928291489886, + "tokens_seen": 1875574784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043907070707070706, + "loss": 2.431, + "theoretical_loss": 3.448918116337085, + "tokens_seen": 1875640320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439050505050505, + "loss": 2.662, + "theoretical_loss": 3.448907941639346, + "tokens_seen": 1875705856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390303030303031, + "loss": 2.4266, + "theoretical_loss": 3.448897767396635, + "tokens_seen": 1875771392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043901010101010104, + "loss": 2.7484, + "theoretical_loss": 3.448887593608914, + "tokens_seen": 1875836928 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438989898989899, + "loss": 2.6149, + "theoretical_loss": 3.448877420276147, + "tokens_seen": 1875902464 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043896969696969695, + "loss": 2.6773, + "theoretical_loss": 3.448867247398298, + "tokens_seen": 1875968000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389494949494949, + "loss": 2.6709, + "theoretical_loss": 3.448857074975332, + "tokens_seen": 1876033536 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438929292929293, + "loss": 2.669, + "theoretical_loss": 3.448846903007211, + "tokens_seen": 1876099072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043890909090909093, + "loss": 2.5903, + "theoretical_loss": 3.448836731493899, + "tokens_seen": 1876164608 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1033016, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4885294437408447, + "objective/train/theoretical_loss": 3.44882656043536, + "objective/train/tokens_used": 235089376, + "theoretical_loss": 3.44882656043536, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388888888888889, + "loss": 2.6458, + "theoretical_loss": 3.44882656043536, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043886868686868685, + "loss": 2.7158, + "theoretical_loss": 3.4488163898315585, + "tokens_seen": 1876295680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388484848484848, + "loss": 2.576, + "theoretical_loss": 3.4488062196824574, + "tokens_seen": 1876361216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043882828282828287, + "loss": 2.5016, + "theoretical_loss": 3.448796049988021, + "tokens_seen": 1876426752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388080808080808, + "loss": 2.5476, + "theoretical_loss": 3.448785880748213, + "tokens_seen": 1876492288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387878787878788, + "loss": 2.7925, + "theoretical_loss": 3.4487757119629965, + "tokens_seen": 1876557824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043876767676767674, + "loss": 2.7359, + "theoretical_loss": 3.4487655436323363, + "tokens_seen": 1876623360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387474747474748, + "loss": 2.5205, + "theoretical_loss": 3.448755375756196, + "tokens_seen": 1876688896 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043872727272727276, + "loss": 2.7716, + "theoretical_loss": 3.4487452083345396, + "tokens_seen": 1876754432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387070707070707, + "loss": 2.6872, + "theoretical_loss": 3.44873504136733, + "tokens_seen": 1876819968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386868686868687, + "loss": 2.5491, + "theoretical_loss": 3.4487248748545314, + "tokens_seen": 1876885504 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043866666666666663, + "loss": 2.7096, + "theoretical_loss": 3.448714708796108, + "tokens_seen": 1876951040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386464646464647, + "loss": 2.6948, + "theoretical_loss": 3.4487045431920236, + "tokens_seen": 1877016576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043862626262626265, + "loss": 2.4952, + "theoretical_loss": 3.4486943780422417, + "tokens_seen": 1877082112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386060606060606, + "loss": 2.6549, + "theoretical_loss": 3.4486842133467266, + "tokens_seen": 1877147648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043858585858585857, + "loss": 2.4912, + "theoretical_loss": 3.4486740491054415, + "tokens_seen": 1877213184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385656565656566, + "loss": 2.7272, + "theoretical_loss": 3.448663885318351, + "tokens_seen": 1877278720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385454545454546, + "loss": 2.9654, + "theoretical_loss": 3.448653721985418, + "tokens_seen": 1877344256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043852525252525255, + "loss": 2.7963, + "theoretical_loss": 3.448643559106607, + "tokens_seen": 1877409792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385050505050505, + "loss": 2.6008, + "theoretical_loss": 3.448633396681882, + "tokens_seen": 1877475328 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043848484848484846, + "loss": 2.6553, + "theoretical_loss": 3.4486232347112065, + "tokens_seen": 1877540864 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043846464646464647, + "loss": 2.7418, + "theoretical_loss": 3.448613073194544, + "tokens_seen": 1877606400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384444444444445, + "loss": 2.9507, + "theoretical_loss": 3.4486029121318595, + "tokens_seen": 1877671936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043842424242424244, + "loss": 2.7075, + "theoretical_loss": 3.448592751523116, + "tokens_seen": 1877737472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384040404040404, + "loss": 2.6087, + "theoretical_loss": 3.4485825913682775, + "tokens_seen": 1877803008 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1034169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.757983922958374, + "objective/train/theoretical_loss": 3.448572431667308, + "objective/train/tokens_used": 236727776, + "theoretical_loss": 3.448572431667308, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383838383838384, + "loss": 2.4029, + "theoretical_loss": 3.448572431667308, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043836363636363636, + "loss": 2.6623, + "theoretical_loss": 3.448562272420171, + "tokens_seen": 1877934080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383434343434344, + "loss": 2.6641, + "theoretical_loss": 3.4485521136268313, + "tokens_seen": 1877999616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043832323232323233, + "loss": 2.6791, + "theoretical_loss": 3.4485419552872525, + "tokens_seen": 1878065152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383030303030303, + "loss": 2.5486, + "theoretical_loss": 3.4485317974013974, + "tokens_seen": 1878130688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382828282828283, + "loss": 2.7649, + "theoretical_loss": 3.4485216399692313, + "tokens_seen": 1878196224 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043826262626262626, + "loss": 2.7451, + "theoretical_loss": 3.4485114829907175, + "tokens_seen": 1878261760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043824242424242427, + "loss": 2.7333, + "theoretical_loss": 3.44850132646582, + "tokens_seen": 1878327296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382222222222222, + "loss": 2.6926, + "theoretical_loss": 3.4484911703945027, + "tokens_seen": 1878392832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043820202020202024, + "loss": 2.4957, + "theoretical_loss": 3.4484810147767293, + "tokens_seen": 1878458368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381818181818182, + "loss": 2.5296, + "theoretical_loss": 3.448470859612464, + "tokens_seen": 1878523904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043816161616161615, + "loss": 2.9351, + "theoretical_loss": 3.4484607049016702, + "tokens_seen": 1878589440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043814141414141416, + "loss": 2.7707, + "theoretical_loss": 3.448450550644313, + "tokens_seen": 1878654976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381212121212121, + "loss": 2.5848, + "theoretical_loss": 3.4484403968403554, + "tokens_seen": 1878720512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043810101010101013, + "loss": 2.7145, + "theoretical_loss": 3.448430243489762, + "tokens_seen": 1878786048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380808080808081, + "loss": 2.7403, + "theoretical_loss": 3.4484200905924953, + "tokens_seen": 1878851584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043806060606060604, + "loss": 2.778, + "theoretical_loss": 3.448409938148521, + "tokens_seen": 1878917120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043804040404040405, + "loss": 2.4632, + "theoretical_loss": 3.448399786157802, + "tokens_seen": 1878982656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043802020202020206, + "loss": 2.5384, + "theoretical_loss": 3.4483896346203027, + "tokens_seen": 1879048192 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438, + "loss": 2.6639, + "theoretical_loss": 3.448379483535987, + "tokens_seen": 1879113728 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437979797979798, + "loss": 2.6435, + "theoretical_loss": 3.4483693329048184, + "tokens_seen": 1879179264 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043795959595959593, + "loss": 2.61, + "theoretical_loss": 3.4483591827267617, + "tokens_seen": 1879244800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043793939393939395, + "loss": 2.7294, + "theoretical_loss": 3.4483490330017803, + "tokens_seen": 1879310336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043791919191919196, + "loss": 2.4736, + "theoretical_loss": 3.4483388837298383, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378989898989899, + "loss": 2.7058, + "theoretical_loss": 3.4483287349108993, + "tokens_seen": 1879441408 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1034876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5990076065063477, + "objective/train/theoretical_loss": 3.448318586544928, + "objective/train/tokens_used": 238366176, + "theoretical_loss": 3.448318586544928, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043787878787878787, + "loss": 2.8493, + "theoretical_loss": 3.448318586544928, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785858585858583, + "loss": 2.5927, + "theoretical_loss": 3.448308438631888, + "tokens_seen": 1879572480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378383838383839, + "loss": 2.7262, + "theoretical_loss": 3.4482982911717435, + "tokens_seen": 1879638016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781818181818185, + "loss": 2.8021, + "theoretical_loss": 3.4482881441644584, + "tokens_seen": 1879703552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377979797979798, + "loss": 2.8252, + "theoretical_loss": 3.4482779976099964, + "tokens_seen": 1879769088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777777777777776, + "loss": 2.7085, + "theoretical_loss": 3.448267851508322, + "tokens_seen": 1879834624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377575757575757, + "loss": 2.636, + "theoretical_loss": 3.448257705859399, + "tokens_seen": 1879900160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377373737373738, + "loss": 2.7353, + "theoretical_loss": 3.4482475606631913, + "tokens_seen": 1879965696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043771717171717174, + "loss": 2.6714, + "theoretical_loss": 3.448237415919663, + "tokens_seen": 1880031232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376969696969697, + "loss": 2.5129, + "theoretical_loss": 3.448227271628778, + "tokens_seen": 1880096768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043767676767676766, + "loss": 2.812, + "theoretical_loss": 3.4482171277905005, + "tokens_seen": 1880162304 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043765656565656567, + "loss": 2.6062, + "theoretical_loss": 3.448206984404795, + "tokens_seen": 1880227840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376363636363637, + "loss": 2.7784, + "theoretical_loss": 3.448196841471624, + "tokens_seen": 1880293376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043761616161616164, + "loss": 2.6684, + "theoretical_loss": 3.4481866989909533, + "tokens_seen": 1880358912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375959595959596, + "loss": 2.5526, + "theoretical_loss": 3.4481765569627463, + "tokens_seen": 1880424448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043757575757575755, + "loss": 2.494, + "theoretical_loss": 3.448166415386966, + "tokens_seen": 1880489984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043755555555555556, + "loss": 2.8102, + "theoretical_loss": 3.448156274263578, + "tokens_seen": 1880555520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043753535353535357, + "loss": 2.7983, + "theoretical_loss": 3.4481461335925463, + "tokens_seen": 1880621056 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043751515151515153, + "loss": 2.645, + "theoretical_loss": 3.4481359933738336, + "tokens_seen": 1880686592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374949494949495, + "loss": 2.5337, + "theoretical_loss": 3.4481258536074053, + "tokens_seen": 1880752128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374747474747475, + "loss": 2.8128, + "theoretical_loss": 3.448115714293225, + "tokens_seen": 1880817664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043745454545454545, + "loss": 2.5904, + "theoretical_loss": 3.4481055754312564, + "tokens_seen": 1880883200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043743434343434346, + "loss": 2.8262, + "theoretical_loss": 3.448095437021464, + "tokens_seen": 1880948736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374141414141414, + "loss": 2.548, + "theoretical_loss": 3.448085299063812, + "tokens_seen": 1881014272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373939393939394, + "loss": 2.2551, + "theoretical_loss": 3.448075161558264, + "tokens_seen": 1881079808 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1035977, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.632066488265991, + "objective/train/theoretical_loss": 3.448065024504784, + "objective/train/tokens_used": 240004576, + "theoretical_loss": 3.448065024504784, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373737373737374, + "loss": 2.8682, + "theoretical_loss": 3.448065024504784, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043735353535353535, + "loss": 2.567, + "theoretical_loss": 3.448054887903337, + "tokens_seen": 1881210880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043733333333333336, + "loss": 2.6562, + "theoretical_loss": 3.448044751753886, + "tokens_seen": 1881276416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373131313131313, + "loss": 2.5353, + "theoretical_loss": 3.448034616056396, + "tokens_seen": 1881341952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372929292929293, + "loss": 2.6606, + "theoretical_loss": 3.448024480810831, + "tokens_seen": 1881407488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372727272727273, + "loss": 2.7143, + "theoretical_loss": 3.4480143460171546, + "tokens_seen": 1881473024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043725252525252524, + "loss": 2.723, + "theoretical_loss": 3.448004211675331, + "tokens_seen": 1881538560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043723232323232325, + "loss": 2.8467, + "theoretical_loss": 3.447994077785325, + "tokens_seen": 1881604096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372121212121212, + "loss": 2.727, + "theoretical_loss": 3.4479839443471, + "tokens_seen": 1881669632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371919191919192, + "loss": 2.7398, + "theoretical_loss": 3.4479738113606198, + "tokens_seen": 1881735168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371717171717172, + "loss": 2.4879, + "theoretical_loss": 3.4479636788258494, + "tokens_seen": 1881800704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043715151515151513, + "loss": 2.8289, + "theoretical_loss": 3.4479535467427525, + "tokens_seen": 1881866240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043713131313131314, + "loss": 2.7452, + "theoretical_loss": 3.4479434151112933, + "tokens_seen": 1881931776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043711111111111115, + "loss": 2.546, + "theoretical_loss": 3.4479332839314365, + "tokens_seen": 1881997312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370909090909091, + "loss": 2.6797, + "theoretical_loss": 3.447923153203145, + "tokens_seen": 1882062848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043707070707070707, + "loss": 2.7142, + "theoretical_loss": 3.447913022926384, + "tokens_seen": 1882128384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370505050505051, + "loss": 2.75, + "theoretical_loss": 3.4479028931011175, + "tokens_seen": 1882193920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043703030303030303, + "loss": 2.8057, + "theoretical_loss": 3.4478927637273094, + "tokens_seen": 1882259456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043701010101010105, + "loss": 2.827, + "theoretical_loss": 3.447882634804923, + "tokens_seen": 1882324992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000436989898989899, + "loss": 2.4703, + "theoretical_loss": 3.4478725063339244, + "tokens_seen": 1882390528 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043696969696969696, + "loss": 2.749, + "theoretical_loss": 3.4478623783142766, + "tokens_seen": 1882456064 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043694949494949497, + "loss": 2.6755, + "theoretical_loss": 3.447852250745944, + "tokens_seen": 1882521600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043692929292929293, + "loss": 2.6679, + "theoretical_loss": 3.4478421236288908, + "tokens_seen": 1882587136 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043690909090909094, + "loss": 2.7283, + "theoretical_loss": 3.447831996963081, + "tokens_seen": 1882652672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004368888888888889, + "loss": 2.8893, + "theoretical_loss": 3.447821870748479, + "tokens_seen": 1882718208 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1036495, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.311638116836548, + "objective/train/theoretical_loss": 3.4478117449850485, + "objective/train/tokens_used": 241642976, + "theoretical_loss": 3.4478117449850485, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043686868686868685, + "loss": 2.602, + "theoretical_loss": 3.4478117449850485, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043684848484848486, + "loss": 2.7245, + "theoretical_loss": 3.447801619672754, + "tokens_seen": 1882849280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368282828282829, + "loss": 2.6415, + "theoretical_loss": 3.44779149481156, + "tokens_seen": 1882914816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043680808080808083, + "loss": 2.8219, + "theoretical_loss": 3.4477813704014313, + "tokens_seen": 1882980352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367878787878788, + "loss": 2.4011, + "theoretical_loss": 3.44777124644233, + "tokens_seen": 1883045888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043676767676767674, + "loss": 2.6543, + "theoretical_loss": 3.4477611229342227, + "tokens_seen": 1883111424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043674747474747476, + "loss": 2.66, + "theoretical_loss": 3.447750999877072, + "tokens_seen": 1883176960 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043672727272727277, + "loss": 3.0482, + "theoretical_loss": 3.447740877270842, + "tokens_seen": 1883242496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367070707070707, + "loss": 2.9646, + "theoretical_loss": 3.4477307551154985, + "tokens_seen": 1883308032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366868686868687, + "loss": 2.713, + "theoretical_loss": 3.4477206334110044, + "tokens_seen": 1883373568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043666666666666664, + "loss": 2.5537, + "theoretical_loss": 3.4477105121573244, + "tokens_seen": 1883439104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366464646464647, + "loss": 2.7314, + "theoretical_loss": 3.447700391354423, + "tokens_seen": 1883504640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043662626262626266, + "loss": 2.5515, + "theoretical_loss": 3.4476902710022634, + "tokens_seen": 1883570176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366060606060606, + "loss": 2.6342, + "theoretical_loss": 3.447680151100811, + "tokens_seen": 1883635712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365858585858586, + "loss": 2.5765, + "theoretical_loss": 3.4476700316500293, + "tokens_seen": 1883701248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656565656565653, + "loss": 2.624, + "theoretical_loss": 3.447659912649883, + "tokens_seen": 1883766784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365454545454546, + "loss": 2.7815, + "theoretical_loss": 3.447649794100336, + "tokens_seen": 1883832320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043652525252525255, + "loss": 2.4913, + "theoretical_loss": 3.4476396760013532, + "tokens_seen": 1883897856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365050505050505, + "loss": 2.7279, + "theoretical_loss": 3.447629558352898, + "tokens_seen": 1883963392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648484848484847, + "loss": 2.5147, + "theoretical_loss": 3.4476194411549352, + "tokens_seen": 1884028928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364646464646465, + "loss": 2.669, + "theoretical_loss": 3.447609324407429, + "tokens_seen": 1884094464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364444444444445, + "loss": 2.5508, + "theoretical_loss": 3.4475992081103435, + "tokens_seen": 1884160000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043642424242424244, + "loss": 2.7432, + "theoretical_loss": 3.4475890922636436, + "tokens_seen": 1884225536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364040404040404, + "loss": 2.6209, + "theoretical_loss": 3.447578976867293, + "tokens_seen": 1884291072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043638383838383836, + "loss": 2.4768, + "theoretical_loss": 3.4475688619212557, + "tokens_seen": 1884356608 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1037793, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5640203952789307, + "objective/train/theoretical_loss": 3.4475587474254965, + "objective/train/tokens_used": 243281376, + "theoretical_loss": 3.4475587474254965, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043636363636363637, + "loss": 2.8154, + "theoretical_loss": 3.4475587474254965, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363434343434344, + "loss": 2.7487, + "theoretical_loss": 3.4475486333799794, + "tokens_seen": 1884487680 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632323232323234, + "loss": 2.6674, + "theoretical_loss": 3.4475385197846693, + "tokens_seen": 1884553216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363030303030303, + "loss": 2.6732, + "theoretical_loss": 3.4475284066395293, + "tokens_seen": 1884618752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362828282828283, + "loss": 2.7621, + "theoretical_loss": 3.4475182939445252, + "tokens_seen": 1884684288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043626262626262626, + "loss": 2.6823, + "theoretical_loss": 3.4475081816996207, + "tokens_seen": 1884749824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362424242424243, + "loss": 2.9771, + "theoretical_loss": 3.4474980699047792, + "tokens_seen": 1884815360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043622222222222223, + "loss": 2.6468, + "theoretical_loss": 3.4474879585599667, + "tokens_seen": 1884880896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362020202020202, + "loss": 2.6748, + "theoretical_loss": 3.447477847665146, + "tokens_seen": 1884946432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361818181818182, + "loss": 2.6497, + "theoretical_loss": 3.4474677372202827, + "tokens_seen": 1885011968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043616161616161616, + "loss": 2.4553, + "theoretical_loss": 3.4474576272253405, + "tokens_seen": 1885077504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043614141414141417, + "loss": 2.7674, + "theoretical_loss": 3.4474475176802835, + "tokens_seen": 1885143040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361212121212121, + "loss": 2.7152, + "theoretical_loss": 3.4474374085850763, + "tokens_seen": 1885208576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043610101010101013, + "loss": 2.5882, + "theoretical_loss": 3.447427299939683, + "tokens_seen": 1885274112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360808080808081, + "loss": 2.7256, + "theoretical_loss": 3.447417191744069, + "tokens_seen": 1885339648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043606060606060605, + "loss": 2.6481, + "theoretical_loss": 3.4474070839981974, + "tokens_seen": 1885405184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043604040404040406, + "loss": 2.6377, + "theoretical_loss": 3.447396976702033, + "tokens_seen": 1885470720 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436020202020202, + "loss": 2.8219, + "theoretical_loss": 3.4473868698555403, + "tokens_seen": 1885536256 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436, + "loss": 2.7823, + "theoretical_loss": 3.447376763458683, + "tokens_seen": 1885601792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435979797979798, + "loss": 2.7027, + "theoretical_loss": 3.4473666575114263, + "tokens_seen": 1885667328 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043595959595959594, + "loss": 2.7249, + "theoretical_loss": 3.447356552013735, + "tokens_seen": 1885732864 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043593939393939395, + "loss": 2.7471, + "theoretical_loss": 3.4473464469655717, + "tokens_seen": 1885798400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043591919191919196, + "loss": 2.6487, + "theoretical_loss": 3.447336342366903, + "tokens_seen": 1885863936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358989898989899, + "loss": 2.4869, + "theoretical_loss": 3.4473262382176912, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358787878787879, + "loss": 2.766, + "theoretical_loss": 3.447316134517902, + "tokens_seen": 1885995008 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1038946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.652374267578125, + "objective/train/theoretical_loss": 3.447306031267499, + "objective/train/tokens_used": 244919776, + "theoretical_loss": 3.447306031267499, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043585858585858583, + "loss": 2.7403, + "theoretical_loss": 3.447306031267499, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043583838383838384, + "loss": 2.7421, + "theoretical_loss": 3.4472959284664473, + "tokens_seen": 1886126080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043581818181818186, + "loss": 2.7329, + "theoretical_loss": 3.4472858261147112, + "tokens_seen": 1886191616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357979797979798, + "loss": 2.6731, + "theoretical_loss": 3.447275724212255, + "tokens_seen": 1886257152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043577777777777777, + "loss": 2.684, + "theoretical_loss": 3.4472656227590424, + "tokens_seen": 1886322688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357575757575757, + "loss": 2.4135, + "theoretical_loss": 3.4472555217550394, + "tokens_seen": 1886388224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357373737373738, + "loss": 2.6845, + "theoretical_loss": 3.4472454212002086, + "tokens_seen": 1886453760 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043571717171717175, + "loss": 2.8128, + "theoretical_loss": 3.4472353210945155, + "tokens_seen": 1886519296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356969696969697, + "loss": 2.6487, + "theoretical_loss": 3.4472252214379244, + "tokens_seen": 1886584832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043567676767676766, + "loss": 2.8778, + "theoretical_loss": 3.4472151222304, + "tokens_seen": 1886650368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356565656565656, + "loss": 2.8099, + "theoretical_loss": 3.447205023471906, + "tokens_seen": 1886715904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356363636363637, + "loss": 2.6793, + "theoretical_loss": 3.4471949251624077, + "tokens_seen": 1886781440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043561616161616164, + "loss": 2.8376, + "theoretical_loss": 3.4471848273018684, + "tokens_seen": 1886846976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355959595959596, + "loss": 2.816, + "theoretical_loss": 3.4471747298902535, + "tokens_seen": 1886912512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043557575757575755, + "loss": 2.5422, + "theoretical_loss": 3.447164632927527, + "tokens_seen": 1886978048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355555555555555, + "loss": 2.7872, + "theoretical_loss": 3.447154536413654, + "tokens_seen": 1887043584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355353535353536, + "loss": 2.5499, + "theoretical_loss": 3.4471444403485982, + "tokens_seen": 1887109120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043551515151515153, + "loss": 2.8599, + "theoretical_loss": 3.447134344732324, + "tokens_seen": 1887174656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354949494949495, + "loss": 2.5081, + "theoretical_loss": 3.447124249564797, + "tokens_seen": 1887240192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043547474747474745, + "loss": 2.5106, + "theoretical_loss": 3.44711415484598, + "tokens_seen": 1887305728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043545454545454546, + "loss": 2.8259, + "theoretical_loss": 3.447104060575839, + "tokens_seen": 1887371264 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043543434343434347, + "loss": 2.6445, + "theoretical_loss": 3.4470939667543377, + "tokens_seen": 1887436800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354141414141414, + "loss": 2.6661, + "theoretical_loss": 3.44708387338144, + "tokens_seen": 1887502336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353939393939394, + "loss": 2.4127, + "theoretical_loss": 3.447073780457112, + "tokens_seen": 1887567872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043537373737373734, + "loss": 2.5971, + "theoretical_loss": 3.4470636879813172, + "tokens_seen": 1887633408 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1039523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.781736135482788, + "objective/train/theoretical_loss": 3.44705359595402, + "objective/train/tokens_used": 246558176, + "theoretical_loss": 3.44705359595402, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043535353535353535, + "loss": 2.7614, + "theoretical_loss": 3.44705359595402, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043533333333333336, + "loss": 2.8807, + "theoretical_loss": 3.447043504375185, + "tokens_seen": 1887764480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353131313131313, + "loss": 2.7627, + "theoretical_loss": 3.447033413244777, + "tokens_seen": 1887830016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352929292929293, + "loss": 2.763, + "theoretical_loss": 3.4470233225627602, + "tokens_seen": 1887895552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352727272727273, + "loss": 2.7779, + "theoretical_loss": 3.447013232329099, + "tokens_seen": 1887961088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352525252525253, + "loss": 2.7414, + "theoretical_loss": 3.4470031425437586, + "tokens_seen": 1888026624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043523232323232325, + "loss": 2.8604, + "theoretical_loss": 3.446993053206703, + "tokens_seen": 1888092160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352121212121212, + "loss": 2.4655, + "theoretical_loss": 3.4469829643178964, + "tokens_seen": 1888157696 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043519191919191917, + "loss": 2.7379, + "theoretical_loss": 3.4469728758773037, + "tokens_seen": 1888223232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351717171717172, + "loss": 2.8034, + "theoretical_loss": 3.44696278788489, + "tokens_seen": 1888288768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351515151515152, + "loss": 2.7015, + "theoretical_loss": 3.446952700340619, + "tokens_seen": 1888354304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043513131313131315, + "loss": 2.6724, + "theoretical_loss": 3.446942613244455, + "tokens_seen": 1888419840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351111111111111, + "loss": 2.9569, + "theoretical_loss": 3.4469325265963637, + "tokens_seen": 1888485376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350909090909091, + "loss": 2.6612, + "theoretical_loss": 3.4469224403963086, + "tokens_seen": 1888550912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043507070707070707, + "loss": 2.868, + "theoretical_loss": 3.4469123546442546, + "tokens_seen": 1888616448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350505050505051, + "loss": 2.6853, + "theoretical_loss": 3.4469022693401667, + "tokens_seen": 1888681984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043503030303030304, + "loss": 2.698, + "theoretical_loss": 3.446892184484009, + "tokens_seen": 1888747520 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435010101010101, + "loss": 2.7729, + "theoretical_loss": 3.4468821000757464, + "tokens_seen": 1888813056 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434989898989899, + "loss": 2.7598, + "theoretical_loss": 3.4468720161153428, + "tokens_seen": 1888878592 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043496969696969696, + "loss": 2.5709, + "theoretical_loss": 3.446861932602763, + "tokens_seen": 1888944128 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434949494949495, + "loss": 2.7712, + "theoretical_loss": 3.446851849537972, + "tokens_seen": 1889009664 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043492929292929293, + "loss": 2.7276, + "theoretical_loss": 3.446841766920934, + "tokens_seen": 1889075200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043490909090909094, + "loss": 2.6929, + "theoretical_loss": 3.446831684751614, + "tokens_seen": 1889140736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348888888888889, + "loss": 2.6453, + "theoretical_loss": 3.446821603029976, + "tokens_seen": 1889206272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043486868686868686, + "loss": 2.5445, + "theoretical_loss": 3.4468115217559854, + "tokens_seen": 1889271808 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1040960, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6786797046661377, + "objective/train/theoretical_loss": 3.446801440929606, + "objective/train/tokens_used": 248196576, + "theoretical_loss": 3.446801440929606, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043484848484848487, + "loss": 2.8896, + "theoretical_loss": 3.446801440929606, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348282828282828, + "loss": 2.7437, + "theoretical_loss": 3.4467913605508027, + "tokens_seen": 1889402880 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480808080808084, + "loss": 2.7, + "theoretical_loss": 3.4467812806195397, + "tokens_seen": 1889468416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347878787878788, + "loss": 2.7376, + "theoretical_loss": 3.4467712011357827, + "tokens_seen": 1889533952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476767676767675, + "loss": 2.7908, + "theoretical_loss": 3.4467611220994954, + "tokens_seen": 1889599488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043474747474747476, + "loss": 2.7504, + "theoretical_loss": 3.446751043510642, + "tokens_seen": 1889665024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472727272727277, + "loss": 2.4802, + "theoretical_loss": 3.4467409653691887, + "tokens_seen": 1889730560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043470707070707073, + "loss": 2.7347, + "theoretical_loss": 3.446730887675099, + "tokens_seen": 1889796096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346868686868687, + "loss": 2.6914, + "theoretical_loss": 3.4467208104283373, + "tokens_seen": 1889861632 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043466666666666664, + "loss": 2.8039, + "theoretical_loss": 3.446710733628869, + "tokens_seen": 1889927168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043464646464646465, + "loss": 2.9309, + "theoretical_loss": 3.446700657276658, + "tokens_seen": 1889992704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043462626262626267, + "loss": 2.7041, + "theoretical_loss": 3.4466905813716697, + "tokens_seen": 1890058240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346060606060606, + "loss": 2.8169, + "theoretical_loss": 3.446680505913868, + "tokens_seen": 1890123776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345858585858586, + "loss": 2.67, + "theoretical_loss": 3.446670430903218, + "tokens_seen": 1890189312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043456565656565654, + "loss": 2.7819, + "theoretical_loss": 3.4466603563396845, + "tokens_seen": 1890254848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345454545454546, + "loss": 2.817, + "theoretical_loss": 3.4466502822232323, + "tokens_seen": 1890320384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043452525252525256, + "loss": 2.8273, + "theoretical_loss": 3.4466402085538252, + "tokens_seen": 1890385920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345050505050505, + "loss": 2.863, + "theoretical_loss": 3.446630135331428, + "tokens_seen": 1890451456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043448484848484847, + "loss": 2.6555, + "theoretical_loss": 3.4466200625560064, + "tokens_seen": 1890516992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043446464646464643, + "loss": 2.7843, + "theoretical_loss": 3.446609990227524, + "tokens_seen": 1890582528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344444444444445, + "loss": 2.6445, + "theoretical_loss": 3.446599918345946, + "tokens_seen": 1890648064 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043442424242424245, + "loss": 2.6006, + "theoretical_loss": 3.4465898469112366, + "tokens_seen": 1890713600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344040404040404, + "loss": 2.928, + "theoretical_loss": 3.446579775923362, + "tokens_seen": 1890779136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043438383838383836, + "loss": 2.6915, + "theoretical_loss": 3.4465697053822844, + "tokens_seen": 1890844672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343636363636364, + "loss": 2.9646, + "theoretical_loss": 3.4465596352879704, + "tokens_seen": 1890910208 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1041567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.472703456878662, + "objective/train/theoretical_loss": 3.446549565640384, + "objective/train/tokens_used": 249834976, + "theoretical_loss": 3.446549565640384, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343434343434344, + "loss": 2.969, + "theoretical_loss": 3.446549565640384, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043432323232323234, + "loss": 2.5333, + "theoretical_loss": 3.44653949643949, + "tokens_seen": 1891041280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343030303030303, + "loss": 2.7445, + "theoretical_loss": 3.4465294276852534, + "tokens_seen": 1891106816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428282828282826, + "loss": 2.5441, + "theoretical_loss": 3.446519359377638, + "tokens_seen": 1891172352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043426262626262627, + "loss": 2.5551, + "theoretical_loss": 3.44650929151661, + "tokens_seen": 1891237888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342424242424243, + "loss": 2.7024, + "theoretical_loss": 3.4464992241021326, + "tokens_seen": 1891303424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043422222222222224, + "loss": 2.6171, + "theoretical_loss": 3.446489157134171, + "tokens_seen": 1891368960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342020202020202, + "loss": 2.7406, + "theoretical_loss": 3.4464790906126908, + "tokens_seen": 1891434496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341818181818182, + "loss": 2.6309, + "theoretical_loss": 3.446469024537656, + "tokens_seen": 1891500032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043416161616161616, + "loss": 2.6551, + "theoretical_loss": 3.446458958909031, + "tokens_seen": 1891565568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043414141414141417, + "loss": 2.6236, + "theoretical_loss": 3.4464488937267808, + "tokens_seen": 1891631104 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043412121212121213, + "loss": 2.6514, + "theoretical_loss": 3.4464388289908703, + "tokens_seen": 1891696640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341010101010101, + "loss": 2.6576, + "theoretical_loss": 3.4464287647012646, + "tokens_seen": 1891762176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340808080808081, + "loss": 2.6109, + "theoretical_loss": 3.4464187008579277, + "tokens_seen": 1891827712 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043406060606060605, + "loss": 2.5924, + "theoretical_loss": 3.446408637460825, + "tokens_seen": 1891893248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043404040404040406, + "loss": 2.4555, + "theoretical_loss": 3.446398574509921, + "tokens_seen": 1891958784 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434020202020202, + "loss": 2.7578, + "theoretical_loss": 3.44638851200518, + "tokens_seen": 1892024320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043400000000000003, + "loss": 2.4286, + "theoretical_loss": 3.4463784499465673, + "tokens_seen": 1892089856 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433979797979798, + "loss": 2.4674, + "theoretical_loss": 3.4463683883340472, + "tokens_seen": 1892155392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043395959595959595, + "loss": 2.7483, + "theoretical_loss": 3.4463583271675855, + "tokens_seen": 1892220928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043393939393939396, + "loss": 2.8933, + "theoretical_loss": 3.446348266447146, + "tokens_seen": 1892286464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339191919191919, + "loss": 2.6796, + "theoretical_loss": 3.446338206172694, + "tokens_seen": 1892352000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338989898989899, + "loss": 2.6755, + "theoretical_loss": 3.446328146344194, + "tokens_seen": 1892417536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338787878787879, + "loss": 2.6965, + "theoretical_loss": 3.44631808696161, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043385858585858584, + "loss": 2.6102, + "theoretical_loss": 3.4463080280249088, + "tokens_seen": 1892548608 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1042675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5914690494537354, + "objective/train/theoretical_loss": 3.4462979695340534, + "objective/train/tokens_used": 251473376, + "theoretical_loss": 3.4462979695340534, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043383838383838385, + "loss": 2.5992, + "theoretical_loss": 3.4462979695340534, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043381818181818186, + "loss": 2.7988, + "theoretical_loss": 3.4462879114890095, + "tokens_seen": 1892679680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337979797979798, + "loss": 2.7278, + "theoretical_loss": 3.446277853889742, + "tokens_seen": 1892745216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337777777777778, + "loss": 2.7914, + "theoretical_loss": 3.4462677967362145, + "tokens_seen": 1892810752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043375757575757573, + "loss": 2.5553, + "theoretical_loss": 3.4462577400283934, + "tokens_seen": 1892876288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043373737373737374, + "loss": 2.8135, + "theoretical_loss": 3.4462476837662424, + "tokens_seen": 1892941824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043371717171717175, + "loss": 2.7649, + "theoretical_loss": 3.446237627949727, + "tokens_seen": 1893007360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336969696969697, + "loss": 2.454, + "theoretical_loss": 3.4462275725788114, + "tokens_seen": 1893072896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043367676767676767, + "loss": 2.9034, + "theoretical_loss": 3.4462175176534613, + "tokens_seen": 1893138432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336565656565656, + "loss": 2.8346, + "theoretical_loss": 3.44620746317364, + "tokens_seen": 1893203968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043363636363636364, + "loss": 2.3368, + "theoretical_loss": 3.446197409139314, + "tokens_seen": 1893269504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043361616161616165, + "loss": 2.8816, + "theoretical_loss": 3.446187355550448, + "tokens_seen": 1893335040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335959595959596, + "loss": 2.591, + "theoretical_loss": 3.4461773024070057, + "tokens_seen": 1893400576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043357575757575756, + "loss": 2.8308, + "theoretical_loss": 3.4461672497089526, + "tokens_seen": 1893466112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043355555555555557, + "loss": 2.7431, + "theoretical_loss": 3.4461571974562535, + "tokens_seen": 1893531648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335353535353536, + "loss": 2.6631, + "theoretical_loss": 3.4461471456488733, + "tokens_seen": 1893597184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043351515151515154, + "loss": 2.6796, + "theoretical_loss": 3.446137094286777, + "tokens_seen": 1893662720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334949494949495, + "loss": 2.5672, + "theoretical_loss": 3.4461270433699296, + "tokens_seen": 1893728256 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043347474747474745, + "loss": 2.5745, + "theoretical_loss": 3.446116992898295, + "tokens_seen": 1893793792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043345454545454546, + "loss": 2.7892, + "theoretical_loss": 3.4461069428718387, + "tokens_seen": 1893859328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334343434343435, + "loss": 2.7141, + "theoretical_loss": 3.4460968932905263, + "tokens_seen": 1893924864 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043341414141414143, + "loss": 2.7473, + "theoretical_loss": 3.4460868441543218, + "tokens_seen": 1893990400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333939393939394, + "loss": 2.5743, + "theoretical_loss": 3.44607679546319, + "tokens_seen": 1894055936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043337373737373735, + "loss": 2.7776, + "theoretical_loss": 3.4460667472170963, + "tokens_seen": 1894121472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333535353535354, + "loss": 2.7774, + "theoretical_loss": 3.446056699416005, + "tokens_seen": 1894187008 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1043835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.616513967514038, + "objective/train/theoretical_loss": 3.446046652059882, + "objective/train/tokens_used": 253111776, + "theoretical_loss": 3.446046652059882, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043333333333333337, + "loss": 2.6525, + "theoretical_loss": 3.446046652059882, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333131313131313, + "loss": 2.6293, + "theoretical_loss": 3.4460366051486915, + "tokens_seen": 1894318080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332929292929293, + "loss": 2.7239, + "theoretical_loss": 3.446026558682398, + "tokens_seen": 1894383616 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327272727272724, + "loss": 2.7973, + "theoretical_loss": 3.446016512660967, + "tokens_seen": 1894449152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332525252525253, + "loss": 2.6836, + "theoretical_loss": 3.4460064670843638, + "tokens_seen": 1894514688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323232323232326, + "loss": 2.7825, + "theoretical_loss": 3.445996421952552, + "tokens_seen": 1894580224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332121212121212, + "loss": 2.7624, + "theoretical_loss": 3.445986377265498, + "tokens_seen": 1894645760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331919191919192, + "loss": 2.5716, + "theoretical_loss": 3.445976333023166, + "tokens_seen": 1894711296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331717171717172, + "loss": 2.7031, + "theoretical_loss": 3.4459662892255203, + "tokens_seen": 1894776832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331515151515152, + "loss": 2.5949, + "theoretical_loss": 3.4459562458725275, + "tokens_seen": 1894842368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043313131313131315, + "loss": 2.5145, + "theoretical_loss": 3.445946202964151, + "tokens_seen": 1894907904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331111111111111, + "loss": 2.6517, + "theoretical_loss": 3.4459361605003567, + "tokens_seen": 1894973440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043309090909090907, + "loss": 2.5671, + "theoretical_loss": 3.4459261184811085, + "tokens_seen": 1895038976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330707070707071, + "loss": 2.6993, + "theoretical_loss": 3.4459160769063724, + "tokens_seen": 1895104512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330505050505051, + "loss": 2.6661, + "theoretical_loss": 3.445906035776113, + "tokens_seen": 1895170048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043303030303030305, + "loss": 2.8961, + "theoretical_loss": 3.4458959950902948, + "tokens_seen": 1895235584 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433010101010101, + "loss": 2.7048, + "theoretical_loss": 3.445885954848884, + "tokens_seen": 1895301120 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432989898989899, + "loss": 2.6647, + "theoretical_loss": 3.445875915051844, + "tokens_seen": 1895366656 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043296969696969697, + "loss": 2.5512, + "theoretical_loss": 3.4458658756991407, + "tokens_seen": 1895432192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432949494949495, + "loss": 2.8466, + "theoretical_loss": 3.445855836790739, + "tokens_seen": 1895497728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043292929292929294, + "loss": 2.7774, + "theoretical_loss": 3.4458457983266033, + "tokens_seen": 1895563264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329090909090909, + "loss": 2.8034, + "theoretical_loss": 3.4458357603066996, + "tokens_seen": 1895628800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328888888888889, + "loss": 2.6688, + "theoretical_loss": 3.445825722730992, + "tokens_seen": 1895694336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043286868686868686, + "loss": 2.2792, + "theoretical_loss": 3.4458156855994457, + "tokens_seen": 1895759872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328484848484849, + "loss": 2.6926, + "theoretical_loss": 3.4458056489120263, + "tokens_seen": 1895825408 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1044447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.802863836288452, + "objective/train/theoretical_loss": 3.4457956126686975, + "objective/train/tokens_used": 254750176, + "theoretical_loss": 3.4457956126686975, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043282828282828283, + "loss": 2.6796, + "theoretical_loss": 3.4457956126686975, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043280808080808084, + "loss": 2.7162, + "theoretical_loss": 3.445785576869426, + "tokens_seen": 1895956480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327878787878788, + "loss": 2.6145, + "theoretical_loss": 3.445775541514175, + "tokens_seen": 1896022016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043276767676767676, + "loss": 2.8484, + "theoretical_loss": 3.445765506602911, + "tokens_seen": 1896087552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043274747474747477, + "loss": 2.6471, + "theoretical_loss": 3.4457554721355983, + "tokens_seen": 1896153088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327272727272727, + "loss": 2.6083, + "theoretical_loss": 3.445745438112202, + "tokens_seen": 1896218624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043270707070707073, + "loss": 2.5806, + "theoretical_loss": 3.4457354045326873, + "tokens_seen": 1896284160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326868686868687, + "loss": 2.8685, + "theoretical_loss": 3.4457253713970184, + "tokens_seen": 1896349696 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043266666666666665, + "loss": 2.6143, + "theoretical_loss": 3.4457153387051616, + "tokens_seen": 1896415232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043264646464646466, + "loss": 2.7779, + "theoretical_loss": 3.4457053064570813, + "tokens_seen": 1896480768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043262626262626267, + "loss": 2.8286, + "theoretical_loss": 3.4456952746527425, + "tokens_seen": 1896546304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043260606060606063, + "loss": 2.8636, + "theoretical_loss": 3.44568524329211, + "tokens_seen": 1896611840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325858585858586, + "loss": 2.7182, + "theoretical_loss": 3.44567521237515, + "tokens_seen": 1896677376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043256565656565654, + "loss": 2.7157, + "theoretical_loss": 3.445665181901826, + "tokens_seen": 1896742912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043254545454545455, + "loss": 2.5607, + "theoretical_loss": 3.4456551518721037, + "tokens_seen": 1896808448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043252525252525256, + "loss": 2.6054, + "theoretical_loss": 3.4456451222859483, + "tokens_seen": 1896873984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325050505050505, + "loss": 2.5935, + "theoretical_loss": 3.4456350931433253, + "tokens_seen": 1896939520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324848484848485, + "loss": 2.6931, + "theoretical_loss": 3.445625064444199, + "tokens_seen": 1897005056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043246464646464643, + "loss": 2.7926, + "theoretical_loss": 3.445615036188534, + "tokens_seen": 1897070592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324444444444445, + "loss": 2.783, + "theoretical_loss": 3.445605008376297, + "tokens_seen": 1897136128 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043242424242424246, + "loss": 2.4811, + "theoretical_loss": 3.4455949810074515, + "tokens_seen": 1897201664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324040404040404, + "loss": 2.6182, + "theoretical_loss": 3.445584954081964, + "tokens_seen": 1897267200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043238383838383837, + "loss": 2.4329, + "theoretical_loss": 3.4455749275997976, + "tokens_seen": 1897332736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323636363636363, + "loss": 2.8432, + "theoretical_loss": 3.4455649015609193, + "tokens_seen": 1897398272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323434343434344, + "loss": 2.5067, + "theoretical_loss": 3.4455548759652936, + "tokens_seen": 1897463808 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1045650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5606865882873535, + "objective/train/theoretical_loss": 3.4455448508128854, + "objective/train/tokens_used": 256388576, + "theoretical_loss": 3.4455448508128854, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043232323232323235, + "loss": 2.4806, + "theoretical_loss": 3.4455448508128854, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323030303030303, + "loss": 2.5365, + "theoretical_loss": 3.4455348261036596, + "tokens_seen": 1897594880 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043228282828282826, + "loss": 2.4085, + "theoretical_loss": 3.445524801837582, + "tokens_seen": 1897660416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322626262626262, + "loss": 2.7238, + "theoretical_loss": 3.4455147780146174, + "tokens_seen": 1897725952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322424242424243, + "loss": 2.5818, + "theoretical_loss": 3.4455047546347304, + "tokens_seen": 1897791488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043222222222222224, + "loss": 2.8515, + "theoretical_loss": 3.4454947316978863, + "tokens_seen": 1897857024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322020202020202, + "loss": 2.5814, + "theoretical_loss": 3.445484709204051, + "tokens_seen": 1897922560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043218181818181816, + "loss": 3.0203, + "theoretical_loss": 3.445474687153189, + "tokens_seen": 1897988096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043216161616161617, + "loss": 3.0408, + "theoretical_loss": 3.4454646655452654, + "tokens_seen": 1898053632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321414141414142, + "loss": 2.7143, + "theoretical_loss": 3.4454546443802454, + "tokens_seen": 1898119168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043212121212121213, + "loss": 2.8371, + "theoretical_loss": 3.4454446236580942, + "tokens_seen": 1898184704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321010101010101, + "loss": 2.7068, + "theoretical_loss": 3.445434603378777, + "tokens_seen": 1898250240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043208080808080805, + "loss": 2.6067, + "theoretical_loss": 3.445424583542258, + "tokens_seen": 1898315776 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043206060606060606, + "loss": 2.607, + "theoretical_loss": 3.4454145641485043, + "tokens_seen": 1898381312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043204040404040407, + "loss": 2.4594, + "theoretical_loss": 3.445404545197479, + "tokens_seen": 1898446848 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432020202020202, + "loss": 2.6032, + "theoretical_loss": 3.4453945266891486, + "tokens_seen": 1898512384 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432, + "loss": 2.5731, + "theoretical_loss": 3.4453845086234782, + "tokens_seen": 1898577920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000431979797979798, + "loss": 2.7252, + "theoretical_loss": 3.4453744910004325, + "tokens_seen": 1898643456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043195959595959595, + "loss": 2.5493, + "theoretical_loss": 3.4453644738199767, + "tokens_seen": 1898708992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043193939393939396, + "loss": 2.7383, + "theoretical_loss": 3.4453544570820758, + "tokens_seen": 1898774528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319191919191919, + "loss": 2.5631, + "theoretical_loss": 3.4453444407866956, + "tokens_seen": 1898840064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318989898989899, + "loss": 2.5864, + "theoretical_loss": 3.4453344249338005, + "tokens_seen": 1898905600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318787878787879, + "loss": 2.5916, + "theoretical_loss": 3.445324409523356, + "tokens_seen": 1898971136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318585858585859, + "loss": 2.7156, + "theoretical_loss": 3.4453143945553277, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183838383838386, + "loss": 2.8104, + "theoretical_loss": 3.4453043800296808, + "tokens_seen": 1899102208 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1046191, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9813382625579834, + "objective/train/theoretical_loss": 3.4452943659463795, + "objective/train/tokens_used": 258026976, + "theoretical_loss": 3.4452943659463795, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318181818181818, + "loss": 2.7552, + "theoretical_loss": 3.4452943659463795, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317979797979798, + "loss": 2.7213, + "theoretical_loss": 3.4452843523053898, + "tokens_seen": 1899233280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317777777777778, + "loss": 2.5726, + "theoretical_loss": 3.445274339106677, + "tokens_seen": 1899298816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317575757575758, + "loss": 2.6529, + "theoretical_loss": 3.445264326350206, + "tokens_seen": 1899364352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043173737373737375, + "loss": 2.6968, + "theoretical_loss": 3.4452543140359415, + "tokens_seen": 1899429888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317171717171717, + "loss": 2.6248, + "theoretical_loss": 3.44524430216385, + "tokens_seen": 1899495424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316969696969697, + "loss": 2.8309, + "theoretical_loss": 3.4452342907338958, + "tokens_seen": 1899560960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316767676767677, + "loss": 2.5161, + "theoretical_loss": 3.445224279746044, + "tokens_seen": 1899626496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316565656565657, + "loss": 2.6626, + "theoretical_loss": 3.4452142692002603, + "tokens_seen": 1899692032 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043163636363636364, + "loss": 2.6857, + "theoretical_loss": 3.44520425909651, + "tokens_seen": 1899757568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043161616161616165, + "loss": 2.7009, + "theoretical_loss": 3.445194249434758, + "tokens_seen": 1899823104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315959595959596, + "loss": 2.4982, + "theoretical_loss": 3.44518424021497, + "tokens_seen": 1899888640 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043157575757575757, + "loss": 2.698, + "theoretical_loss": 3.4451742314371105, + "tokens_seen": 1899954176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315555555555556, + "loss": 2.7291, + "theoretical_loss": 3.4451642231011452, + "tokens_seen": 1900019712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043153535353535353, + "loss": 2.6778, + "theoretical_loss": 3.4451542152070393, + "tokens_seen": 1900085248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043151515151515154, + "loss": 2.676, + "theoretical_loss": 3.4451442077547583, + "tokens_seen": 1900150784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314949494949495, + "loss": 2.7419, + "theoretical_loss": 3.445134200744267, + "tokens_seen": 1900216320 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043147474747474746, + "loss": 2.5326, + "theoretical_loss": 3.44512419417553, + "tokens_seen": 1900281856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043145454545454547, + "loss": 2.5492, + "theoretical_loss": 3.4451141880485148, + "tokens_seen": 1900347392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314343434343435, + "loss": 2.4373, + "theoretical_loss": 3.4451041823631843, + "tokens_seen": 1900412928 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043141414141414144, + "loss": 2.5494, + "theoretical_loss": 3.445094177119505, + "tokens_seen": 1900478464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313939393939394, + "loss": 2.5809, + "theoretical_loss": 3.4450841723174426, + "tokens_seen": 1900544000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043137373737373735, + "loss": 2.4989, + "theoretical_loss": 3.4450741679569608, + "tokens_seen": 1900609536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043135353535353536, + "loss": 2.677, + "theoretical_loss": 3.445064164038026, + "tokens_seen": 1900675072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313333333333334, + "loss": 2.6094, + "theoretical_loss": 3.4450541605606038, + "tokens_seen": 1900740608 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1047224, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5529582500457764, + "objective/train/theoretical_loss": 3.4450441575246584, + "objective/train/tokens_used": 259665376, + "theoretical_loss": 3.4450441575246584, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043131313131313133, + "loss": 2.4374, + "theoretical_loss": 3.4450441575246584, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312929292929293, + "loss": 2.5186, + "theoretical_loss": 3.4450341549301564, + "tokens_seen": 1900871680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127272727272724, + "loss": 2.7014, + "theoretical_loss": 3.4450241527770613, + "tokens_seen": 1900937216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312525252525253, + "loss": 2.8584, + "theoretical_loss": 3.4450141510653403, + "tokens_seen": 1901002752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043123232323232327, + "loss": 2.5654, + "theoretical_loss": 3.4450041497949577, + "tokens_seen": 1901068288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312121212121212, + "loss": 2.6917, + "theoretical_loss": 3.444994148965879, + "tokens_seen": 1901133824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311919191919192, + "loss": 2.5157, + "theoretical_loss": 3.4449841485780697, + "tokens_seen": 1901199360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043117171717171714, + "loss": 2.457, + "theoretical_loss": 3.444974148631495, + "tokens_seen": 1901264896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311515151515152, + "loss": 2.7993, + "theoretical_loss": 3.4449641491261196, + "tokens_seen": 1901330432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043113131313131316, + "loss": 2.3877, + "theoretical_loss": 3.4449541500619096, + "tokens_seen": 1901395968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311111111111111, + "loss": 2.762, + "theoretical_loss": 3.4449441514388304, + "tokens_seen": 1901461504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043109090909090907, + "loss": 2.5943, + "theoretical_loss": 3.4449341532568467, + "tokens_seen": 1901527040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310707070707071, + "loss": 2.8475, + "theoretical_loss": 3.4449241555159245, + "tokens_seen": 1901592576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310505050505051, + "loss": 2.5425, + "theoretical_loss": 3.4449141582160285, + "tokens_seen": 1901658112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043103030303030305, + "loss": 2.6034, + "theoretical_loss": 3.4449041613571247, + "tokens_seen": 1901723648 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431010101010101, + "loss": 2.6465, + "theoretical_loss": 3.4448941649391784, + "tokens_seen": 1901789184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043098989898989897, + "loss": 2.8249, + "theoretical_loss": 3.444884168962154, + "tokens_seen": 1901854720 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430969696969697, + "loss": 2.6515, + "theoretical_loss": 3.444874173426018, + "tokens_seen": 1901920256 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430949494949495, + "loss": 2.7534, + "theoretical_loss": 3.444864178330735, + "tokens_seen": 1901985792 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043092929292929294, + "loss": 2.5677, + "theoretical_loss": 3.444854183676271, + "tokens_seen": 1902051328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309090909090909, + "loss": 2.5041, + "theoretical_loss": 3.444844189462591, + "tokens_seen": 1902116864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308888888888889, + "loss": 2.4229, + "theoretical_loss": 3.44483419568966, + "tokens_seen": 1902182400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043086868686868687, + "loss": 2.8729, + "theoretical_loss": 3.4448242023574447, + "tokens_seen": 1902247936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308484848484849, + "loss": 2.726, + "theoretical_loss": 3.444814209465909, + "tokens_seen": 1902313472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043082828282828284, + "loss": 2.6824, + "theoretical_loss": 3.444804217015019, + "tokens_seen": 1902379008 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1047755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.809091329574585, + "objective/train/theoretical_loss": 3.44479422500474, + "objective/train/tokens_used": 261303776, + "theoretical_loss": 3.44479422500474, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308080808080808, + "loss": 2.5754, + "theoretical_loss": 3.44479422500474, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307878787878788, + "loss": 2.6735, + "theoretical_loss": 3.4447842334350374, + "tokens_seen": 1902510080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043076767676767676, + "loss": 2.7893, + "theoretical_loss": 3.4447742423058765, + "tokens_seen": 1902575616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043074747474747477, + "loss": 2.7023, + "theoretical_loss": 3.4447642516172223, + "tokens_seen": 1902641152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043072727272727273, + "loss": 2.6068, + "theoretical_loss": 3.4447542613690416, + "tokens_seen": 1902706688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043070707070707074, + "loss": 2.6704, + "theoretical_loss": 3.444744271561298, + "tokens_seen": 1902772224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306868686868687, + "loss": 2.5841, + "theoretical_loss": 3.4447342821939584, + "tokens_seen": 1902837760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043066666666666665, + "loss": 2.5968, + "theoretical_loss": 3.4447242932669875, + "tokens_seen": 1902903296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043064646464646467, + "loss": 2.4953, + "theoretical_loss": 3.4447143047803506, + "tokens_seen": 1902968832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306262626262626, + "loss": 2.7003, + "theoretical_loss": 3.444704316734013, + "tokens_seen": 1903034368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043060606060606063, + "loss": 2.6531, + "theoretical_loss": 3.444694329127941, + "tokens_seen": 1903099904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305858585858586, + "loss": 2.6429, + "theoretical_loss": 3.4446843419621, + "tokens_seen": 1903165440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043056565656565655, + "loss": 2.6939, + "theoretical_loss": 3.4446743552364545, + "tokens_seen": 1903230976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043054545454545456, + "loss": 2.5613, + "theoretical_loss": 3.4446643689509697, + "tokens_seen": 1903296512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043052525252525257, + "loss": 2.8203, + "theoretical_loss": 3.444654383105613, + "tokens_seen": 1903362048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305050505050505, + "loss": 2.6728, + "theoretical_loss": 3.4446443977003476, + "tokens_seen": 1903427584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304848484848485, + "loss": 2.6016, + "theoretical_loss": 3.44463441273514, + "tokens_seen": 1903493120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043046464646464644, + "loss": 2.7691, + "theoretical_loss": 3.444624428209956, + "tokens_seen": 1903558656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043044444444444445, + "loss": 2.7279, + "theoretical_loss": 3.4446144441247606, + "tokens_seen": 1903624192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043042424242424246, + "loss": 2.7133, + "theoretical_loss": 3.4446044604795194, + "tokens_seen": 1903689728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304040404040404, + "loss": 2.7337, + "theoretical_loss": 3.4445944772741974, + "tokens_seen": 1903755264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303838383838384, + "loss": 2.7528, + "theoretical_loss": 3.444584494508761, + "tokens_seen": 1903820800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043036363636363633, + "loss": 2.6488, + "theoretical_loss": 3.444574512183175, + "tokens_seen": 1903886336 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043034343434343434, + "loss": 2.686, + "theoretical_loss": 3.444564530297405, + "tokens_seen": 1903951872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043032323232323235, + "loss": 2.7676, + "theoretical_loss": 3.444554548851416, + "tokens_seen": 1904017408 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1048883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5599634647369385, + "objective/train/theoretical_loss": 3.4445445678451745, + "objective/train/tokens_used": 262942176, + "theoretical_loss": 3.4445445678451745, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303030303030303, + "loss": 2.8532, + "theoretical_loss": 3.4445445678451745, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043028282828282827, + "loss": 2.8837, + "theoretical_loss": 3.4445345872786453, + "tokens_seen": 1904148480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302626262626262, + "loss": 2.6033, + "theoretical_loss": 3.444524607151794, + "tokens_seen": 1904214016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302424242424243, + "loss": 2.7305, + "theoretical_loss": 3.444514627464587, + "tokens_seen": 1904279552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043022222222222225, + "loss": 2.7995, + "theoretical_loss": 3.444504648216988, + "tokens_seen": 1904345088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302020202020202, + "loss": 2.8029, + "theoretical_loss": 3.4444946694089635, + "tokens_seen": 1904410624 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043018181818181816, + "loss": 2.6914, + "theoretical_loss": 3.4444846910404796, + "tokens_seen": 1904476160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043016161616161617, + "loss": 2.6082, + "theoretical_loss": 3.444474713111501, + "tokens_seen": 1904541696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301414141414142, + "loss": 2.8255, + "theoretical_loss": 3.4444647356219935, + "tokens_seen": 1904607232 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043012121212121214, + "loss": 2.5401, + "theoretical_loss": 3.444454758571922, + "tokens_seen": 1904672768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301010101010101, + "loss": 2.7175, + "theoretical_loss": 3.4444447819612534, + "tokens_seen": 1904738304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043008080808080805, + "loss": 2.5902, + "theoretical_loss": 3.444434805789952, + "tokens_seen": 1904803840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300606060606061, + "loss": 2.8099, + "theoretical_loss": 3.444424830057984, + "tokens_seen": 1904869376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300404040404041, + "loss": 2.6037, + "theoretical_loss": 3.444414854765314, + "tokens_seen": 1904934912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043002020202020203, + "loss": 2.7313, + "theoretical_loss": 3.444404879911909, + "tokens_seen": 1905000448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043, + "loss": 2.8642, + "theoretical_loss": 3.4443949054977336, + "tokens_seen": 1905065984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042997979797979795, + "loss": 2.777, + "theoretical_loss": 3.4443849315227535, + "tokens_seen": 1905131520 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429959595959596, + "loss": 2.5088, + "theoretical_loss": 3.444374957986934, + "tokens_seen": 1905197056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042993939393939397, + "loss": 2.6678, + "theoretical_loss": 3.444364984890241, + "tokens_seen": 1905262592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299191919191919, + "loss": 2.7726, + "theoretical_loss": 3.4443550122326405, + "tokens_seen": 1905328128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298989898989899, + "loss": 2.6621, + "theoretical_loss": 3.444345040014097, + "tokens_seen": 1905393664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298787878787879, + "loss": 2.8235, + "theoretical_loss": 3.444335068234577, + "tokens_seen": 1905459200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298585858585859, + "loss": 2.807, + "theoretical_loss": 3.4443250968940458, + "tokens_seen": 1905524736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042983838383838386, + "loss": 2.785, + "theoretical_loss": 3.4443151259924685, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298181818181818, + "loss": 2.69, + "theoretical_loss": 3.444305155529811, + "tokens_seen": 1905655808 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1049481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5614824295043945, + "objective/train/theoretical_loss": 3.4442951855060393, + "objective/train/tokens_used": 264580576, + "theoretical_loss": 3.4442951855060393, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297979797979798, + "loss": 2.5091, + "theoretical_loss": 3.4442951855060393, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297777777777778, + "loss": 2.7097, + "theoretical_loss": 3.4442852159211186, + "tokens_seen": 1905786880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297575757575758, + "loss": 2.5279, + "theoretical_loss": 3.444275246775015, + "tokens_seen": 1905852416 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042973737373737375, + "loss": 2.6529, + "theoretical_loss": 3.4442652780676926, + "tokens_seen": 1905917952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297171717171717, + "loss": 2.6415, + "theoretical_loss": 3.444255309799119, + "tokens_seen": 1905983488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296969696969697, + "loss": 2.7135, + "theoretical_loss": 3.444245341969258, + "tokens_seen": 1906049024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296767676767677, + "loss": 2.7133, + "theoretical_loss": 3.4442353745780765, + "tokens_seen": 1906114560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296565656565657, + "loss": 2.5431, + "theoretical_loss": 3.44422540762554, + "tokens_seen": 1906180096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042963636363636365, + "loss": 2.6903, + "theoretical_loss": 3.444215441111613, + "tokens_seen": 1906245632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296161616161616, + "loss": 2.6747, + "theoretical_loss": 3.444205475036262, + "tokens_seen": 1906311168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295959595959596, + "loss": 2.6883, + "theoretical_loss": 3.444195509399453, + "tokens_seen": 1906376704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042957575757575757, + "loss": 2.5868, + "theoretical_loss": 3.444185544201151, + "tokens_seen": 1906442240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295555555555556, + "loss": 2.8323, + "theoretical_loss": 3.444175579441322, + "tokens_seen": 1906507776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042953535353535354, + "loss": 2.766, + "theoretical_loss": 3.444165615119931, + "tokens_seen": 1906573312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042951515151515155, + "loss": 2.665, + "theoretical_loss": 3.4441556512369442, + "tokens_seen": 1906638848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294949494949495, + "loss": 2.7014, + "theoretical_loss": 3.4441456877923273, + "tokens_seen": 1906704384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042947474747474746, + "loss": 2.6946, + "theoretical_loss": 3.4441357247860456, + "tokens_seen": 1906769920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294545454545455, + "loss": 2.4496, + "theoretical_loss": 3.4441257622180648, + "tokens_seen": 1906835456 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042943434343434343, + "loss": 2.6594, + "theoretical_loss": 3.4441158000883507, + "tokens_seen": 1906900992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042941414141414144, + "loss": 2.3002, + "theoretical_loss": 3.444105838396869, + "tokens_seen": 1906966528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293939393939394, + "loss": 2.5308, + "theoretical_loss": 3.4440958771435852, + "tokens_seen": 1907032064 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042937373737373736, + "loss": 2.5182, + "theoretical_loss": 3.444085916328465, + "tokens_seen": 1907097600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042935353535353537, + "loss": 2.5564, + "theoretical_loss": 3.444075955951474, + "tokens_seen": 1907163136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293333333333334, + "loss": 2.5707, + "theoretical_loss": 3.4440659960125783, + "tokens_seen": 1907228672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042931313131313134, + "loss": 2.6122, + "theoretical_loss": 3.4440560365117427, + "tokens_seen": 1907294208 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1050597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.497748374938965, + "objective/train/theoretical_loss": 3.444046077448934, + "objective/train/tokens_used": 266218976, + "theoretical_loss": 3.444046077448934, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292929292929293, + "loss": 2.6615, + "theoretical_loss": 3.444046077448934, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042927272727272725, + "loss": 2.7089, + "theoretical_loss": 3.4440361188241173, + "tokens_seen": 1907425280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042925252525252526, + "loss": 2.6821, + "theoretical_loss": 3.444026160637258, + "tokens_seen": 1907490816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042923232323232327, + "loss": 2.6433, + "theoretical_loss": 3.4440162028883226, + "tokens_seen": 1907556352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042921212121212123, + "loss": 2.7893, + "theoretical_loss": 3.444006245577276, + "tokens_seen": 1907621888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291919191919192, + "loss": 2.4906, + "theoretical_loss": 3.443996288704084, + "tokens_seen": 1907687424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042917171717171714, + "loss": 2.745, + "theoretical_loss": 3.443986332268713, + "tokens_seen": 1907752960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291515151515152, + "loss": 2.4249, + "theoretical_loss": 3.443976376271128, + "tokens_seen": 1907818496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042913131313131316, + "loss": 2.7227, + "theoretical_loss": 3.4439664207112948, + "tokens_seen": 1907884032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291111111111111, + "loss": 2.762, + "theoretical_loss": 3.4439564655891797, + "tokens_seen": 1907949568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290909090909091, + "loss": 2.7586, + "theoretical_loss": 3.4439465109047473, + "tokens_seen": 1908015104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042907070707070703, + "loss": 2.616, + "theoretical_loss": 3.4439365566579645, + "tokens_seen": 1908080640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290505050505051, + "loss": 2.6937, + "theoretical_loss": 3.4439266028487965, + "tokens_seen": 1908146176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042903030303030306, + "loss": 2.6258, + "theoretical_loss": 3.4439166494772087, + "tokens_seen": 1908211712 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429010101010101, + "loss": 2.7516, + "theoretical_loss": 3.4439066965431677, + "tokens_seen": 1908277248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042898989898989897, + "loss": 2.7227, + "theoretical_loss": 3.4438967440466386, + "tokens_seen": 1908342784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042896969696969693, + "loss": 2.4991, + "theoretical_loss": 3.443886791987587, + "tokens_seen": 1908408320 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428949494949495, + "loss": 2.5172, + "theoretical_loss": 3.443876840365979, + "tokens_seen": 1908473856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042892929292929295, + "loss": 2.6511, + "theoretical_loss": 3.4438668891817805, + "tokens_seen": 1908539392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289090909090909, + "loss": 2.6616, + "theoretical_loss": 3.443856938434957, + "tokens_seen": 1908604928 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042888888888888886, + "loss": 2.7852, + "theoretical_loss": 3.4438469881254745, + "tokens_seen": 1908670464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288686868686869, + "loss": 2.6391, + "theoretical_loss": 3.443837038253298, + "tokens_seen": 1908736000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288484848484849, + "loss": 2.6146, + "theoretical_loss": 3.443827088818394, + "tokens_seen": 1908801536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042882828282828284, + "loss": 2.6307, + "theoretical_loss": 3.4438171398207285, + "tokens_seen": 1908867072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288080808080808, + "loss": 2.4644, + "theoretical_loss": 3.443807191260267, + "tokens_seen": 1908932608 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1051477, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.080481767654419, + "objective/train/theoretical_loss": 3.4437972431369746, + "objective/train/tokens_used": 267857376, + "theoretical_loss": 3.4437972431369746, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878787878787876, + "loss": 2.712, + "theoretical_loss": 3.4437972431369746, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042876767676767677, + "loss": 2.772, + "theoretical_loss": 3.4437872954508175, + "tokens_seen": 1909063680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287474747474748, + "loss": 2.712, + "theoretical_loss": 3.443777348201762, + "tokens_seen": 1909129216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042872727272727274, + "loss": 2.7066, + "theoretical_loss": 3.443767401389773, + "tokens_seen": 1909194752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287070707070707, + "loss": 2.7274, + "theoretical_loss": 3.4437574550148176, + "tokens_seen": 1909260288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286868686868687, + "loss": 2.6661, + "theoretical_loss": 3.4437475090768603, + "tokens_seen": 1909325824 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042866666666666666, + "loss": 2.663, + "theoretical_loss": 3.4437375635758674, + "tokens_seen": 1909391360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042864646464646467, + "loss": 2.6362, + "theoretical_loss": 3.4437276185118053, + "tokens_seen": 1909456896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042862626262626263, + "loss": 2.514, + "theoretical_loss": 3.4437176738846382, + "tokens_seen": 1909522432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286060606060606, + "loss": 2.5079, + "theoretical_loss": 3.4437077296943337, + "tokens_seen": 1909587968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285858585858586, + "loss": 2.801, + "theoretical_loss": 3.4436977859408566, + "tokens_seen": 1909653504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042856565656565655, + "loss": 2.6743, + "theoretical_loss": 3.443687842624173, + "tokens_seen": 1909719040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854545454545456, + "loss": 2.6999, + "theoretical_loss": 3.443677899744249, + "tokens_seen": 1909784576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285252525252525, + "loss": 2.603, + "theoretical_loss": 3.4436679573010496, + "tokens_seen": 1909850112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042850505050505053, + "loss": 2.7413, + "theoretical_loss": 3.4436580152945413, + "tokens_seen": 1909915648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284848484848485, + "loss": 2.3256, + "theoretical_loss": 3.44364807372469, + "tokens_seen": 1909981184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846464646464645, + "loss": 2.6853, + "theoretical_loss": 3.4436381325914613, + "tokens_seen": 1910046720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042844444444444446, + "loss": 2.4406, + "theoretical_loss": 3.443628191894821, + "tokens_seen": 1910112256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284242424242424, + "loss": 2.7554, + "theoretical_loss": 3.443618251634735, + "tokens_seen": 1910177792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284040404040404, + "loss": 2.4338, + "theoretical_loss": 3.443608311811169, + "tokens_seen": 1910243328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283838383838384, + "loss": 2.5513, + "theoretical_loss": 3.443598372424089, + "tokens_seen": 1910308864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283636363636364, + "loss": 2.9312, + "theoretical_loss": 3.443588433473461, + "tokens_seen": 1910374400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042834343434343435, + "loss": 2.6278, + "theoretical_loss": 3.443578494959251, + "tokens_seen": 1910439936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042832323232323236, + "loss": 2.8923, + "theoretical_loss": 3.4435685568814245, + "tokens_seen": 1910505472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283030303030303, + "loss": 2.7032, + "theoretical_loss": 3.4435586192399477, + "tokens_seen": 1910571008 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1051849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.255452871322632, + "objective/train/theoretical_loss": 3.443548682034786, + "objective/train/tokens_used": 269495776, + "theoretical_loss": 3.443548682034786, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282828282828283, + "loss": 2.7182, + "theoretical_loss": 3.443548682034786, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282626262626263, + "loss": 2.5879, + "theoretical_loss": 3.443538745265905, + "tokens_seen": 1910702080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042824242424242424, + "loss": 2.5467, + "theoretical_loss": 3.443528808933272, + "tokens_seen": 1910767616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042822222222222225, + "loss": 2.8346, + "theoretical_loss": 3.443518873036852, + "tokens_seen": 1910833152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282020202020202, + "loss": 2.3565, + "theoretical_loss": 3.44350893757661, + "tokens_seen": 1910898688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042818181818181817, + "loss": 2.7003, + "theoretical_loss": 3.443499002552514, + "tokens_seen": 1910964224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281616161616162, + "loss": 2.3673, + "theoretical_loss": 3.443489067964528, + "tokens_seen": 1911029760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281414141414142, + "loss": 2.8019, + "theoretical_loss": 3.4434791338126187, + "tokens_seen": 1911095296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042812121212121215, + "loss": 2.7316, + "theoretical_loss": 3.443469200096752, + "tokens_seen": 1911160832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281010101010101, + "loss": 2.8181, + "theoretical_loss": 3.4434592668168933, + "tokens_seen": 1911226368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042808080808080806, + "loss": 2.7831, + "theoretical_loss": 3.4434493339730095, + "tokens_seen": 1911291904 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042806060606060607, + "loss": 2.8366, + "theoretical_loss": 3.4434394015650653, + "tokens_seen": 1911357440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280404040404041, + "loss": 2.5423, + "theoretical_loss": 3.4434294695930276, + "tokens_seen": 1911422976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802020202020204, + "loss": 2.5892, + "theoretical_loss": 3.443419538056862, + "tokens_seen": 1911488512 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428, + "loss": 2.5842, + "theoretical_loss": 3.4434096069565348, + "tokens_seen": 1911554048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042797979797979795, + "loss": 2.7545, + "theoretical_loss": 3.4433996762920107, + "tokens_seen": 1911619584 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427959595959596, + "loss": 2.591, + "theoretical_loss": 3.443389746063257, + "tokens_seen": 1911685120 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427939393939394, + "loss": 2.8032, + "theoretical_loss": 3.443379816270239, + "tokens_seen": 1911750656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042791919191919193, + "loss": 2.6334, + "theoretical_loss": 3.4433698869129232, + "tokens_seen": 1911816192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278989898989899, + "loss": 2.7227, + "theoretical_loss": 3.443359957991275, + "tokens_seen": 1911881728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042787878787878784, + "loss": 2.7052, + "theoretical_loss": 3.4433500295052597, + "tokens_seen": 1911947264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278585858585859, + "loss": 2.5298, + "theoretical_loss": 3.4433401014548446, + "tokens_seen": 1912012800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042783838383838387, + "loss": 2.5337, + "theoretical_loss": 3.4433301738399953, + "tokens_seen": 1912078336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278181818181818, + "loss": 2.5673, + "theoretical_loss": 3.4433202466606767, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277979797979798, + "loss": 2.8075, + "theoretical_loss": 3.4433103199168564, + "tokens_seen": 1912209408 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1053170, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4347217082977295, + "objective/train/theoretical_loss": 3.4433003936084994, + "objective/train/tokens_used": 271134176, + "theoretical_loss": 3.4433003936084994, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277777777777778, + "loss": 2.6555, + "theoretical_loss": 3.4433003936084994, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277575757575758, + "loss": 2.5505, + "theoretical_loss": 3.443290467735572, + "tokens_seen": 1912340480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042773737373737376, + "loss": 2.7639, + "theoretical_loss": 3.44328054229804, + "tokens_seen": 1912406016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277171717171717, + "loss": 2.8434, + "theoretical_loss": 3.4432706172958687, + "tokens_seen": 1912471552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276969696969697, + "loss": 2.7693, + "theoretical_loss": 3.4432606927290257, + "tokens_seen": 1912537088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276767676767677, + "loss": 2.601, + "theoretical_loss": 3.4432507685974754, + "tokens_seen": 1912602624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276565656565657, + "loss": 2.3856, + "theoretical_loss": 3.443240844901185, + "tokens_seen": 1912668160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042763636363636365, + "loss": 2.6203, + "theoretical_loss": 3.4432309216401196, + "tokens_seen": 1912733696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276161616161616, + "loss": 2.762, + "theoretical_loss": 3.4432209988142457, + "tokens_seen": 1912799232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275959595959596, + "loss": 2.5514, + "theoretical_loss": 3.4432110764235295, + "tokens_seen": 1912864768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275757575757576, + "loss": 2.6362, + "theoretical_loss": 3.4432011544679364, + "tokens_seen": 1912930304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275555555555556, + "loss": 2.6411, + "theoretical_loss": 3.443191232947433, + "tokens_seen": 1912995840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042753535353535354, + "loss": 2.7044, + "theoretical_loss": 3.4431813118619847, + "tokens_seen": 1913061376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275151515151515, + "loss": 2.5536, + "theoretical_loss": 3.443171391211558, + "tokens_seen": 1913126912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274949494949495, + "loss": 2.7948, + "theoretical_loss": 3.4431614709961185, + "tokens_seen": 1913192448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042747474747474747, + "loss": 2.7015, + "theoretical_loss": 3.4431515512156325, + "tokens_seen": 1913257984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274545454545455, + "loss": 2.7094, + "theoretical_loss": 3.4431416318700663, + "tokens_seen": 1913323520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042743434343434344, + "loss": 2.7221, + "theoretical_loss": 3.443131712959386, + "tokens_seen": 1913389056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042741414141414145, + "loss": 2.7083, + "theoretical_loss": 3.443121794483557, + "tokens_seen": 1913454592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273939393939394, + "loss": 2.5024, + "theoretical_loss": 3.4431118764425452, + "tokens_seen": 1913520128 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042737373737373736, + "loss": 2.5784, + "theoretical_loss": 3.4431019588363174, + "tokens_seen": 1913585664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273535353535354, + "loss": 2.864, + "theoretical_loss": 3.4430920416648396, + "tokens_seen": 1913651200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042733333333333333, + "loss": 2.8075, + "theoretical_loss": 3.443082124928077, + "tokens_seen": 1913716736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042731313131313134, + "loss": 2.7673, + "theoretical_loss": 3.443072208625997, + "tokens_seen": 1913782272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272929292929293, + "loss": 2.662, + "theoretical_loss": 3.4430622927585643, + "tokens_seen": 1913847808 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1053822, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.910274028778076, + "objective/train/theoretical_loss": 3.443052377325746, + "objective/train/tokens_used": 272772576, + "theoretical_loss": 3.443052377325746, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042727272727272726, + "loss": 2.6898, + "theoretical_loss": 3.443052377325746, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042725252525252527, + "loss": 2.5891, + "theoretical_loss": 3.4430424623275075, + "tokens_seen": 1913978880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272323232323233, + "loss": 2.6262, + "theoretical_loss": 3.4430325477638153, + "tokens_seen": 1914044416 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042721212121212123, + "loss": 2.5299, + "theoretical_loss": 3.443022633634635, + "tokens_seen": 1914109952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271919191919192, + "loss": 2.6919, + "theoretical_loss": 3.4430127199399334, + "tokens_seen": 1914175488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042717171717171715, + "loss": 2.5389, + "theoretical_loss": 3.443002806679676, + "tokens_seen": 1914241024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042715151515151516, + "loss": 2.4234, + "theoretical_loss": 3.442992893853829, + "tokens_seen": 1914306560 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042713131313131317, + "loss": 2.7233, + "theoretical_loss": 3.4429829814623583, + "tokens_seen": 1914372096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271111111111111, + "loss": 2.4909, + "theoretical_loss": 3.4429730695052303, + "tokens_seen": 1914437632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270909090909091, + "loss": 2.7478, + "theoretical_loss": 3.442963157982412, + "tokens_seen": 1914503168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042707070707070704, + "loss": 2.5509, + "theoretical_loss": 3.4429532468938673, + "tokens_seen": 1914568704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042705050505050505, + "loss": 2.525, + "theoretical_loss": 3.4429433362395643, + "tokens_seen": 1914634240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042703030303030306, + "loss": 2.7176, + "theoretical_loss": 3.442933426019468, + "tokens_seen": 1914699776 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427010101010101, + "loss": 2.7536, + "theoretical_loss": 3.442923516233545, + "tokens_seen": 1914765312 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426989898989899, + "loss": 2.6505, + "theoretical_loss": 3.4429136068817616, + "tokens_seen": 1914830848 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042696969696969693, + "loss": 2.54, + "theoretical_loss": 3.4429036979640832, + "tokens_seen": 1914896384 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426949494949495, + "loss": 2.7148, + "theoretical_loss": 3.4428937894804763, + "tokens_seen": 1914961920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042692929292929296, + "loss": 2.4606, + "theoretical_loss": 3.442883881430908, + "tokens_seen": 1915027456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269090909090909, + "loss": 2.6427, + "theoretical_loss": 3.4428739738153427, + "tokens_seen": 1915092992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042688888888888887, + "loss": 2.5241, + "theoretical_loss": 3.442864066633747, + "tokens_seen": 1915158528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268686868686868, + "loss": 2.7025, + "theoretical_loss": 3.442854159886088, + "tokens_seen": 1915224064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268484848484849, + "loss": 2.8231, + "theoretical_loss": 3.442844253572331, + "tokens_seen": 1915289600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042682828282828285, + "loss": 2.6401, + "theoretical_loss": 3.4428343476924432, + "tokens_seen": 1915355136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268080808080808, + "loss": 2.7003, + "theoretical_loss": 3.4428244422463887, + "tokens_seen": 1915420672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042678787878787876, + "loss": 2.4902, + "theoretical_loss": 3.4428145372341357, + "tokens_seen": 1915486208 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1054940, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.356700897216797, + "objective/train/theoretical_loss": 3.4428046326556494, + "objective/train/tokens_used": 274410976, + "theoretical_loss": 3.4428046326556494, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042676767676767677, + "loss": 2.4488, + "theoretical_loss": 3.4428046326556494, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267474747474748, + "loss": 2.7953, + "theoretical_loss": 3.442794728510896, + "tokens_seen": 1915617280 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042672727272727274, + "loss": 2.6184, + "theoretical_loss": 3.4427848247998423, + "tokens_seen": 1915682816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267070707070707, + "loss": 2.4235, + "theoretical_loss": 3.4427749215224535, + "tokens_seen": 1915748352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042668686868686865, + "loss": 2.7641, + "theoretical_loss": 3.4427650186786964, + "tokens_seen": 1915813888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266666666666667, + "loss": 2.7819, + "theoretical_loss": 3.442755116268537, + "tokens_seen": 1915879424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266464646464647, + "loss": 2.6544, + "theoretical_loss": 3.4427452142919415, + "tokens_seen": 1915944960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042662626262626263, + "loss": 2.6319, + "theoretical_loss": 3.442735312748876, + "tokens_seen": 1916010496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266060606060606, + "loss": 2.7644, + "theoretical_loss": 3.442725411639307, + "tokens_seen": 1916076032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265858585858586, + "loss": 2.6702, + "theoretical_loss": 3.4427155109632004, + "tokens_seen": 1916141568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265656565656566, + "loss": 2.5994, + "theoretical_loss": 3.4427056107205223, + "tokens_seen": 1916207104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042654545454545457, + "loss": 2.7542, + "theoretical_loss": 3.4426957109112393, + "tokens_seen": 1916272640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265252525252525, + "loss": 2.4553, + "theoretical_loss": 3.442685811535317, + "tokens_seen": 1916338176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265050505050505, + "loss": 2.6332, + "theoretical_loss": 3.4426759125927227, + "tokens_seen": 1916403712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264848484848485, + "loss": 2.6213, + "theoretical_loss": 3.4426660140834215, + "tokens_seen": 1916469248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264646464646465, + "loss": 2.6587, + "theoretical_loss": 3.44265611600738, + "tokens_seen": 1916534784 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042644444444444446, + "loss": 2.6661, + "theoretical_loss": 3.442646218364565, + "tokens_seen": 1916600320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264242424242424, + "loss": 2.4961, + "theoretical_loss": 3.442636321154942, + "tokens_seen": 1916665856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042640404040404043, + "loss": 2.3362, + "theoretical_loss": 3.442626424378477, + "tokens_seen": 1916731392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263838383838384, + "loss": 2.6343, + "theoretical_loss": 3.4426165280351366, + "tokens_seen": 1916796928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263636363636364, + "loss": 2.6596, + "theoretical_loss": 3.4426066321248876, + "tokens_seen": 1916862464 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042634343434343435, + "loss": 2.8295, + "theoretical_loss": 3.4425967366476957, + "tokens_seen": 1916928000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263232323232323, + "loss": 2.6982, + "theoretical_loss": 3.442586841603527, + "tokens_seen": 1916993536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263030303030303, + "loss": 2.4981, + "theoretical_loss": 3.4425769469923475, + "tokens_seen": 1917059072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262828282828283, + "loss": 2.7179, + "theoretical_loss": 3.442567052814124, + "tokens_seen": 1917124608 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1055508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.873598575592041, + "objective/train/theoretical_loss": 3.4425571590688233, + "objective/train/tokens_used": 276049376, + "theoretical_loss": 3.4425571590688233, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262626262626263, + "loss": 2.8046, + "theoretical_loss": 3.4425571590688233, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042624242424242425, + "loss": 2.6275, + "theoretical_loss": 3.442547265756411, + "tokens_seen": 1917255680 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042622222222222226, + "loss": 2.5438, + "theoretical_loss": 3.4425373728768527, + "tokens_seen": 1917321216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262020202020202, + "loss": 2.5688, + "theoretical_loss": 3.4425274804301154, + "tokens_seen": 1917386752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042618181818181817, + "loss": 2.5858, + "theoretical_loss": 3.442517588416165, + "tokens_seen": 1917452288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261616161616162, + "loss": 2.3179, + "theoretical_loss": 3.442507696834969, + "tokens_seen": 1917517824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042614141414141414, + "loss": 2.6822, + "theoretical_loss": 3.442497805686492, + "tokens_seen": 1917583360 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042612121212121215, + "loss": 2.8473, + "theoretical_loss": 3.4424879149707017, + "tokens_seen": 1917648896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261010101010101, + "loss": 2.7661, + "theoretical_loss": 3.442478024687563, + "tokens_seen": 1917714432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042608080808080806, + "loss": 2.6435, + "theoretical_loss": 3.442468134837043, + "tokens_seen": 1917779968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260606060606061, + "loss": 2.5611, + "theoretical_loss": 3.4424582454191084, + "tokens_seen": 1917845504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260404040404041, + "loss": 2.9118, + "theoretical_loss": 3.4424483564337245, + "tokens_seen": 1917911040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042602020202020204, + "loss": 2.5781, + "theoretical_loss": 3.4424384678808586, + "tokens_seen": 1917976576 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426, + "loss": 2.6566, + "theoretical_loss": 3.4424285797604757, + "tokens_seen": 1918042112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042597979797979796, + "loss": 2.5477, + "theoretical_loss": 3.4424186920725437, + "tokens_seen": 1918107648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042595959595959597, + "loss": 2.6297, + "theoretical_loss": 3.4424088048170276, + "tokens_seen": 1918173184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425939393939394, + "loss": 2.5819, + "theoretical_loss": 3.4423989179938945, + "tokens_seen": 1918238720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042591919191919194, + "loss": 2.8178, + "theoretical_loss": 3.4423890316031107, + "tokens_seen": 1918304256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258989898989899, + "loss": 2.5821, + "theoretical_loss": 3.442379145644642, + "tokens_seen": 1918369792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042587878787878785, + "loss": 2.6713, + "theoretical_loss": 3.442369260118455, + "tokens_seen": 1918435328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258585858585859, + "loss": 2.6253, + "theoretical_loss": 3.442359375024516, + "tokens_seen": 1918500864 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042583838383838387, + "loss": 2.7187, + "theoretical_loss": 3.4423494903627914, + "tokens_seen": 1918566400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042581818181818183, + "loss": 2.6976, + "theoretical_loss": 3.442339606133247, + "tokens_seen": 1918631936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257979797979798, + "loss": 2.7475, + "theoretical_loss": 3.4423297223358507, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042577777777777774, + "loss": 2.7081, + "theoretical_loss": 3.442319838970567, + "tokens_seen": 1918763008 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1056802, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.889559268951416, + "objective/train/theoretical_loss": 3.4423099560373633, + "objective/train/tokens_used": 277687776, + "theoretical_loss": 3.4423099560373633, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257575757575758, + "loss": 2.8355, + "theoretical_loss": 3.4423099560373633, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042573737373737377, + "loss": 2.7127, + "theoretical_loss": 3.442300073536206, + "tokens_seen": 1918894080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257171717171717, + "loss": 2.5629, + "theoretical_loss": 3.4422901914670607, + "tokens_seen": 1918959616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256969696969697, + "loss": 2.5977, + "theoretical_loss": 3.4422803098298944, + "tokens_seen": 1919025152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042567676767676764, + "loss": 2.4683, + "theoretical_loss": 3.4422704286246733, + "tokens_seen": 1919090688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256565656565657, + "loss": 2.502, + "theoretical_loss": 3.442260547851364, + "tokens_seen": 1919156224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042563636363636366, + "loss": 2.6394, + "theoretical_loss": 3.4422506675099322, + "tokens_seen": 1919221760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256161616161616, + "loss": 2.5128, + "theoretical_loss": 3.442240787600345, + "tokens_seen": 1919287296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042559595959595957, + "loss": 2.5185, + "theoretical_loss": 3.4422309081225686, + "tokens_seen": 1919352832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255757575757576, + "loss": 2.6894, + "theoretical_loss": 3.4422210290765687, + "tokens_seen": 1919418368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255555555555556, + "loss": 2.5627, + "theoretical_loss": 3.442211150462313, + "tokens_seen": 1919483904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042553535353535355, + "loss": 2.8682, + "theoretical_loss": 3.4422012722797666, + "tokens_seen": 1919549440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255151515151515, + "loss": 2.5446, + "theoretical_loss": 3.442191394528897, + "tokens_seen": 1919614976 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549494949494946, + "loss": 2.6554, + "theoretical_loss": 3.4421815172096695, + "tokens_seen": 1919680512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254747474747475, + "loss": 2.808, + "theoretical_loss": 3.4421716403220515, + "tokens_seen": 1919746048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254545454545455, + "loss": 2.5093, + "theoretical_loss": 3.4421617638660083, + "tokens_seen": 1919811584 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042543434343434344, + "loss": 2.6735, + "theoretical_loss": 3.442151887841508, + "tokens_seen": 1919877120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254141414141414, + "loss": 2.6459, + "theoretical_loss": 3.442142012248515, + "tokens_seen": 1919942656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253939393939394, + "loss": 2.4776, + "theoretical_loss": 3.442132137086997, + "tokens_seen": 1920008192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042537373737373737, + "loss": 2.5164, + "theoretical_loss": 3.4421222623569205, + "tokens_seen": 1920073728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253535353535354, + "loss": 2.7869, + "theoretical_loss": 3.442112388058251, + "tokens_seen": 1920139264 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533333333333334, + "loss": 2.5343, + "theoretical_loss": 3.442102514190956, + "tokens_seen": 1920204800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253131313131313, + "loss": 2.6066, + "theoretical_loss": 3.4420926407550008, + "tokens_seen": 1920270336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252929292929293, + "loss": 2.7356, + "theoretical_loss": 3.442082767750353, + "tokens_seen": 1920335872 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042527272727272726, + "loss": 2.6266, + "theoretical_loss": 3.4420728951769783, + "tokens_seen": 1920401408 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1058271, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8692092895507812, + "objective/train/theoretical_loss": 3.4420630230348435, + "objective/train/tokens_used": 279326176, + "theoretical_loss": 3.4420630230348435, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042525252525252527, + "loss": 2.789, + "theoretical_loss": 3.4420630230348435, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042523232323232323, + "loss": 2.5816, + "theoretical_loss": 3.4420531513239148, + "tokens_seen": 1920532480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042521212121212124, + "loss": 2.4545, + "theoretical_loss": 3.4420432800441585, + "tokens_seen": 1920598016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251919191919192, + "loss": 2.5497, + "theoretical_loss": 3.4420334091955413, + "tokens_seen": 1920663552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042517171717171715, + "loss": 2.4662, + "theoretical_loss": 3.44202353877803, + "tokens_seen": 1920729088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042515151515151516, + "loss": 2.581, + "theoretical_loss": 3.4420136687915903, + "tokens_seen": 1920794624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251313131313131, + "loss": 2.5163, + "theoretical_loss": 3.442003799236189, + "tokens_seen": 1920860160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042511111111111113, + "loss": 2.6854, + "theoretical_loss": 3.441993930111793, + "tokens_seen": 1920925696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250909090909091, + "loss": 2.6518, + "theoretical_loss": 3.4419840614183688, + "tokens_seen": 1920991232 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042507070707070705, + "loss": 2.7454, + "theoretical_loss": 3.4419741931558816, + "tokens_seen": 1921056768 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042505050505050506, + "loss": 2.4956, + "theoretical_loss": 3.4419643253242995, + "tokens_seen": 1921122304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042503030303030307, + "loss": 2.7089, + "theoretical_loss": 3.441954457923588, + "tokens_seen": 1921187840 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425010101010101, + "loss": 2.9059, + "theoretical_loss": 3.441944590953714, + "tokens_seen": 1921253376 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424989898989899, + "loss": 2.7142, + "theoretical_loss": 3.4419347244146437, + "tokens_seen": 1921318912 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424969696969697, + "loss": 2.66, + "theoretical_loss": 3.4419248583063435, + "tokens_seen": 1921384448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042494949494949495, + "loss": 2.5657, + "theoretical_loss": 3.4419149926287806, + "tokens_seen": 1921449984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042492929292929296, + "loss": 2.5889, + "theoretical_loss": 3.441905127381921, + "tokens_seen": 1921515520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249090909090909, + "loss": 2.8425, + "theoretical_loss": 3.4418952625657306, + "tokens_seen": 1921581056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248888888888889, + "loss": 2.642, + "theoretical_loss": 3.4418853981801774, + "tokens_seen": 1921646592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248686868686869, + "loss": 2.6522, + "theoretical_loss": 3.4418755342252267, + "tokens_seen": 1921712128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248484848484849, + "loss": 2.6508, + "theoretical_loss": 3.441865670700846, + "tokens_seen": 1921777664 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042482828282828285, + "loss": 2.7685, + "theoretical_loss": 3.4418558076070003, + "tokens_seen": 1921843200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248080808080808, + "loss": 2.5812, + "theoretical_loss": 3.4418459449436574, + "tokens_seen": 1921908736 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042478787878787877, + "loss": 2.419, + "theoretical_loss": 3.4418360827107835, + "tokens_seen": 1921974272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247676767676768, + "loss": 2.5664, + "theoretical_loss": 3.4418262209083457, + "tokens_seen": 1922039808 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1058853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.736966609954834, + "objective/train/theoretical_loss": 3.4418163595363094, + "objective/train/tokens_used": 280964576, + "theoretical_loss": 3.4418163595363094, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247474747474748, + "loss": 2.7834, + "theoretical_loss": 3.4418163595363094, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042472727272727275, + "loss": 2.6645, + "theoretical_loss": 3.4418064985946417, + "tokens_seen": 1922170880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247070707070707, + "loss": 2.6125, + "theoretical_loss": 3.441796638083309, + "tokens_seen": 1922236416 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042468686868686866, + "loss": 2.6596, + "theoretical_loss": 3.441786778002278, + "tokens_seen": 1922301952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246666666666667, + "loss": 2.631, + "theoretical_loss": 3.4417769183515157, + "tokens_seen": 1922367488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246464646464647, + "loss": 2.5757, + "theoretical_loss": 3.4417670591309877, + "tokens_seen": 1922433024 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042462626262626264, + "loss": 2.5161, + "theoretical_loss": 3.441757200340662, + "tokens_seen": 1922498560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246060606060606, + "loss": 2.6606, + "theoretical_loss": 3.4417473419805034, + "tokens_seen": 1922564096 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042458585858585855, + "loss": 2.7274, + "theoretical_loss": 3.4417374840504795, + "tokens_seen": 1922629632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245656565656566, + "loss": 2.6871, + "theoretical_loss": 3.4417276265505565, + "tokens_seen": 1922695168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245454545454546, + "loss": 2.8311, + "theoretical_loss": 3.441717769480701, + "tokens_seen": 1922760704 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042452525252525253, + "loss": 2.5027, + "theoretical_loss": 3.4417079128408803, + "tokens_seen": 1922826240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245050505050505, + "loss": 2.7313, + "theoretical_loss": 3.4416980566310604, + "tokens_seen": 1922891776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244848484848485, + "loss": 2.6135, + "theoretical_loss": 3.4416882008512073, + "tokens_seen": 1922957312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244646464646465, + "loss": 2.6602, + "theoretical_loss": 3.4416783455012885, + "tokens_seen": 1923022848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042444444444444447, + "loss": 2.6453, + "theoretical_loss": 3.4416684905812707, + "tokens_seen": 1923088384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244242424242424, + "loss": 2.57, + "theoretical_loss": 3.4416586360911197, + "tokens_seen": 1923153920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244040404040404, + "loss": 2.7243, + "theoretical_loss": 3.441648782030802, + "tokens_seen": 1923219456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243838383838384, + "loss": 2.8261, + "theoretical_loss": 3.4416389284002853, + "tokens_seen": 1923284992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243636363636364, + "loss": 2.4296, + "theoretical_loss": 3.4416290751995353, + "tokens_seen": 1923350528 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042434343434343436, + "loss": 2.5943, + "theoretical_loss": 3.4416192224285194, + "tokens_seen": 1923416064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243232323232323, + "loss": 2.6942, + "theoretical_loss": 3.4416093700872032, + "tokens_seen": 1923481600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042430303030303033, + "loss": 2.6438, + "theoretical_loss": 3.441599518175554, + "tokens_seen": 1923547136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242828282828283, + "loss": 2.5493, + "theoretical_loss": 3.441589666693538, + "tokens_seen": 1923612672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242626262626263, + "loss": 2.6233, + "theoretical_loss": 3.4415798156411226, + "tokens_seen": 1923678208 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1059748, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.226703643798828, + "objective/train/theoretical_loss": 3.4415699650182736, + "objective/train/tokens_used": 282602976, + "theoretical_loss": 3.4415699650182736, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042424242424242425, + "loss": 2.5937, + "theoretical_loss": 3.4415699650182736, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242222222222222, + "loss": 2.6444, + "theoretical_loss": 3.4415601148249584, + "tokens_seen": 1923809280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242020202020202, + "loss": 2.6106, + "theoretical_loss": 3.4415502650611427, + "tokens_seen": 1923874816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241818181818182, + "loss": 2.6631, + "theoretical_loss": 3.441540415726794, + "tokens_seen": 1923940352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241616161616162, + "loss": 2.6376, + "theoretical_loss": 3.4415305668218785, + "tokens_seen": 1924005888 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042414141414141415, + "loss": 2.7537, + "theoretical_loss": 3.4415207183463625, + "tokens_seen": 1924071424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042412121212121216, + "loss": 2.858, + "theoretical_loss": 3.4415108703002137, + "tokens_seen": 1924136960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241010101010101, + "loss": 2.7537, + "theoretical_loss": 3.441501022683398, + "tokens_seen": 1924202496 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042408080808080807, + "loss": 2.7596, + "theoretical_loss": 3.441491175495882, + "tokens_seen": 1924268032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240606060606061, + "loss": 2.5014, + "theoretical_loss": 3.4414813287376327, + "tokens_seen": 1924333568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042404040404040404, + "loss": 2.6273, + "theoretical_loss": 3.4414714824086166, + "tokens_seen": 1924399104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042402020202020205, + "loss": 2.4565, + "theoretical_loss": 3.4414616365088007, + "tokens_seen": 1924464640 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424, + "loss": 2.5366, + "theoretical_loss": 3.4414517910381512, + "tokens_seen": 1924530176 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042397979797979796, + "loss": 2.5527, + "theoretical_loss": 3.441441945996635, + "tokens_seen": 1924595712 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423959595959596, + "loss": 2.6158, + "theoretical_loss": 3.441432101384218, + "tokens_seen": 1924661248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042393939393939393, + "loss": 2.5519, + "theoretical_loss": 3.441422257200869, + "tokens_seen": 1924726784 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042391919191919194, + "loss": 2.6452, + "theoretical_loss": 3.4414124134465522, + "tokens_seen": 1924792320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238989898989899, + "loss": 2.4951, + "theoretical_loss": 3.4414025701212365, + "tokens_seen": 1924857856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042387878787878786, + "loss": 2.6911, + "theoretical_loss": 3.4413927272248865, + "tokens_seen": 1924923392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042385858585858587, + "loss": 2.6374, + "theoretical_loss": 3.441382884757471, + "tokens_seen": 1924988928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238383838383839, + "loss": 2.995, + "theoretical_loss": 3.4413730427189546, + "tokens_seen": 1925054464 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042381818181818183, + "loss": 2.9582, + "theoretical_loss": 3.441363201109305, + "tokens_seen": 1925120000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237979797979798, + "loss": 2.7288, + "theoretical_loss": 3.44135335992849, + "tokens_seen": 1925185536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042377777777777775, + "loss": 2.5508, + "theoretical_loss": 3.4413435191764745, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042375757575757576, + "loss": 2.5517, + "theoretical_loss": 3.4413336788532263, + "tokens_seen": 1925316608 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1060184, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.955732583999634, + "objective/train/theoretical_loss": 3.441323838958712, + "objective/train/tokens_used": 284241376, + "theoretical_loss": 3.441323838958712, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042373737373737377, + "loss": 2.7418, + "theoretical_loss": 3.441323838958712, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042371717171717173, + "loss": 2.5483, + "theoretical_loss": 3.4413139994928974, + "tokens_seen": 1925447680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236969696969697, + "loss": 2.646, + "theoretical_loss": 3.4413041604557506, + "tokens_seen": 1925513216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042367676767676764, + "loss": 2.7451, + "theoretical_loss": 3.441294321847238, + "tokens_seen": 1925578752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236565656565657, + "loss": 2.6215, + "theoretical_loss": 3.4412844836673253, + "tokens_seen": 1925644288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042363636363636366, + "loss": 2.6979, + "theoretical_loss": 3.44127464591598, + "tokens_seen": 1925709824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236161616161616, + "loss": 2.5959, + "theoretical_loss": 3.4412648085931696, + "tokens_seen": 1925775360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235959595959596, + "loss": 2.5962, + "theoretical_loss": 3.44125497169886, + "tokens_seen": 1925840896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042357575757575753, + "loss": 2.7386, + "theoretical_loss": 3.4412451352330176, + "tokens_seen": 1925906432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235555555555556, + "loss": 2.6526, + "theoretical_loss": 3.4412352991956094, + "tokens_seen": 1925971968 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042353535353535356, + "loss": 2.552, + "theoretical_loss": 3.441225463586603, + "tokens_seen": 1926037504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235151515151515, + "loss": 2.6995, + "theoretical_loss": 3.4412156284059643, + "tokens_seen": 1926103040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042349494949494947, + "loss": 2.7203, + "theoretical_loss": 3.44120579365366, + "tokens_seen": 1926168576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234747474747475, + "loss": 2.6077, + "theoretical_loss": 3.441195959329658, + "tokens_seen": 1926234112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234545454545455, + "loss": 2.7539, + "theoretical_loss": 3.4411861254339238, + "tokens_seen": 1926299648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042343434343434345, + "loss": 2.5654, + "theoretical_loss": 3.4411762919664244, + "tokens_seen": 1926365184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234141414141414, + "loss": 2.6057, + "theoretical_loss": 3.441166458927127, + "tokens_seen": 1926430720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042339393939393936, + "loss": 2.713, + "theoretical_loss": 3.4411566263159985, + "tokens_seen": 1926496256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233737373737374, + "loss": 2.6969, + "theoretical_loss": 3.441146794133005, + "tokens_seen": 1926561792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233535353535354, + "loss": 2.7568, + "theoretical_loss": 3.4411369623781134, + "tokens_seen": 1926627328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042333333333333334, + "loss": 2.5438, + "theoretical_loss": 3.4411271310512914, + "tokens_seen": 1926692864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233131313131313, + "loss": 2.6664, + "theoretical_loss": 3.441117300152505, + "tokens_seen": 1926758400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232929292929293, + "loss": 2.6309, + "theoretical_loss": 3.441107469681721, + "tokens_seen": 1926823936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232727272727273, + "loss": 2.4625, + "theoretical_loss": 3.441097639638907, + "tokens_seen": 1926889472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232525252525253, + "loss": 2.6909, + "theoretical_loss": 3.4410878100240287, + "tokens_seen": 1926955008 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1061402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8198060989379883, + "objective/train/theoretical_loss": 3.4410779808370533, + "objective/train/tokens_used": 285879776, + "theoretical_loss": 3.4410779808370533, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042323232323232323, + "loss": 2.7919, + "theoretical_loss": 3.4410779808370533, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232121212121212, + "loss": 2.6017, + "theoretical_loss": 3.441068152077948, + "tokens_seen": 1927086080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231919191919192, + "loss": 2.7291, + "theoretical_loss": 3.4410583237466796, + "tokens_seen": 1927151616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231717171717172, + "loss": 2.5611, + "theoretical_loss": 3.441048495843215, + "tokens_seen": 1927217152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042315151515151517, + "loss": 2.5936, + "theoretical_loss": 3.4410386683675203, + "tokens_seen": 1927282688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231313131313131, + "loss": 2.8128, + "theoretical_loss": 3.4410288413195627, + "tokens_seen": 1927348224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042311111111111114, + "loss": 2.6127, + "theoretical_loss": 3.441019014699309, + "tokens_seen": 1927413760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230909090909091, + "loss": 2.662, + "theoretical_loss": 3.4410091885067264, + "tokens_seen": 1927479296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230707070707071, + "loss": 2.8147, + "theoretical_loss": 3.440999362741781, + "tokens_seen": 1927544832 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042305050505050506, + "loss": 2.7461, + "theoretical_loss": 3.440989537404441, + "tokens_seen": 1927610368 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423030303030303, + "loss": 2.6908, + "theoretical_loss": 3.4409797124946726, + "tokens_seen": 1927675904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042301010101010103, + "loss": 2.7072, + "theoretical_loss": 3.4409698880124413, + "tokens_seen": 1927741440 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422989898989899, + "loss": 2.3188, + "theoretical_loss": 3.440960063957716, + "tokens_seen": 1927806976 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422969696969697, + "loss": 2.7596, + "theoretical_loss": 3.4409502403304626, + "tokens_seen": 1927872512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042294949494949496, + "loss": 2.6218, + "theoretical_loss": 3.4409404171306477, + "tokens_seen": 1927938048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042292929292929297, + "loss": 2.5852, + "theoretical_loss": 3.440930594358239, + "tokens_seen": 1928003584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229090909090909, + "loss": 2.6241, + "theoretical_loss": 3.440920772013203, + "tokens_seen": 1928069120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228888888888889, + "loss": 2.7613, + "theoretical_loss": 3.440910950095506, + "tokens_seen": 1928134656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228686868686869, + "loss": 2.7854, + "theoretical_loss": 3.440901128605116, + "tokens_seen": 1928200192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284848484848485, + "loss": 2.5871, + "theoretical_loss": 3.4408913075419987, + "tokens_seen": 1928265728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042282828282828286, + "loss": 2.6012, + "theoretical_loss": 3.4408814869061217, + "tokens_seen": 1928331264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228080808080808, + "loss": 2.663, + "theoretical_loss": 3.440871666697452, + "tokens_seen": 1928396800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042278787878787877, + "loss": 2.6573, + "theoretical_loss": 3.440861846915956, + "tokens_seen": 1928462336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227676767676768, + "loss": 2.5695, + "theoretical_loss": 3.4408520275616006, + "tokens_seen": 1928527872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227474747474748, + "loss": 2.8241, + "theoretical_loss": 3.4408422086343533, + "tokens_seen": 1928593408 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1062041, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.024728298187256, + "objective/train/theoretical_loss": 3.4408323901341813, + "objective/train/tokens_used": 287518176, + "theoretical_loss": 3.4408323901341813, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042272727272727275, + "loss": 2.6987, + "theoretical_loss": 3.4408323901341813, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227070707070707, + "loss": 2.6473, + "theoretical_loss": 3.44082257206105, + "tokens_seen": 1928724480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042268686868686867, + "loss": 2.7776, + "theoretical_loss": 3.440812754414927, + "tokens_seen": 1928790016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226666666666667, + "loss": 2.7868, + "theoretical_loss": 3.4408029371957802, + "tokens_seen": 1928855552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226464646464647, + "loss": 2.6503, + "theoretical_loss": 3.4407931204035753, + "tokens_seen": 1928921088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042262626262626264, + "loss": 2.5474, + "theoretical_loss": 3.44078330403828, + "tokens_seen": 1928986624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226060606060606, + "loss": 2.6479, + "theoretical_loss": 3.440773488099861, + "tokens_seen": 1929052160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042258585858585856, + "loss": 2.6409, + "theoretical_loss": 3.4407636725882846, + "tokens_seen": 1929117696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225656565656566, + "loss": 2.5776, + "theoretical_loss": 3.440753857503519, + "tokens_seen": 1929183232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225454545454546, + "loss": 2.6831, + "theoretical_loss": 3.44074404284553, + "tokens_seen": 1929248768 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252525252525254, + "loss": 2.7039, + "theoretical_loss": 3.440734228614285, + "tokens_seen": 1929314304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225050505050505, + "loss": 2.5256, + "theoretical_loss": 3.4407244148097504, + "tokens_seen": 1929379840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248484848484845, + "loss": 2.8189, + "theoretical_loss": 3.4407146014318943, + "tokens_seen": 1929445376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224646464646465, + "loss": 2.7982, + "theoretical_loss": 3.4407047884806827, + "tokens_seen": 1929510912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224444444444445, + "loss": 2.5438, + "theoretical_loss": 3.4406949759560836, + "tokens_seen": 1929576448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042242424242424243, + "loss": 2.5654, + "theoretical_loss": 3.4406851638580624, + "tokens_seen": 1929641984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224040404040404, + "loss": 2.5339, + "theoretical_loss": 3.440675352186587, + "tokens_seen": 1929707520 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042238383838383834, + "loss": 2.3803, + "theoretical_loss": 3.440665540941625, + "tokens_seen": 1929773056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223636363636364, + "loss": 2.5308, + "theoretical_loss": 3.4406557301231424, + "tokens_seen": 1929838592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042234343434343437, + "loss": 2.5711, + "theoretical_loss": 3.4406459197311063, + "tokens_seen": 1929904128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223232323232323, + "loss": 2.6102, + "theoretical_loss": 3.440636109765484, + "tokens_seen": 1929969664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223030303030303, + "loss": 2.5921, + "theoretical_loss": 3.440626300226242, + "tokens_seen": 1930035200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222828282828283, + "loss": 2.5174, + "theoretical_loss": 3.440616491113348, + "tokens_seen": 1930100736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222626262626263, + "loss": 2.7388, + "theoretical_loss": 3.4406066824267683, + "tokens_seen": 1930166272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042224242424242426, + "loss": 2.7482, + "theoretical_loss": 3.44059687416647, + "tokens_seen": 1930231808 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1063329, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.848512649536133, + "objective/train/theoretical_loss": 3.440587066332421, + "objective/train/tokens_used": 289156576, + "theoretical_loss": 3.440587066332421, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222222222222222, + "loss": 2.5179, + "theoretical_loss": 3.440587066332421, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042220202020202017, + "loss": 2.451, + "theoretical_loss": 3.4405772589245873, + "tokens_seen": 1930362880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221818181818182, + "loss": 2.9025, + "theoretical_loss": 3.440567451942936, + "tokens_seen": 1930428416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221616161616162, + "loss": 2.5348, + "theoretical_loss": 3.440557645387435, + "tokens_seen": 1930493952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042214141414141415, + "loss": 2.5781, + "theoretical_loss": 3.44054783925805, + "tokens_seen": 1930559488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221212121212121, + "loss": 2.7627, + "theoretical_loss": 3.4405380335547493, + "tokens_seen": 1930625024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221010101010101, + "loss": 2.5685, + "theoretical_loss": 3.4405282282774987, + "tokens_seen": 1930690560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220808080808081, + "loss": 2.5545, + "theoretical_loss": 3.4405184234262656, + "tokens_seen": 1930756096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220606060606061, + "loss": 2.519, + "theoretical_loss": 3.440508619001018, + "tokens_seen": 1930821632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042204040404040404, + "loss": 2.6168, + "theoretical_loss": 3.440498815001722, + "tokens_seen": 1930887168 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422020202020202, + "loss": 2.513, + "theoretical_loss": 3.440489011428345, + "tokens_seen": 1930952704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422, + "loss": 2.6129, + "theoretical_loss": 3.440479208280853, + "tokens_seen": 1931018240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042197979797979797, + "loss": 2.8086, + "theoretical_loss": 3.440469405559215, + "tokens_seen": 1931083776 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421959595959596, + "loss": 2.7246, + "theoretical_loss": 3.4404596032633963, + "tokens_seen": 1931149312 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042193939393939394, + "loss": 2.5076, + "theoretical_loss": 3.440449801393364, + "tokens_seen": 1931214848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042191919191919195, + "loss": 2.6279, + "theoretical_loss": 3.440439999949087, + "tokens_seen": 1931280384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218989898989899, + "loss": 2.7267, + "theoretical_loss": 3.440430198930531, + "tokens_seen": 1931345920 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042187878787878786, + "loss": 2.6061, + "theoretical_loss": 3.440420398337663, + "tokens_seen": 1931411456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042185858585858587, + "loss": 2.6032, + "theoretical_loss": 3.44041059817045, + "tokens_seen": 1931476992 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042183838383838383, + "loss": 2.4926, + "theoretical_loss": 3.4404007984288594, + "tokens_seen": 1931542528 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042181818181818184, + "loss": 2.5209, + "theoretical_loss": 3.440390999112858, + "tokens_seen": 1931608064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217979797979798, + "loss": 2.4727, + "theoretical_loss": 3.4403812002224137, + "tokens_seen": 1931673600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042177777777777775, + "loss": 2.8118, + "theoretical_loss": 3.440371401757493, + "tokens_seen": 1931739136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042175757575757577, + "loss": 2.6462, + "theoretical_loss": 3.4403616037180624, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217373737373738, + "loss": 2.7645, + "theoretical_loss": 3.44035180610409, + "tokens_seen": 1931870208 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1064052, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8039486408233643, + "objective/train/theoretical_loss": 3.440342008915542, + "objective/train/tokens_used": 290794976, + "theoretical_loss": 3.440342008915542, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042171717171717173, + "loss": 2.663, + "theoretical_loss": 3.440342008915542, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216969696969697, + "loss": 2.5882, + "theoretical_loss": 3.440332212152386, + "tokens_seen": 1932001280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042167676767676765, + "loss": 2.931, + "theoretical_loss": 3.440322415814589, + "tokens_seen": 1932066816 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042165656565656566, + "loss": 2.5881, + "theoretical_loss": 3.4403126199021186, + "tokens_seen": 1932132352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042163636363636367, + "loss": 2.6762, + "theoretical_loss": 3.4403028244149416, + "tokens_seen": 1932197888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216161616161616, + "loss": 2.4638, + "theoretical_loss": 3.4402930293530245, + "tokens_seen": 1932263424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215959595959596, + "loss": 2.5095, + "theoretical_loss": 3.440283234716335, + "tokens_seen": 1932328960 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042157575757575754, + "loss": 2.6447, + "theoretical_loss": 3.44027344050484, + "tokens_seen": 1932394496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215555555555556, + "loss": 2.6329, + "theoretical_loss": 3.440263646718507, + "tokens_seen": 1932460032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042153535353535356, + "loss": 2.497, + "theoretical_loss": 3.4402538533573024, + "tokens_seen": 1932525568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215151515151515, + "loss": 2.6166, + "theoretical_loss": 3.440244060421194, + "tokens_seen": 1932591104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214949494949495, + "loss": 2.5755, + "theoretical_loss": 3.440234267910149, + "tokens_seen": 1932656640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214747474747475, + "loss": 2.7671, + "theoretical_loss": 3.4402244758241336, + "tokens_seen": 1932722176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214545454545455, + "loss": 2.7073, + "theoretical_loss": 3.4402146841631165, + "tokens_seen": 1932787712 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042143434343434345, + "loss": 2.6975, + "theoretical_loss": 3.4402048929270634, + "tokens_seen": 1932853248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214141414141414, + "loss": 2.7931, + "theoretical_loss": 3.4401951021159416, + "tokens_seen": 1932918784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042139393939393937, + "loss": 2.6141, + "theoretical_loss": 3.4401853117297194, + "tokens_seen": 1932984320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042137373737373743, + "loss": 2.6676, + "theoretical_loss": 3.4401755217683627, + "tokens_seen": 1933049856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213535353535354, + "loss": 2.5838, + "theoretical_loss": 3.440165732231839, + "tokens_seen": 1933115392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042133333333333335, + "loss": 2.7085, + "theoretical_loss": 3.440155943120116, + "tokens_seen": 1933180928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213131313131313, + "loss": 2.6783, + "theoretical_loss": 3.440146154433161, + "tokens_seen": 1933246464 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042129292929292926, + "loss": 2.6153, + "theoretical_loss": 3.4401363661709397, + "tokens_seen": 1933312000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212727272727273, + "loss": 2.6217, + "theoretical_loss": 3.4401265783334205, + "tokens_seen": 1933377536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212525252525253, + "loss": 2.6967, + "theoretical_loss": 3.4401167909205705, + "tokens_seen": 1933443072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042123232323232324, + "loss": 2.7279, + "theoretical_loss": 3.440107003932357, + "tokens_seen": 1933508608 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1065201, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.028951406478882, + "objective/train/theoretical_loss": 3.440097217368746, + "objective/train/tokens_used": 292433376, + "theoretical_loss": 3.440097217368746, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212121212121212, + "loss": 2.831, + "theoretical_loss": 3.440097217368746, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211919191919192, + "loss": 2.6147, + "theoretical_loss": 3.4400874312297063, + "tokens_seen": 1933639680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211717171717172, + "loss": 2.6807, + "theoretical_loss": 3.4400776455152036, + "tokens_seen": 1933705216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211515151515152, + "loss": 2.5114, + "theoretical_loss": 3.4400678602252066, + "tokens_seen": 1933770752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042113131313131313, + "loss": 2.3262, + "theoretical_loss": 3.4400580753596817, + "tokens_seen": 1933836288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211111111111111, + "loss": 2.5457, + "theoretical_loss": 3.440048290918596, + "tokens_seen": 1933901824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210909090909091, + "loss": 2.5826, + "theoretical_loss": 3.4400385069019164, + "tokens_seen": 1933967360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210707070707071, + "loss": 2.4071, + "theoretical_loss": 3.4400287233096107, + "tokens_seen": 1934032896 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042105050505050507, + "loss": 2.6196, + "theoretical_loss": 3.4400189401416466, + "tokens_seen": 1934098432 + }, + { + "epoch": 0.17, + "learning_rate": 0.000421030303030303, + "loss": 2.5962, + "theoretical_loss": 3.4400091573979896, + "tokens_seen": 1934163968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042101010101010104, + "loss": 2.669, + "theoretical_loss": 3.439999375078609, + "tokens_seen": 1934229504 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420989898989899, + "loss": 2.4876, + "theoretical_loss": 3.439989593183471, + "tokens_seen": 1934295040 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420969696969697, + "loss": 2.4545, + "theoretical_loss": 3.4399798117125426, + "tokens_seen": 1934360576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042094949494949496, + "loss": 2.6715, + "theoretical_loss": 3.439970030665791, + "tokens_seen": 1934426112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209292929292929, + "loss": 2.3414, + "theoretical_loss": 3.439960250043184, + "tokens_seen": 1934491648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042090909090909093, + "loss": 2.5181, + "theoretical_loss": 3.4399504698446886, + "tokens_seen": 1934557184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208888888888889, + "loss": 2.4947, + "theoretical_loss": 3.439940690070272, + "tokens_seen": 1934622720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208686868686869, + "loss": 2.554, + "theoretical_loss": 3.4399309107199016, + "tokens_seen": 1934688256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042084848484848485, + "loss": 2.7057, + "theoretical_loss": 3.439921131793544, + "tokens_seen": 1934753792 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042082828282828286, + "loss": 2.5841, + "theoretical_loss": 3.439911353291168, + "tokens_seen": 1934819328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208080808080808, + "loss": 2.5806, + "theoretical_loss": 3.4399015752127386, + "tokens_seen": 1934884864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207878787878788, + "loss": 2.7166, + "theoretical_loss": 3.439891797558225, + "tokens_seen": 1934950400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207676767676768, + "loss": 2.9193, + "theoretical_loss": 3.4398820203275937, + "tokens_seen": 1935015936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042074747474747475, + "loss": 2.5482, + "theoretical_loss": 3.439872243520812, + "tokens_seen": 1935081472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042072727272727276, + "loss": 2.641, + "theoretical_loss": 3.4398624671378473, + "tokens_seen": 1935147008 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1066155, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.908733367919922, + "objective/train/theoretical_loss": 3.4398526911786664, + "objective/train/tokens_used": 294071776, + "theoretical_loss": 3.4398526911786664, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207070707070707, + "loss": 2.8247, + "theoretical_loss": 3.4398526911786664, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042068686868686867, + "loss": 2.7459, + "theoretical_loss": 3.4398429156432373, + "tokens_seen": 1935278080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206666666666667, + "loss": 2.5958, + "theoretical_loss": 3.439833140531527, + "tokens_seen": 1935343616 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042064646464646464, + "loss": 2.6365, + "theoretical_loss": 3.4398233658435027, + "tokens_seen": 1935409152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042062626262626265, + "loss": 2.5277, + "theoretical_loss": 3.4398135915791315, + "tokens_seen": 1935474688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206060606060606, + "loss": 2.5039, + "theoretical_loss": 3.4398038177383805, + "tokens_seen": 1935540224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042058585858585856, + "loss": 2.6533, + "theoretical_loss": 3.439794044321218, + "tokens_seen": 1935605760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205656565656566, + "loss": 2.7235, + "theoretical_loss": 3.4397842713276106, + "tokens_seen": 1935671296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205454545454546, + "loss": 2.5335, + "theoretical_loss": 3.439774498757526, + "tokens_seen": 1935736832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042052525252525254, + "loss": 2.4197, + "theoretical_loss": 3.439764726610931, + "tokens_seen": 1935802368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205050505050505, + "loss": 2.6907, + "theoretical_loss": 3.4397549548877926, + "tokens_seen": 1935867904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042048484848484846, + "loss": 2.481, + "theoretical_loss": 3.4397451835880792, + "tokens_seen": 1935933440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042046464646464647, + "loss": 2.4428, + "theoretical_loss": 3.439735412711757, + "tokens_seen": 1935998976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204444444444445, + "loss": 2.7433, + "theoretical_loss": 3.4397256422587947, + "tokens_seen": 1936064512 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042042424242424244, + "loss": 2.5442, + "theoretical_loss": 3.4397158722291583, + "tokens_seen": 1936130048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204040404040404, + "loss": 2.4129, + "theoretical_loss": 3.4397061026228157, + "tokens_seen": 1936195584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042038383838383835, + "loss": 2.5154, + "theoretical_loss": 3.439696333439734, + "tokens_seen": 1936261120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203636363636364, + "loss": 2.4758, + "theoretical_loss": 3.439686564679881, + "tokens_seen": 1936326656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042034343434343437, + "loss": 2.7579, + "theoretical_loss": 3.4396767963432238, + "tokens_seen": 1936392192 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042032323232323233, + "loss": 2.5475, + "theoretical_loss": 3.4396670284297297, + "tokens_seen": 1936457728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203030303030303, + "loss": 2.5622, + "theoretical_loss": 3.4396572609393656, + "tokens_seen": 1936523264 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042028282828282824, + "loss": 2.6701, + "theoretical_loss": 3.4396474938720996, + "tokens_seen": 1936588800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202626262626263, + "loss": 2.8353, + "theoretical_loss": 3.439637727227899, + "tokens_seen": 1936654336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042024242424242426, + "loss": 2.7227, + "theoretical_loss": 3.439627961006731, + "tokens_seen": 1936719872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202222222222222, + "loss": 2.5612, + "theoretical_loss": 3.4396181952085625, + "tokens_seen": 1936785408 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1066685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.676300048828125, + "objective/train/theoretical_loss": 3.4396084298333616, + "objective/train/tokens_used": 295710176, + "theoretical_loss": 3.4396084298333616, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202020202020202, + "loss": 2.6837, + "theoretical_loss": 3.4396084298333616, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201818181818182, + "loss": 2.4526, + "theoretical_loss": 3.4395986648810943, + "tokens_seen": 1936916480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201616161616162, + "loss": 2.698, + "theoretical_loss": 3.43958890035173, + "tokens_seen": 1936982016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042014141414141416, + "loss": 2.6901, + "theoretical_loss": 3.439579136245235, + "tokens_seen": 1937047552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201212121212121, + "loss": 2.6405, + "theoretical_loss": 3.439569372561577, + "tokens_seen": 1937113088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042010101010101007, + "loss": 2.7245, + "theoretical_loss": 3.439559609300722, + "tokens_seen": 1937178624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200808080808081, + "loss": 2.7824, + "theoretical_loss": 3.439549846462639, + "tokens_seen": 1937244160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200606060606061, + "loss": 2.3995, + "theoretical_loss": 3.4395400840472954, + "tokens_seen": 1937309696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042004040404040405, + "loss": 2.558, + "theoretical_loss": 3.439530322054658, + "tokens_seen": 1937375232 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420020202020202, + "loss": 2.6701, + "theoretical_loss": 3.439520560484694, + "tokens_seen": 1937440768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042, + "loss": 2.4783, + "theoretical_loss": 3.4395107993373712, + "tokens_seen": 1937506304 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419979797979798, + "loss": 2.6693, + "theoretical_loss": 3.439501038612657, + "tokens_seen": 1937571840 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419959595959596, + "loss": 2.5367, + "theoretical_loss": 3.4394912783105185, + "tokens_seen": 1937637376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041993939393939394, + "loss": 2.6768, + "theoretical_loss": 3.4394815184309238, + "tokens_seen": 1937702912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199191919191919, + "loss": 2.794, + "theoretical_loss": 3.4394717589738395, + "tokens_seen": 1937768448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198989898989899, + "loss": 2.5629, + "theoretical_loss": 3.4394619999392333, + "tokens_seen": 1937833984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041987878787878787, + "loss": 2.8272, + "theoretical_loss": 3.4394522413270727, + "tokens_seen": 1937899520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198585858585859, + "loss": 2.8337, + "theoretical_loss": 3.4394424831373254, + "tokens_seen": 1937965056 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041983838383838383, + "loss": 2.5516, + "theoretical_loss": 3.439432725369958, + "tokens_seen": 1938030592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041981818181818185, + "loss": 2.689, + "theoretical_loss": 3.439422968024939, + "tokens_seen": 1938096128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197979797979798, + "loss": 2.6582, + "theoretical_loss": 3.4394132111022353, + "tokens_seen": 1938161664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197777777777778, + "loss": 2.449, + "theoretical_loss": 3.4394034546018144, + "tokens_seen": 1938227200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197777777777778, + "loss": 2.9064, + "theoretical_loss": 3.4393936985236433, + "tokens_seen": 1938292736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041975757575757577, + "loss": 2.8677, + "theoretical_loss": 3.439383942867691, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041973737373737373, + "loss": 2.6937, + "theoretical_loss": 3.4393741876339226, + "tokens_seen": 1938423808 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1067842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.121251344680786, + "objective/train/theoretical_loss": 3.439364432822307, + "objective/train/tokens_used": 297348576, + "theoretical_loss": 3.439364432822307, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041971717171717174, + "loss": 2.7767, + "theoretical_loss": 3.439364432822307, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196969696969697, + "loss": 2.7315, + "theoretical_loss": 3.439354678432812, + "tokens_seen": 1938554880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196767676767677, + "loss": 2.7844, + "theoretical_loss": 3.439344924465404, + "tokens_seen": 1938620416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041965656565656566, + "loss": 2.6795, + "theoretical_loss": 3.4393351709200513, + "tokens_seen": 1938685952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196363636363637, + "loss": 2.6743, + "theoretical_loss": 3.439325417796721, + "tokens_seen": 1938751488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041961616161616163, + "loss": 2.605, + "theoretical_loss": 3.4393156650953802, + "tokens_seen": 1938817024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195959595959596, + "loss": 2.6641, + "theoretical_loss": 3.439305912815997, + "tokens_seen": 1938882560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195757575757576, + "loss": 2.8231, + "theoretical_loss": 3.4392961609585386, + "tokens_seen": 1938948096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041955555555555556, + "loss": 2.7356, + "theoretical_loss": 3.439286409522973, + "tokens_seen": 1939013632 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041953535353535357, + "loss": 2.6914, + "theoretical_loss": 3.439276658509267, + "tokens_seen": 1939079168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195151515151515, + "loss": 2.6307, + "theoretical_loss": 3.4392669079173883, + "tokens_seen": 1939144704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194949494949495, + "loss": 2.339, + "theoretical_loss": 3.4392571577473046, + "tokens_seen": 1939210240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194747474747475, + "loss": 2.614, + "theoretical_loss": 3.439247407998983, + "tokens_seen": 1939275776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194545454545455, + "loss": 2.5329, + "theoretical_loss": 3.4392376586723916, + "tokens_seen": 1939341312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943434343434346, + "loss": 2.789, + "theoretical_loss": 3.439227909767497, + "tokens_seen": 1939406848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194141414141414, + "loss": 2.7102, + "theoretical_loss": 3.439218161284268, + "tokens_seen": 1939472384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193939393939394, + "loss": 2.7777, + "theoretical_loss": 3.439208413222671, + "tokens_seen": 1939537920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193737373737374, + "loss": 2.593, + "theoretical_loss": 3.4391986655826736, + "tokens_seen": 1939603456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193535353535354, + "loss": 2.662, + "theoretical_loss": 3.439188918364244, + "tokens_seen": 1939668992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041933333333333335, + "loss": 2.8318, + "theoretical_loss": 3.439179171567349, + "tokens_seen": 1939734528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193131313131313, + "loss": 2.8346, + "theoretical_loss": 3.439169425191957, + "tokens_seen": 1939800064 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041929292929292927, + "loss": 2.4422, + "theoretical_loss": 3.439159679238035, + "tokens_seen": 1939865600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927272727272733, + "loss": 2.5032, + "theoretical_loss": 3.4391499337055498, + "tokens_seen": 1939931136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192525252525253, + "loss": 2.6996, + "theoretical_loss": 3.4391401885944703, + "tokens_seen": 1939996672 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041923232323232325, + "loss": 2.7708, + "theoretical_loss": 3.439130443904763, + "tokens_seen": 1940062208 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1068281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4002110958099365, + "objective/train/theoretical_loss": 3.4391206996363968, + "objective/train/tokens_used": 298986976, + "theoretical_loss": 3.4391206996363968, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192121212121212, + "loss": 2.7074, + "theoretical_loss": 3.4391206996363968, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041919191919191916, + "loss": 2.7498, + "theoretical_loss": 3.439110955789338, + "tokens_seen": 1940193280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191717171717172, + "loss": 2.583, + "theoretical_loss": 3.4391012123635534, + "tokens_seen": 1940258816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191515151515152, + "loss": 2.6829, + "theoretical_loss": 3.4390914693590124, + "tokens_seen": 1940324352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041913131313131314, + "loss": 2.6073, + "theoretical_loss": 3.439081726775682, + "tokens_seen": 1940389888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191111111111111, + "loss": 2.8032, + "theoretical_loss": 3.4390719846135296, + "tokens_seen": 1940455424 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041909090909090905, + "loss": 2.6993, + "theoretical_loss": 3.4390622428725224, + "tokens_seen": 1940520960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190707070707071, + "loss": 2.5315, + "theoretical_loss": 3.439052501552628, + "tokens_seen": 1940586496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190505050505051, + "loss": 2.6999, + "theoretical_loss": 3.4390427606538148, + "tokens_seen": 1940652032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041903030303030303, + "loss": 2.6889, + "theoretical_loss": 3.4390330201760495, + "tokens_seen": 1940717568 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419010101010101, + "loss": 2.6122, + "theoretical_loss": 3.439023280119301, + "tokens_seen": 1940783104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418989898989899, + "loss": 2.7696, + "theoretical_loss": 3.439013540483535, + "tokens_seen": 1940848640 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418969696969697, + "loss": 2.644, + "theoretical_loss": 3.4390038012687203, + "tokens_seen": 1940914176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041894949494949497, + "loss": 2.4996, + "theoretical_loss": 3.4389940624748236, + "tokens_seen": 1940979712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189292929292929, + "loss": 2.6993, + "theoretical_loss": 3.438984324101814, + "tokens_seen": 1941045248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189090909090909, + "loss": 2.4784, + "theoretical_loss": 3.4389745861496577, + "tokens_seen": 1941110784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188888888888889, + "loss": 2.6716, + "theoretical_loss": 3.438964848618323, + "tokens_seen": 1941176320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188686868686869, + "loss": 2.7038, + "theoretical_loss": 3.438955111507777, + "tokens_seen": 1941241856 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041884848484848486, + "loss": 2.4977, + "theoretical_loss": 3.438945374817988, + "tokens_seen": 1941307392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188282828282828, + "loss": 2.8805, + "theoretical_loss": 3.4389356385489234, + "tokens_seen": 1941372928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041880808080808083, + "loss": 2.6442, + "theoretical_loss": 3.43892590270055, + "tokens_seen": 1941438464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187878787878788, + "loss": 2.3895, + "theoretical_loss": 3.4389161672728368, + "tokens_seen": 1941504000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187676767676768, + "loss": 2.9731, + "theoretical_loss": 3.4389064322657505, + "tokens_seen": 1941569536 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041874747474747475, + "loss": 2.5621, + "theoretical_loss": 3.438896697679259, + "tokens_seen": 1941635072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187272727272727, + "loss": 2.6273, + "theoretical_loss": 3.4388869635133297, + "tokens_seen": 1941700608 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1069559, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2460758686065674, + "objective/train/theoretical_loss": 3.4388772297679306, + "objective/train/tokens_used": 300625376, + "theoretical_loss": 3.4388772297679306, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187070707070707, + "loss": 2.6064, + "theoretical_loss": 3.4388772297679306, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186868686868687, + "loss": 2.589, + "theoretical_loss": 3.438867496443029, + "tokens_seen": 1941831680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186666666666667, + "loss": 2.6783, + "theoretical_loss": 3.438857763538593, + "tokens_seen": 1941897216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041864646464646464, + "loss": 2.7044, + "theoretical_loss": 3.4388480310545892, + "tokens_seen": 1941962752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041862626262626266, + "loss": 2.364, + "theoretical_loss": 3.4388382989909863, + "tokens_seen": 1942028288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186060606060606, + "loss": 2.6038, + "theoretical_loss": 3.438828567347752, + "tokens_seen": 1942093824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041858585858585857, + "loss": 2.5249, + "theoretical_loss": 3.4388188361248533, + "tokens_seen": 1942159360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185656565656566, + "loss": 2.7292, + "theoretical_loss": 3.4388091053222585, + "tokens_seen": 1942224896 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041854545454545454, + "loss": 2.9845, + "theoretical_loss": 3.4387993749399346, + "tokens_seen": 1942290432 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041852525252525255, + "loss": 2.6296, + "theoretical_loss": 3.4387896449778497, + "tokens_seen": 1942355968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185050505050505, + "loss": 2.507, + "theoretical_loss": 3.4387799154359717, + "tokens_seen": 1942421504 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041848484848484846, + "loss": 2.5466, + "theoretical_loss": 3.4387701863142675, + "tokens_seen": 1942487040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184646464646465, + "loss": 2.6209, + "theoretical_loss": 3.4387604576127053, + "tokens_seen": 1942552576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184444444444445, + "loss": 2.6257, + "theoretical_loss": 3.438750729331253, + "tokens_seen": 1942618112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041842424242424244, + "loss": 2.5931, + "theoretical_loss": 3.438741001469878, + "tokens_seen": 1942683648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184040404040404, + "loss": 2.6329, + "theoretical_loss": 3.438731274028547, + "tokens_seen": 1942749184 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041838383838383835, + "loss": 2.6169, + "theoretical_loss": 3.43872154700723, + "tokens_seen": 1942814720 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041836363636363637, + "loss": 2.7174, + "theoretical_loss": 3.438711820405893, + "tokens_seen": 1942880256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183434343434344, + "loss": 2.7735, + "theoretical_loss": 3.438702094224504, + "tokens_seen": 1942945792 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041832323232323233, + "loss": 2.5786, + "theoretical_loss": 3.4386923684630304, + "tokens_seen": 1943011328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183030303030303, + "loss": 2.4048, + "theoretical_loss": 3.438682643121441, + "tokens_seen": 1943076864 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041828282828282825, + "loss": 2.4826, + "theoretical_loss": 3.4386729181997024, + "tokens_seen": 1943142400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182626262626263, + "loss": 2.6934, + "theoretical_loss": 3.4386631936977827, + "tokens_seen": 1943207936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041824242424242427, + "loss": 2.691, + "theoretical_loss": 3.43865346961565, + "tokens_seen": 1943273472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182222222222222, + "loss": 2.6558, + "theoretical_loss": 3.4386437459532715, + "tokens_seen": 1943339008 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1070184, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5214946269989014, + "objective/train/theoretical_loss": 3.4386340227106147, + "objective/train/tokens_used": 302263776, + "theoretical_loss": 3.4386340227106147, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182020202020202, + "loss": 2.5596, + "theoretical_loss": 3.4386340227106147, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041818181818181814, + "loss": 2.5195, + "theoretical_loss": 3.438624299887648, + "tokens_seen": 1943470080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181616161616162, + "loss": 2.7051, + "theoretical_loss": 3.438614577484339, + "tokens_seen": 1943535616 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041814141414141416, + "loss": 2.7595, + "theoretical_loss": 3.438604855500655, + "tokens_seen": 1943601152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181212121212121, + "loss": 2.9422, + "theoretical_loss": 3.4385951339365644, + "tokens_seen": 1943666688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181010101010101, + "loss": 2.6183, + "theoretical_loss": 3.438585412792034, + "tokens_seen": 1943732224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041808080808080814, + "loss": 2.8163, + "theoretical_loss": 3.4385756920670327, + "tokens_seen": 1943797760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180606060606061, + "loss": 2.7602, + "theoretical_loss": 3.438565971761528, + "tokens_seen": 1943863296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041804040404040406, + "loss": 2.8253, + "theoretical_loss": 3.4385562518754864, + "tokens_seen": 1943928832 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418020202020202, + "loss": 2.3846, + "theoretical_loss": 3.4385465324088766, + "tokens_seen": 1943994368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041799999999999997, + "loss": 2.5301, + "theoretical_loss": 3.438536813361667, + "tokens_seen": 1944059904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041797979797979803, + "loss": 2.6371, + "theoretical_loss": 3.4385270947338245, + "tokens_seen": 1944125440 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417959595959596, + "loss": 2.5773, + "theoretical_loss": 3.4385173765253167, + "tokens_seen": 1944190976 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041793939393939395, + "loss": 2.4892, + "theoretical_loss": 3.4385076587361123, + "tokens_seen": 1944256512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179191919191919, + "loss": 2.7892, + "theoretical_loss": 3.4384979413661787, + "tokens_seen": 1944322048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178989898989899, + "loss": 2.5332, + "theoretical_loss": 3.438488224415483, + "tokens_seen": 1944387584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178787878787879, + "loss": 2.636, + "theoretical_loss": 3.4384785078839935, + "tokens_seen": 1944453120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178585858585859, + "loss": 2.6614, + "theoretical_loss": 3.438468791771678, + "tokens_seen": 1944518656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783838383838384, + "loss": 2.7253, + "theoretical_loss": 3.4384590760785043, + "tokens_seen": 1944584192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178181818181818, + "loss": 2.7251, + "theoretical_loss": 3.4384493608044404, + "tokens_seen": 1944649728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177979797979798, + "loss": 2.6503, + "theoretical_loss": 3.4384396459494537, + "tokens_seen": 1944715264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177777777777778, + "loss": 2.4993, + "theoretical_loss": 3.4384299315135123, + "tokens_seen": 1944780800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177575757575758, + "loss": 2.7246, + "theoretical_loss": 3.438420217496584, + "tokens_seen": 1944846336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041773737373737373, + "loss": 2.3471, + "theoretical_loss": 3.438410503898636, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041771717171717174, + "loss": 2.6327, + "theoretical_loss": 3.438400790719637, + "tokens_seen": 1944977408 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1071164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9253249168395996, + "objective/train/theoretical_loss": 3.4383910779595537, + "objective/train/tokens_used": 303902176, + "theoretical_loss": 3.4383910779595537, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176969696969697, + "loss": 2.591, + "theoretical_loss": 3.4383910779595537, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176767676767677, + "loss": 2.5417, + "theoretical_loss": 3.4383813656183557, + "tokens_seen": 1945108480 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041765656565656567, + "loss": 2.5524, + "theoretical_loss": 3.438371653696009, + "tokens_seen": 1945174016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176363636363636, + "loss": 2.686, + "theoretical_loss": 3.4383619421924823, + "tokens_seen": 1945239552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041761616161616164, + "loss": 2.652, + "theoretical_loss": 3.4383522311077432, + "tokens_seen": 1945305088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175959595959596, + "loss": 2.5249, + "theoretical_loss": 3.43834252044176, + "tokens_seen": 1945370624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175757575757576, + "loss": 2.7325, + "theoretical_loss": 3.4383328101945, + "tokens_seen": 1945436160 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041755555555555556, + "loss": 2.6777, + "theoretical_loss": 3.438323100365931, + "tokens_seen": 1945501696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041753535353535357, + "loss": 2.8154, + "theoretical_loss": 3.4383133909560213, + "tokens_seen": 1945567232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041751515151515153, + "loss": 2.6971, + "theoretical_loss": 3.4383036819647383, + "tokens_seen": 1945632768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174949494949495, + "loss": 2.6609, + "theoretical_loss": 3.4382939733920503, + "tokens_seen": 1945698304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174747474747475, + "loss": 2.6654, + "theoretical_loss": 3.4382842652379244, + "tokens_seen": 1945763840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041745454545454545, + "loss": 2.6757, + "theoretical_loss": 3.4382745575023295, + "tokens_seen": 1945829376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041743434343434347, + "loss": 2.8648, + "theoretical_loss": 3.4382648501852326, + "tokens_seen": 1945894912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174141414141414, + "loss": 2.7462, + "theoretical_loss": 3.4382551432866024, + "tokens_seen": 1945960448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173939393939394, + "loss": 2.5422, + "theoretical_loss": 3.438245436806406, + "tokens_seen": 1946025984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173737373737374, + "loss": 2.5824, + "theoretical_loss": 3.438235730744611, + "tokens_seen": 1946091520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041735353535353535, + "loss": 2.5466, + "theoretical_loss": 3.4382260251011862, + "tokens_seen": 1946157056 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041733333333333336, + "loss": 2.5127, + "theoretical_loss": 3.4382163198760995, + "tokens_seen": 1946222592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173131313131313, + "loss": 2.6517, + "theoretical_loss": 3.4382066150693182, + "tokens_seen": 1946288128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041729292929292927, + "loss": 2.6034, + "theoretical_loss": 3.43819691068081, + "tokens_seen": 1946353664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172727272727273, + "loss": 2.6726, + "theoretical_loss": 3.438187206710543, + "tokens_seen": 1946419200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172525252525253, + "loss": 2.6317, + "theoretical_loss": 3.4381775031584856, + "tokens_seen": 1946484736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041723232323232325, + "loss": 2.7176, + "theoretical_loss": 3.4381678000246056, + "tokens_seen": 1946550272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172121212121212, + "loss": 2.5587, + "theoretical_loss": 3.43815809730887, + "tokens_seen": 1946615808 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1071829, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5584614276885986, + "objective/train/theoretical_loss": 3.4381483950112477, + "objective/train/tokens_used": 305540576, + "theoretical_loss": 3.4381483950112477, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041719191919191916, + "loss": 2.5635, + "theoretical_loss": 3.4381483950112477, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171717171717172, + "loss": 2.6877, + "theoretical_loss": 3.438138693131706, + "tokens_seen": 1946746880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171515151515152, + "loss": 2.7448, + "theoretical_loss": 3.4381289916702134, + "tokens_seen": 1946812416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041713131313131314, + "loss": 2.5441, + "theoretical_loss": 3.4381192906267373, + "tokens_seen": 1946877952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171111111111111, + "loss": 2.8343, + "theoretical_loss": 3.438109590001246, + "tokens_seen": 1946943488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041709090909090906, + "loss": 2.7814, + "theoretical_loss": 3.4380998897937065, + "tokens_seen": 1947009024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170707070707071, + "loss": 2.5301, + "theoretical_loss": 3.438090190004088, + "tokens_seen": 1947074560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170505050505051, + "loss": 2.6603, + "theoretical_loss": 3.438080490632358, + "tokens_seen": 1947140096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041703030303030304, + "loss": 2.4052, + "theoretical_loss": 3.438070791678484, + "tokens_seen": 1947205632 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417010101010101, + "loss": 2.5384, + "theoretical_loss": 3.4380610931424345, + "tokens_seen": 1947271168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041698989898989895, + "loss": 2.5251, + "theoretical_loss": 3.4380513950241767, + "tokens_seen": 1947336704 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416969696969697, + "loss": 2.754, + "theoretical_loss": 3.438041697323679, + "tokens_seen": 1947402240 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041694949494949497, + "loss": 2.5165, + "theoretical_loss": 3.43803200004091, + "tokens_seen": 1947467776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041692929292929293, + "loss": 2.6118, + "theoretical_loss": 3.4380223031758366, + "tokens_seen": 1947533312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169090909090909, + "loss": 2.6706, + "theoretical_loss": 3.438012606728427, + "tokens_seen": 1947598848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168888888888889, + "loss": 2.5127, + "theoretical_loss": 3.43800291069865, + "tokens_seen": 1947664384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168686868686869, + "loss": 2.6097, + "theoretical_loss": 3.4379932150864723, + "tokens_seen": 1947729920 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041684848484848486, + "loss": 2.7995, + "theoretical_loss": 3.4379835198918625, + "tokens_seen": 1947795456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168282828282828, + "loss": 2.5767, + "theoretical_loss": 3.4379738251147884, + "tokens_seen": 1947860992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168080808080808, + "loss": 2.7866, + "theoretical_loss": 3.4379641307552182, + "tokens_seen": 1947926528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167878787878788, + "loss": 2.3767, + "theoretical_loss": 3.43795443681312, + "tokens_seen": 1947992064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167676767676768, + "loss": 2.6098, + "theoretical_loss": 3.4379447432884613, + "tokens_seen": 1948057600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041674747474747476, + "loss": 2.5764, + "theoretical_loss": 3.4379350501812107, + "tokens_seen": 1948123136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167272727272727, + "loss": 2.7819, + "theoretical_loss": 3.437925357491335, + "tokens_seen": 1948188672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167070707070707, + "loss": 2.6237, + "theoretical_loss": 3.437915665218804, + "tokens_seen": 1948254208 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1073161, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.831909656524658, + "objective/train/theoretical_loss": 3.437905973363584, + "objective/train/tokens_used": 307178976, + "theoretical_loss": 3.437905973363584, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166868686868687, + "loss": 2.6736, + "theoretical_loss": 3.437905973363584, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166666666666667, + "loss": 2.6268, + "theoretical_loss": 3.437896281925644, + "tokens_seen": 1948385280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041664646464646465, + "loss": 2.8293, + "theoretical_loss": 3.4378865909049514, + "tokens_seen": 1948450816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166262626262626, + "loss": 2.5125, + "theoretical_loss": 3.4378769003014744, + "tokens_seen": 1948516352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166060606060606, + "loss": 2.7312, + "theoretical_loss": 3.437867210115181, + "tokens_seen": 1948581888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165858585858586, + "loss": 2.5568, + "theoretical_loss": 3.43785752034604, + "tokens_seen": 1948647424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165656565656566, + "loss": 2.9045, + "theoretical_loss": 3.437847830994018, + "tokens_seen": 1948712960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654545454545454, + "loss": 2.6776, + "theoretical_loss": 3.4378381420590842, + "tokens_seen": 1948778496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041652525252525255, + "loss": 2.5809, + "theoretical_loss": 3.437828453541206, + "tokens_seen": 1948844032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165050505050505, + "loss": 2.5214, + "theoretical_loss": 3.4378187654403516, + "tokens_seen": 1948909568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041648484848484847, + "loss": 2.5708, + "theoretical_loss": 3.437809077756489, + "tokens_seen": 1948975104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164646464646465, + "loss": 2.7088, + "theoretical_loss": 3.437799390489586, + "tokens_seen": 1949040640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041644444444444444, + "loss": 2.5165, + "theoretical_loss": 3.437789703639611, + "tokens_seen": 1949106176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041642424242424245, + "loss": 2.6073, + "theoretical_loss": 3.4377800172065323, + "tokens_seen": 1949171712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164040404040404, + "loss": 2.5349, + "theoretical_loss": 3.4377703311903174, + "tokens_seen": 1949237248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638383838383836, + "loss": 2.5828, + "theoretical_loss": 3.437760645590934, + "tokens_seen": 1949302784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041636363636363637, + "loss": 2.6568, + "theoretical_loss": 3.437750960408351, + "tokens_seen": 1949368320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163434343434344, + "loss": 2.7027, + "theoretical_loss": 3.4377412756425363, + "tokens_seen": 1949433856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041632323232323234, + "loss": 2.5977, + "theoretical_loss": 3.437731591293457, + "tokens_seen": 1949499392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163030303030303, + "loss": 2.5681, + "theoretical_loss": 3.4377219073610825, + "tokens_seen": 1949564928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162828282828283, + "loss": 2.4887, + "theoretical_loss": 3.4377122238453803, + "tokens_seen": 1949630464 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041626262626262626, + "loss": 2.7247, + "theoretical_loss": 3.437702540746318, + "tokens_seen": 1949696000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162424242424243, + "loss": 2.6263, + "theoretical_loss": 3.4376928580638646, + "tokens_seen": 1949761536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041622222222222223, + "loss": 2.7817, + "theoretical_loss": 3.4376831757979875, + "tokens_seen": 1949827072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162020202020202, + "loss": 2.6438, + "theoretical_loss": 3.437673493948655, + "tokens_seen": 1949892608 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1073760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.301966428756714, + "objective/train/theoretical_loss": 3.437663812515835, + "objective/train/tokens_used": 308817376, + "theoretical_loss": 3.437663812515835, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161818181818182, + "loss": 2.5007, + "theoretical_loss": 3.437663812515835, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161616161616162, + "loss": 2.8909, + "theoretical_loss": 3.437654131499496, + "tokens_seen": 1950023680 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041614141414141417, + "loss": 2.6493, + "theoretical_loss": 3.4376444508996054, + "tokens_seen": 1950089216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161212121212121, + "loss": 2.5106, + "theoretical_loss": 3.437634770716132, + "tokens_seen": 1950154752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161010101010101, + "loss": 2.4883, + "theoretical_loss": 3.4376250909490436, + "tokens_seen": 1950220288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160808080808081, + "loss": 2.6714, + "theoretical_loss": 3.4376154115983084, + "tokens_seen": 1950285824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160606060606061, + "loss": 2.4813, + "theoretical_loss": 3.4376057326638945, + "tokens_seen": 1950351360 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041604040404040406, + "loss": 2.7367, + "theoretical_loss": 3.4375960541457697, + "tokens_seen": 1950416896 + }, + { + "epoch": 0.18, + "learning_rate": 0.000416020202020202, + "loss": 2.6954, + "theoretical_loss": 3.437586376043902, + "tokens_seen": 1950482432 + }, + { + "epoch": 0.18, + "learning_rate": 0.000416, + "loss": 2.605, + "theoretical_loss": 3.43757669835826, + "tokens_seen": 1950547968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041597979797979804, + "loss": 2.552, + "theoretical_loss": 3.437567021088812, + "tokens_seen": 1950613504 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415959595959596, + "loss": 2.6863, + "theoretical_loss": 3.4375573442355254, + "tokens_seen": 1950679040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041593939393939395, + "loss": 2.6037, + "theoretical_loss": 3.4375476677983685, + "tokens_seen": 1950744576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159191919191919, + "loss": 2.7154, + "theoretical_loss": 3.43753799177731, + "tokens_seen": 1950810112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041589898989898987, + "loss": 2.7183, + "theoretical_loss": 3.437528316172317, + "tokens_seen": 1950875648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041587878787878793, + "loss": 2.7948, + "theoretical_loss": 3.437518640983359, + "tokens_seen": 1950941184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158585858585859, + "loss": 2.5989, + "theoretical_loss": 3.4375089662104035, + "tokens_seen": 1951006720 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041583838383838385, + "loss": 2.6351, + "theoretical_loss": 3.4374992918534177, + "tokens_seen": 1951072256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158181818181818, + "loss": 2.4972, + "theoretical_loss": 3.4374896179123713, + "tokens_seen": 1951137792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041579797979797976, + "loss": 2.5836, + "theoretical_loss": 3.4374799443872317, + "tokens_seen": 1951203328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157777777777778, + "loss": 2.4592, + "theoretical_loss": 3.4374702712779666, + "tokens_seen": 1951268864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157575757575758, + "loss": 2.6278, + "theoretical_loss": 3.4374605985845452, + "tokens_seen": 1951334400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041573737373737374, + "loss": 2.7606, + "theoretical_loss": 3.4374509263069353, + "tokens_seen": 1951399936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157171717171717, + "loss": 2.5732, + "theoretical_loss": 3.437441254445104, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156969696969697, + "loss": 2.6677, + "theoretical_loss": 3.4374315829990207, + "tokens_seen": 1951531008 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1074960, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.260453701019287, + "objective/train/theoretical_loss": 3.437421911968653, + "objective/train/tokens_used": 310455776, + "theoretical_loss": 3.437421911968653, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156767676767677, + "loss": 2.4721, + "theoretical_loss": 3.437421911968653, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156565656565657, + "loss": 2.5506, + "theoretical_loss": 3.43741224135397, + "tokens_seen": 1951662080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041563636363636363, + "loss": 2.6751, + "theoretical_loss": 3.4374025711549385, + "tokens_seen": 1951727616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156161616161616, + "loss": 2.5199, + "theoretical_loss": 3.4373929013715276, + "tokens_seen": 1951793152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155959595959596, + "loss": 2.6086, + "theoretical_loss": 3.437383232003705, + "tokens_seen": 1951858688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155757575757576, + "loss": 2.5168, + "theoretical_loss": 3.4373735630514393, + "tokens_seen": 1951924224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041555555555555557, + "loss": 2.6447, + "theoretical_loss": 3.4373638945146983, + "tokens_seen": 1951989760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155353535353535, + "loss": 2.4685, + "theoretical_loss": 3.4373542263934507, + "tokens_seen": 1952055296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041551515151515154, + "loss": 2.5438, + "theoretical_loss": 3.4373445586876636, + "tokens_seen": 1952120832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154949494949495, + "loss": 2.5168, + "theoretical_loss": 3.4373348913973065, + "tokens_seen": 1952186368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154747474747475, + "loss": 2.5002, + "theoretical_loss": 3.4373252245223473, + "tokens_seen": 1952251904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041545454545454546, + "loss": 2.6449, + "theoretical_loss": 3.4373155580627532, + "tokens_seen": 1952317440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154343434343434, + "loss": 2.5862, + "theoretical_loss": 3.437305892018494, + "tokens_seen": 1952382976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041541414141414143, + "loss": 2.6386, + "theoretical_loss": 3.437296226389537, + "tokens_seen": 1952448512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153939393939394, + "loss": 2.448, + "theoretical_loss": 3.43728656117585, + "tokens_seen": 1952514048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153737373737374, + "loss": 2.7955, + "theoretical_loss": 3.4372768963774023, + "tokens_seen": 1952579584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041535353535353535, + "loss": 2.5514, + "theoretical_loss": 3.437267231994161, + "tokens_seen": 1952645120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041533333333333336, + "loss": 2.7479, + "theoretical_loss": 3.4372575680260953, + "tokens_seen": 1952710656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153131313131313, + "loss": 2.6988, + "theoretical_loss": 3.437247904473173, + "tokens_seen": 1952776192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152929292929293, + "loss": 2.716, + "theoretical_loss": 3.437238241335362, + "tokens_seen": 1952841728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152727272727273, + "loss": 2.7452, + "theoretical_loss": 3.4372285786126318, + "tokens_seen": 1952907264 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041525252525252525, + "loss": 2.5785, + "theoretical_loss": 3.437218916304949, + "tokens_seen": 1952972800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041523232323232326, + "loss": 2.5327, + "theoretical_loss": 3.4372092544122825, + "tokens_seen": 1953038336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152121212121212, + "loss": 2.6263, + "theoretical_loss": 3.4371995929346006, + "tokens_seen": 1953103872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041519191919191917, + "loss": 2.5292, + "theoretical_loss": 3.437189931871872, + "tokens_seen": 1953169408 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1075533, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5056207180023193, + "objective/train/theoretical_loss": 3.437180271224064, + "objective/train/tokens_used": 312094176, + "theoretical_loss": 3.437180271224064, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151717171717172, + "loss": 2.6481, + "theoretical_loss": 3.437180271224064, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151515151515152, + "loss": 2.801, + "theoretical_loss": 3.437170610991146, + "tokens_seen": 1953300480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041513131313131315, + "loss": 2.3028, + "theoretical_loss": 3.4371609511730856, + "tokens_seen": 1953366016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151111111111111, + "loss": 2.6984, + "theoretical_loss": 3.437151291769851, + "tokens_seen": 1953431552 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041509090909090906, + "loss": 2.6107, + "theoretical_loss": 3.437141632781411, + "tokens_seen": 1953497088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150707070707071, + "loss": 2.8986, + "theoretical_loss": 3.4371319742077326, + "tokens_seen": 1953562624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150505050505051, + "loss": 2.4985, + "theoretical_loss": 3.4371223160487854, + "tokens_seen": 1953628160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041503030303030304, + "loss": 2.4593, + "theoretical_loss": 3.437112658304537, + "tokens_seen": 1953693696 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415010101010101, + "loss": 2.7836, + "theoretical_loss": 3.4371030009749566, + "tokens_seen": 1953759232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498989898989896, + "loss": 2.5408, + "theoretical_loss": 3.437093344060011, + "tokens_seen": 1953824768 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414969696969697, + "loss": 2.7091, + "theoretical_loss": 3.4370836875596695, + "tokens_seen": 1953890304 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414949494949495, + "loss": 2.745, + "theoretical_loss": 3.437074031473901, + "tokens_seen": 1953955840 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041492929292929293, + "loss": 2.6009, + "theoretical_loss": 3.437064375802672, + "tokens_seen": 1954021376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149090909090909, + "loss": 2.3948, + "theoretical_loss": 3.4370547205459516, + "tokens_seen": 1954086912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041488888888888885, + "loss": 2.5832, + "theoretical_loss": 3.4370450657037086, + "tokens_seen": 1954152448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148686868686869, + "loss": 2.6297, + "theoretical_loss": 3.4370354112759114, + "tokens_seen": 1954217984 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041484848484848487, + "loss": 2.5004, + "theoretical_loss": 3.4370257572625276, + "tokens_seen": 1954283520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041482828282828283, + "loss": 2.7232, + "theoretical_loss": 3.4370161036635256, + "tokens_seen": 1954349056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148080808080808, + "loss": 2.7519, + "theoretical_loss": 3.437006450478875, + "tokens_seen": 1954414592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147878787878788, + "loss": 2.6824, + "theoretical_loss": 3.4369967977085416, + "tokens_seen": 1954480128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147676767676768, + "loss": 2.5209, + "theoretical_loss": 3.436987145352496, + "tokens_seen": 1954545664 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041474747474747476, + "loss": 2.6794, + "theoretical_loss": 3.436977493410706, + "tokens_seen": 1954611200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147272727272727, + "loss": 2.577, + "theoretical_loss": 3.436967841883139, + "tokens_seen": 1954676736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147070707070707, + "loss": 2.6933, + "theoretical_loss": 3.436958190769764, + "tokens_seen": 1954742272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146868686868687, + "loss": 2.7894, + "theoretical_loss": 3.4369485400705497, + "tokens_seen": 1954807808 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1076714, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.991243839263916, + "objective/train/theoretical_loss": 3.4369388897854636, + "objective/train/tokens_used": 313732576, + "theoretical_loss": 3.4369388897854636, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146666666666667, + "loss": 2.5591, + "theoretical_loss": 3.4369388897854636, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041464646464646466, + "loss": 2.6222, + "theoretical_loss": 3.436929239914475, + "tokens_seen": 1954938880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146262626262626, + "loss": 2.4555, + "theoretical_loss": 3.436919590457552, + "tokens_seen": 1955004416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146060606060606, + "loss": 2.699, + "theoretical_loss": 3.436909941414662, + "tokens_seen": 1955069952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041458585858585863, + "loss": 2.587, + "theoretical_loss": 3.4369002927857744, + "tokens_seen": 1955135488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145656565656566, + "loss": 2.4276, + "theoretical_loss": 3.436890644570857, + "tokens_seen": 1955201024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454545454545455, + "loss": 2.6416, + "theoretical_loss": 3.436880996769879, + "tokens_seen": 1955266560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145252525252525, + "loss": 2.5871, + "theoretical_loss": 3.436871349382808, + "tokens_seen": 1955332096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145050505050505, + "loss": 2.6925, + "theoretical_loss": 3.436861702409612, + "tokens_seen": 1955397632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041448484848484853, + "loss": 2.7467, + "theoretical_loss": 3.4368520558502604, + "tokens_seen": 1955463168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144646464646465, + "loss": 2.5199, + "theoretical_loss": 3.4368424097047208, + "tokens_seen": 1955528704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041444444444444444, + "loss": 2.6255, + "theoretical_loss": 3.436832763972962, + "tokens_seen": 1955594240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041442424242424245, + "loss": 2.7099, + "theoretical_loss": 3.4368231186549525, + "tokens_seen": 1955659776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144040404040404, + "loss": 2.6867, + "theoretical_loss": 3.43681347375066, + "tokens_seen": 1955725312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143838383838384, + "loss": 2.7098, + "theoretical_loss": 3.436803829260054, + "tokens_seen": 1955790848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143636363636364, + "loss": 2.3875, + "theoretical_loss": 3.4367941851831016, + "tokens_seen": 1955856384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041434343434343433, + "loss": 2.6654, + "theoretical_loss": 3.4367845415197715, + "tokens_seen": 1955921920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041432323232323235, + "loss": 2.6225, + "theoretical_loss": 3.4367748982700332, + "tokens_seen": 1955987456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143030303030303, + "loss": 2.6428, + "theoretical_loss": 3.436765255433854, + "tokens_seen": 1956052992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142828282828283, + "loss": 2.6854, + "theoretical_loss": 3.436755613011203, + "tokens_seen": 1956118528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041426262626262627, + "loss": 2.4692, + "theoretical_loss": 3.436745971002048, + "tokens_seen": 1956184064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142424242424243, + "loss": 2.7986, + "theoretical_loss": 3.4367363294063575, + "tokens_seen": 1956249600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041422222222222224, + "loss": 2.6346, + "theoretical_loss": 3.4367266882241, + "tokens_seen": 1956315136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142020202020202, + "loss": 2.8425, + "theoretical_loss": 3.436717047455245, + "tokens_seen": 1956380672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141818181818182, + "loss": 2.5537, + "theoretical_loss": 3.436707407099759, + "tokens_seen": 1956446208 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1077107, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.886265754699707, + "objective/train/theoretical_loss": 3.4366977671576118, + "objective/train/tokens_used": 315370976, + "theoretical_loss": 3.4366977671576118, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041416161616161616, + "loss": 2.6059, + "theoretical_loss": 3.4366977671576118, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141414141414142, + "loss": 2.7731, + "theoretical_loss": 3.436688127628771, + "tokens_seen": 1956577280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041412121212121213, + "loss": 2.5947, + "theoretical_loss": 3.4366784885132056, + "tokens_seen": 1956642816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141010101010101, + "loss": 2.571, + "theoretical_loss": 3.436668849810884, + "tokens_seen": 1956708352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140808080808081, + "loss": 2.4155, + "theoretical_loss": 3.436659211521774, + "tokens_seen": 1956773888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041406060606060606, + "loss": 2.5718, + "theoretical_loss": 3.4366495736458456, + "tokens_seen": 1956839424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041404040404040407, + "loss": 2.4914, + "theoretical_loss": 3.4366399361830657, + "tokens_seen": 1956904960 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414020202020202, + "loss": 2.7574, + "theoretical_loss": 3.436630299133403, + "tokens_seen": 1956970496 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414, + "loss": 2.5857, + "theoretical_loss": 3.4366206624968267, + "tokens_seen": 1957036032 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413979797979798, + "loss": 2.6283, + "theoretical_loss": 3.4366110262733045, + "tokens_seen": 1957101568 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413959595959596, + "loss": 2.6388, + "theoretical_loss": 3.4366013904628048, + "tokens_seen": 1957167104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041393939393939396, + "loss": 2.4807, + "theoretical_loss": 3.4365917550652973, + "tokens_seen": 1957232640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139191919191919, + "loss": 2.5695, + "theoretical_loss": 3.4365821200807494, + "tokens_seen": 1957298176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041389898989898987, + "loss": 2.7659, + "theoretical_loss": 3.4365724855091293, + "tokens_seen": 1957363712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138787878787879, + "loss": 2.6308, + "theoretical_loss": 3.4365628513504065, + "tokens_seen": 1957429248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138585858585859, + "loss": 2.6777, + "theoretical_loss": 3.4365532176045486, + "tokens_seen": 1957494784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041383838383838385, + "loss": 2.4554, + "theoretical_loss": 3.436543584271524, + "tokens_seen": 1957560320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138181818181818, + "loss": 2.8151, + "theoretical_loss": 3.4365339513513025, + "tokens_seen": 1957625856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041379797979797977, + "loss": 2.473, + "theoretical_loss": 3.4365243188438512, + "tokens_seen": 1957691392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041377777777777783, + "loss": 2.5284, + "theoretical_loss": 3.436514686749139, + "tokens_seen": 1957756928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137575757575758, + "loss": 2.5062, + "theoretical_loss": 3.4365050550671348, + "tokens_seen": 1957822464 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041373737373737374, + "loss": 2.6967, + "theoretical_loss": 3.4364954237978065, + "tokens_seen": 1957888000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137171717171717, + "loss": 2.6102, + "theoretical_loss": 3.436485792941123, + "tokens_seen": 1957953536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041369696969696966, + "loss": 2.6571, + "theoretical_loss": 3.436476162497053, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136767676767677, + "loss": 2.9256, + "theoretical_loss": 3.4364665324655643, + "tokens_seen": 1958084608 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1078288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1963400840759277, + "objective/train/theoretical_loss": 3.4364569028466265, + "objective/train/tokens_used": 317009376, + "theoretical_loss": 3.4364569028466265, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136565656565657, + "loss": 2.6203, + "theoretical_loss": 3.4364569028466265, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041363636363636364, + "loss": 2.7172, + "theoretical_loss": 3.4364472736402067, + "tokens_seen": 1958215680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136161616161616, + "loss": 2.5027, + "theoretical_loss": 3.436437644846275, + "tokens_seen": 1958281216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135959595959596, + "loss": 2.6738, + "theoretical_loss": 3.4364280164647987, + "tokens_seen": 1958346752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135757575757576, + "loss": 2.5267, + "theoretical_loss": 3.4364183884957464, + "tokens_seen": 1958412288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135555555555556, + "loss": 2.4939, + "theoretical_loss": 3.4364087609390874, + "tokens_seen": 1958477824 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041353535353535353, + "loss": 2.5811, + "theoretical_loss": 3.43639913379479, + "tokens_seen": 1958543360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135151515151515, + "loss": 2.3717, + "theoretical_loss": 3.436389507062822, + "tokens_seen": 1958608896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134949494949495, + "loss": 2.5317, + "theoretical_loss": 3.4363798807431527, + "tokens_seen": 1958674432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134747474747475, + "loss": 2.5381, + "theoretical_loss": 3.436370254835751, + "tokens_seen": 1958739968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041345454545454547, + "loss": 2.5417, + "theoretical_loss": 3.4363606293405837, + "tokens_seen": 1958805504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134343434343434, + "loss": 2.6023, + "theoretical_loss": 3.4363510042576215, + "tokens_seen": 1958871040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041341414141414143, + "loss": 2.8136, + "theoretical_loss": 3.436341379586832, + "tokens_seen": 1958936576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133939393939394, + "loss": 2.6125, + "theoretical_loss": 3.4363317553281836, + "tokens_seen": 1959002112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133737373737374, + "loss": 2.5957, + "theoretical_loss": 3.436322131481645, + "tokens_seen": 1959067648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041335353535353536, + "loss": 2.6129, + "theoretical_loss": 3.436312508047185, + "tokens_seen": 1959133184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133333333333333, + "loss": 2.5397, + "theoretical_loss": 3.4363028850247717, + "tokens_seen": 1959198720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133131313131313, + "loss": 2.409, + "theoretical_loss": 3.436293262414374, + "tokens_seen": 1959264256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132929292929293, + "loss": 2.4528, + "theoretical_loss": 3.4362836402159607, + "tokens_seen": 1959329792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132727272727273, + "loss": 2.6568, + "theoretical_loss": 3.4362740184294998, + "tokens_seen": 1959395328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041325252525252525, + "loss": 2.6499, + "theoretical_loss": 3.43626439705496, + "tokens_seen": 1959460864 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041323232323232326, + "loss": 2.3524, + "theoretical_loss": 3.4362547760923103, + "tokens_seen": 1959526400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132121212121212, + "loss": 2.5487, + "theoretical_loss": 3.4362451555415197, + "tokens_seen": 1959591936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131919191919192, + "loss": 2.5426, + "theoretical_loss": 3.436235535402555, + "tokens_seen": 1959657472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131717171717172, + "loss": 2.3804, + "theoretical_loss": 3.4362259156753874, + "tokens_seen": 1959723008 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1079604, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.561680793762207, + "objective/train/theoretical_loss": 3.4362162963599827, + "objective/train/tokens_used": 318647776, + "theoretical_loss": 3.4362162963599827, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041315151515151514, + "loss": 2.7004, + "theoretical_loss": 3.4362162963599827, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041313131313131315, + "loss": 2.4122, + "theoretical_loss": 3.436206677456312, + "tokens_seen": 1959854080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131111111111111, + "loss": 2.7408, + "theoretical_loss": 3.436197058964342, + "tokens_seen": 1959919616 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041309090909090907, + "loss": 2.7353, + "theoretical_loss": 3.4361874408840425, + "tokens_seen": 1959985152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130707070707071, + "loss": 2.7172, + "theoretical_loss": 3.4361778232153815, + "tokens_seen": 1960050688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130505050505051, + "loss": 2.5049, + "theoretical_loss": 3.436168205958328, + "tokens_seen": 1960116224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041303030303030305, + "loss": 2.602, + "theoretical_loss": 3.4361585891128508, + "tokens_seen": 1960181760 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413010101010101, + "loss": 2.6146, + "theoretical_loss": 3.436148972678917, + "tokens_seen": 1960247296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041298989898989896, + "loss": 2.7808, + "theoretical_loss": 3.436139356656498, + "tokens_seen": 1960312832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041296969696969697, + "loss": 2.6078, + "theoretical_loss": 3.43612974104556, + "tokens_seen": 1960378368 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412949494949495, + "loss": 2.7435, + "theoretical_loss": 3.436120125846072, + "tokens_seen": 1960443904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041292929292929294, + "loss": 2.5543, + "theoretical_loss": 3.436110511058004, + "tokens_seen": 1960509440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129090909090909, + "loss": 2.8017, + "theoretical_loss": 3.4361008966813236, + "tokens_seen": 1960574976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128888888888889, + "loss": 2.7139, + "theoretical_loss": 3.4360912827159997, + "tokens_seen": 1960640512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128686868686869, + "loss": 2.7691, + "theoretical_loss": 3.4360816691620006, + "tokens_seen": 1960706048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128484848484849, + "loss": 2.558, + "theoretical_loss": 3.436072056019295, + "tokens_seen": 1960771584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041282828282828283, + "loss": 2.5107, + "theoretical_loss": 3.4360624432878524, + "tokens_seen": 1960837120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128080808080808, + "loss": 2.4678, + "theoretical_loss": 3.4360528309676406, + "tokens_seen": 1960902656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127878787878788, + "loss": 2.6761, + "theoretical_loss": 3.4360432190586288, + "tokens_seen": 1960968192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127676767676768, + "loss": 2.6746, + "theoretical_loss": 3.436033607560785, + "tokens_seen": 1961033728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041274747474747477, + "loss": 2.8905, + "theoretical_loss": 3.4360239964740784, + "tokens_seen": 1961099264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127272727272727, + "loss": 2.3687, + "theoretical_loss": 3.4360143857984777, + "tokens_seen": 1961164800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127070707070707, + "loss": 2.7308, + "theoretical_loss": 3.4360047755339513, + "tokens_seen": 1961230336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041268686868686875, + "loss": 2.6371, + "theoretical_loss": 3.4359951656804677, + "tokens_seen": 1961295872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126666666666667, + "loss": 2.6793, + "theoretical_loss": 3.4359855562379957, + "tokens_seen": 1961361408 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1080156, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.839082717895508, + "objective/train/theoretical_loss": 3.435975947206505, + "objective/train/tokens_used": 320286176, + "theoretical_loss": 3.435975947206505, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041264646464646466, + "loss": 2.7623, + "theoretical_loss": 3.435975947206505, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126262626262626, + "loss": 2.7329, + "theoretical_loss": 3.435966338585963, + "tokens_seen": 1961492480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126060606060606, + "loss": 2.5897, + "theoretical_loss": 3.4359567303763385, + "tokens_seen": 1961558016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041258585858585864, + "loss": 2.4654, + "theoretical_loss": 3.435947122577601, + "tokens_seen": 1961623552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125656565656566, + "loss": 2.7386, + "theoretical_loss": 3.4359375151897185, + "tokens_seen": 1961689088 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041254545454545455, + "loss": 2.6012, + "theoretical_loss": 3.43592790821266, + "tokens_seen": 1961754624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125252525252525, + "loss": 2.6477, + "theoretical_loss": 3.4359183016463946, + "tokens_seen": 1961820160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041250505050505047, + "loss": 2.8662, + "theoretical_loss": 3.43590869549089, + "tokens_seen": 1961885696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041248484848484853, + "loss": 2.455, + "theoretical_loss": 3.435899089746116, + "tokens_seen": 1961951232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124646464646465, + "loss": 2.8418, + "theoretical_loss": 3.4358894844120407, + "tokens_seen": 1962016768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041244444444444445, + "loss": 2.666, + "theoretical_loss": 3.435879879488633, + "tokens_seen": 1962082304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124242424242424, + "loss": 2.6833, + "theoretical_loss": 3.435870274975861, + "tokens_seen": 1962147840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124040404040404, + "loss": 2.5037, + "theoretical_loss": 3.4358606708736943, + "tokens_seen": 1962213376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123838383838384, + "loss": 2.5875, + "theoretical_loss": 3.435851067182101, + "tokens_seen": 1962278912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123636363636364, + "loss": 2.5752, + "theoretical_loss": 3.435841463901051, + "tokens_seen": 1962344448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041234343434343434, + "loss": 2.7171, + "theoretical_loss": 3.435831861030512, + "tokens_seen": 1962409984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123232323232323, + "loss": 2.5322, + "theoretical_loss": 3.4358222585704525, + "tokens_seen": 1962475520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123030303030303, + "loss": 2.621, + "theoretical_loss": 3.4358126565208416, + "tokens_seen": 1962541056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122828282828283, + "loss": 2.6025, + "theoretical_loss": 3.4358030548816485, + "tokens_seen": 1962606592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122626262626263, + "loss": 2.6889, + "theoretical_loss": 3.435793453652842, + "tokens_seen": 1962672128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041224242424242423, + "loss": 2.5616, + "theoretical_loss": 3.4357838528343896, + "tokens_seen": 1962737664 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041222222222222224, + "loss": 2.445, + "theoretical_loss": 3.4357742524262616, + "tokens_seen": 1962803200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122020202020202, + "loss": 2.5863, + "theoretical_loss": 3.4357646524284258, + "tokens_seen": 1962868736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121818181818182, + "loss": 2.5758, + "theoretical_loss": 3.435755052840851, + "tokens_seen": 1962934272 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041216161616161617, + "loss": 2.7752, + "theoretical_loss": 3.4357454536635066, + "tokens_seen": 1962999808 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1081069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6743087768554688, + "objective/train/theoretical_loss": 3.435735854896361, + "objective/train/tokens_used": 321924576, + "theoretical_loss": 3.435735854896361, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121414141414141, + "loss": 2.397, + "theoretical_loss": 3.435735854896361, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041212121212121214, + "loss": 2.7075, + "theoretical_loss": 3.4357262565393825, + "tokens_seen": 1963130880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121010101010101, + "loss": 2.6683, + "theoretical_loss": 3.4357166585925407, + "tokens_seen": 1963196416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120808080808081, + "loss": 2.4466, + "theoretical_loss": 3.4357070610558043, + "tokens_seen": 1963261952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041206060606060606, + "loss": 2.7529, + "theoretical_loss": 3.4356974639291415, + "tokens_seen": 1963327488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041204040404040407, + "loss": 2.6604, + "theoretical_loss": 3.4356878672125215, + "tokens_seen": 1963393024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041202020202020203, + "loss": 2.4729, + "theoretical_loss": 3.435678270905913, + "tokens_seen": 1963458560 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412, + "loss": 2.9114, + "theoretical_loss": 3.4356686750092846, + "tokens_seen": 1963524096 + }, + { + "epoch": 0.18, + "learning_rate": 0.000411979797979798, + "loss": 2.6102, + "theoretical_loss": 3.4356590795226056, + "tokens_seen": 1963589632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041195959595959595, + "loss": 2.5086, + "theoretical_loss": 3.4356494844458445, + "tokens_seen": 1963655168 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041193939393939396, + "loss": 2.7648, + "theoretical_loss": 3.43563988977897, + "tokens_seen": 1963720704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119191919191919, + "loss": 2.4557, + "theoretical_loss": 3.435630295521951, + "tokens_seen": 1963786240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118989898989899, + "loss": 2.5652, + "theoretical_loss": 3.4356207016747566, + "tokens_seen": 1963851776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118787878787879, + "loss": 2.5733, + "theoretical_loss": 3.4356111082373553, + "tokens_seen": 1963917312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118585858585859, + "loss": 2.6456, + "theoretical_loss": 3.435601515209716, + "tokens_seen": 1963982848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041183838383838386, + "loss": 2.5374, + "theoretical_loss": 3.435591922591807, + "tokens_seen": 1964048384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118181818181818, + "loss": 2.4767, + "theoretical_loss": 3.435582330383598, + "tokens_seen": 1964113920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041179797979797977, + "loss": 2.737, + "theoretical_loss": 3.4355727385850576, + "tokens_seen": 1964179456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117777777777778, + "loss": 2.4674, + "theoretical_loss": 3.435563147196155, + "tokens_seen": 1964244992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117575757575758, + "loss": 2.4722, + "theoretical_loss": 3.435553556216858, + "tokens_seen": 1964310528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041173737373737375, + "loss": 2.5974, + "theoretical_loss": 3.4355439656471356, + "tokens_seen": 1964376064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117171717171717, + "loss": 2.8329, + "theoretical_loss": 3.4355343754869576, + "tokens_seen": 1964441600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041169696969696966, + "loss": 2.654, + "theoretical_loss": 3.435524785736292, + "tokens_seen": 1964507136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041167676767676773, + "loss": 2.7161, + "theoretical_loss": 3.4355151963951083, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116565656565657, + "loss": 2.5793, + "theoretical_loss": 3.4355056074633747, + "tokens_seen": 1964638208 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1081685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.539858341217041, + "objective/train/theoretical_loss": 3.4354960189410604, + "objective/train/tokens_used": 323562976, + "theoretical_loss": 3.4354960189410604, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041163636363636364, + "loss": 2.7547, + "theoretical_loss": 3.4354960189410604, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004116161616161616, + "loss": 2.8963, + "theoretical_loss": 3.435486430828134, + "tokens_seen": 1964769280 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041159595959595956, + "loss": 2.6418, + "theoretical_loss": 3.4354768431245652, + "tokens_seen": 1964834816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115757575757576, + "loss": 2.8092, + "theoretical_loss": 3.435467255830322, + "tokens_seen": 1964900352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115555555555556, + "loss": 2.5482, + "theoretical_loss": 3.4354576689453733, + "tokens_seen": 1964965888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041153535353535354, + "loss": 2.6656, + "theoretical_loss": 3.4354480824696885, + "tokens_seen": 1965031424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115151515151515, + "loss": 2.71, + "theoretical_loss": 3.4354384964032363, + "tokens_seen": 1965096960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114949494949495, + "loss": 2.7688, + "theoretical_loss": 3.4354289107459848, + "tokens_seen": 1965162496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114747474747475, + "loss": 2.3955, + "theoretical_loss": 3.435419325497904, + "tokens_seen": 1965228032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041145454545454547, + "loss": 2.6943, + "theoretical_loss": 3.4354097406589625, + "tokens_seen": 1965293568 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041143434343434343, + "loss": 2.6809, + "theoretical_loss": 3.4354001562291288, + "tokens_seen": 1965359104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114141414141414, + "loss": 2.6625, + "theoretical_loss": 3.435390572208372, + "tokens_seen": 1965424640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113939393939394, + "loss": 2.6743, + "theoretical_loss": 3.4353809885966613, + "tokens_seen": 1965490176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113737373737374, + "loss": 2.4488, + "theoretical_loss": 3.4353714053939655, + "tokens_seen": 1965555712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041135353535353536, + "loss": 2.6777, + "theoretical_loss": 3.4353618226002527, + "tokens_seen": 1965621248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113333333333333, + "loss": 2.6546, + "theoretical_loss": 3.435352240215493, + "tokens_seen": 1965686784 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041131313131313133, + "loss": 2.5162, + "theoretical_loss": 3.4353426582396542, + "tokens_seen": 1965752320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112929292929293, + "loss": 2.5366, + "theoretical_loss": 3.4353330766727064, + "tokens_seen": 1965817856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112727272727273, + "loss": 2.6484, + "theoretical_loss": 3.4353234955146177, + "tokens_seen": 1965883392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125252525252526, + "loss": 2.574, + "theoretical_loss": 3.4353139147653575, + "tokens_seen": 1965948928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112323232323232, + "loss": 2.3125, + "theoretical_loss": 3.4353043344248944, + "tokens_seen": 1966014464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112121212121212, + "loss": 2.5873, + "theoretical_loss": 3.4352947544931967, + "tokens_seen": 1966080000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041119191919191924, + "loss": 2.6623, + "theoretical_loss": 3.435285174970235, + "tokens_seen": 1966145536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111717171717172, + "loss": 2.7218, + "theoretical_loss": 3.435275595855977, + "tokens_seen": 1966211072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041115151515151515, + "loss": 2.4976, + "theoretical_loss": 3.4352660171503913, + "tokens_seen": 1966276608 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1082856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7312498092651367, + "objective/train/theoretical_loss": 3.4352564388534477, + "objective/train/tokens_used": 325201376, + "theoretical_loss": 3.4352564388534477, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041113131313131316, + "loss": 2.6919, + "theoretical_loss": 3.4352564388534477, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111111111111111, + "loss": 2.5451, + "theoretical_loss": 3.4352468609651154, + "tokens_seen": 1966407680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041109090909090913, + "loss": 2.4027, + "theoretical_loss": 3.435237283485362, + "tokens_seen": 1966473216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110707070707071, + "loss": 2.6824, + "theoretical_loss": 3.4352277064141585, + "tokens_seen": 1966538752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041105050505050504, + "loss": 2.4505, + "theoretical_loss": 3.4352181297514717, + "tokens_seen": 1966604288 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041103030303030305, + "loss": 2.4184, + "theoretical_loss": 3.4352085534972723, + "tokens_seen": 1966669824 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411010101010101, + "loss": 2.5168, + "theoretical_loss": 3.4351989776515275, + "tokens_seen": 1966735360 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410989898989899, + "loss": 2.5689, + "theoretical_loss": 3.435189402214208, + "tokens_seen": 1966800896 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410969696969697, + "loss": 3.0605, + "theoretical_loss": 3.435179827185282, + "tokens_seen": 1966866432 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410949494949495, + "loss": 2.6067, + "theoretical_loss": 3.4351702525647183, + "tokens_seen": 1966931968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041092929292929295, + "loss": 2.7054, + "theoretical_loss": 3.435160678352486, + "tokens_seen": 1966997504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109090909090909, + "loss": 2.3814, + "theoretical_loss": 3.4351511045485545, + "tokens_seen": 1967063040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108888888888889, + "loss": 2.8035, + "theoretical_loss": 3.435141531152892, + "tokens_seen": 1967128576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041086868686868687, + "loss": 2.5896, + "theoretical_loss": 3.4351319581654685, + "tokens_seen": 1967194112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108484848484849, + "loss": 2.9124, + "theoretical_loss": 3.4351223855862525, + "tokens_seen": 1967259648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041082828282828284, + "loss": 2.7392, + "theoretical_loss": 3.4351128134152122, + "tokens_seen": 1967325184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108080808080808, + "loss": 2.5456, + "theoretical_loss": 3.435103241652318, + "tokens_seen": 1967390720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107878787878788, + "loss": 2.6759, + "theoretical_loss": 3.4350936702975376, + "tokens_seen": 1967456256 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041076767676767676, + "loss": 2.6334, + "theoretical_loss": 3.435084099350841, + "tokens_seen": 1967521792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107474747474748, + "loss": 2.409, + "theoretical_loss": 3.435074528812197, + "tokens_seen": 1967587328 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041072727272727273, + "loss": 2.4961, + "theoretical_loss": 3.4350649586815742, + "tokens_seen": 1967652864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107070707070707, + "loss": 2.6618, + "theoretical_loss": 3.435055388958942, + "tokens_seen": 1967718400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106868686868687, + "loss": 2.6335, + "theoretical_loss": 3.4350458196442695, + "tokens_seen": 1967783936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106666666666667, + "loss": 2.6031, + "theoretical_loss": 3.435036250737525, + "tokens_seen": 1967849472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041064646464646467, + "loss": 2.2975, + "theoretical_loss": 3.4350266822386786, + "tokens_seen": 1967915008 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1083372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8083508014678955, + "objective/train/theoretical_loss": 3.4350171141476986, + "objective/train/tokens_used": 326839776, + "theoretical_loss": 3.4350171141476986, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106262626262626, + "loss": 2.7665, + "theoretical_loss": 3.4350171141476986, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106060606060606, + "loss": 2.6247, + "theoretical_loss": 3.435007546464554, + "tokens_seen": 1968046080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105858585858586, + "loss": 2.6699, + "theoretical_loss": 3.4349979791892142, + "tokens_seen": 1968111616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105656565656566, + "loss": 2.6583, + "theoretical_loss": 3.434988412321648, + "tokens_seen": 1968177152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041054545454545456, + "loss": 2.6518, + "theoretical_loss": 3.4349788458618247, + "tokens_seen": 1968242688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105252525252525, + "loss": 2.5485, + "theoretical_loss": 3.434969279809713, + "tokens_seen": 1968308224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105050505050505, + "loss": 2.3527, + "theoretical_loss": 3.434959714165282, + "tokens_seen": 1968373760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041048484848484854, + "loss": 2.6313, + "theoretical_loss": 3.4349501489285013, + "tokens_seen": 1968439296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104646464646465, + "loss": 2.6005, + "theoretical_loss": 3.4349405840993392, + "tokens_seen": 1968504832 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041044444444444445, + "loss": 2.6, + "theoretical_loss": 3.434931019677765, + "tokens_seen": 1968570368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104242424242424, + "loss": 2.7116, + "theoretical_loss": 3.434921455663748, + "tokens_seen": 1968635904 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041040404040404037, + "loss": 2.48, + "theoretical_loss": 3.4349118920572574, + "tokens_seen": 1968701440 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041038383838383843, + "loss": 2.661, + "theoretical_loss": 3.4349023288582616, + "tokens_seen": 1968766976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103636363636364, + "loss": 2.5923, + "theoretical_loss": 3.4348927660667306, + "tokens_seen": 1968832512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041034343434343435, + "loss": 2.3861, + "theoretical_loss": 3.4348832036826322, + "tokens_seen": 1968898048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103232323232323, + "loss": 2.492, + "theoretical_loss": 3.434873641705937, + "tokens_seen": 1968963584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103030303030303, + "loss": 2.7685, + "theoretical_loss": 3.434864080136612, + "tokens_seen": 1969029120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102828282828283, + "loss": 2.8386, + "theoretical_loss": 3.434854518974629, + "tokens_seen": 1969094656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102626262626263, + "loss": 2.8557, + "theoretical_loss": 3.434844958219955, + "tokens_seen": 1969160192 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041024242424242424, + "loss": 2.5892, + "theoretical_loss": 3.4348353978725603, + "tokens_seen": 1969225728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102222222222222, + "loss": 2.5535, + "theoretical_loss": 3.434825837932413, + "tokens_seen": 1969291264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102020202020202, + "loss": 2.66, + "theoretical_loss": 3.4348162783994827, + "tokens_seen": 1969356800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101818181818182, + "loss": 2.5879, + "theoretical_loss": 3.4348067192737384, + "tokens_seen": 1969422336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101616161616162, + "loss": 2.7818, + "theoretical_loss": 3.4347971605551493, + "tokens_seen": 1969487872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041014141414141413, + "loss": 2.5091, + "theoretical_loss": 3.4347876022436843, + "tokens_seen": 1969553408 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1084308, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9995901584625244, + "objective/train/theoretical_loss": 3.434778044339313, + "objective/train/tokens_used": 328478176, + "theoretical_loss": 3.434778044339313, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041012121212121214, + "loss": 2.6201, + "theoretical_loss": 3.434778044339313, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101010101010101, + "loss": 2.7089, + "theoretical_loss": 3.434768486842005, + "tokens_seen": 1969684480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100808080808081, + "loss": 2.6555, + "theoretical_loss": 3.4347589297517276, + "tokens_seen": 1969750016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041006060606060607, + "loss": 2.3727, + "theoretical_loss": 3.434749373068451, + "tokens_seen": 1969815552 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410040404040404, + "loss": 2.6942, + "theoretical_loss": 3.4347398167921446, + "tokens_seen": 1969881088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041002020202020203, + "loss": 2.5807, + "theoretical_loss": 3.4347302609227772, + "tokens_seen": 1969946624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041, + "loss": 2.6507, + "theoretical_loss": 3.434720705460318, + "tokens_seen": 1970012160 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409979797979798, + "loss": 2.5104, + "theoretical_loss": 3.434711150404736, + "tokens_seen": 1970077696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040995959595959596, + "loss": 2.57, + "theoretical_loss": 3.4347015957560005, + "tokens_seen": 1970143232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040993939393939397, + "loss": 2.3079, + "theoretical_loss": 3.4346920415140803, + "tokens_seen": 1970208768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099191919191919, + "loss": 2.6171, + "theoretical_loss": 3.4346824876789452, + "tokens_seen": 1970274304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098989898989899, + "loss": 2.5322, + "theoretical_loss": 3.434672934250564, + "tokens_seen": 1970339840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098787878787879, + "loss": 2.6325, + "theoretical_loss": 3.434663381228906, + "tokens_seen": 1970405376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040985858585858585, + "loss": 2.6111, + "theoretical_loss": 3.434653828613939, + "tokens_seen": 1970470912 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040983838383838386, + "loss": 2.6792, + "theoretical_loss": 3.434644276405635, + "tokens_seen": 1970536448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098181818181818, + "loss": 2.6454, + "theoretical_loss": 3.434634724603961, + "tokens_seen": 1970601984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097979797979798, + "loss": 2.408, + "theoretical_loss": 3.434625173208886, + "tokens_seen": 1970667520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097777777777778, + "loss": 2.5299, + "theoretical_loss": 3.43461562222038, + "tokens_seen": 1970733056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097575757575758, + "loss": 2.701, + "theoretical_loss": 3.4346060716384126, + "tokens_seen": 1970798592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040973737373737376, + "loss": 2.6637, + "theoretical_loss": 3.434596521462952, + "tokens_seen": 1970864128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097171717171717, + "loss": 2.4817, + "theoretical_loss": 3.434586971693968, + "tokens_seen": 1970929664 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040969696969696967, + "loss": 2.6051, + "theoretical_loss": 3.4345774223314294, + "tokens_seen": 1970995200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096767676767677, + "loss": 2.4224, + "theoretical_loss": 3.434567873375306, + "tokens_seen": 1971060736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096565656565657, + "loss": 2.8169, + "theoretical_loss": 3.4345583248255656, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040963636363636365, + "loss": 2.439, + "theoretical_loss": 3.4345487766821794, + "tokens_seen": 1971191808 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1085454, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7234983444213867, + "objective/train/theoretical_loss": 3.434539228945115, + "objective/train/tokens_used": 330116576, + "theoretical_loss": 3.434539228945115, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096161616161616, + "loss": 2.4934, + "theoretical_loss": 3.434539228945115, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040959595959595956, + "loss": 2.5968, + "theoretical_loss": 3.4345296816143422, + "tokens_seen": 1971322880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040957575757575763, + "loss": 2.8397, + "theoretical_loss": 3.43452013468983, + "tokens_seen": 1971388416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095555555555556, + "loss": 2.6118, + "theoretical_loss": 3.4345105881715483, + "tokens_seen": 1971453952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040953535353535354, + "loss": 2.785, + "theoretical_loss": 3.4345010420594653, + "tokens_seen": 1971519488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095151515151515, + "loss": 2.6511, + "theoretical_loss": 3.4344914963535507, + "tokens_seen": 1971585024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040949494949494945, + "loss": 2.5009, + "theoretical_loss": 3.434481951053774, + "tokens_seen": 1971650560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094747474747475, + "loss": 2.7502, + "theoretical_loss": 3.4344724061601037, + "tokens_seen": 1971716096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094545454545455, + "loss": 2.4277, + "theoretical_loss": 3.4344628616725097, + "tokens_seen": 1971781632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040943434343434343, + "loss": 2.6531, + "theoretical_loss": 3.434453317590961, + "tokens_seen": 1971847168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094141414141414, + "loss": 2.5949, + "theoretical_loss": 3.4344437739154268, + "tokens_seen": 1971912704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040939393939393946, + "loss": 2.7611, + "theoretical_loss": 3.434434230645876, + "tokens_seen": 1971978240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093737373737374, + "loss": 2.6381, + "theoretical_loss": 3.434424687782279, + "tokens_seen": 1972043776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040935353535353537, + "loss": 2.4472, + "theoretical_loss": 3.4344151453246035, + "tokens_seen": 1972109312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093333333333333, + "loss": 2.448, + "theoretical_loss": 3.4344056032728196, + "tokens_seen": 1972174848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093131313131313, + "loss": 2.6905, + "theoretical_loss": 3.4343960616268965, + "tokens_seen": 1972240384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040929292929292935, + "loss": 2.8451, + "theoretical_loss": 3.4343865203868034, + "tokens_seen": 1972305920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092727272727273, + "loss": 2.4575, + "theoretical_loss": 3.4343769795525096, + "tokens_seen": 1972371456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040925252525252526, + "loss": 2.8152, + "theoretical_loss": 3.4343674391239842, + "tokens_seen": 1972436992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092323232323232, + "loss": 2.4402, + "theoretical_loss": 3.4343578991011965, + "tokens_seen": 1972502528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092121212121212, + "loss": 2.4436, + "theoretical_loss": 3.4343483594841158, + "tokens_seen": 1972568064 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040919191919191924, + "loss": 2.845, + "theoretical_loss": 3.4343388202727114, + "tokens_seen": 1972633600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091717171717172, + "loss": 2.7507, + "theoretical_loss": 3.4343292814669524, + "tokens_seen": 1972699136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040915151515151515, + "loss": 2.6888, + "theoretical_loss": 3.4343197430668084, + "tokens_seen": 1972764672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091313131313131, + "loss": 2.6904, + "theoretical_loss": 3.4343102050722485, + "tokens_seen": 1972830208 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1086100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.58170747756958, + "objective/train/theoretical_loss": 3.434300667483242, + "objective/train/tokens_used": 331754976, + "theoretical_loss": 3.434300667483242, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091111111111111, + "loss": 2.5805, + "theoretical_loss": 3.434300667483242, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040909090909090913, + "loss": 2.5536, + "theoretical_loss": 3.434291130299758, + "tokens_seen": 1972961280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090707070707071, + "loss": 2.4578, + "theoretical_loss": 3.4342815935217663, + "tokens_seen": 1973026816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040905050505050505, + "loss": 2.7724, + "theoretical_loss": 3.4342720571492356, + "tokens_seen": 1973092352 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409030303030303, + "loss": 2.7268, + "theoretical_loss": 3.434262521182135, + "tokens_seen": 1973157888 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409010101010101, + "loss": 2.3653, + "theoretical_loss": 3.434252985620435, + "tokens_seen": 1973223424 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408989898989899, + "loss": 2.8847, + "theoretical_loss": 3.4342434504641037, + "tokens_seen": 1973288960 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408969696969697, + "loss": 2.5669, + "theoretical_loss": 3.4342339157131114, + "tokens_seen": 1973354496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040894949494949494, + "loss": 2.6521, + "theoretical_loss": 3.434224381367426, + "tokens_seen": 1973420032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040892929292929295, + "loss": 2.7036, + "theoretical_loss": 3.434214847427018, + "tokens_seen": 1973485568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089090909090909, + "loss": 2.5247, + "theoretical_loss": 3.434205313891857, + "tokens_seen": 1973551104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088888888888889, + "loss": 2.5833, + "theoretical_loss": 3.434195780761911, + "tokens_seen": 1973616640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088686868686869, + "loss": 2.6482, + "theoretical_loss": 3.4341862480371503, + "tokens_seen": 1973682176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040884848484848483, + "loss": 2.8507, + "theoretical_loss": 3.434176715717544, + "tokens_seen": 1973747712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040882828282828284, + "loss": 2.4306, + "theoretical_loss": 3.434167183803061, + "tokens_seen": 1973813248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088080808080808, + "loss": 2.6021, + "theoretical_loss": 3.4341576522936714, + "tokens_seen": 1973878784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087878787878788, + "loss": 2.7193, + "theoretical_loss": 3.4341481211893443, + "tokens_seen": 1973944320 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040876767676767677, + "loss": 2.6304, + "theoretical_loss": 3.4341385904900483, + "tokens_seen": 1974009856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087474747474748, + "loss": 2.5909, + "theoretical_loss": 3.4341290601957537, + "tokens_seen": 1974075392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040872727272727274, + "loss": 2.6612, + "theoretical_loss": 3.4341195303064294, + "tokens_seen": 1974140928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087070707070707, + "loss": 2.5381, + "theoretical_loss": 3.4341100008220447, + "tokens_seen": 1974206464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086868686868687, + "loss": 2.8058, + "theoretical_loss": 3.4341004717425694, + "tokens_seen": 1974272000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040866666666666666, + "loss": 2.653, + "theoretical_loss": 3.4340909430679725, + "tokens_seen": 1974337536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040864646464646467, + "loss": 2.6647, + "theoretical_loss": 3.4340814147982233, + "tokens_seen": 1974403072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040862626262626263, + "loss": 2.6635, + "theoretical_loss": 3.4340718869332907, + "tokens_seen": 1974468608 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1087285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9132044315338135, + "objective/train/theoretical_loss": 3.4340623594731454, + "objective/train/tokens_used": 333393376, + "theoretical_loss": 3.4340623594731454, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086060606060606, + "loss": 2.8076, + "theoretical_loss": 3.4340623594731454, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085858585858586, + "loss": 2.4489, + "theoretical_loss": 3.4340528324177555, + "tokens_seen": 1974599680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085656565656566, + "loss": 2.5364, + "theoretical_loss": 3.4340433057670907, + "tokens_seen": 1974665216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040854545454545457, + "loss": 2.5742, + "theoretical_loss": 3.434033779521121, + "tokens_seen": 1974730752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085252525252525, + "loss": 2.5656, + "theoretical_loss": 3.434024253679815, + "tokens_seen": 1974796288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085050505050505, + "loss": 2.7908, + "theoretical_loss": 3.4340147282431426, + "tokens_seen": 1974861824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084848484848485, + "loss": 2.8222, + "theoretical_loss": 3.434005203211073, + "tokens_seen": 1974927360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084646464646465, + "loss": 2.6662, + "theoretical_loss": 3.4339956785835755, + "tokens_seen": 1974992896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040844444444444446, + "loss": 2.673, + "theoretical_loss": 3.4339861543606194, + "tokens_seen": 1975058432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084242424242424, + "loss": 2.5198, + "theoretical_loss": 3.4339766305421744, + "tokens_seen": 1975123968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040840404040404037, + "loss": 2.5674, + "theoretical_loss": 3.4339671071282094, + "tokens_seen": 1975189504 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040838383838383844, + "loss": 2.503, + "theoretical_loss": 3.433957584118695, + "tokens_seen": 1975255040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083636363636364, + "loss": 2.6536, + "theoretical_loss": 3.4339480615135987, + "tokens_seen": 1975320576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040834343434343435, + "loss": 2.5693, + "theoretical_loss": 3.4339385393128916, + "tokens_seen": 1975386112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083232323232323, + "loss": 2.4123, + "theoretical_loss": 3.4339290175165424, + "tokens_seen": 1975451648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040830303030303026, + "loss": 2.7539, + "theoretical_loss": 3.4339194961245205, + "tokens_seen": 1975517184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040828282828282833, + "loss": 2.8219, + "theoretical_loss": 3.4339099751367956, + "tokens_seen": 1975582720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082626262626263, + "loss": 2.4702, + "theoretical_loss": 3.4339004545533367, + "tokens_seen": 1975648256 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040824242424242424, + "loss": 2.5782, + "theoretical_loss": 3.433890934374114, + "tokens_seen": 1975713792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082222222222222, + "loss": 2.6308, + "theoretical_loss": 3.433881414599096, + "tokens_seen": 1975779328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082020202020202, + "loss": 2.5198, + "theoretical_loss": 3.4338718952282523, + "tokens_seen": 1975844864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081818181818182, + "loss": 2.5395, + "theoretical_loss": 3.4338623762615526, + "tokens_seen": 1975910400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081616161616162, + "loss": 2.5631, + "theoretical_loss": 3.4338528576989664, + "tokens_seen": 1975975936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040814141414141414, + "loss": 2.5006, + "theoretical_loss": 3.433843339540463, + "tokens_seen": 1976041472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081212121212121, + "loss": 2.6091, + "theoretical_loss": 3.4338338217860116, + "tokens_seen": 1976107008 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1088636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3727774620056152, + "objective/train/theoretical_loss": 3.4338243044355825, + "objective/train/tokens_used": 335031776, + "theoretical_loss": 3.4338243044355825, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081010101010101, + "loss": 2.5955, + "theoretical_loss": 3.4338243044355825, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080808080808081, + "loss": 2.664, + "theoretical_loss": 3.433814787489144, + "tokens_seen": 1976238080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040806060606060607, + "loss": 2.7374, + "theoretical_loss": 3.4338052709466664, + "tokens_seen": 1976303616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804040404040403, + "loss": 2.6428, + "theoretical_loss": 3.433795754808119, + "tokens_seen": 1976369152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040802020202020204, + "loss": 2.6984, + "theoretical_loss": 3.4337862390734717, + "tokens_seen": 1976434688 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408, + "loss": 2.7755, + "theoretical_loss": 3.4337767237426924, + "tokens_seen": 1976500224 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407979797979798, + "loss": 2.5779, + "theoretical_loss": 3.433767208815752, + "tokens_seen": 1976565760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040795959595959596, + "loss": 2.5811, + "theoretical_loss": 3.43375769429262, + "tokens_seen": 1976631296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079393939393939, + "loss": 2.634, + "theoretical_loss": 3.4337481801732648, + "tokens_seen": 1976696832 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040791919191919193, + "loss": 2.7384, + "theoretical_loss": 3.4337386664576566, + "tokens_seen": 1976762368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078989898989899, + "loss": 2.517, + "theoretical_loss": 3.433729153145765, + "tokens_seen": 1976827904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078787878787879, + "loss": 2.4324, + "theoretical_loss": 3.433719640237559, + "tokens_seen": 1976893440 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040785858585858586, + "loss": 2.6609, + "theoretical_loss": 3.4337101277330087, + "tokens_seen": 1976958976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040783838383838387, + "loss": 2.5041, + "theoretical_loss": 3.433700615632083, + "tokens_seen": 1977024512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078181818181818, + "loss": 2.4608, + "theoretical_loss": 3.433691103934752, + "tokens_seen": 1977090048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077979797979798, + "loss": 2.6694, + "theoretical_loss": 3.4336815926409843, + "tokens_seen": 1977155584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077777777777778, + "loss": 2.6661, + "theoretical_loss": 3.4336720817507507, + "tokens_seen": 1977221120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040775757575757575, + "loss": 2.6757, + "theoretical_loss": 3.433662571264019, + "tokens_seen": 1977286656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040773737373737376, + "loss": 2.6244, + "theoretical_loss": 3.4336530611807605, + "tokens_seen": 1977352192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077171717171717, + "loss": 2.7615, + "theoretical_loss": 3.4336435515009436, + "tokens_seen": 1977417728 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040769696969696973, + "loss": 2.8321, + "theoretical_loss": 3.433634042224538, + "tokens_seen": 1977483264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076767676767677, + "loss": 2.6079, + "theoretical_loss": 3.433624533351513, + "tokens_seen": 1977548800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076565656565657, + "loss": 2.5141, + "theoretical_loss": 3.4336150248818393, + "tokens_seen": 1977614336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040763636363636365, + "loss": 2.5957, + "theoretical_loss": 3.4336055168154846, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076161616161616, + "loss": 2.5912, + "theoretical_loss": 3.43359600915242, + "tokens_seen": 1977745408 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1089194, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.324873208999634, + "objective/train/theoretical_loss": 3.4335865018926146, + "objective/train/tokens_used": 336670176, + "theoretical_loss": 3.4335865018926146, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075959595959596, + "loss": 2.7609, + "theoretical_loss": 3.4335865018926146, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075757575757576, + "loss": 2.6968, + "theoretical_loss": 3.433576995036037, + "tokens_seen": 1977876480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075555555555556, + "loss": 2.7448, + "theoretical_loss": 3.433567488582658, + "tokens_seen": 1977942016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040753535353535355, + "loss": 2.5191, + "theoretical_loss": 3.433557982532447, + "tokens_seen": 1978007552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075151515151515, + "loss": 2.62, + "theoretical_loss": 3.433548476885372, + "tokens_seen": 1978073088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074949494949495, + "loss": 2.632, + "theoretical_loss": 3.433538971641405, + "tokens_seen": 1978138624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040747474747474747, + "loss": 2.6667, + "theoretical_loss": 3.433529466800514, + "tokens_seen": 1978204160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074545454545455, + "loss": 2.6104, + "theoretical_loss": 3.433519962362668, + "tokens_seen": 1978269696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040743434343434344, + "loss": 2.6996, + "theoretical_loss": 3.433510458327838, + "tokens_seen": 1978335232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074141414141414, + "loss": 2.6062, + "theoretical_loss": 3.433500954695993, + "tokens_seen": 1978400768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073939393939394, + "loss": 2.5305, + "theoretical_loss": 3.4334914514671024, + "tokens_seen": 1978466304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073737373737374, + "loss": 2.5134, + "theoretical_loss": 3.433481948641136, + "tokens_seen": 1978531840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073535353535354, + "loss": 2.7429, + "theoretical_loss": 3.433472446218063, + "tokens_seen": 1978597376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040733333333333333, + "loss": 2.6326, + "theoretical_loss": 3.4334629441978537, + "tokens_seen": 1978662912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073131313131313, + "loss": 2.7723, + "theoretical_loss": 3.4334534425804764, + "tokens_seen": 1978728448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072929292929293, + "loss": 2.8548, + "theoretical_loss": 3.433443941365902, + "tokens_seen": 1978793984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072727272727273, + "loss": 2.289, + "theoretical_loss": 3.4334344405541, + "tokens_seen": 1978859520 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040725252525252527, + "loss": 2.4712, + "theoretical_loss": 3.4334249401450387, + "tokens_seen": 1978925056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072323232323232, + "loss": 2.5277, + "theoretical_loss": 3.433415440138689, + "tokens_seen": 1978990592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072121212121212, + "loss": 2.5607, + "theoretical_loss": 3.43340594053502, + "tokens_seen": 1979056128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040719191919191925, + "loss": 2.6904, + "theoretical_loss": 3.4333964413340015, + "tokens_seen": 1979121664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071717171717172, + "loss": 2.6751, + "theoretical_loss": 3.4333869425356025, + "tokens_seen": 1979187200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040715151515151516, + "loss": 2.2723, + "theoretical_loss": 3.4333774441397935, + "tokens_seen": 1979252736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071313131313131, + "loss": 2.6722, + "theoretical_loss": 3.4333679461465434, + "tokens_seen": 1979318272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071111111111111, + "loss": 2.723, + "theoretical_loss": 3.433358448555822, + "tokens_seen": 1979383808 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1090532, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0842976570129395, + "objective/train/theoretical_loss": 3.4333489513675994, + "objective/train/tokens_used": 338308576, + "theoretical_loss": 3.4333489513675994, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040709090909090914, + "loss": 2.5824, + "theoretical_loss": 3.4333489513675994, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070707070707071, + "loss": 2.7518, + "theoretical_loss": 3.4333394545818443, + "tokens_seen": 1979514880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040705050505050505, + "loss": 2.7846, + "theoretical_loss": 3.4333299581985273, + "tokens_seen": 1979580416 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407030303030303, + "loss": 2.6402, + "theoretical_loss": 3.433320462217617, + "tokens_seen": 1979645952 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407010101010101, + "loss": 2.6241, + "theoretical_loss": 3.433310966639084, + "tokens_seen": 1979711488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040698989898989903, + "loss": 2.7387, + "theoretical_loss": 3.4333014714628973, + "tokens_seen": 1979777024 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406969696969697, + "loss": 2.5149, + "theoretical_loss": 3.433291976689027, + "tokens_seen": 1979842560 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040694949494949495, + "loss": 2.7728, + "theoretical_loss": 3.433282482317442, + "tokens_seen": 1979908096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069292929292929, + "loss": 2.5955, + "theoretical_loss": 3.4332729883481123, + "tokens_seen": 1979973632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069090909090909, + "loss": 2.6294, + "theoretical_loss": 3.4332634947810083, + "tokens_seen": 1980039168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068888888888889, + "loss": 2.6673, + "theoretical_loss": 3.4332540016160986, + "tokens_seen": 1980104704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068686868686869, + "loss": 2.6137, + "theoretical_loss": 3.4332445088533534, + "tokens_seen": 1980170240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040684848484848484, + "loss": 2.6255, + "theoretical_loss": 3.433235016492742, + "tokens_seen": 1980235776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040682828282828285, + "loss": 2.3272, + "theoretical_loss": 3.433225524534235, + "tokens_seen": 1980301312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068080808080808, + "loss": 2.7693, + "theoretical_loss": 3.4332160329778003, + "tokens_seen": 1980366848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067878787878788, + "loss": 2.7496, + "theoretical_loss": 3.433206541823409, + "tokens_seen": 1980432384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067676767676768, + "loss": 2.5695, + "theoretical_loss": 3.4331970510710303, + "tokens_seen": 1980497920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040674747474747473, + "loss": 2.5576, + "theoretical_loss": 3.433187560720634, + "tokens_seen": 1980563456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040672727272727274, + "loss": 2.7761, + "theoretical_loss": 3.4331780707721897, + "tokens_seen": 1980628992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067070707070707, + "loss": 2.7222, + "theoretical_loss": 3.433168581225667, + "tokens_seen": 1980694528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066868686868687, + "loss": 2.6214, + "theoretical_loss": 3.433159092081036, + "tokens_seen": 1980760064 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040666666666666667, + "loss": 2.6567, + "theoretical_loss": 3.433149603338266, + "tokens_seen": 1980825600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066464646464647, + "loss": 2.5001, + "theoretical_loss": 3.433140114997326, + "tokens_seen": 1980891136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040662626262626264, + "loss": 2.5997, + "theoretical_loss": 3.433130627058187, + "tokens_seen": 1980956672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066060606060606, + "loss": 2.5609, + "theoretical_loss": 3.433121139520818, + "tokens_seen": 1981022208 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1091139, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3586647510528564, + "objective/train/theoretical_loss": 3.433111652385189, + "objective/train/tokens_used": 339946976, + "theoretical_loss": 3.433111652385189, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065858585858586, + "loss": 2.4624, + "theoretical_loss": 3.433111652385189, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040656565656565656, + "loss": 2.6518, + "theoretical_loss": 3.4331021656512695, + "tokens_seen": 1981153280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040654545454545457, + "loss": 2.4553, + "theoretical_loss": 3.433092679319029, + "tokens_seen": 1981218816 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040652525252525253, + "loss": 2.6482, + "theoretical_loss": 3.4330831933884367, + "tokens_seen": 1981284352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065050505050505, + "loss": 2.6432, + "theoretical_loss": 3.4330737078594638, + "tokens_seen": 1981349888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064848484848485, + "loss": 2.4884, + "theoretical_loss": 3.4330642227320793, + "tokens_seen": 1981415424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064646464646465, + "loss": 2.748, + "theoretical_loss": 3.433054738006253, + "tokens_seen": 1981480960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040644444444444446, + "loss": 2.5247, + "theoretical_loss": 3.433045253681954, + "tokens_seen": 1981546496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064242424242424, + "loss": 2.5808, + "theoretical_loss": 3.4330357697591527, + "tokens_seen": 1981612032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064040404040404, + "loss": 2.5666, + "theoretical_loss": 3.4330262862378182, + "tokens_seen": 1981677568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063838383838384, + "loss": 2.5371, + "theoretical_loss": 3.4330168031179213, + "tokens_seen": 1981743104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063636363636364, + "loss": 2.6204, + "theoretical_loss": 3.433007320399431, + "tokens_seen": 1981808640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040634343434343436, + "loss": 2.6785, + "theoretical_loss": 3.4329978380823167, + "tokens_seen": 1981874176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063232323232323, + "loss": 2.4334, + "theoretical_loss": 3.4329883561665486, + "tokens_seen": 1981939712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040630303030303027, + "loss": 2.8279, + "theoretical_loss": 3.432978874652097, + "tokens_seen": 1982005248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040628282828282834, + "loss": 2.5103, + "theoretical_loss": 3.4329693935389303, + "tokens_seen": 1982070784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062626262626263, + "loss": 2.7507, + "theoretical_loss": 3.4329599128270196, + "tokens_seen": 1982136320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040624242424242425, + "loss": 2.8613, + "theoretical_loss": 3.432950432516334, + "tokens_seen": 1982201856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062222222222222, + "loss": 2.3773, + "theoretical_loss": 3.4329409526068426, + "tokens_seen": 1982267392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040620202020202016, + "loss": 2.5538, + "theoretical_loss": 3.4329314730985163, + "tokens_seen": 1982332928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040618181818181823, + "loss": 2.7964, + "theoretical_loss": 3.4329219939913247, + "tokens_seen": 1982398464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061616161616162, + "loss": 2.7069, + "theoretical_loss": 3.4329125152852367, + "tokens_seen": 1982464000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040614141414141414, + "loss": 2.6791, + "theoretical_loss": 3.432903036980223, + "tokens_seen": 1982529536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061212121212121, + "loss": 2.5445, + "theoretical_loss": 3.4328935590762533, + "tokens_seen": 1982595072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040610101010101006, + "loss": 2.6502, + "theoretical_loss": 3.4328840815732966, + "tokens_seen": 1982660608 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1091959, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3474152088165283, + "objective/train/theoretical_loss": 3.4328746044713236, + "objective/train/tokens_used": 341585376, + "theoretical_loss": 3.4328746044713236, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060808080808081, + "loss": 2.7832, + "theoretical_loss": 3.4328746044713236, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060606060606061, + "loss": 2.7179, + "theoretical_loss": 3.4328651277703033, + "tokens_seen": 1982791680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040604040404040403, + "loss": 2.835, + "theoretical_loss": 3.4328556514702058, + "tokens_seen": 1982857216 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406020202020202, + "loss": 2.4766, + "theoretical_loss": 3.432846175571001, + "tokens_seen": 1982922752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040600000000000006, + "loss": 2.4752, + "theoretical_loss": 3.432836700072659, + "tokens_seen": 1982988288 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405979797979798, + "loss": 2.816, + "theoretical_loss": 3.4328272249751492, + "tokens_seen": 1983053824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040595959595959597, + "loss": 2.5177, + "theoretical_loss": 3.432817750278441, + "tokens_seen": 1983119360 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040593939393939393, + "loss": 2.6334, + "theoretical_loss": 3.432808275982505, + "tokens_seen": 1983184896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059191919191919, + "loss": 2.5028, + "theoretical_loss": 3.4327988020873104, + "tokens_seen": 1983250432 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040589898989898995, + "loss": 2.5369, + "theoretical_loss": 3.4327893285928273, + "tokens_seen": 1983315968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058787878787879, + "loss": 2.7305, + "theoretical_loss": 3.4327798554990254, + "tokens_seen": 1983381504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040585858585858586, + "loss": 2.7252, + "theoretical_loss": 3.432770382805874, + "tokens_seen": 1983447040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058383838383838, + "loss": 2.6841, + "theoretical_loss": 3.4327609105133443, + "tokens_seen": 1983512576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040581818181818183, + "loss": 2.7614, + "theoretical_loss": 3.432751438621405, + "tokens_seen": 1983578112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040579797979797984, + "loss": 2.857, + "theoretical_loss": 3.4327419671300263, + "tokens_seen": 1983643648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057777777777778, + "loss": 2.7226, + "theoretical_loss": 3.4327324960391783, + "tokens_seen": 1983709184 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040575757575757576, + "loss": 2.3924, + "theoretical_loss": 3.43272302534883, + "tokens_seen": 1983774720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057373737373737, + "loss": 2.7689, + "theoretical_loss": 3.4327135550589514, + "tokens_seen": 1983840256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057171717171717, + "loss": 2.5132, + "theoretical_loss": 3.4327040851695134, + "tokens_seen": 1983905792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040569696969696973, + "loss": 2.5889, + "theoretical_loss": 3.432694615680485, + "tokens_seen": 1983971328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056767676767677, + "loss": 2.7441, + "theoretical_loss": 3.4326851465918358, + "tokens_seen": 1984036864 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040565656565656565, + "loss": 2.6829, + "theoretical_loss": 3.4326756779035357, + "tokens_seen": 1984102400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040563636363636366, + "loss": 2.701, + "theoretical_loss": 3.4326662096155554, + "tokens_seen": 1984167936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056161616161616, + "loss": 2.8375, + "theoretical_loss": 3.4326567417278637, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040559595959595963, + "loss": 2.5529, + "theoretical_loss": 3.4326472742404315, + "tokens_seen": 1984299008 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1092471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.014146327972412, + "objective/train/theoretical_loss": 3.432637807153228, + "objective/train/tokens_used": 343223776, + "theoretical_loss": 3.432637807153228, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055757575757576, + "loss": 2.7457, + "theoretical_loss": 3.432637807153228, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040555555555555554, + "loss": 2.7626, + "theoretical_loss": 3.4326283404662226, + "tokens_seen": 1984430080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040553535353535355, + "loss": 2.7682, + "theoretical_loss": 3.4326188741793864, + "tokens_seen": 1984495616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055151515151515, + "loss": 2.6067, + "theoretical_loss": 3.432609408292689, + "tokens_seen": 1984561152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054949494949495, + "loss": 2.6628, + "theoretical_loss": 3.4325999428060987, + "tokens_seen": 1984626688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054747474747475, + "loss": 2.8009, + "theoretical_loss": 3.432590477719587, + "tokens_seen": 1984692224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054545454545455, + "loss": 2.7039, + "theoretical_loss": 3.432581013033124, + "tokens_seen": 1984757760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040543434343434344, + "loss": 2.6553, + "theoretical_loss": 3.4325715487466777, + "tokens_seen": 1984823296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054141414141414, + "loss": 2.5869, + "theoretical_loss": 3.43256208486022, + "tokens_seen": 1984888832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053939393939394, + "loss": 2.6068, + "theoretical_loss": 3.43255262137372, + "tokens_seen": 1984954368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040537373737373737, + "loss": 2.5967, + "theoretical_loss": 3.4325431582871473, + "tokens_seen": 1985019904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053535353535354, + "loss": 2.4943, + "theoretical_loss": 3.432533695600472, + "tokens_seen": 1985085440 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040533333333333334, + "loss": 2.9349, + "theoretical_loss": 3.4325242333136643, + "tokens_seen": 1985150976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053131313131313, + "loss": 2.7874, + "theoretical_loss": 3.4325147714266944, + "tokens_seen": 1985216512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052929292929293, + "loss": 2.7921, + "theoretical_loss": 3.432505309939531, + "tokens_seen": 1985282048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052727272727273, + "loss": 2.419, + "theoretical_loss": 3.432495848852145, + "tokens_seen": 1985347584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052525252525253, + "loss": 2.7875, + "theoretical_loss": 3.432486388164506, + "tokens_seen": 1985413120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040523232323232323, + "loss": 2.7653, + "theoretical_loss": 3.4324769278765834, + "tokens_seen": 1985478656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052121212121212, + "loss": 2.9169, + "theoretical_loss": 3.432467467988348, + "tokens_seen": 1985544192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051919191919192, + "loss": 2.6095, + "theoretical_loss": 3.4324580084997693, + "tokens_seen": 1985609728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051717171717172, + "loss": 2.718, + "theoretical_loss": 3.4324485494108177, + "tokens_seen": 1985675264 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040515151515151517, + "loss": 2.6026, + "theoretical_loss": 3.4324390907214624, + "tokens_seen": 1985740800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051313131313131, + "loss": 2.6342, + "theoretical_loss": 3.4324296324316736, + "tokens_seen": 1985806336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051111111111111, + "loss": 2.5302, + "theoretical_loss": 3.432420174541422, + "tokens_seen": 1985871872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040509090909090915, + "loss": 2.8093, + "theoretical_loss": 3.4324107170506757, + "tokens_seen": 1985937408 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1093318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.581864595413208, + "objective/train/theoretical_loss": 3.4324012599594065, + "objective/train/tokens_used": 344862176, + "theoretical_loss": 3.4324012599594065, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050707070707071, + "loss": 2.4938, + "theoretical_loss": 3.4324012599594065, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040505050505050506, + "loss": 2.6114, + "theoretical_loss": 3.432391803267583, + "tokens_seen": 1986068480 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405030303030303, + "loss": 2.6796, + "theoretical_loss": 3.4323823469751766, + "tokens_seen": 1986134016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040501010101010097, + "loss": 2.6682, + "theoretical_loss": 3.432372891082156, + "tokens_seen": 1986199552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040498989898989904, + "loss": 2.6571, + "theoretical_loss": 3.4323634355884916, + "tokens_seen": 1986265088 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404969696969697, + "loss": 2.5284, + "theoretical_loss": 3.4323539804941534, + "tokens_seen": 1986330624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040494949494949495, + "loss": 2.685, + "theoretical_loss": 3.432344525799111, + "tokens_seen": 1986396160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049292929292929, + "loss": 2.7383, + "theoretical_loss": 3.432335071503335, + "tokens_seen": 1986461696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049090909090909, + "loss": 2.6443, + "theoretical_loss": 3.4323256176067947, + "tokens_seen": 1986527232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040488888888888893, + "loss": 2.7833, + "theoretical_loss": 3.4323161641094604, + "tokens_seen": 1986592768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048686868686869, + "loss": 2.7787, + "theoretical_loss": 3.4323067110113024, + "tokens_seen": 1986658304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040484848484848484, + "loss": 2.7694, + "theoretical_loss": 3.4322972583122904, + "tokens_seen": 1986723840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048282828282828, + "loss": 2.6806, + "theoretical_loss": 3.4322878060123942, + "tokens_seen": 1986789376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048080808080808, + "loss": 2.8119, + "theoretical_loss": 3.4322783541115838, + "tokens_seen": 1986854912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047878787878788, + "loss": 2.59, + "theoretical_loss": 3.432268902609829, + "tokens_seen": 1986920448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047676767676768, + "loss": 2.721, + "theoretical_loss": 3.4322594515071003, + "tokens_seen": 1986985984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040474747474747474, + "loss": 2.5832, + "theoretical_loss": 3.432250000803368, + "tokens_seen": 1987051520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040472727272727275, + "loss": 2.7105, + "theoretical_loss": 3.4322405504986007, + "tokens_seen": 1987117056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047070707070707, + "loss": 2.779, + "theoretical_loss": 3.4322311005927704, + "tokens_seen": 1987182592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046868686868687, + "loss": 2.8089, + "theoretical_loss": 3.432221651085845, + "tokens_seen": 1987248128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040466666666666667, + "loss": 2.6261, + "theoretical_loss": 3.432212201977796, + "tokens_seen": 1987313664 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040464646464646463, + "loss": 2.5074, + "theoretical_loss": 3.4322027532685926, + "tokens_seen": 1987379200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040462626262626264, + "loss": 2.4476, + "theoretical_loss": 3.432193304958205, + "tokens_seen": 1987444736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046060606060606, + "loss": 2.7534, + "theoretical_loss": 3.4321838570466037, + "tokens_seen": 1987510272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045858585858586, + "loss": 2.6793, + "theoretical_loss": 3.432174409533758, + "tokens_seen": 1987575808 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1094373, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.851362943649292, + "objective/train/theoretical_loss": 3.4321649624196384, + "objective/train/tokens_used": 346500576, + "theoretical_loss": 3.4321649624196384, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040456565656565657, + "loss": 2.5657, + "theoretical_loss": 3.4321649624196384, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045454545454546, + "loss": 2.3815, + "theoretical_loss": 3.432155515704215, + "tokens_seen": 1987706880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040452525252525253, + "loss": 2.7743, + "theoretical_loss": 3.432146069387458, + "tokens_seen": 1987772416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045050505050505, + "loss": 2.7407, + "theoretical_loss": 3.4321366234693365, + "tokens_seen": 1987837952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044848484848485, + "loss": 2.5287, + "theoretical_loss": 3.432127177949821, + "tokens_seen": 1987903488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040446464646464646, + "loss": 2.6323, + "theoretical_loss": 3.432117732828882, + "tokens_seen": 1987969024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040444444444444447, + "loss": 2.6108, + "theoretical_loss": 3.4321082881064884, + "tokens_seen": 1988034560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044242424242424, + "loss": 2.4885, + "theoretical_loss": 3.4320988437826117, + "tokens_seen": 1988100096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044040404040404, + "loss": 2.5306, + "theoretical_loss": 3.4320893998572215, + "tokens_seen": 1988165632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043838383838384, + "loss": 2.5742, + "theoretical_loss": 3.432079956330287, + "tokens_seen": 1988231168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040436363636363635, + "loss": 2.7162, + "theoretical_loss": 3.4320705132017792, + "tokens_seen": 1988296704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040434343434343436, + "loss": 2.7681, + "theoretical_loss": 3.4320610704716676, + "tokens_seen": 1988362240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043232323232323, + "loss": 2.8476, + "theoretical_loss": 3.4320516281399227, + "tokens_seen": 1988427776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040430303030303033, + "loss": 2.7233, + "theoretical_loss": 3.4320421862065147, + "tokens_seen": 1988493312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042828282828283, + "loss": 2.949, + "theoretical_loss": 3.432032744671413, + "tokens_seen": 1988558848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042626262626263, + "loss": 2.7295, + "theoretical_loss": 3.4320233035345877, + "tokens_seen": 1988624384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040424242424242425, + "loss": 2.7111, + "theoretical_loss": 3.4320138627960097, + "tokens_seen": 1988689920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042222222222222, + "loss": 2.7302, + "theoretical_loss": 3.4320044224556483, + "tokens_seen": 1988755456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042020202020202, + "loss": 2.7109, + "theoretical_loss": 3.4319949825134737, + "tokens_seen": 1988820992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041818181818182, + "loss": 2.9131, + "theoretical_loss": 3.4319855429694566, + "tokens_seen": 1988886528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041616161616162, + "loss": 2.6445, + "theoretical_loss": 3.4319761038235663, + "tokens_seen": 1988952064 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040414141414141415, + "loss": 2.6379, + "theoretical_loss": 3.431966665075773, + "tokens_seen": 1989017600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041212121212121, + "loss": 2.6443, + "theoretical_loss": 3.4319572267260474, + "tokens_seen": 1989083136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041010101010101, + "loss": 2.6639, + "theoretical_loss": 3.431947788774359, + "tokens_seen": 1989148672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040808080808081, + "loss": 2.9648, + "theoretical_loss": 3.431938351220678, + "tokens_seen": 1989214208 + }, + { + "debugging/Self-BLEU-5": 0.4028481195939984, + "debugging/distinct-1-grams": 0.8260221595150598, + "debugging/distinct-2-grams": 0.9811327641385946, + "debugging/entropy-1-grams": 5.420510909354855, + "debugging/entropy-2-grams": 5.998582774683988, + "debugging/length": 472.57142857142856, + "debugging/num_segments": 7, + "epoch": 0.2, + "objective/train/docs_used": 1094923, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8087246417999268, + "objective/train/theoretical_loss": 3.431928914064975, + "objective/train/tokens_used": 348138976, + "theoretical_loss": 3.431928914064975, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040606060606061, + "loss": 2.7047, + "theoretical_loss": 3.431928914064975, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040404040404040404, + "loss": 2.6149, + "theoretical_loss": 3.4319194773072192, + "tokens_seen": 1989345280 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404020202020202, + "loss": 2.6278, + "theoretical_loss": 3.4319100409473813, + "tokens_seen": 1989410816 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404, + "loss": 2.6593, + "theoretical_loss": 3.4319006049854313, + "tokens_seen": 1989476352 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403979797979798, + "loss": 2.6405, + "theoretical_loss": 3.4318911694213394, + "tokens_seen": 1989541888 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403959595959596, + "loss": 2.745, + "theoretical_loss": 3.431881734255076, + "tokens_seen": 1989607424 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040393939393939393, + "loss": 2.841, + "theoretical_loss": 3.431872299486611, + "tokens_seen": 1989672960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039191919191919, + "loss": 2.6359, + "theoretical_loss": 3.4318628651159138, + "tokens_seen": 1989738496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040389898989898995, + "loss": 2.7627, + "theoretical_loss": 3.4318534311429554, + "tokens_seen": 1989804032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038787878787879, + "loss": 2.4968, + "theoretical_loss": 3.4318439975677055, + "tokens_seen": 1989869568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040385858585858587, + "loss": 2.5069, + "theoretical_loss": 3.431834564390135, + "tokens_seen": 1989935104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038383838383838, + "loss": 2.7199, + "theoretical_loss": 3.4318251316102124, + "tokens_seen": 1990000640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038181818181818, + "loss": 2.3488, + "theoretical_loss": 3.4318156992279096, + "tokens_seen": 1990066176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040379797979797985, + "loss": 2.7606, + "theoretical_loss": 3.4318062672431964, + "tokens_seen": 1990131712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037777777777778, + "loss": 2.6489, + "theoretical_loss": 3.431796835656042, + "tokens_seen": 1990197248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040375757575757576, + "loss": 2.8979, + "theoretical_loss": 3.431787404466417, + "tokens_seen": 1990262784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037373737373737, + "loss": 2.6212, + "theoretical_loss": 3.431777973674292, + "tokens_seen": 1990328320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040371717171717173, + "loss": 2.8688, + "theoretical_loss": 3.431768543279637, + "tokens_seen": 1990393856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040369696969696974, + "loss": 2.858, + "theoretical_loss": 3.431759113282422, + "tokens_seen": 1990459392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036767676767677, + "loss": 2.5182, + "theoretical_loss": 3.431749683682617, + "tokens_seen": 1990524928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040365656565656565, + "loss": 2.5937, + "theoretical_loss": 3.4317402544801925, + "tokens_seen": 1990590464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036363636363636, + "loss": 2.6897, + "theoretical_loss": 3.4317308256751184, + "tokens_seen": 1990656000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036161616161616, + "loss": 2.7109, + "theoretical_loss": 3.4317213972673652, + "tokens_seen": 1990721536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040359595959595963, + "loss": 2.7822, + "theoretical_loss": 3.4317119692569023, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035757575757576, + "loss": 2.7954, + "theoretical_loss": 3.431702541643701, + "tokens_seen": 1990852608 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1096094, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.850614547729492, + "objective/train/theoretical_loss": 3.431693114427731, + "objective/train/tokens_used": 349777376, + "theoretical_loss": 3.431693114427731, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040355555555555555, + "loss": 2.7385, + "theoretical_loss": 3.431693114427731, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040353535353535356, + "loss": 2.9012, + "theoretical_loss": 3.4316836876089623, + "tokens_seen": 1990983680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035151515151515, + "loss": 2.6501, + "theoretical_loss": 3.431674261187365, + "tokens_seen": 1991049216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034949494949495, + "loss": 2.6113, + "theoretical_loss": 3.4316648351629095, + "tokens_seen": 1991114752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034747474747475, + "loss": 2.6352, + "theoretical_loss": 3.4316554095355665, + "tokens_seen": 1991180288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040345454545454544, + "loss": 2.5908, + "theoretical_loss": 3.4316459843053053, + "tokens_seen": 1991245824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040343434343434345, + "loss": 2.466, + "theoretical_loss": 3.4316365594720963, + "tokens_seen": 1991311360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034141414141414, + "loss": 2.9378, + "theoretical_loss": 3.43162713503591, + "tokens_seen": 1991376896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033939393939394, + "loss": 2.8731, + "theoretical_loss": 3.431617710996717, + "tokens_seen": 1991442432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033737373737374, + "loss": 2.5793, + "theoretical_loss": 3.431608287354487, + "tokens_seen": 1991507968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033535353535354, + "loss": 2.8316, + "theoretical_loss": 3.4315988641091906, + "tokens_seen": 1991573504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040333333333333334, + "loss": 2.7059, + "theoretical_loss": 3.4315894412607966, + "tokens_seen": 1991639040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033131313131313, + "loss": 2.9928, + "theoretical_loss": 3.431580018809277, + "tokens_seen": 1991704576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032929292929293, + "loss": 2.709, + "theoretical_loss": 3.431570596754601, + "tokens_seen": 1991770112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040327272727272727, + "loss": 2.7009, + "theoretical_loss": 3.4315611750967396, + "tokens_seen": 1991835648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032525252525253, + "loss": 2.4751, + "theoretical_loss": 3.431551753835662, + "tokens_seen": 1991901184 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040323232323232324, + "loss": 2.5752, + "theoretical_loss": 3.4315423329713397, + "tokens_seen": 1991966720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032121212121212, + "loss": 2.6393, + "theoretical_loss": 3.4315329125037417, + "tokens_seen": 1992032256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031919191919192, + "loss": 2.8421, + "theoretical_loss": 3.4315234924328393, + "tokens_seen": 1992097792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031717171717172, + "loss": 2.65, + "theoretical_loss": 3.431514072758602, + "tokens_seen": 1992163328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040315151515151517, + "loss": 2.6166, + "theoretical_loss": 3.4315046534810003, + "tokens_seen": 1992228864 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040313131313131313, + "loss": 2.6158, + "theoretical_loss": 3.4314952346000047, + "tokens_seen": 1992294400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031111111111111, + "loss": 2.7504, + "theoretical_loss": 3.4314858161155852, + "tokens_seen": 1992359936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030909090909091, + "loss": 2.5385, + "theoretical_loss": 3.431476398027712, + "tokens_seen": 1992425472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030707070707071, + "loss": 2.4957, + "theoretical_loss": 3.431466980336355, + "tokens_seen": 1992491008 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1096583, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7095284461975098, + "objective/train/theoretical_loss": 3.4314575630414854, + "objective/train/tokens_used": 351415776, + "theoretical_loss": 3.4314575630414854, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040305050505050506, + "loss": 2.6214, + "theoretical_loss": 3.4314575630414854, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403030303030303, + "loss": 2.807, + "theoretical_loss": 3.4314481461430724, + "tokens_seen": 1992622080 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403010101010101, + "loss": 2.634, + "theoretical_loss": 3.4314387296410875, + "tokens_seen": 1992687616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040298989898989904, + "loss": 2.7925, + "theoretical_loss": 3.4314293135355003, + "tokens_seen": 1992753152 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402969696969697, + "loss": 2.4985, + "theoretical_loss": 3.431419897826281, + "tokens_seen": 1992818688 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040294949494949496, + "loss": 2.8183, + "theoretical_loss": 3.4314104825133995, + "tokens_seen": 1992884224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029292929292929, + "loss": 2.7924, + "theoretical_loss": 3.431401067596827, + "tokens_seen": 1992949760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040290909090909087, + "loss": 2.6961, + "theoretical_loss": 3.431391653076533, + "tokens_seen": 1993015296 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040288888888888894, + "loss": 2.4038, + "theoretical_loss": 3.431382238952488, + "tokens_seen": 1993080832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028686868686869, + "loss": 2.7434, + "theoretical_loss": 3.4313728252246634, + "tokens_seen": 1993146368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040284848484848485, + "loss": 2.718, + "theoretical_loss": 3.4313634118930274, + "tokens_seen": 1993211904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028282828282828, + "loss": 2.7766, + "theoretical_loss": 3.431353998957552, + "tokens_seen": 1993277440 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040280808080808076, + "loss": 2.7098, + "theoretical_loss": 3.4313445864182066, + "tokens_seen": 1993342976 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040278787878787883, + "loss": 2.7335, + "theoretical_loss": 3.4313351742749623, + "tokens_seen": 1993408512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027676767676768, + "loss": 2.6668, + "theoretical_loss": 3.4313257625277886, + "tokens_seen": 1993474048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040274747474747474, + "loss": 2.5202, + "theoretical_loss": 3.4313163511766565, + "tokens_seen": 1993539584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027272727272727, + "loss": 2.7599, + "theoretical_loss": 3.4313069402215355, + "tokens_seen": 1993605120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027070707070707, + "loss": 2.6586, + "theoretical_loss": 3.431297529662397, + "tokens_seen": 1993670656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026868686868687, + "loss": 2.8594, + "theoretical_loss": 3.43128811949921, + "tokens_seen": 1993736192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026666666666667, + "loss": 2.6204, + "theoretical_loss": 3.431278709731946, + "tokens_seen": 1993801728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040264646464646464, + "loss": 2.6484, + "theoretical_loss": 3.431269300360575, + "tokens_seen": 1993867264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026262626262626, + "loss": 2.6423, + "theoretical_loss": 3.431259891385067, + "tokens_seen": 1993932800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026060606060606, + "loss": 2.6703, + "theoretical_loss": 3.431250482805392, + "tokens_seen": 1993998336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025858585858586, + "loss": 2.5785, + "theoretical_loss": 3.4312410746215214, + "tokens_seen": 1994063872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040256565656565657, + "loss": 2.655, + "theoretical_loss": 3.4312316668334253, + "tokens_seen": 1994129408 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1097915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4799556732177734, + "objective/train/theoretical_loss": 3.431222259441073, + "objective/train/tokens_used": 353054176, + "theoretical_loss": 3.431222259441073, + "tokens_seen": 1994194944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040254545454545453, + "loss": 2.6861, + "theoretical_loss": 3.431222259441073, + "tokens_seen": 1994194944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040252525252525254, + "loss": 2.6419, + "theoretical_loss": 3.4312128524444363, + "tokens_seen": 1994260480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040250505050505055, + "loss": 2.548, + "theoretical_loss": 3.431203445843485, + "tokens_seen": 1994326016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024848484848485, + "loss": 2.5519, + "theoretical_loss": 3.4311940396381884, + "tokens_seen": 1994391552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040246464646464646, + "loss": 2.6708, + "theoretical_loss": 3.4311846338285186, + "tokens_seen": 1994457088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024444444444444, + "loss": 2.6083, + "theoretical_loss": 3.4311752284144443, + "tokens_seen": 1994522624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040242424242424243, + "loss": 2.5466, + "theoretical_loss": 3.4311658233959372, + "tokens_seen": 1994588160 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040240404040404044, + "loss": 2.5841, + "theoretical_loss": 3.4311564187729675, + "tokens_seen": 1994653696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023838383838384, + "loss": 2.6176, + "theoretical_loss": 3.431147014545505, + "tokens_seen": 1994719232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040236363636363636, + "loss": 2.6555, + "theoretical_loss": 3.4311376107135203, + "tokens_seen": 1994784768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040234343434343437, + "loss": 2.681, + "theoretical_loss": 3.4311282072769833, + "tokens_seen": 1994850304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023232323232323, + "loss": 2.5542, + "theoretical_loss": 3.4311188042358656, + "tokens_seen": 1994915840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040230303030303034, + "loss": 2.8682, + "theoretical_loss": 3.4311094015901364, + "tokens_seen": 1994981376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022828282828283, + "loss": 2.7279, + "theoretical_loss": 3.431099999339767, + "tokens_seen": 1995046912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040226262626262625, + "loss": 2.8612, + "theoretical_loss": 3.431090597484727, + "tokens_seen": 1995112448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040224242424242426, + "loss": 2.6292, + "theoretical_loss": 3.4310811960249867, + "tokens_seen": 1995177984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022222222222222, + "loss": 2.5724, + "theoretical_loss": 3.431071794960517, + "tokens_seen": 1995243520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040220202020202023, + "loss": 2.6812, + "theoretical_loss": 3.4310623942912892, + "tokens_seen": 1995309056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021818181818182, + "loss": 2.9322, + "theoretical_loss": 3.431052994017272, + "tokens_seen": 1995374592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021616161616162, + "loss": 2.7268, + "theoretical_loss": 3.431043594138436, + "tokens_seen": 1995440128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040214141414141415, + "loss": 2.7061, + "theoretical_loss": 3.431034194654753, + "tokens_seen": 1995505664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021212121212121, + "loss": 2.7505, + "theoretical_loss": 3.4310247955661923, + "tokens_seen": 1995571200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021010101010101, + "loss": 2.5533, + "theoretical_loss": 3.4310153968727244, + "tokens_seen": 1995636736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020808080808081, + "loss": 2.5434, + "theoretical_loss": 3.43100599857432, + "tokens_seen": 1995702272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020606060606061, + "loss": 2.7473, + "theoretical_loss": 3.4309966006709494, + "tokens_seen": 1995767808 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1098508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5174620151519775, + "objective/train/theoretical_loss": 3.4309872031625828, + "objective/train/tokens_used": 354692576, + "theoretical_loss": 3.4309872031625828, + "tokens_seen": 1995833344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040204040404040405, + "loss": 2.7295, + "theoretical_loss": 3.4309872031625828, + "tokens_seen": 1995833344 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402020202020202, + "loss": 2.3933, + "theoretical_loss": 3.4309778060491913, + "tokens_seen": 1995898880 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402, + "loss": 2.7622, + "theoretical_loss": 3.4309684093307444, + "tokens_seen": 1995964416 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401979797979798, + "loss": 2.8072, + "theoretical_loss": 3.430959013007213, + "tokens_seen": 1996029952 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401959595959596, + "loss": 2.4624, + "theoretical_loss": 3.430949617078568, + "tokens_seen": 1996095488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040193939393939394, + "loss": 2.6096, + "theoretical_loss": 3.4309402215447795, + "tokens_seen": 1996161024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019191919191919, + "loss": 2.8545, + "theoretical_loss": 3.430930826405817, + "tokens_seen": 1996226560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018989898989899, + "loss": 2.6149, + "theoretical_loss": 3.4309214316616528, + "tokens_seen": 1996292096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018787878787879, + "loss": 2.621, + "theoretical_loss": 3.4309120373122557, + "tokens_seen": 1996357632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018585858585859, + "loss": 2.8016, + "theoretical_loss": 3.4309026433575966, + "tokens_seen": 1996423168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040183838383838383, + "loss": 2.7612, + "theoretical_loss": 3.4308932497976468, + "tokens_seen": 1996488704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018181818181818, + "loss": 2.5173, + "theoretical_loss": 3.4308838566323754, + "tokens_seen": 1996554240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040179797979797985, + "loss": 2.5817, + "theoretical_loss": 3.4308744638617537, + "tokens_seen": 1996619776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017777777777778, + "loss": 2.5235, + "theoretical_loss": 3.4308650714857523, + "tokens_seen": 1996685312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040175757575757577, + "loss": 2.5699, + "theoretical_loss": 3.4308556795043414, + "tokens_seen": 1996750848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017373737373737, + "loss": 2.7443, + "theoretical_loss": 3.4308462879174915, + "tokens_seen": 1996816384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017171717171717, + "loss": 2.5264, + "theoretical_loss": 3.4308368967251726, + "tokens_seen": 1996881920 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040169696969696975, + "loss": 2.647, + "theoretical_loss": 3.4308275059273563, + "tokens_seen": 1996947456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016767676767677, + "loss": 2.6705, + "theoretical_loss": 3.430818115524012, + "tokens_seen": 1997012992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040165656565656566, + "loss": 2.7531, + "theoretical_loss": 3.430808725515111, + "tokens_seen": 1997078528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016363636363636, + "loss": 2.7024, + "theoretical_loss": 3.430799335900623, + "tokens_seen": 1997144064 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040161616161616163, + "loss": 2.7327, + "theoretical_loss": 3.4307899466805187, + "tokens_seen": 1997209600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040159595959595964, + "loss": 2.746, + "theoretical_loss": 3.430780557854769, + "tokens_seen": 1997275136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015757575757576, + "loss": 2.7495, + "theoretical_loss": 3.4307711694233443, + "tokens_seen": 1997340672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040155555555555555, + "loss": 2.608, + "theoretical_loss": 3.430761781386215, + "tokens_seen": 1997406208 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1099394, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8477187156677246, + "objective/train/theoretical_loss": 3.430752393743351, + "objective/train/tokens_used": 356330976, + "theoretical_loss": 3.430752393743351, + "tokens_seen": 1997471744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015353535353535, + "loss": 2.63, + "theoretical_loss": 3.430752393743351, + "tokens_seen": 1997471744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015151515151515, + "loss": 2.8339, + "theoretical_loss": 3.430743006494724, + "tokens_seen": 1997537280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040149494949494953, + "loss": 2.9765, + "theoretical_loss": 3.4307336196403035, + "tokens_seen": 1997602816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014747474747475, + "loss": 2.9089, + "theoretical_loss": 3.4307242331800607, + "tokens_seen": 1997668352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040145454545454545, + "loss": 2.7895, + "theoretical_loss": 3.4307148471139657, + "tokens_seen": 1997733888 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040143434343434346, + "loss": 2.7117, + "theoretical_loss": 3.430705461441989, + "tokens_seen": 1997799424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014141414141414, + "loss": 2.6922, + "theoretical_loss": 3.4306960761641014, + "tokens_seen": 1997864960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013939393939394, + "loss": 2.6537, + "theoretical_loss": 3.4306866912802736, + "tokens_seen": 1997930496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013737373737374, + "loss": 2.6435, + "theoretical_loss": 3.430677306790475, + "tokens_seen": 1997996032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040135353535353534, + "loss": 2.7962, + "theoretical_loss": 3.4306679226946777, + "tokens_seen": 1998061568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040133333333333335, + "loss": 2.5895, + "theoretical_loss": 3.430658538992851, + "tokens_seen": 1998127104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013131313131313, + "loss": 2.7134, + "theoretical_loss": 3.430649155684966, + "tokens_seen": 1998192640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012929292929293, + "loss": 2.6309, + "theoretical_loss": 3.4306397727709936, + "tokens_seen": 1998258176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012727272727273, + "loss": 2.573, + "theoretical_loss": 3.4306303902509034, + "tokens_seen": 1998323712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012525252525253, + "loss": 2.8616, + "theoretical_loss": 3.4306210081246666, + "tokens_seen": 1998389248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040123232323232324, + "loss": 2.587, + "theoretical_loss": 3.430611626392254, + "tokens_seen": 1998454784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012121212121212, + "loss": 2.7992, + "theoretical_loss": 3.4306022450536355, + "tokens_seen": 1998520320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011919191919192, + "loss": 2.5505, + "theoretical_loss": 3.4305928641087817, + "tokens_seen": 1998585856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040117171717171717, + "loss": 2.7951, + "theoretical_loss": 3.4305834835576636, + "tokens_seen": 1998651392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011515151515152, + "loss": 2.6681, + "theoretical_loss": 3.430574103400252, + "tokens_seen": 1998716928 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040113131313131313, + "loss": 2.5223, + "theoretical_loss": 3.4305647236365164, + "tokens_seen": 1998782464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011111111111111, + "loss": 2.7467, + "theoretical_loss": 3.430555344266428, + "tokens_seen": 1998848000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010909090909091, + "loss": 2.7558, + "theoretical_loss": 3.4305459652899577, + "tokens_seen": 1998913536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040107070707070706, + "loss": 2.6164, + "theoretical_loss": 3.4305365867070754, + "tokens_seen": 1998979072 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040105050505050507, + "loss": 2.878, + "theoretical_loss": 3.430527208517752, + "tokens_seen": 1999044608 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1100643, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.998358726501465, + "objective/train/theoretical_loss": 3.4305178307219584, + "objective/train/tokens_used": 357969376, + "theoretical_loss": 3.4305178307219584, + "tokens_seen": 1999110144 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401030303030303, + "loss": 2.8925, + "theoretical_loss": 3.4305178307219584, + "tokens_seen": 1999110144 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401010101010101, + "loss": 2.8018, + "theoretical_loss": 3.430508453319665, + "tokens_seen": 1999175680 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400989898989899, + "loss": 2.7074, + "theoretical_loss": 3.430499076310842, + "tokens_seen": 1999241216 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400969696969697, + "loss": 2.6598, + "theoretical_loss": 3.4304896996954604, + "tokens_seen": 1999306752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040094949494949496, + "loss": 2.555, + "theoretical_loss": 3.4304803234734904, + "tokens_seen": 1999372288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009292929292929, + "loss": 2.7593, + "theoretical_loss": 3.430470947644903, + "tokens_seen": 1999437824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009090909090909, + "loss": 3.0023, + "theoretical_loss": 3.4304615722096687, + "tokens_seen": 1999503360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008888888888889, + "loss": 2.7062, + "theoretical_loss": 3.4304521971677584, + "tokens_seen": 1999568896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008686868686869, + "loss": 2.711, + "theoretical_loss": 3.430442822519142, + "tokens_seen": 1999634432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040084848484848486, + "loss": 2.6887, + "theoretical_loss": 3.4304334482637904, + "tokens_seen": 1999699968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008282828282828, + "loss": 2.6289, + "theoretical_loss": 3.4304240744016745, + "tokens_seen": 1999765504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008080808080808, + "loss": 2.6162, + "theoretical_loss": 3.430414700932765, + "tokens_seen": 1999831040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040078787878787883, + "loss": 2.7507, + "theoretical_loss": 3.4304053278570317, + "tokens_seen": 1999896576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007676767676768, + "loss": 2.6243, + "theoretical_loss": 3.430395955174446, + "tokens_seen": 1999962112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040074747474747475, + "loss": 2.6822, + "theoretical_loss": 3.4303865828849784, + "tokens_seen": 2000027648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007272727272727, + "loss": 2.5374, + "theoretical_loss": 3.4303772109885995, + "tokens_seen": 2000093184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007070707070707, + "loss": 2.8277, + "theoretical_loss": 3.43036783948528, + "tokens_seen": 2000158720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040068686868686873, + "loss": 2.9846, + "theoretical_loss": 3.43035846837499, + "tokens_seen": 2000224256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006666666666667, + "loss": 2.718, + "theoretical_loss": 3.4303490976577007, + "tokens_seen": 2000289792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040064646464646464, + "loss": 2.4615, + "theoretical_loss": 3.430339727333383, + "tokens_seen": 2000355328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006262626262626, + "loss": 2.7228, + "theoretical_loss": 3.4303303574020063, + "tokens_seen": 2000420864 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040060606060606066, + "loss": 2.5237, + "theoretical_loss": 3.4303209878635426, + "tokens_seen": 2000486400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005858585858586, + "loss": 2.866, + "theoretical_loss": 3.430311618717962, + "tokens_seen": 2000551936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005656565656566, + "loss": 2.6097, + "theoretical_loss": 3.4303022499652354, + "tokens_seen": 2000617472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040054545454545453, + "loss": 2.7682, + "theoretical_loss": 3.4302928816053324, + "tokens_seen": 2000683008 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1101270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.287954092025757, + "objective/train/theoretical_loss": 3.4302835136382255, + "objective/train/tokens_used": 359607776, + "theoretical_loss": 3.4302835136382255, + "tokens_seen": 2000748544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005252525252525, + "loss": 2.6407, + "theoretical_loss": 3.4302835136382255, + "tokens_seen": 2000748544 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040050505050505056, + "loss": 2.6081, + "theoretical_loss": 3.4302741460638844, + "tokens_seen": 2000814080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004848484848485, + "loss": 2.9468, + "theoretical_loss": 3.430264778882279, + "tokens_seen": 2000879616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040046464646464647, + "loss": 2.6506, + "theoretical_loss": 3.430255412093381, + "tokens_seen": 2000945152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004444444444444, + "loss": 2.7632, + "theoretical_loss": 3.430246045697161, + "tokens_seen": 2001010688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040042424242424244, + "loss": 2.7678, + "theoretical_loss": 3.4302366796935893, + "tokens_seen": 2001076224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040040404040404045, + "loss": 2.7699, + "theoretical_loss": 3.4302273140826367, + "tokens_seen": 2001141760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003838383838384, + "loss": 2.8298, + "theoretical_loss": 3.430217948864274, + "tokens_seen": 2001207296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040036363636363636, + "loss": 2.6612, + "theoretical_loss": 3.430208584038472, + "tokens_seen": 2001272832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003434343434343, + "loss": 2.684, + "theoretical_loss": 3.430199219605201, + "tokens_seen": 2001338368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040032323232323233, + "loss": 2.6532, + "theoretical_loss": 3.430189855564432, + "tokens_seen": 2001403904 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040030303030303034, + "loss": 2.8199, + "theoretical_loss": 3.4301804919161354, + "tokens_seen": 2001469440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002828282828283, + "loss": 2.7311, + "theoretical_loss": 3.430171128660283, + "tokens_seen": 2001534976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040026262626262625, + "loss": 2.6164, + "theoretical_loss": 3.4301617657968437, + "tokens_seen": 2001600512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040024242424242427, + "loss": 2.7514, + "theoretical_loss": 3.4301524033257893, + "tokens_seen": 2001666048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002222222222222, + "loss": 2.7445, + "theoretical_loss": 3.4301430412470904, + "tokens_seen": 2001731584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040020202020202023, + "loss": 2.5826, + "theoretical_loss": 3.4301336795607176, + "tokens_seen": 2001797120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001818181818182, + "loss": 2.6789, + "theoretical_loss": 3.4301243182666417, + "tokens_seen": 2001862656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040016161616161615, + "loss": 2.6448, + "theoretical_loss": 3.4301149573648333, + "tokens_seen": 2001928192 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040014141414141416, + "loss": 2.6389, + "theoretical_loss": 3.430105596855263, + "tokens_seen": 2001993728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001212121212121, + "loss": 2.4755, + "theoretical_loss": 3.430096236737902, + "tokens_seen": 2002059264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001010101010101, + "loss": 2.7416, + "theoretical_loss": 3.4300868770127213, + "tokens_seen": 2002124800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000808080808081, + "loss": 2.7288, + "theoretical_loss": 3.43007751767969, + "tokens_seen": 2002190336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000606060606061, + "loss": 2.7819, + "theoretical_loss": 3.4300681587387807, + "tokens_seen": 2002255872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040004040404040405, + "loss": 2.7553, + "theoretical_loss": 3.430058800189963, + "tokens_seen": 2002321408 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1102668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.553368330001831, + "objective/train/theoretical_loss": 3.430049442033208, + "objective/train/tokens_used": 361246176, + "theoretical_loss": 3.430049442033208, + "tokens_seen": 2002386944 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400020202020202, + "loss": 2.6432, + "theoretical_loss": 3.430049442033208, + "tokens_seen": 2002386944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004, + "loss": 2.7453, + "theoretical_loss": 3.4300400842684864, + "tokens_seen": 2002452480 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399979797979798, + "loss": 2.5433, + "theoretical_loss": 3.4300307268957693, + "tokens_seen": 2002518016 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399959595959596, + "loss": 2.645, + "theoretical_loss": 3.430021369915027, + "tokens_seen": 2002583552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039993939393939394, + "loss": 2.7866, + "theoretical_loss": 3.4300120133262304, + "tokens_seen": 2002649088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999191919191919, + "loss": 2.7484, + "theoretical_loss": 3.43000265712935, + "tokens_seen": 2002714624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998989898989899, + "loss": 2.4926, + "theoretical_loss": 3.429993301324357, + "tokens_seen": 2002780160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998787878787879, + "loss": 2.6038, + "theoretical_loss": 3.4299839459112222, + "tokens_seen": 2002845696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998585858585859, + "loss": 2.7284, + "theoretical_loss": 3.4299745908899157, + "tokens_seen": 2002911232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039983838383838384, + "loss": 2.8709, + "theoretical_loss": 3.4299652362604087, + "tokens_seen": 2002976768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998181818181818, + "loss": 2.6466, + "theoretical_loss": 3.4299558820226723, + "tokens_seen": 2003042304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997979797979798, + "loss": 2.5868, + "theoretical_loss": 3.429946528176677, + "tokens_seen": 2003107840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997777777777778, + "loss": 2.6806, + "theoretical_loss": 3.4299371747223937, + "tokens_seen": 2003173376 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039975757575757577, + "loss": 2.7211, + "theoretical_loss": 3.429927821659793, + "tokens_seen": 2003238912 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039973737373737373, + "loss": 2.6575, + "theoretical_loss": 3.429918468988845, + "tokens_seen": 2003304448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997171717171717, + "loss": 2.3557, + "theoretical_loss": 3.4299091167095215, + "tokens_seen": 2003369984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039969696969696975, + "loss": 2.5517, + "theoretical_loss": 3.4298997648217933, + "tokens_seen": 2003435520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996767676767677, + "loss": 2.876, + "theoretical_loss": 3.42989041332563, + "tokens_seen": 2003501056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039965656565656567, + "loss": 2.6888, + "theoretical_loss": 3.4298810622210043, + "tokens_seen": 2003566592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996363636363636, + "loss": 2.7384, + "theoretical_loss": 3.4298717115078854, + "tokens_seen": 2003632128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996161616161616, + "loss": 2.4843, + "theoretical_loss": 3.429862361186245, + "tokens_seen": 2003697664 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039959595959595964, + "loss": 2.6384, + "theoretical_loss": 3.4298530112560535, + "tokens_seen": 2003763200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995757575757576, + "loss": 2.7286, + "theoretical_loss": 3.429843661717282, + "tokens_seen": 2003828736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039955555555555556, + "loss": 2.8028, + "theoretical_loss": 3.4298343125699002, + "tokens_seen": 2003894272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995353535353535, + "loss": 2.7432, + "theoretical_loss": 3.4298249638138807, + "tokens_seen": 2003959808 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1103172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3909668922424316, + "objective/train/theoretical_loss": 3.4298156154491934, + "objective/train/tokens_used": 362884576, + "theoretical_loss": 3.4298156154491934, + "tokens_seen": 2004025344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039951515151515147, + "loss": 2.5196, + "theoretical_loss": 3.4298156154491934, + "tokens_seen": 2004025344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039949494949494954, + "loss": 2.7479, + "theoretical_loss": 3.4298062674758087, + "tokens_seen": 2004090880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994747474747475, + "loss": 2.5021, + "theoretical_loss": 3.429796919893698, + "tokens_seen": 2004156416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039945454545454545, + "loss": 2.791, + "theoretical_loss": 3.4297875727028324, + "tokens_seen": 2004221952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994343434343434, + "loss": 2.8064, + "theoretical_loss": 3.429778225903182, + "tokens_seen": 2004287488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994141414141414, + "loss": 2.7233, + "theoretical_loss": 3.429768879494718, + "tokens_seen": 2004353024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039939393939393943, + "loss": 2.6588, + "theoretical_loss": 3.4297595334774114, + "tokens_seen": 2004418560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993737373737374, + "loss": 2.6098, + "theoretical_loss": 3.4297501878512326, + "tokens_seen": 2004484096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039935353535353534, + "loss": 2.7278, + "theoretical_loss": 3.429740842616153, + "tokens_seen": 2004549632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993333333333333, + "loss": 2.5958, + "theoretical_loss": 3.429731497772143, + "tokens_seen": 2004615168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993131313131313, + "loss": 2.8843, + "theoretical_loss": 3.4297221533191737, + "tokens_seen": 2004680704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992929292929293, + "loss": 2.6212, + "theoretical_loss": 3.429712809257216, + "tokens_seen": 2004746240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992727272727273, + "loss": 2.7094, + "theoretical_loss": 3.4297034655862406, + "tokens_seen": 2004811776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039925252525252524, + "loss": 2.7256, + "theoretical_loss": 3.429694122306218, + "tokens_seen": 2004877312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039923232323232325, + "loss": 2.8226, + "theoretical_loss": 3.4296847794171197, + "tokens_seen": 2004942848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992121212121212, + "loss": 2.7651, + "theoretical_loss": 3.4296754369189166, + "tokens_seen": 2005008384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991919191919192, + "loss": 2.7283, + "theoretical_loss": 3.4296660948115782, + "tokens_seen": 2005073920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039917171717171717, + "loss": 2.6783, + "theoretical_loss": 3.4296567530950774, + "tokens_seen": 2005139456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039915151515151513, + "loss": 2.7694, + "theoretical_loss": 3.429647411769384, + "tokens_seen": 2005204992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039913131313131314, + "loss": 2.855, + "theoretical_loss": 3.4296380708344687, + "tokens_seen": 2005270528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039911111111111115, + "loss": 2.6749, + "theoretical_loss": 3.429628730290303, + "tokens_seen": 2005336064 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990909090909091, + "loss": 2.4842, + "theoretical_loss": 3.4296193901368577, + "tokens_seen": 2005401600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039907070707070706, + "loss": 2.9729, + "theoretical_loss": 3.429610050374103, + "tokens_seen": 2005467136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990505050505051, + "loss": 2.7454, + "theoretical_loss": 3.4296007110020104, + "tokens_seen": 2005532672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039903030303030303, + "loss": 2.6091, + "theoretical_loss": 3.4295913720205506, + "tokens_seen": 2005598208 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1104281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4382286071777344, + "objective/train/theoretical_loss": 3.4295820334296945, + "objective/train/tokens_used": 364522976, + "theoretical_loss": 3.4295820334296945, + "tokens_seen": 2005663744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039901010101010104, + "loss": 2.7794, + "theoretical_loss": 3.4295820334296945, + "tokens_seen": 2005663744 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398989898989899, + "loss": 2.5314, + "theoretical_loss": 3.429572695229413, + "tokens_seen": 2005729280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039896969696969696, + "loss": 2.5465, + "theoretical_loss": 3.429563357419677, + "tokens_seen": 2005794816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039894949494949497, + "loss": 2.6578, + "theoretical_loss": 3.4295540200004577, + "tokens_seen": 2005860352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989292929292929, + "loss": 2.7723, + "theoretical_loss": 3.4295446829717258, + "tokens_seen": 2005925888 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039890909090909094, + "loss": 2.5896, + "theoretical_loss": 3.429535346333452, + "tokens_seen": 2005991424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988888888888889, + "loss": 2.4996, + "theoretical_loss": 3.429526010085607, + "tokens_seen": 2006056960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988686868686869, + "loss": 2.8004, + "theoretical_loss": 3.4295166742281626, + "tokens_seen": 2006122496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039884848484848486, + "loss": 2.5072, + "theoretical_loss": 3.4295073387610895, + "tokens_seen": 2006188032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988282828282828, + "loss": 2.6983, + "theoretical_loss": 3.4294980036843583, + "tokens_seen": 2006253568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039880808080808083, + "loss": 2.9369, + "theoretical_loss": 3.4294886689979394, + "tokens_seen": 2006319104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987878787878788, + "loss": 2.5722, + "theoretical_loss": 3.4294793347018047, + "tokens_seen": 2006384640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987676767676768, + "loss": 2.586, + "theoretical_loss": 3.4294700007959245, + "tokens_seen": 2006450176 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039874747474747475, + "loss": 2.8116, + "theoretical_loss": 3.4294606672802703, + "tokens_seen": 2006515712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987272727272727, + "loss": 2.7575, + "theoretical_loss": 3.4294513341548125, + "tokens_seen": 2006581248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987070707070707, + "loss": 2.8497, + "theoretical_loss": 3.4294420014195226, + "tokens_seen": 2006646784 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039868686868686873, + "loss": 2.8441, + "theoretical_loss": 3.4294326690743713, + "tokens_seen": 2006712320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986666666666667, + "loss": 2.559, + "theoretical_loss": 3.429423337119329, + "tokens_seen": 2006777856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039864646464646465, + "loss": 2.4556, + "theoretical_loss": 3.429414005554367, + "tokens_seen": 2006843392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986262626262626, + "loss": 2.5888, + "theoretical_loss": 3.429404674379457, + "tokens_seen": 2006908928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986060606060606, + "loss": 2.6556, + "theoretical_loss": 3.429395343594569, + "tokens_seen": 2006974464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985858585858586, + "loss": 2.6575, + "theoretical_loss": 3.4293860131996743, + "tokens_seen": 2007040000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985656565656566, + "loss": 2.597, + "theoretical_loss": 3.429376683194744, + "tokens_seen": 2007105536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039854545454545454, + "loss": 2.6176, + "theoretical_loss": 3.4293673535797486, + "tokens_seen": 2007171072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985252525252525, + "loss": 2.7297, + "theoretical_loss": 3.4293580243546598, + "tokens_seen": 2007236608 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1105495, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7771756649017334, + "objective/train/theoretical_loss": 3.4293486955194483, + "objective/train/tokens_used": 366161376, + "theoretical_loss": 3.4293486955194483, + "tokens_seen": 2007302144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039850505050505056, + "loss": 2.7023, + "theoretical_loss": 3.4293486955194483, + "tokens_seen": 2007302144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984848484848485, + "loss": 2.7546, + "theoretical_loss": 3.4293393670740846, + "tokens_seen": 2007367680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984646464646465, + "loss": 2.6322, + "theoretical_loss": 3.42933003901854, + "tokens_seen": 2007433216 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039844444444444443, + "loss": 2.806, + "theoretical_loss": 3.429320711352786, + "tokens_seen": 2007498752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984242424242424, + "loss": 2.5904, + "theoretical_loss": 3.429311384076793, + "tokens_seen": 2007564288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039840404040404045, + "loss": 2.8624, + "theoretical_loss": 3.4293020571905317, + "tokens_seen": 2007629824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983838383838384, + "loss": 2.6415, + "theoretical_loss": 3.429292730693974, + "tokens_seen": 2007695360 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039836363636363637, + "loss": 2.5293, + "theoretical_loss": 3.42928340458709, + "tokens_seen": 2007760896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983434343434343, + "loss": 2.5056, + "theoretical_loss": 3.4292740788698515, + "tokens_seen": 2007826432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039832323232323234, + "loss": 2.6948, + "theoretical_loss": 3.4292647535422285, + "tokens_seen": 2007891968 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039830303030303035, + "loss": 2.602, + "theoretical_loss": 3.4292554286041934, + "tokens_seen": 2007957504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982828282828283, + "loss": 2.6045, + "theoretical_loss": 3.429246104055716, + "tokens_seen": 2008023040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039826262626262626, + "loss": 2.8573, + "theoretical_loss": 3.4292367798967676, + "tokens_seen": 2008088576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982424242424242, + "loss": 2.5131, + "theoretical_loss": 3.4292274561273195, + "tokens_seen": 2008154112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039822222222222223, + "loss": 2.6732, + "theoretical_loss": 3.4292181327473426, + "tokens_seen": 2008219648 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039820202020202024, + "loss": 2.591, + "theoretical_loss": 3.429208809756808, + "tokens_seen": 2008285184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981818181818182, + "loss": 2.5137, + "theoretical_loss": 3.429199487155686, + "tokens_seen": 2008350720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039816161616161615, + "loss": 2.5927, + "theoretical_loss": 3.4291901649439493, + "tokens_seen": 2008416256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039814141414141416, + "loss": 2.8069, + "theoretical_loss": 3.429180843121567, + "tokens_seen": 2008481792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981212121212121, + "loss": 2.6222, + "theoretical_loss": 3.4291715216885112, + "tokens_seen": 2008547328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039810101010101013, + "loss": 2.7161, + "theoretical_loss": 3.429162200644753, + "tokens_seen": 2008612864 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980808080808081, + "loss": 2.7619, + "theoretical_loss": 3.429152879990263, + "tokens_seen": 2008678400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039806060606060605, + "loss": 2.697, + "theoretical_loss": 3.429143559725013, + "tokens_seen": 2008743936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039804040404040406, + "loss": 2.5508, + "theoretical_loss": 3.429134239848973, + "tokens_seen": 2008809472 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398020202020202, + "loss": 2.6974, + "theoretical_loss": 3.4291249203621144, + "tokens_seen": 2008875008 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1105995, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0905230045318604, + "objective/train/theoretical_loss": 3.4291156012644084, + "objective/train/tokens_used": 367799776, + "theoretical_loss": 3.4291156012644084, + "tokens_seen": 2008940544 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398, + "loss": 2.7571, + "theoretical_loss": 3.4291156012644084, + "tokens_seen": 2008940544 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397979797979798, + "loss": 2.4411, + "theoretical_loss": 3.429106282555826, + "tokens_seen": 2009006080 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397959595959596, + "loss": 2.5121, + "theoretical_loss": 3.4290969642363387, + "tokens_seen": 2009071616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039793939393939395, + "loss": 2.521, + "theoretical_loss": 3.429087646305917, + "tokens_seen": 2009137152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979191919191919, + "loss": 2.6003, + "theoretical_loss": 3.4290783287645317, + "tokens_seen": 2009202688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978989898989899, + "loss": 2.8013, + "theoretical_loss": 3.429069011612155, + "tokens_seen": 2009268224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978787878787879, + "loss": 2.8218, + "theoretical_loss": 3.429059694848757, + "tokens_seen": 2009333760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978585858585859, + "loss": 2.5237, + "theoretical_loss": 3.4290503784743085, + "tokens_seen": 2009399296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039783838383838384, + "loss": 2.6163, + "theoretical_loss": 3.429041062488782, + "tokens_seen": 2009464832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978181818181818, + "loss": 2.7317, + "theoretical_loss": 3.429031746892147, + "tokens_seen": 2009530368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977979797979798, + "loss": 2.5755, + "theoretical_loss": 3.429022431684375, + "tokens_seen": 2009595904 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039777777777777777, + "loss": 2.5838, + "theoretical_loss": 3.4290131168654385, + "tokens_seen": 2009661440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977575757575758, + "loss": 2.6078, + "theoretical_loss": 3.4290038024353064, + "tokens_seen": 2009726976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039773737373737373, + "loss": 2.7659, + "theoretical_loss": 3.4289944883939514, + "tokens_seen": 2009792512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977171717171717, + "loss": 2.5651, + "theoretical_loss": 3.428985174741344, + "tokens_seen": 2009858048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976969696969697, + "loss": 2.3688, + "theoretical_loss": 3.428975861477455, + "tokens_seen": 2009923584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976767676767677, + "loss": 2.7842, + "theoretical_loss": 3.4289665486022565, + "tokens_seen": 2009989120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039765656565656567, + "loss": 2.6497, + "theoretical_loss": 3.4289572361157186, + "tokens_seen": 2010054656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039763636363636363, + "loss": 2.5598, + "theoretical_loss": 3.428947924017813, + "tokens_seen": 2010120192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976161616161616, + "loss": 2.7141, + "theoretical_loss": 3.42893861230851, + "tokens_seen": 2010185728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975959595959596, + "loss": 2.6488, + "theoretical_loss": 3.428929300987782, + "tokens_seen": 2010251264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975757575757576, + "loss": 2.6941, + "theoretical_loss": 3.428919990055599, + "tokens_seen": 2010316800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039755555555555556, + "loss": 2.6565, + "theoretical_loss": 3.428910679511932, + "tokens_seen": 2010382336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975353535353535, + "loss": 2.5045, + "theoretical_loss": 3.428901369356754, + "tokens_seen": 2010447872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975151515151515, + "loss": 2.8396, + "theoretical_loss": 3.4288920595900336, + "tokens_seen": 2010513408 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1107334, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7171061038970947, + "objective/train/theoretical_loss": 3.4288827502117436, + "objective/train/tokens_used": 369438176, + "theoretical_loss": 3.4288827502117436, + "tokens_seen": 2010578944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039749494949494954, + "loss": 2.6364, + "theoretical_loss": 3.4288827502117436, + "tokens_seen": 2010578944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974747474747475, + "loss": 2.616, + "theoretical_loss": 3.4288734412218544, + "tokens_seen": 2010644480 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039745454545454546, + "loss": 2.9022, + "theoretical_loss": 3.4288641326203377, + "tokens_seen": 2010710016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974343434343434, + "loss": 2.6037, + "theoretical_loss": 3.428854824407164, + "tokens_seen": 2010775552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039741414141414137, + "loss": 2.5749, + "theoretical_loss": 3.428845516582305, + "tokens_seen": 2010841088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039739393939393944, + "loss": 2.8528, + "theoretical_loss": 3.4288362091457314, + "tokens_seen": 2010906624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973737373737374, + "loss": 2.7218, + "theoretical_loss": 3.428826902097415, + "tokens_seen": 2010972160 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039735353535353535, + "loss": 2.8239, + "theoretical_loss": 3.428817595437326, + "tokens_seen": 2011037696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973333333333333, + "loss": 2.7298, + "theoretical_loss": 3.4288082891654366, + "tokens_seen": 2011103232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039731313131313137, + "loss": 2.6982, + "theoretical_loss": 3.428798983281717, + "tokens_seen": 2011168768 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039729292929292933, + "loss": 2.8055, + "theoretical_loss": 3.4287896777861384, + "tokens_seen": 2011234304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972727272727273, + "loss": 2.5813, + "theoretical_loss": 3.4287803726786734, + "tokens_seen": 2011299840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039725252525252524, + "loss": 2.6005, + "theoretical_loss": 3.4287710679592913, + "tokens_seen": 2011365376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972323232323232, + "loss": 2.8153, + "theoretical_loss": 3.428761763627964, + "tokens_seen": 2011430912 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039721212121212126, + "loss": 2.2978, + "theoretical_loss": 3.4287524596846635, + "tokens_seen": 2011496448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971919191919192, + "loss": 2.5187, + "theoretical_loss": 3.42874315612936, + "tokens_seen": 2011561984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971717171717172, + "loss": 2.6217, + "theoretical_loss": 3.4287338529620244, + "tokens_seen": 2011627520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039715151515151513, + "loss": 2.6869, + "theoretical_loss": 3.428724550182629, + "tokens_seen": 2011693056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039713131313131315, + "loss": 2.6536, + "theoretical_loss": 3.4287152477911436, + "tokens_seen": 2011758592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039711111111111116, + "loss": 2.6725, + "theoretical_loss": 3.4287059457875406, + "tokens_seen": 2011824128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970909090909091, + "loss": 2.7913, + "theoretical_loss": 3.428696644171791, + "tokens_seen": 2011889664 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039707070707070707, + "loss": 2.7083, + "theoretical_loss": 3.428687342943866, + "tokens_seen": 2011955200 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397050505050505, + "loss": 2.7214, + "theoretical_loss": 3.428678042103736, + "tokens_seen": 2012020736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039703030303030304, + "loss": 2.839, + "theoretical_loss": 3.428668741651373, + "tokens_seen": 2012086272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039701010101010105, + "loss": 2.6332, + "theoretical_loss": 3.4286594415867477, + "tokens_seen": 2012151808 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1107920, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5978739261627197, + "objective/train/theoretical_loss": 3.4286501419098316, + "objective/train/tokens_used": 371076576, + "theoretical_loss": 3.4286501419098316, + "tokens_seen": 2012217344 + }, + { + "epoch": 0.21, + "learning_rate": 0.000396989898989899, + "loss": 2.5902, + "theoretical_loss": 3.4286501419098316, + "tokens_seen": 2012217344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039696969696969696, + "loss": 2.9242, + "theoretical_loss": 3.428640842620596, + "tokens_seen": 2012282880 + }, + { + "epoch": 0.21, + "learning_rate": 0.000396949494949495, + "loss": 2.8167, + "theoretical_loss": 3.4286315437190122, + "tokens_seen": 2012348416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039692929292929293, + "loss": 2.5715, + "theoretical_loss": 3.4286222452050508, + "tokens_seen": 2012413952 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039690909090909094, + "loss": 2.6004, + "theoretical_loss": 3.428612947078684, + "tokens_seen": 2012479488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968888888888889, + "loss": 2.6496, + "theoretical_loss": 3.428603649339882, + "tokens_seen": 2012545024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039686868686868686, + "loss": 2.3663, + "theoretical_loss": 3.4285943519886164, + "tokens_seen": 2012610560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039684848484848487, + "loss": 2.7802, + "theoretical_loss": 3.4285850550248584, + "tokens_seen": 2012676096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968282828282828, + "loss": 2.6398, + "theoretical_loss": 3.42857575844858, + "tokens_seen": 2012741632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039680808080808083, + "loss": 2.457, + "theoretical_loss": 3.4285664622597514, + "tokens_seen": 2012807168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967878787878788, + "loss": 2.5914, + "theoretical_loss": 3.428557166458344, + "tokens_seen": 2012872704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967676767676768, + "loss": 2.4352, + "theoretical_loss": 3.4285478710443296, + "tokens_seen": 2012938240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039674747474747476, + "loss": 2.3116, + "theoretical_loss": 3.428538576017679, + "tokens_seen": 2013003776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967272727272727, + "loss": 2.6761, + "theoretical_loss": 3.4285292813783634, + "tokens_seen": 2013069312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039670707070707073, + "loss": 2.6778, + "theoretical_loss": 3.4285199871263545, + "tokens_seen": 2013134848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966868686868687, + "loss": 2.6637, + "theoretical_loss": 3.4285106932616225, + "tokens_seen": 2013200384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966666666666667, + "loss": 2.6304, + "theoretical_loss": 3.4285013997841403, + "tokens_seen": 2013265920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039664646464646465, + "loss": 2.6416, + "theoretical_loss": 3.428492106693878, + "tokens_seen": 2013331456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966262626262626, + "loss": 2.6832, + "theoretical_loss": 3.428482813990807, + "tokens_seen": 2013396992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966060606060606, + "loss": 2.7265, + "theoretical_loss": 3.4284735216748983, + "tokens_seen": 2013462528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039658585858585863, + "loss": 2.2403, + "theoretical_loss": 3.4284642297461243, + "tokens_seen": 2013528064 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965656565656566, + "loss": 2.5304, + "theoretical_loss": 3.428454938204455, + "tokens_seen": 2013593600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039654545454545454, + "loss": 2.6138, + "theoretical_loss": 3.4284456470498625, + "tokens_seen": 2013659136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965252525252525, + "loss": 2.2908, + "theoretical_loss": 3.4284363562823175, + "tokens_seen": 2013724672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965050505050505, + "loss": 2.5686, + "theoretical_loss": 3.4284270659017917, + "tokens_seen": 2013790208 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1109221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5685839653015137, + "objective/train/theoretical_loss": 3.4284177759082564, + "objective/train/tokens_used": 372714976, + "theoretical_loss": 3.4284177759082564, + "tokens_seen": 2013855744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003964848484848485, + "loss": 2.6221, + "theoretical_loss": 3.4284177759082564, + "tokens_seen": 2013855744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964646464646465, + "loss": 2.5984, + "theoretical_loss": 3.4284084863016826, + "tokens_seen": 2013921280 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039644444444444444, + "loss": 2.4075, + "theoretical_loss": 3.428399197082042, + "tokens_seen": 2013986816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964242424242424, + "loss": 2.5087, + "theoretical_loss": 3.4283899082493052, + "tokens_seen": 2014052352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039640404040404046, + "loss": 2.5658, + "theoretical_loss": 3.428380619803444, + "tokens_seen": 2014117888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963838383838384, + "loss": 2.7391, + "theoretical_loss": 3.4283713317444295, + "tokens_seen": 2014183424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963636363636364, + "loss": 2.7968, + "theoretical_loss": 3.4283620440722333, + "tokens_seen": 2014248960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039634343434343433, + "loss": 2.6103, + "theoretical_loss": 3.4283527567868264, + "tokens_seen": 2014314496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963232323232323, + "loss": 2.7116, + "theoretical_loss": 3.4283434698881807, + "tokens_seen": 2014380032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039630303030303035, + "loss": 2.7277, + "theoretical_loss": 3.4283341833762666, + "tokens_seen": 2014445568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962828282828283, + "loss": 2.7381, + "theoretical_loss": 3.4283248972510556, + "tokens_seen": 2014511104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039626262626262627, + "loss": 2.6238, + "theoretical_loss": 3.42831561151252, + "tokens_seen": 2014576640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962424242424242, + "loss": 2.3142, + "theoretical_loss": 3.42830632616063, + "tokens_seen": 2014642176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962222222222222, + "loss": 2.741, + "theoretical_loss": 3.428297041195357, + "tokens_seen": 2014707712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039620202020202024, + "loss": 2.7352, + "theoretical_loss": 3.4282877566166734, + "tokens_seen": 2014773248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961818181818182, + "loss": 2.6948, + "theoretical_loss": 3.428278472424549, + "tokens_seen": 2014838784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039616161616161616, + "loss": 2.7039, + "theoretical_loss": 3.4282691886189562, + "tokens_seen": 2014904320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961414141414141, + "loss": 2.4815, + "theoretical_loss": 3.4282599051998663, + "tokens_seen": 2014969856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961212121212121, + "loss": 2.6714, + "theoretical_loss": 3.4282506221672504, + "tokens_seen": 2015035392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039610101010101014, + "loss": 2.7503, + "theoretical_loss": 3.4282413395210796, + "tokens_seen": 2015100928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960808080808081, + "loss": 2.5656, + "theoretical_loss": 3.4282320572613254, + "tokens_seen": 2015166464 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039606060606060605, + "loss": 2.3771, + "theoretical_loss": 3.428222775387959, + "tokens_seen": 2015232000 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396040404040404, + "loss": 2.5924, + "theoretical_loss": 3.4282134939009525, + "tokens_seen": 2015297536 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396020202020202, + "loss": 2.5664, + "theoretical_loss": 3.4282042128002765, + "tokens_seen": 2015363072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039600000000000003, + "loss": 2.6235, + "theoretical_loss": 3.4281949320859026, + "tokens_seen": 2015428608 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1110352, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.888545274734497, + "objective/train/theoretical_loss": 3.4281856517578024, + "objective/train/tokens_used": 374353376, + "theoretical_loss": 3.4281856517578024, + "tokens_seen": 2015494144 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395979797979798, + "loss": 2.527, + "theoretical_loss": 3.4281856517578024, + "tokens_seen": 2015494144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039595959595959594, + "loss": 2.8353, + "theoretical_loss": 3.4281763718159466, + "tokens_seen": 2015559680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039593939393939396, + "loss": 2.3749, + "theoretical_loss": 3.428167092260307, + "tokens_seen": 2015625216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959191919191919, + "loss": 2.5688, + "theoretical_loss": 3.4281578130908548, + "tokens_seen": 2015690752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958989898989899, + "loss": 2.6914, + "theoretical_loss": 3.428148534307562, + "tokens_seen": 2015756288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958787878787879, + "loss": 2.6239, + "theoretical_loss": 3.4281392559103994, + "tokens_seen": 2015821824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039585858585858584, + "loss": 2.3188, + "theoretical_loss": 3.428129977899338, + "tokens_seen": 2015887360 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039583838383838385, + "loss": 2.5494, + "theoretical_loss": 3.4281207002743503, + "tokens_seen": 2015952896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958181818181818, + "loss": 2.5859, + "theoretical_loss": 3.4281114230354066, + "tokens_seen": 2016018432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957979797979798, + "loss": 2.6417, + "theoretical_loss": 3.428102146182479, + "tokens_seen": 2016083968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039577777777777777, + "loss": 2.3921, + "theoretical_loss": 3.428092869715538, + "tokens_seen": 2016149504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957575757575758, + "loss": 2.8298, + "theoretical_loss": 3.4280835936345566, + "tokens_seen": 2016215040 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039573737373737374, + "loss": 2.6659, + "theoretical_loss": 3.4280743179395046, + "tokens_seen": 2016280576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957171717171717, + "loss": 2.5375, + "theoretical_loss": 3.4280650426303536, + "tokens_seen": 2016346112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956969696969697, + "loss": 2.6967, + "theoretical_loss": 3.428055767707076, + "tokens_seen": 2016411648 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039567676767676767, + "loss": 2.5046, + "theoretical_loss": 3.428046493169643, + "tokens_seen": 2016477184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956565656565657, + "loss": 2.6307, + "theoretical_loss": 3.428037219018025, + "tokens_seen": 2016542720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039563636363636363, + "loss": 2.6712, + "theoretical_loss": 3.428027945252194, + "tokens_seen": 2016608256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039561616161616164, + "loss": 2.6101, + "theoretical_loss": 3.4280186718721217, + "tokens_seen": 2016673792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955959595959596, + "loss": 2.6178, + "theoretical_loss": 3.428009398877779, + "tokens_seen": 2016739328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955757575757576, + "loss": 2.6002, + "theoretical_loss": 3.428000126269138, + "tokens_seen": 2016804864 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039555555555555557, + "loss": 2.4496, + "theoretical_loss": 3.427990854046169, + "tokens_seen": 2016870400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955353535353535, + "loss": 2.6548, + "theoretical_loss": 3.4279815822088455, + "tokens_seen": 2016935936 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039551515151515154, + "loss": 2.4536, + "theoretical_loss": 3.427972310757136, + "tokens_seen": 2017001472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954949494949495, + "loss": 2.6407, + "theoretical_loss": 3.4279630396910146, + "tokens_seen": 2017067008 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1110721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7697877883911133, + "objective/train/theoretical_loss": 3.4279537690104513, + "objective/train/tokens_used": 375991776, + "theoretical_loss": 3.4279537690104513, + "tokens_seen": 2017132544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954747474747475, + "loss": 2.5957, + "theoretical_loss": 3.4279537690104513, + "tokens_seen": 2017132544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039545454545454546, + "loss": 2.5364, + "theoretical_loss": 3.427944498715418, + "tokens_seen": 2017198080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954343434343434, + "loss": 2.8179, + "theoretical_loss": 3.427935228805886, + "tokens_seen": 2017263616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039541414141414143, + "loss": 2.6458, + "theoretical_loss": 3.4279259592818265, + "tokens_seen": 2017329152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039539393939393944, + "loss": 2.58, + "theoretical_loss": 3.4279166901432117, + "tokens_seen": 2017394688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953737373737374, + "loss": 2.2848, + "theoretical_loss": 3.427907421390012, + "tokens_seen": 2017460224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039535353535353535, + "loss": 2.5158, + "theoretical_loss": 3.4278981530221997, + "tokens_seen": 2017525760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953333333333333, + "loss": 2.5589, + "theoretical_loss": 3.4278888850397458, + "tokens_seen": 2017591296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953131313131313, + "loss": 2.5954, + "theoretical_loss": 3.4278796174426223, + "tokens_seen": 2017656832 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039529292929292933, + "loss": 2.7181, + "theoretical_loss": 3.4278703502308003, + "tokens_seen": 2017722368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952727272727273, + "loss": 2.5333, + "theoretical_loss": 3.427861083404251, + "tokens_seen": 2017787904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039525252525252525, + "loss": 2.4276, + "theoretical_loss": 3.4278518169629466, + "tokens_seen": 2017853440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952323232323232, + "loss": 2.5602, + "theoretical_loss": 3.4278425509068575, + "tokens_seen": 2017918976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039521212121212127, + "loss": 2.7124, + "theoretical_loss": 3.4278332852359563, + "tokens_seen": 2017984512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951919191919192, + "loss": 2.6338, + "theoretical_loss": 3.4278240199502137, + "tokens_seen": 2018050048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951717171717172, + "loss": 2.5743, + "theoretical_loss": 3.427814755049601, + "tokens_seen": 2018115584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039515151515151514, + "loss": 2.7207, + "theoretical_loss": 3.4278054905340913, + "tokens_seen": 2018181120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951313131313131, + "loss": 2.7285, + "theoretical_loss": 3.4277962264036543, + "tokens_seen": 2018246656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039511111111111116, + "loss": 2.3587, + "theoretical_loss": 3.427786962658262, + "tokens_seen": 2018312192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950909090909091, + "loss": 2.4289, + "theoretical_loss": 3.427777699297886, + "tokens_seen": 2018377728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950707070707071, + "loss": 2.6449, + "theoretical_loss": 3.4277684363224985, + "tokens_seen": 2018443264 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039505050505050503, + "loss": 2.3899, + "theoretical_loss": 3.4277591737320696, + "tokens_seen": 2018508800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039503030303030304, + "loss": 2.6567, + "theoretical_loss": 3.427749911526572, + "tokens_seen": 2018574336 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039501010101010105, + "loss": 2.7126, + "theoretical_loss": 3.427740649705976, + "tokens_seen": 2018639872 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394989898989899, + "loss": 2.6026, + "theoretical_loss": 3.4277313882702547, + "tokens_seen": 2018705408 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1111912, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.503018856048584, + "objective/train/theoretical_loss": 3.4277221272193783, + "objective/train/tokens_used": 377630176, + "theoretical_loss": 3.4277221272193783, + "tokens_seen": 2018770944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039496969696969697, + "loss": 2.6544, + "theoretical_loss": 3.4277221272193783, + "tokens_seen": 2018770944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949494949494949, + "loss": 2.7159, + "theoretical_loss": 3.4277128665533194, + "tokens_seen": 2018836480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039492929292929294, + "loss": 2.6576, + "theoretical_loss": 3.427703606272048, + "tokens_seen": 2018902016 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039490909090909095, + "loss": 2.9404, + "theoretical_loss": 3.427694346375537, + "tokens_seen": 2018967552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948888888888889, + "loss": 2.5293, + "theoretical_loss": 3.4276850868637574, + "tokens_seen": 2019033088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039486868686868686, + "loss": 2.7324, + "theoretical_loss": 3.4276758277366803, + "tokens_seen": 2019098624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039484848484848487, + "loss": 2.5299, + "theoretical_loss": 3.4276665689942782, + "tokens_seen": 2019164160 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039482828282828283, + "loss": 2.6959, + "theoretical_loss": 3.427657310636522, + "tokens_seen": 2019229696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039480808080808084, + "loss": 2.5669, + "theoretical_loss": 3.4276480526633835, + "tokens_seen": 2019295232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947878787878788, + "loss": 2.7492, + "theoretical_loss": 3.4276387950748335, + "tokens_seen": 2019360768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039476767676767675, + "loss": 2.3663, + "theoretical_loss": 3.4276295378708452, + "tokens_seen": 2019426304 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039474747474747476, + "loss": 2.4772, + "theoretical_loss": 3.427620281051388, + "tokens_seen": 2019491840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947272727272727, + "loss": 2.5573, + "theoretical_loss": 3.4276110246164353, + "tokens_seen": 2019557376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039470707070707073, + "loss": 2.5873, + "theoretical_loss": 3.4276017685659577, + "tokens_seen": 2019622912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946868686868687, + "loss": 2.6352, + "theoretical_loss": 3.4275925128999267, + "tokens_seen": 2019688448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946666666666667, + "loss": 2.3906, + "theoretical_loss": 3.4275832576183145, + "tokens_seen": 2019753984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039464646464646466, + "loss": 2.5083, + "theoretical_loss": 3.4275740027210917, + "tokens_seen": 2019819520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946262626262626, + "loss": 2.5144, + "theoretical_loss": 3.4275647482082308, + "tokens_seen": 2019885056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946060606060606, + "loss": 2.6141, + "theoretical_loss": 3.4275554940797033, + "tokens_seen": 2019950592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945858585858586, + "loss": 2.5868, + "theoretical_loss": 3.42754624033548, + "tokens_seen": 2020016128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945656565656566, + "loss": 2.3907, + "theoretical_loss": 3.427536986975533, + "tokens_seen": 2020081664 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039454545454545455, + "loss": 2.5512, + "theoretical_loss": 3.427527733999834, + "tokens_seen": 2020147200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945252525252525, + "loss": 2.4261, + "theoretical_loss": 3.427518481408354, + "tokens_seen": 2020212736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945050505050505, + "loss": 2.4513, + "theoretical_loss": 3.427509229201066, + "tokens_seen": 2020278272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944848484848485, + "loss": 2.4909, + "theoretical_loss": 3.4274999773779395, + "tokens_seen": 2020343808 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1112399, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.484231472015381, + "objective/train/theoretical_loss": 3.4274907259389478, + "objective/train/tokens_used": 379268576, + "theoretical_loss": 3.4274907259389478, + "tokens_seen": 2020409344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944646464646465, + "loss": 2.6462, + "theoretical_loss": 3.4274907259389478, + "tokens_seen": 2020409344 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039444444444444444, + "loss": 2.5147, + "theoretical_loss": 3.427481474884061, + "tokens_seen": 2020474880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944242424242424, + "loss": 2.5727, + "theoretical_loss": 3.4274722242132523, + "tokens_seen": 2020540416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944040404040404, + "loss": 2.646, + "theoretical_loss": 3.4274629739264926, + "tokens_seen": 2020605952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943838383838384, + "loss": 2.3592, + "theoretical_loss": 3.427453724023753, + "tokens_seen": 2020671488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943636363636364, + "loss": 2.5641, + "theoretical_loss": 3.427444474505006, + "tokens_seen": 2020737024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039434343434343434, + "loss": 2.6167, + "theoretical_loss": 3.427435225370222, + "tokens_seen": 2020802560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943232323232323, + "loss": 2.5598, + "theoretical_loss": 3.427425976619374, + "tokens_seen": 2020868096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943030303030303, + "loss": 2.7719, + "theoretical_loss": 3.427416728252433, + "tokens_seen": 2020933632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942828282828283, + "loss": 2.5257, + "theoretical_loss": 3.4274074802693706, + "tokens_seen": 2020999168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039426262626262627, + "loss": 2.7512, + "theoretical_loss": 3.427398232670158, + "tokens_seen": 2021064704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039424242424242423, + "loss": 2.4711, + "theoretical_loss": 3.4273889854547672, + "tokens_seen": 2021130240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942222222222222, + "loss": 2.6363, + "theoretical_loss": 3.4273797386231704, + "tokens_seen": 2021195776 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039420202020202025, + "loss": 2.66, + "theoretical_loss": 3.4273704921753385, + "tokens_seen": 2021261312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941818181818182, + "loss": 2.5107, + "theoretical_loss": 3.427361246111243, + "tokens_seen": 2021326848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039416161616161616, + "loss": 2.5211, + "theoretical_loss": 3.427352000430856, + "tokens_seen": 2021392384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941414141414141, + "loss": 2.6448, + "theoretical_loss": 3.4273427551341493, + "tokens_seen": 2021457920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941212121212121, + "loss": 2.4784, + "theoretical_loss": 3.427333510221094, + "tokens_seen": 2021523456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039410101010101014, + "loss": 2.6683, + "theoretical_loss": 3.4273242656916616, + "tokens_seen": 2021588992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940808080808081, + "loss": 2.4676, + "theoretical_loss": 3.4273150215458243, + "tokens_seen": 2021654528 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039406060606060606, + "loss": 2.4458, + "theoretical_loss": 3.4273057777835536, + "tokens_seen": 2021720064 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394040404040404, + "loss": 2.4926, + "theoretical_loss": 3.4272965344048214, + "tokens_seen": 2021785600 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394020202020202, + "loss": 2.4265, + "theoretical_loss": 3.427287291409599, + "tokens_seen": 2021851136 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039400000000000004, + "loss": 2.54, + "theoretical_loss": 3.4272780487978576, + "tokens_seen": 2021916672 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393979797979798, + "loss": 2.4683, + "theoretical_loss": 3.42726880656957, + "tokens_seen": 2021982208 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1113663, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.153092622756958, + "objective/train/theoretical_loss": 3.4272595647247073, + "objective/train/tokens_used": 380906976, + "theoretical_loss": 3.4272595647247073, + "tokens_seen": 2022047744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039395959595959595, + "loss": 2.5295, + "theoretical_loss": 3.4272595647247073, + "tokens_seen": 2022047744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939393939393939, + "loss": 2.4037, + "theoretical_loss": 3.42725032326324, + "tokens_seen": 2022113280 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039391919191919197, + "loss": 2.6336, + "theoretical_loss": 3.4272410821851422, + "tokens_seen": 2022178816 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039389898989898993, + "loss": 2.4541, + "theoretical_loss": 3.427231841490384, + "tokens_seen": 2022244352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938787878787879, + "loss": 2.4697, + "theoretical_loss": 3.427222601178937, + "tokens_seen": 2022309888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039385858585858584, + "loss": 2.6019, + "theoretical_loss": 3.4272133612507734, + "tokens_seen": 2022375424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039383838383838385, + "loss": 2.6577, + "theoretical_loss": 3.427204121705864, + "tokens_seen": 2022440960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039381818181818186, + "loss": 2.6264, + "theoretical_loss": 3.427194882544182, + "tokens_seen": 2022506496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937979797979798, + "loss": 2.5512, + "theoretical_loss": 3.4271856437656982, + "tokens_seen": 2022572032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937777777777778, + "loss": 2.5572, + "theoretical_loss": 3.4271764053703837, + "tokens_seen": 2022637568 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039375757575757574, + "loss": 2.7579, + "theoretical_loss": 3.4271671673582116, + "tokens_seen": 2022703104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039373737373737375, + "loss": 2.4911, + "theoretical_loss": 3.4271579297291526, + "tokens_seen": 2022768640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039371717171717176, + "loss": 2.6171, + "theoretical_loss": 3.4271486924831787, + "tokens_seen": 2022834176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936969696969697, + "loss": 2.635, + "theoretical_loss": 3.4271394556202615, + "tokens_seen": 2022899712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039367676767676767, + "loss": 2.4454, + "theoretical_loss": 3.4271302191403725, + "tokens_seen": 2022965248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936565656565657, + "loss": 2.5407, + "theoretical_loss": 3.4271209830434843, + "tokens_seen": 2023030784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039363636363636364, + "loss": 2.4783, + "theoretical_loss": 3.427111747329567, + "tokens_seen": 2023096320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039361616161616165, + "loss": 2.815, + "theoretical_loss": 3.4271025119985943, + "tokens_seen": 2023161856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935959595959596, + "loss": 2.2448, + "theoretical_loss": 3.4270932770505365, + "tokens_seen": 2023227392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039357575757575756, + "loss": 2.5646, + "theoretical_loss": 3.4270840424853657, + "tokens_seen": 2023292928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935555555555556, + "loss": 2.5724, + "theoretical_loss": 3.427074808303054, + "tokens_seen": 2023358464 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039353535353535353, + "loss": 2.5373, + "theoretical_loss": 3.427065574503572, + "tokens_seen": 2023424000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039351515151515154, + "loss": 2.6068, + "theoretical_loss": 3.427056341086893, + "tokens_seen": 2023489536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934949494949495, + "loss": 2.4802, + "theoretical_loss": 3.427047108052988, + "tokens_seen": 2023555072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934747474747475, + "loss": 2.5053, + "theoretical_loss": 3.427037875401828, + "tokens_seen": 2023620608 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1114958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.473273754119873, + "objective/train/theoretical_loss": 3.4270286431333856, + "objective/train/tokens_used": 382545376, + "theoretical_loss": 3.4270286431333856, + "tokens_seen": 2023686144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039345454545454547, + "loss": 2.3698, + "theoretical_loss": 3.4270286431333856, + "tokens_seen": 2023686144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934343434343434, + "loss": 2.5781, + "theoretical_loss": 3.4270194112476324, + "tokens_seen": 2023751680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039341414141414144, + "loss": 2.6656, + "theoretical_loss": 3.4270101797445403, + "tokens_seen": 2023817216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933939393939394, + "loss": 2.383, + "theoretical_loss": 3.4270009486240807, + "tokens_seen": 2023882752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933737373737374, + "loss": 2.6131, + "theoretical_loss": 3.4269917178862253, + "tokens_seen": 2023948288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039335353535353536, + "loss": 2.5708, + "theoretical_loss": 3.426982487530947, + "tokens_seen": 2024013824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933333333333333, + "loss": 2.3579, + "theoretical_loss": 3.426973257558216, + "tokens_seen": 2024079360 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039331313131313133, + "loss": 2.57, + "theoretical_loss": 3.4269640279680043, + "tokens_seen": 2024144896 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039329292929292934, + "loss": 2.662, + "theoretical_loss": 3.4269547987602844, + "tokens_seen": 2024210432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932727272727273, + "loss": 2.6302, + "theoretical_loss": 3.4269455699350275, + "tokens_seen": 2024275968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039325252525252525, + "loss": 2.3562, + "theoretical_loss": 3.4269363414922056, + "tokens_seen": 2024341504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932323232323232, + "loss": 2.6728, + "theoretical_loss": 3.426927113431791, + "tokens_seen": 2024407040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932121212121212, + "loss": 2.5875, + "theoretical_loss": 3.4269178857537543, + "tokens_seen": 2024472576 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039319191919191923, + "loss": 2.8369, + "theoretical_loss": 3.426908658458068, + "tokens_seen": 2024538112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931717171717172, + "loss": 2.7961, + "theoretical_loss": 3.426899431544704, + "tokens_seen": 2024603648 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039315151515151515, + "loss": 2.401, + "theoretical_loss": 3.426890205013634, + "tokens_seen": 2024669184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931313131313131, + "loss": 2.5557, + "theoretical_loss": 3.4268809788648293, + "tokens_seen": 2024734720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039311111111111117, + "loss": 2.4, + "theoretical_loss": 3.426871753098262, + "tokens_seen": 2024800256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930909090909091, + "loss": 2.7028, + "theoretical_loss": 3.4268625277139044, + "tokens_seen": 2024865792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930707070707071, + "loss": 2.5791, + "theoretical_loss": 3.426853302711727, + "tokens_seen": 2024931328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039305050505050504, + "loss": 2.6614, + "theoretical_loss": 3.4268440780917038, + "tokens_seen": 2024996864 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393030303030303, + "loss": 2.5811, + "theoretical_loss": 3.426834853853804, + "tokens_seen": 2025062400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039301010101010106, + "loss": 2.6949, + "theoretical_loss": 3.4268256299980013, + "tokens_seen": 2025127936 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392989898989899, + "loss": 2.6975, + "theoretical_loss": 3.4268164065242663, + "tokens_seen": 2025193472 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392969696969697, + "loss": 2.5428, + "theoretical_loss": 3.4268071834325715, + "tokens_seen": 2025259008 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1115580, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.886110544204712, + "objective/train/theoretical_loss": 3.4267979607228893, + "objective/train/tokens_used": 384183776, + "theoretical_loss": 3.4267979607228893, + "tokens_seen": 2025324544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039294949494949493, + "loss": 2.5216, + "theoretical_loss": 3.4267979607228893, + "tokens_seen": 2025324544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929292929292929, + "loss": 2.5705, + "theoretical_loss": 3.42678873839519, + "tokens_seen": 2025390080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039290909090909095, + "loss": 2.735, + "theoretical_loss": 3.4267795164494466, + "tokens_seen": 2025455616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928888888888889, + "loss": 2.6675, + "theoretical_loss": 3.42677029488563, + "tokens_seen": 2025521152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039286868686868687, + "loss": 2.453, + "theoretical_loss": 3.4267610737037133, + "tokens_seen": 2025586688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928484848484848, + "loss": 2.6699, + "theoretical_loss": 3.426751852903667, + "tokens_seen": 2025652224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039282828282828283, + "loss": 2.5889, + "theoretical_loss": 3.426742632485464, + "tokens_seen": 2025717760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039280808080808085, + "loss": 2.5707, + "theoretical_loss": 3.4267334124490754, + "tokens_seen": 2025783296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927878787878788, + "loss": 2.5891, + "theoretical_loss": 3.426724192794473, + "tokens_seen": 2025848832 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039276767676767676, + "loss": 2.5181, + "theoretical_loss": 3.426714973521629, + "tokens_seen": 2025914368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927474747474747, + "loss": 2.4398, + "theoretical_loss": 3.4267057546305155, + "tokens_seen": 2025979904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039272727272727273, + "loss": 2.5029, + "theoretical_loss": 3.426696536121104, + "tokens_seen": 2026045440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039270707070707074, + "loss": 2.6772, + "theoretical_loss": 3.426687317993366, + "tokens_seen": 2026110976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926868686868687, + "loss": 2.4527, + "theoretical_loss": 3.4266781002472744, + "tokens_seen": 2026176512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039266666666666665, + "loss": 2.5978, + "theoretical_loss": 3.4266688828828, + "tokens_seen": 2026242048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039264646464646466, + "loss": 2.4343, + "theoretical_loss": 3.4266596658999147, + "tokens_seen": 2026307584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926262626262626, + "loss": 2.6968, + "theoretical_loss": 3.426650449298591, + "tokens_seen": 2026373120 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039260606060606063, + "loss": 2.589, + "theoretical_loss": 3.4266412330788008, + "tokens_seen": 2026438656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925858585858586, + "loss": 2.604, + "theoretical_loss": 3.4266320172405154, + "tokens_seen": 2026504192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039256565656565654, + "loss": 2.5602, + "theoretical_loss": 3.426622801783707, + "tokens_seen": 2026569728 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039254545454545456, + "loss": 2.5963, + "theoretical_loss": 3.426613586708347, + "tokens_seen": 2026635264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925252525252525, + "loss": 2.4481, + "theoretical_loss": 3.426604372014408, + "tokens_seen": 2026700800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925050505050505, + "loss": 2.5384, + "theoretical_loss": 3.426595157701861, + "tokens_seen": 2026766336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924848484848485, + "loss": 2.496, + "theoretical_loss": 3.426585943770679, + "tokens_seen": 2026831872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924646464646465, + "loss": 2.6017, + "theoretical_loss": 3.426576730220833, + "tokens_seen": 2026897408 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1116847, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7708523273468018, + "objective/train/theoretical_loss": 3.4265675170522956, + "objective/train/tokens_used": 385822176, + "theoretical_loss": 3.4265675170522956, + "tokens_seen": 2026962944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039244444444444445, + "loss": 2.6355, + "theoretical_loss": 3.4265675170522956, + "tokens_seen": 2026962944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924242424242424, + "loss": 2.7949, + "theoretical_loss": 3.426558304265038, + "tokens_seen": 2027028480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924040404040404, + "loss": 2.6705, + "theoretical_loss": 3.426549091859032, + "tokens_seen": 2027094016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923838383838384, + "loss": 2.7159, + "theoretical_loss": 3.426539879834251, + "tokens_seen": 2027159552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923636363636364, + "loss": 2.689, + "theoretical_loss": 3.4265306681906647, + "tokens_seen": 2027225088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039234343434343434, + "loss": 2.5202, + "theoretical_loss": 3.4265214569282465, + "tokens_seen": 2027290624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923232323232323, + "loss": 2.5135, + "theoretical_loss": 3.4265122460469675, + "tokens_seen": 2027356160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923030303030303, + "loss": 2.5385, + "theoretical_loss": 3.4265030355468005, + "tokens_seen": 2027421696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922828282828283, + "loss": 2.635, + "theoretical_loss": 3.4264938254277166, + "tokens_seen": 2027487232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922626262626263, + "loss": 2.5788, + "theoretical_loss": 3.426484615689688, + "tokens_seen": 2027552768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039224242424242423, + "loss": 2.5681, + "theoretical_loss": 3.4264754063326865, + "tokens_seen": 2027618304 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039222222222222225, + "loss": 2.441, + "theoretical_loss": 3.426466197356685, + "tokens_seen": 2027683840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922020202020202, + "loss": 2.5216, + "theoretical_loss": 3.426456988761654, + "tokens_seen": 2027749376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921818181818182, + "loss": 2.4985, + "theoretical_loss": 3.4264477805475657, + "tokens_seen": 2027814912 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039216161616161617, + "loss": 2.5328, + "theoretical_loss": 3.426438572714393, + "tokens_seen": 2027880448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921414141414141, + "loss": 2.3552, + "theoretical_loss": 3.4264293652621065, + "tokens_seen": 2027945984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039212121212121214, + "loss": 2.5907, + "theoretical_loss": 3.4264201581906795, + "tokens_seen": 2028011520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039210101010101015, + "loss": 2.4451, + "theoretical_loss": 3.426410951500083, + "tokens_seen": 2028077056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920808080808081, + "loss": 2.6231, + "theoretical_loss": 3.4264017451902893, + "tokens_seen": 2028142592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039206060606060606, + "loss": 2.6823, + "theoretical_loss": 3.4263925392612697, + "tokens_seen": 2028208128 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392040404040404, + "loss": 2.5544, + "theoretical_loss": 3.426383333712997, + "tokens_seen": 2028273664 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039202020202020203, + "loss": 2.832, + "theoretical_loss": 3.4263741285454428, + "tokens_seen": 2028339200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039200000000000004, + "loss": 2.3735, + "theoretical_loss": 3.4263649237585794, + "tokens_seen": 2028404736 + }, + { + "epoch": 0.22, + "learning_rate": 0.000391979797979798, + "loss": 2.6263, + "theoretical_loss": 3.426355719352378, + "tokens_seen": 2028470272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039195959595959596, + "loss": 2.4902, + "theoretical_loss": 3.4263465153268116, + "tokens_seen": 2028535808 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1117380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6172401905059814, + "objective/train/theoretical_loss": 3.426337311681851, + "objective/train/tokens_used": 387460576, + "theoretical_loss": 3.426337311681851, + "tokens_seen": 2028601344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919393939393939, + "loss": 2.4793, + "theoretical_loss": 3.426337311681851, + "tokens_seen": 2028601344 + }, + { + "epoch": 0.22, + "learning_rate": 0.000391919191919192, + "loss": 2.4009, + "theoretical_loss": 3.4263281084174695, + "tokens_seen": 2028666880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039189898989898993, + "loss": 2.492, + "theoretical_loss": 3.4263189055336376, + "tokens_seen": 2028732416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918787878787879, + "loss": 2.4839, + "theoretical_loss": 3.426309703030328, + "tokens_seen": 2028797952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039185858585858585, + "loss": 2.7173, + "theoretical_loss": 3.426300500907513, + "tokens_seen": 2028863488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918383838383838, + "loss": 2.5024, + "theoretical_loss": 3.4262912991651637, + "tokens_seen": 2028929024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039181818181818187, + "loss": 2.6256, + "theoretical_loss": 3.426282097803253, + "tokens_seen": 2028994560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917979797979798, + "loss": 2.638, + "theoretical_loss": 3.426272896821753, + "tokens_seen": 2029060096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917777777777778, + "loss": 2.3982, + "theoretical_loss": 3.4262636962206345, + "tokens_seen": 2029125632 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039175757575757574, + "loss": 2.3516, + "theoretical_loss": 3.42625449599987, + "tokens_seen": 2029191168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039173737373737375, + "loss": 2.4793, + "theoretical_loss": 3.4262452961594323, + "tokens_seen": 2029256704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039171717171717176, + "loss": 2.5811, + "theoretical_loss": 3.4262360966992924, + "tokens_seen": 2029322240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916969696969697, + "loss": 2.6168, + "theoretical_loss": 3.426226897619423, + "tokens_seen": 2029387776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916767676767677, + "loss": 2.4779, + "theoretical_loss": 3.426217698919795, + "tokens_seen": 2029453312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039165656565656563, + "loss": 2.4986, + "theoretical_loss": 3.426208500600382, + "tokens_seen": 2029518848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039163636363636364, + "loss": 2.7759, + "theoretical_loss": 3.426199302661155, + "tokens_seen": 2029584384 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039161616161616166, + "loss": 2.4451, + "theoretical_loss": 3.4261901051020858, + "tokens_seen": 2029649920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915959595959596, + "loss": 2.4326, + "theoretical_loss": 3.426180907923147, + "tokens_seen": 2029715456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039157575757575757, + "loss": 2.4658, + "theoretical_loss": 3.4261717111243106, + "tokens_seen": 2029780992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915555555555556, + "loss": 2.4223, + "theoretical_loss": 3.426162514705548, + "tokens_seen": 2029846528 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039153535353535354, + "loss": 2.7919, + "theoretical_loss": 3.426153318666832, + "tokens_seen": 2029912064 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039151515151515155, + "loss": 2.6321, + "theoretical_loss": 3.426144123008134, + "tokens_seen": 2029977600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914949494949495, + "loss": 2.5509, + "theoretical_loss": 3.426134927729427, + "tokens_seen": 2030043136 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039147474747474746, + "loss": 2.5512, + "theoretical_loss": 3.426125732830682, + "tokens_seen": 2030108672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914545454545455, + "loss": 2.5388, + "theoretical_loss": 3.426116538311871, + "tokens_seen": 2030174208 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1118494, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7309558391571045, + "objective/train/theoretical_loss": 3.4261073441729666, + "objective/train/tokens_used": 389098976, + "theoretical_loss": 3.4261073441729666, + "tokens_seen": 2030239744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039143434343434343, + "loss": 2.6567, + "theoretical_loss": 3.4261073441729666, + "tokens_seen": 2030239744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039141414141414144, + "loss": 2.4831, + "theoretical_loss": 3.426098150413941, + "tokens_seen": 2030305280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913939393939394, + "loss": 2.6372, + "theoretical_loss": 3.4260889570347652, + "tokens_seen": 2030370816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913737373737374, + "loss": 2.5891, + "theoretical_loss": 3.426079764035413, + "tokens_seen": 2030436352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039135353535353537, + "loss": 2.5801, + "theoretical_loss": 3.4260705714158544, + "tokens_seen": 2030501888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913333333333333, + "loss": 2.8033, + "theoretical_loss": 3.426061379176063, + "tokens_seen": 2030567424 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039131313131313133, + "loss": 2.4674, + "theoretical_loss": 3.42605218731601, + "tokens_seen": 2030632960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912929292929293, + "loss": 2.469, + "theoretical_loss": 3.426042995835668, + "tokens_seen": 2030698496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912727272727273, + "loss": 2.672, + "theoretical_loss": 3.4260338047350087, + "tokens_seen": 2030764032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039125252525252526, + "loss": 2.5594, + "theoretical_loss": 3.426024614014004, + "tokens_seen": 2030829568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912323232323232, + "loss": 2.5454, + "theoretical_loss": 3.426015423672627, + "tokens_seen": 2030895104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912121212121212, + "loss": 2.48, + "theoretical_loss": 3.426006233710848, + "tokens_seen": 2030960640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911919191919192, + "loss": 2.5707, + "theoretical_loss": 3.425997044128641, + "tokens_seen": 2031026176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911717171717172, + "loss": 2.5987, + "theoretical_loss": 3.425987854925977, + "tokens_seen": 2031091712 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039115151515151515, + "loss": 2.5819, + "theoretical_loss": 3.4259786661028278, + "tokens_seen": 2031157248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911313131313131, + "loss": 2.6741, + "theoretical_loss": 3.4259694776591663, + "tokens_seen": 2031222784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911111111111111, + "loss": 2.6763, + "theoretical_loss": 3.425960289594964, + "tokens_seen": 2031288320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039109090909090913, + "loss": 2.6218, + "theoretical_loss": 3.4259511019101936, + "tokens_seen": 2031353856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910707070707071, + "loss": 2.5182, + "theoretical_loss": 3.4259419146048264, + "tokens_seen": 2031419392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039105050505050504, + "loss": 2.7424, + "theoretical_loss": 3.425932727678835, + "tokens_seen": 2031484928 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391030303030303, + "loss": 2.5233, + "theoretical_loss": 3.425923541132191, + "tokens_seen": 2031550464 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391010101010101, + "loss": 2.8571, + "theoretical_loss": 3.4259143549648674, + "tokens_seen": 2031616000 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390989898989899, + "loss": 2.4963, + "theoretical_loss": 3.4259051691768354, + "tokens_seen": 2031681536 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390969696969697, + "loss": 2.6932, + "theoretical_loss": 3.425895983768068, + "tokens_seen": 2031747072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039094949494949494, + "loss": 2.5079, + "theoretical_loss": 3.425886798738537, + "tokens_seen": 2031812608 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1119152, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6228444576263428, + "objective/train/theoretical_loss": 3.425877614088213, + "objective/train/tokens_used": 390737376, + "theoretical_loss": 3.425877614088213, + "tokens_seen": 2031878144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909292929292929, + "loss": 2.5856, + "theoretical_loss": 3.425877614088213, + "tokens_seen": 2031878144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039090909090909096, + "loss": 2.6037, + "theoretical_loss": 3.4258684298170703, + "tokens_seen": 2031943680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908888888888889, + "loss": 2.8355, + "theoretical_loss": 3.4258592459250803, + "tokens_seen": 2032009216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039086868686868687, + "loss": 2.6795, + "theoretical_loss": 3.4258500624122146, + "tokens_seen": 2032074752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039084848484848483, + "loss": 2.496, + "theoretical_loss": 3.4258408792784456, + "tokens_seen": 2032140288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908282828282828, + "loss": 2.5187, + "theoretical_loss": 3.425831696523746, + "tokens_seen": 2032205824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039080808080808085, + "loss": 2.5352, + "theoretical_loss": 3.425822514148087, + "tokens_seen": 2032271360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907878787878788, + "loss": 2.571, + "theoretical_loss": 3.4258133321514412, + "tokens_seen": 2032336896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039076767676767677, + "loss": 2.4906, + "theoretical_loss": 3.4258041505337813, + "tokens_seen": 2032402432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907474747474747, + "loss": 2.6286, + "theoretical_loss": 3.425794969295078, + "tokens_seen": 2032467968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039072727272727273, + "loss": 2.4928, + "theoretical_loss": 3.425785788435305, + "tokens_seen": 2032533504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039070707070707074, + "loss": 2.7237, + "theoretical_loss": 3.4257766079544334, + "tokens_seen": 2032599040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906868686868687, + "loss": 2.4682, + "theoretical_loss": 3.4257674278524353, + "tokens_seen": 2032664576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039066666666666666, + "loss": 2.624, + "theoretical_loss": 3.4257582481292834, + "tokens_seen": 2032730112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906464646464646, + "loss": 2.5292, + "theoretical_loss": 3.42574906878495, + "tokens_seen": 2032795648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906262626262626, + "loss": 2.6357, + "theoretical_loss": 3.4257398898194067, + "tokens_seen": 2032861184 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039060606060606064, + "loss": 2.6523, + "theoretical_loss": 3.425730711232626, + "tokens_seen": 2032926720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905858585858586, + "loss": 2.5825, + "theoretical_loss": 3.4257215330245794, + "tokens_seen": 2032992256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039056565656565655, + "loss": 2.5612, + "theoretical_loss": 3.42571235519524, + "tokens_seen": 2033057792 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039054545454545456, + "loss": 2.4326, + "theoretical_loss": 3.4257031777445803, + "tokens_seen": 2033123328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905252525252525, + "loss": 2.8784, + "theoretical_loss": 3.425694000672571, + "tokens_seen": 2033188864 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039050505050505053, + "loss": 2.4251, + "theoretical_loss": 3.425684823979185, + "tokens_seen": 2033254400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904848484848485, + "loss": 2.722, + "theoretical_loss": 3.4256756476643946, + "tokens_seen": 2033319936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039046464646464644, + "loss": 2.7486, + "theoretical_loss": 3.4256664717281717, + "tokens_seen": 2033385472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039044444444444445, + "loss": 2.4272, + "theoretical_loss": 3.425657296170489, + "tokens_seen": 2033451008 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1120297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3779661655426025, + "objective/train/theoretical_loss": 3.425648120991318, + "objective/train/tokens_used": 392375776, + "theoretical_loss": 3.425648120991318, + "tokens_seen": 2033516544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039042424242424247, + "loss": 2.5723, + "theoretical_loss": 3.425648120991318, + "tokens_seen": 2033516544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904040404040404, + "loss": 2.9231, + "theoretical_loss": 3.425638946190632, + "tokens_seen": 2033582080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903838383838384, + "loss": 2.6368, + "theoretical_loss": 3.4256297717684014, + "tokens_seen": 2033647616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903636363636364, + "loss": 2.6167, + "theoretical_loss": 3.4256205977246, + "tokens_seen": 2033713152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039034343434343435, + "loss": 2.5206, + "theoretical_loss": 3.4256114240591993, + "tokens_seen": 2033778688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039032323232323236, + "loss": 2.6265, + "theoretical_loss": 3.4256022507721715, + "tokens_seen": 2033844224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903030303030303, + "loss": 2.5845, + "theoretical_loss": 3.425593077863489, + "tokens_seen": 2033909760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039028282828282827, + "loss": 2.7537, + "theoretical_loss": 3.4255839053331236, + "tokens_seen": 2033975296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902626262626263, + "loss": 2.8376, + "theoretical_loss": 3.4255747331810484, + "tokens_seen": 2034040832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039024242424242424, + "loss": 2.4043, + "theoretical_loss": 3.4255655614072342, + "tokens_seen": 2034106368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039022222222222225, + "loss": 2.5765, + "theoretical_loss": 3.4255563900116552, + "tokens_seen": 2034171904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902020202020202, + "loss": 2.7122, + "theoretical_loss": 3.4255472189942813, + "tokens_seen": 2034237440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901818181818182, + "loss": 2.5745, + "theoretical_loss": 3.4255380483550866, + "tokens_seen": 2034302976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901616161616162, + "loss": 2.4317, + "theoretical_loss": 3.4255288780940427, + "tokens_seen": 2034368512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039014141414141413, + "loss": 2.5789, + "theoretical_loss": 3.4255197082111213, + "tokens_seen": 2034434048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039012121212121214, + "loss": 2.5604, + "theoretical_loss": 3.425510538706295, + "tokens_seen": 2034499584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901010101010101, + "loss": 2.6917, + "theoretical_loss": 3.4255013695795364, + "tokens_seen": 2034565120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900808080808081, + "loss": 2.6604, + "theoretical_loss": 3.4254922008308175, + "tokens_seen": 2034630656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039006060606060607, + "loss": 2.5923, + "theoretical_loss": 3.4254830324601104, + "tokens_seen": 2034696192 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390040404040404, + "loss": 2.6612, + "theoretical_loss": 3.425473864467387, + "tokens_seen": 2034761728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039002020202020204, + "loss": 2.7573, + "theoretical_loss": 3.425464696852621, + "tokens_seen": 2034827264 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039000000000000005, + "loss": 2.567, + "theoretical_loss": 3.4254555296157827, + "tokens_seen": 2034892800 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389979797979798, + "loss": 2.8421, + "theoretical_loss": 3.425446362756845, + "tokens_seen": 2034958336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038995959595959596, + "loss": 2.605, + "theoretical_loss": 3.425437196275781, + "tokens_seen": 2035023872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899393939393939, + "loss": 2.7614, + "theoretical_loss": 3.4254280301725624, + "tokens_seen": 2035089408 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1121395, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6363847255706787, + "objective/train/theoretical_loss": 3.425418864447161, + "objective/train/tokens_used": 394014176, + "theoretical_loss": 3.425418864447161, + "tokens_seen": 2035154944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038991919191919193, + "loss": 2.5577, + "theoretical_loss": 3.425418864447161, + "tokens_seen": 2035154944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038989898989898994, + "loss": 2.5876, + "theoretical_loss": 3.42540969909955, + "tokens_seen": 2035220480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898787878787879, + "loss": 2.4677, + "theoretical_loss": 3.4254005341297007, + "tokens_seen": 2035286016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038985858585858585, + "loss": 2.5843, + "theoretical_loss": 3.4253913695375857, + "tokens_seen": 2035351552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898383838383838, + "loss": 2.536, + "theoretical_loss": 3.425382205323178, + "tokens_seen": 2035417088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898181818181819, + "loss": 2.59, + "theoretical_loss": 3.4253730414864485, + "tokens_seen": 2035482624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038979797979797983, + "loss": 2.6873, + "theoretical_loss": 3.4253638780273707, + "tokens_seen": 2035548160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897777777777778, + "loss": 2.8808, + "theoretical_loss": 3.4253547149459163, + "tokens_seen": 2035613696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038975757575757575, + "loss": 2.7569, + "theoretical_loss": 3.4253455522420575, + "tokens_seen": 2035679232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897373737373737, + "loss": 2.608, + "theoretical_loss": 3.425336389915767, + "tokens_seen": 2035744768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038971717171717177, + "loss": 2.6626, + "theoretical_loss": 3.4253272279670166, + "tokens_seen": 2035810304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896969696969697, + "loss": 2.7393, + "theoretical_loss": 3.4253180663957794, + "tokens_seen": 2035875840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896767676767677, + "loss": 2.4374, + "theoretical_loss": 3.4253089052020265, + "tokens_seen": 2035941376 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038965656565656564, + "loss": 2.5749, + "theoretical_loss": 3.425299744385731, + "tokens_seen": 2036006912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896363636363636, + "loss": 2.7259, + "theoretical_loss": 3.4252905839468655, + "tokens_seen": 2036072448 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038961616161616166, + "loss": 2.6453, + "theoretical_loss": 3.425281423885401, + "tokens_seen": 2036137984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895959595959596, + "loss": 2.7484, + "theoretical_loss": 3.425272264201311, + "tokens_seen": 2036203520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895757575757576, + "loss": 2.701, + "theoretical_loss": 3.425263104894568, + "tokens_seen": 2036269056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038955555555555553, + "loss": 2.6245, + "theoretical_loss": 3.425253945965143, + "tokens_seen": 2036334592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038953535353535354, + "loss": 2.8589, + "theoretical_loss": 3.4252447874130096, + "tokens_seen": 2036400128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038951515151515155, + "loss": 2.7356, + "theoretical_loss": 3.425235629238139, + "tokens_seen": 2036465664 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894949494949495, + "loss": 2.572, + "theoretical_loss": 3.4252264714405047, + "tokens_seen": 2036531200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038947474747474747, + "loss": 2.6346, + "theoretical_loss": 3.4252173140200783, + "tokens_seen": 2036596736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894545454545454, + "loss": 2.6631, + "theoretical_loss": 3.425208156976832, + "tokens_seen": 2036662272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038943434343434344, + "loss": 2.7026, + "theoretical_loss": 3.4251990003107387, + "tokens_seen": 2036727808 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1121841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7600228786468506, + "objective/train/theoretical_loss": 3.4251898440217703, + "objective/train/tokens_used": 395652576, + "theoretical_loss": 3.4251898440217703, + "tokens_seen": 2036793344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038941414141414145, + "loss": 2.7747, + "theoretical_loss": 3.4251898440217703, + "tokens_seen": 2036793344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893939393939394, + "loss": 2.5716, + "theoretical_loss": 3.425180688109899, + "tokens_seen": 2036858880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038937373737373736, + "loss": 2.3788, + "theoretical_loss": 3.4251715325750975, + "tokens_seen": 2036924416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038935353535353537, + "loss": 2.5345, + "theoretical_loss": 3.4251623774173385, + "tokens_seen": 2036989952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038933333333333333, + "loss": 2.5381, + "theoretical_loss": 3.4251532226365935, + "tokens_seen": 2037055488 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038931313131313134, + "loss": 2.7236, + "theoretical_loss": 3.425144068232835, + "tokens_seen": 2037121024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892929292929293, + "loss": 2.685, + "theoretical_loss": 3.425134914206036, + "tokens_seen": 2037186560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038927272727272725, + "loss": 2.528, + "theoretical_loss": 3.425125760556168, + "tokens_seen": 2037252096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038925252525252526, + "loss": 2.6716, + "theoretical_loss": 3.425116607283204, + "tokens_seen": 2037317632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892323232323232, + "loss": 2.7653, + "theoretical_loss": 3.425107454387116, + "tokens_seen": 2037383168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038921212121212123, + "loss": 2.6616, + "theoretical_loss": 3.4250983018678767, + "tokens_seen": 2037448704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891919191919192, + "loss": 2.7058, + "theoretical_loss": 3.425089149725458, + "tokens_seen": 2037514240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891717171717172, + "loss": 2.8981, + "theoretical_loss": 3.4250799979598328, + "tokens_seen": 2037579776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038915151515151516, + "loss": 2.6551, + "theoretical_loss": 3.4250708465709727, + "tokens_seen": 2037645312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891313131313131, + "loss": 2.4252, + "theoretical_loss": 3.425061695558851, + "tokens_seen": 2037710848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891111111111111, + "loss": 2.7006, + "theoretical_loss": 3.4250525449234397, + "tokens_seen": 2037776384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890909090909091, + "loss": 2.4131, + "theoretical_loss": 3.4250433946647107, + "tokens_seen": 2037841920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890707070707071, + "loss": 2.5942, + "theoretical_loss": 3.425034244782637, + "tokens_seen": 2037907456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038905050505050505, + "loss": 2.6065, + "theoretical_loss": 3.4250250952771912, + "tokens_seen": 2037972992 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389030303030303, + "loss": 2.6553, + "theoretical_loss": 3.4250159461483447, + "tokens_seen": 2038038528 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389010101010101, + "loss": 2.7689, + "theoretical_loss": 3.4250067973960707, + "tokens_seen": 2038104064 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038898989898989903, + "loss": 2.6705, + "theoretical_loss": 3.424997649020341, + "tokens_seen": 2038169600 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388969696969697, + "loss": 2.5966, + "theoretical_loss": 3.4249885010211285, + "tokens_seen": 2038235136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038894949494949494, + "loss": 2.5179, + "theoretical_loss": 3.4249793533984056, + "tokens_seen": 2038300672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889292929292929, + "loss": 2.5465, + "theoretical_loss": 3.4249702061521443, + "tokens_seen": 2038366208 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1123215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.411255121231079, + "objective/train/theoretical_loss": 3.424961059282318, + "objective/train/tokens_used": 397290976, + "theoretical_loss": 3.424961059282318, + "tokens_seen": 2038431744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889090909090909, + "loss": 2.5545, + "theoretical_loss": 3.424961059282318, + "tokens_seen": 2038431744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888888888888889, + "loss": 2.5991, + "theoretical_loss": 3.4249519127888974, + "tokens_seen": 2038497280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888686868686869, + "loss": 2.6284, + "theoretical_loss": 3.4249427666718564, + "tokens_seen": 2038562816 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038884848484848483, + "loss": 2.661, + "theoretical_loss": 3.4249336209311667, + "tokens_seen": 2038628352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888282828282828, + "loss": 2.5987, + "theoretical_loss": 3.424924475566801, + "tokens_seen": 2038693888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038880808080808086, + "loss": 2.6224, + "theoretical_loss": 3.4249153305787314, + "tokens_seen": 2038759424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887878787878788, + "loss": 2.4543, + "theoretical_loss": 3.4249061859669307, + "tokens_seen": 2038824960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038876767676767677, + "loss": 2.6108, + "theoretical_loss": 3.424897041731371, + "tokens_seen": 2038890496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038874747474747473, + "loss": 2.4736, + "theoretical_loss": 3.424887897872025, + "tokens_seen": 2038956032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038872727272727274, + "loss": 2.8445, + "theoretical_loss": 3.4248787543888652, + "tokens_seen": 2039021568 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038870707070707075, + "loss": 2.6671, + "theoretical_loss": 3.4248696112818635, + "tokens_seen": 2039087104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886868686868687, + "loss": 2.6505, + "theoretical_loss": 3.424860468550993, + "tokens_seen": 2039152640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038866666666666666, + "loss": 2.3294, + "theoretical_loss": 3.4248513261962255, + "tokens_seen": 2039218176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886464646464646, + "loss": 2.5899, + "theoretical_loss": 3.4248421842175336, + "tokens_seen": 2039283712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886262626262627, + "loss": 2.9221, + "theoretical_loss": 3.4248330426148907, + "tokens_seen": 2039349248 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038860606060606064, + "loss": 2.5795, + "theoretical_loss": 3.424823901388268, + "tokens_seen": 2039414784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885858585858586, + "loss": 2.593, + "theoretical_loss": 3.424814760537638, + "tokens_seen": 2039480320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038856565656565656, + "loss": 2.592, + "theoretical_loss": 3.4248056200629744, + "tokens_seen": 2039545856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885454545454545, + "loss": 2.6182, + "theoretical_loss": 3.424796479964248, + "tokens_seen": 2039611392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885252525252526, + "loss": 2.8057, + "theoretical_loss": 3.4247873402414326, + "tokens_seen": 2039676928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038850505050505054, + "loss": 2.5074, + "theoretical_loss": 3.4247782008945, + "tokens_seen": 2039742464 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884848484848485, + "loss": 2.8014, + "theoretical_loss": 3.4247690619234223, + "tokens_seen": 2039808000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038846464646464645, + "loss": 2.5553, + "theoretical_loss": 3.424759923328173, + "tokens_seen": 2039873536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038844444444444446, + "loss": 2.6588, + "theoretical_loss": 3.424750785108724, + "tokens_seen": 2039939072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038842424242424247, + "loss": 2.6811, + "theoretical_loss": 3.424741647265048, + "tokens_seen": 2040004608 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1124379, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8337912559509277, + "objective/train/theoretical_loss": 3.424732509797117, + "objective/train/tokens_used": 398929376, + "theoretical_loss": 3.424732509797117, + "tokens_seen": 2040070144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038840404040404043, + "loss": 2.8508, + "theoretical_loss": 3.424732509797117, + "tokens_seen": 2040070144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883838383838384, + "loss": 2.7414, + "theoretical_loss": 3.4247233727049036, + "tokens_seen": 2040135680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038836363636363634, + "loss": 2.5021, + "theoretical_loss": 3.4247142359883807, + "tokens_seen": 2040201216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038834343434343435, + "loss": 2.6071, + "theoretical_loss": 3.424705099647521, + "tokens_seen": 2040266752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038832323232323236, + "loss": 2.665, + "theoretical_loss": 3.4246959636822956, + "tokens_seen": 2040332288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883030303030303, + "loss": 2.6956, + "theoretical_loss": 3.4246868280926783, + "tokens_seen": 2040397824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882828282828283, + "loss": 2.6893, + "theoretical_loss": 3.4246776928786415, + "tokens_seen": 2040463360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882626262626263, + "loss": 2.4344, + "theoretical_loss": 3.4246685580401577, + "tokens_seen": 2040528896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038824242424242425, + "loss": 2.6557, + "theoretical_loss": 3.4246594235771983, + "tokens_seen": 2040594432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038822222222222226, + "loss": 2.6863, + "theoretical_loss": 3.424650289489737, + "tokens_seen": 2040659968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882020202020202, + "loss": 2.6083, + "theoretical_loss": 3.4246411557777456, + "tokens_seen": 2040725504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038818181818181817, + "loss": 2.397, + "theoretical_loss": 3.4246320224411972, + "tokens_seen": 2040791040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881616161616162, + "loss": 2.54, + "theoretical_loss": 3.424622889480064, + "tokens_seen": 2040856576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038814141414141414, + "loss": 2.4706, + "theoretical_loss": 3.4246137568943187, + "tokens_seen": 2040922112 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038812121212121215, + "loss": 2.5738, + "theoretical_loss": 3.4246046246839335, + "tokens_seen": 2040987648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881010101010101, + "loss": 2.5798, + "theoretical_loss": 3.424595492848881, + "tokens_seen": 2041053184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880808080808081, + "loss": 2.5736, + "theoretical_loss": 3.4245863613891343, + "tokens_seen": 2041118720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880606060606061, + "loss": 2.5231, + "theoretical_loss": 3.424577230304665, + "tokens_seen": 2041184256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038804040404040403, + "loss": 2.4575, + "theoretical_loss": 3.424568099595446, + "tokens_seen": 2041249792 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038802020202020204, + "loss": 2.5567, + "theoretical_loss": 3.42455896926145, + "tokens_seen": 2041315328 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388, + "loss": 2.5012, + "theoretical_loss": 3.4245498393026494, + "tokens_seen": 2041380864 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387979797979798, + "loss": 2.6823, + "theoretical_loss": 3.4245407097190173, + "tokens_seen": 2041446400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038795959595959597, + "loss": 2.5196, + "theoretical_loss": 3.424531580510525, + "tokens_seen": 2041511936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879393939393939, + "loss": 2.5629, + "theoretical_loss": 3.424522451677146, + "tokens_seen": 2041577472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038791919191919193, + "loss": 2.5199, + "theoretical_loss": 3.4245133232188527, + "tokens_seen": 2041643008 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1125100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.047553062438965, + "objective/train/theoretical_loss": 3.4245041951356177, + "objective/train/tokens_used": 400567776, + "theoretical_loss": 3.4245041951356177, + "tokens_seen": 2041708544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878989898989899, + "loss": 2.7422, + "theoretical_loss": 3.4245041951356177, + "tokens_seen": 2041708544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878787878787879, + "loss": 2.6362, + "theoretical_loss": 3.424495067427413, + "tokens_seen": 2041774080 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038785858585858586, + "loss": 2.6407, + "theoretical_loss": 3.4244859400942116, + "tokens_seen": 2041839616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878383838383838, + "loss": 2.5186, + "theoretical_loss": 3.424476813135986, + "tokens_seen": 2041905152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878181818181818, + "loss": 2.421, + "theoretical_loss": 3.424467686552709, + "tokens_seen": 2041970688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038779797979797984, + "loss": 2.5184, + "theoretical_loss": 3.4244585603443527, + "tokens_seen": 2042036224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877777777777778, + "loss": 2.6667, + "theoretical_loss": 3.4244494345108905, + "tokens_seen": 2042101760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038775757575757575, + "loss": 2.564, + "theoretical_loss": 3.4244403090522937, + "tokens_seen": 2042167296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877373737373737, + "loss": 2.5341, + "theoretical_loss": 3.4244311839685357, + "tokens_seen": 2042232832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877171717171717, + "loss": 2.4584, + "theoretical_loss": 3.424422059259589, + "tokens_seen": 2042298368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038769696969696973, + "loss": 2.5476, + "theoretical_loss": 3.424412934925426, + "tokens_seen": 2042363904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876767676767677, + "loss": 2.4415, + "theoretical_loss": 3.4244038109660195, + "tokens_seen": 2042429440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038765656565656564, + "loss": 2.5881, + "theoretical_loss": 3.4243946873813425, + "tokens_seen": 2042494976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876363636363636, + "loss": 2.6939, + "theoretical_loss": 3.4243855641713665, + "tokens_seen": 2042560512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038761616161616167, + "loss": 2.2827, + "theoretical_loss": 3.4243764413360642, + "tokens_seen": 2042626048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875959595959596, + "loss": 2.7188, + "theoretical_loss": 3.4243673188754093, + "tokens_seen": 2042691584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875757575757576, + "loss": 2.4194, + "theoretical_loss": 3.4243581967893735, + "tokens_seen": 2042757120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038755555555555554, + "loss": 2.6265, + "theoretical_loss": 3.42434907507793, + "tokens_seen": 2042822656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875353535353535, + "loss": 2.5645, + "theoretical_loss": 3.4243399537410504, + "tokens_seen": 2042888192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038751515151515156, + "loss": 2.5277, + "theoretical_loss": 3.424330832778708, + "tokens_seen": 2042953728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874949494949495, + "loss": 2.7011, + "theoretical_loss": 3.424321712190876, + "tokens_seen": 2043019264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874747474747475, + "loss": 2.5774, + "theoretical_loss": 3.4243125919775257, + "tokens_seen": 2043084800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038745454545454543, + "loss": 2.6009, + "theoretical_loss": 3.4243034721386305, + "tokens_seen": 2043150336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038743434343434344, + "loss": 2.6962, + "theoretical_loss": 3.424294352674163, + "tokens_seen": 2043215872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038741414141414145, + "loss": 2.8634, + "theoretical_loss": 3.4242852335840954, + "tokens_seen": 2043281408 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1126113, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.695091962814331, + "objective/train/theoretical_loss": 3.424276114868401, + "objective/train/tokens_used": 402206176, + "theoretical_loss": 3.424276114868401, + "tokens_seen": 2043346944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873939393939394, + "loss": 2.7604, + "theoretical_loss": 3.424276114868401, + "tokens_seen": 2043346944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038737373737373737, + "loss": 2.4762, + "theoretical_loss": 3.424266996527052, + "tokens_seen": 2043412480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873535353535353, + "loss": 2.6634, + "theoretical_loss": 3.424257878560021, + "tokens_seen": 2043478016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038733333333333333, + "loss": 2.4992, + "theoretical_loss": 3.4242487609672807, + "tokens_seen": 2043543552 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038731313131313134, + "loss": 2.8986, + "theoretical_loss": 3.4242396437488036, + "tokens_seen": 2043609088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872929292929293, + "loss": 2.6341, + "theoretical_loss": 3.4242305269045623, + "tokens_seen": 2043674624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038727272727272726, + "loss": 2.5154, + "theoretical_loss": 3.42422141043453, + "tokens_seen": 2043740160 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038725252525252527, + "loss": 2.5922, + "theoretical_loss": 3.4242122943386786, + "tokens_seen": 2043805696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872323232323232, + "loss": 2.5824, + "theoretical_loss": 3.4242031786169815, + "tokens_seen": 2043871232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038721212121212124, + "loss": 2.6942, + "theoretical_loss": 3.424194063269411, + "tokens_seen": 2043936768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871919191919192, + "loss": 2.6585, + "theoretical_loss": 3.424184948295939, + "tokens_seen": 2044002304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038717171717171715, + "loss": 2.4877, + "theoretical_loss": 3.4241758336965393, + "tokens_seen": 2044067840 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038715151515151516, + "loss": 2.5554, + "theoretical_loss": 3.4241667194711845, + "tokens_seen": 2044133376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871313131313131, + "loss": 2.6856, + "theoretical_loss": 3.424157605619846, + "tokens_seen": 2044198912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038711111111111113, + "loss": 2.4869, + "theoretical_loss": 3.424148492142498, + "tokens_seen": 2044264448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870909090909091, + "loss": 2.5563, + "theoretical_loss": 3.4241393790391124, + "tokens_seen": 2044329984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870707070707071, + "loss": 2.5721, + "theoretical_loss": 3.4241302663096613, + "tokens_seen": 2044395520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038705050505050505, + "loss": 2.4328, + "theoretical_loss": 3.4241211539541188, + "tokens_seen": 2044461056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038703030303030307, + "loss": 2.4588, + "theoretical_loss": 3.4241120419724567, + "tokens_seen": 2044526592 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387010101010101, + "loss": 2.4166, + "theoretical_loss": 3.424102930364647, + "tokens_seen": 2044592128 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386989898989899, + "loss": 2.4133, + "theoretical_loss": 3.424093819130664, + "tokens_seen": 2044657664 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386969696969697, + "loss": 2.6739, + "theoretical_loss": 3.4240847082704793, + "tokens_seen": 2044723200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038694949494949495, + "loss": 2.4544, + "theoretical_loss": 3.4240755977840656, + "tokens_seen": 2044788736 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038692929292929296, + "loss": 2.4786, + "theoretical_loss": 3.4240664876713955, + "tokens_seen": 2044854272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869090909090909, + "loss": 2.6681, + "theoretical_loss": 3.424057377932443, + "tokens_seen": 2044919808 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1126722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8349125385284424, + "objective/train/theoretical_loss": 3.424048268567179, + "objective/train/tokens_used": 403844576, + "theoretical_loss": 3.424048268567179, + "tokens_seen": 2044985344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868888888888889, + "loss": 2.7018, + "theoretical_loss": 3.424048268567179, + "tokens_seen": 2044985344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868686868686869, + "loss": 2.6153, + "theoretical_loss": 3.4240391595755773, + "tokens_seen": 2045050880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038684848484848484, + "loss": 2.6144, + "theoretical_loss": 3.42403005095761, + "tokens_seen": 2045116416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038682828282828285, + "loss": 2.3565, + "theoretical_loss": 3.42402094271325, + "tokens_seen": 2045181952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868080808080808, + "loss": 2.513, + "theoretical_loss": 3.424011834842471, + "tokens_seen": 2045247488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867878787878788, + "loss": 2.8146, + "theoretical_loss": 3.424002727345244, + "tokens_seen": 2045313024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867676767676768, + "loss": 2.6684, + "theoretical_loss": 3.4239936202215424, + "tokens_seen": 2045378560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038674747474747473, + "loss": 2.3647, + "theoretical_loss": 3.423984513471339, + "tokens_seen": 2045444096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038672727272727274, + "loss": 2.5581, + "theoretical_loss": 3.423975407094607, + "tokens_seen": 2045509632 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038670707070707076, + "loss": 2.6833, + "theoretical_loss": 3.423966301091318, + "tokens_seen": 2045575168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866868686868687, + "loss": 2.6042, + "theoretical_loss": 3.423957195461446, + "tokens_seen": 2045640704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038666666666666667, + "loss": 2.7104, + "theoretical_loss": 3.423948090204963, + "tokens_seen": 2045706240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866464646464646, + "loss": 2.6974, + "theoretical_loss": 3.4239389853218416, + "tokens_seen": 2045771776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038662626262626264, + "loss": 2.5431, + "theoretical_loss": 3.4239298808120546, + "tokens_seen": 2045837312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038660606060606065, + "loss": 2.4468, + "theoretical_loss": 3.4239207766755753, + "tokens_seen": 2045902848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865858585858586, + "loss": 2.3854, + "theoretical_loss": 3.423911672912376, + "tokens_seen": 2045968384 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038656565656565656, + "loss": 2.5403, + "theoretical_loss": 3.423902569522429, + "tokens_seen": 2046033920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865454545454545, + "loss": 2.4529, + "theoretical_loss": 3.4238934665057084, + "tokens_seen": 2046099456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865252525252526, + "loss": 2.5285, + "theoretical_loss": 3.4238843638621854, + "tokens_seen": 2046164992 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038650505050505054, + "loss": 2.7578, + "theoretical_loss": 3.4238752615918333, + "tokens_seen": 2046230528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864848484848485, + "loss": 2.6517, + "theoretical_loss": 3.423866159694625, + "tokens_seen": 2046296064 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038646464646464645, + "loss": 2.595, + "theoretical_loss": 3.4238570581705337, + "tokens_seen": 2046361600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864444444444444, + "loss": 2.5574, + "theoretical_loss": 3.4238479570195315, + "tokens_seen": 2046427136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864242424242425, + "loss": 2.3707, + "theoretical_loss": 3.423838856241591, + "tokens_seen": 2046492672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038640404040404043, + "loss": 2.7492, + "theoretical_loss": 3.4238297558366853, + "tokens_seen": 2046558208 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1127892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5316739082336426, + "objective/train/theoretical_loss": 3.4238206558047874, + "objective/train/tokens_used": 405482976, + "theoretical_loss": 3.4238206558047874, + "tokens_seen": 2046623744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003863838383838384, + "loss": 2.4914, + "theoretical_loss": 3.4238206558047874, + "tokens_seen": 2046623744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038636363636363635, + "loss": 2.6275, + "theoretical_loss": 3.4238115561458695, + "tokens_seen": 2046689280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863434343434343, + "loss": 2.3453, + "theoretical_loss": 3.423802456859905, + "tokens_seen": 2046754816 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038632323232323237, + "loss": 2.355, + "theoretical_loss": 3.423793357946866, + "tokens_seen": 2046820352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863030303030303, + "loss": 2.4346, + "theoretical_loss": 3.423784259406726, + "tokens_seen": 2046885888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862828282828283, + "loss": 2.4405, + "theoretical_loss": 3.423775161239457, + "tokens_seen": 2046951424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038626262626262624, + "loss": 2.6577, + "theoretical_loss": 3.4237660634450324, + "tokens_seen": 2047016960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038624242424242425, + "loss": 2.4727, + "theoretical_loss": 3.423756966023425, + "tokens_seen": 2047082496 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038622222222222226, + "loss": 2.6148, + "theoretical_loss": 3.4237478689746066, + "tokens_seen": 2047148032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862020202020202, + "loss": 2.5595, + "theoretical_loss": 3.4237387722985515, + "tokens_seen": 2047213568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861818181818182, + "loss": 2.6433, + "theoretical_loss": 3.4237296759952316, + "tokens_seen": 2047279104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038616161616161613, + "loss": 2.4541, + "theoretical_loss": 3.4237205800646198, + "tokens_seen": 2047344640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038614141414141414, + "loss": 2.6971, + "theoretical_loss": 3.423711484506689, + "tokens_seen": 2047410176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038612121212121215, + "loss": 2.3779, + "theoretical_loss": 3.423702389321412, + "tokens_seen": 2047475712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861010101010101, + "loss": 2.7949, + "theoretical_loss": 3.4236932945087615, + "tokens_seen": 2047541248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038608080808080807, + "loss": 2.5279, + "theoretical_loss": 3.42368420006871, + "tokens_seen": 2047606784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860606060606061, + "loss": 2.692, + "theoretical_loss": 3.423675106001231, + "tokens_seen": 2047672320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038604040404040404, + "loss": 2.4909, + "theoretical_loss": 3.423666012306297, + "tokens_seen": 2047737856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038602020202020205, + "loss": 2.4489, + "theoretical_loss": 3.423656918983881, + "tokens_seen": 2047803392 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386, + "loss": 2.6628, + "theoretical_loss": 3.423647826033955, + "tokens_seen": 2047868928 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038597979797979796, + "loss": 2.4424, + "theoretical_loss": 3.4236387334564933, + "tokens_seen": 2047934464 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038595959595959597, + "loss": 2.7376, + "theoretical_loss": 3.423629641251467, + "tokens_seen": 2048000000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038593939393939393, + "loss": 2.4449, + "theoretical_loss": 3.42362054941885, + "tokens_seen": 2048065536 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038591919191919194, + "loss": 2.4178, + "theoretical_loss": 3.423611457958615, + "tokens_seen": 2048131072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858989898989899, + "loss": 2.3096, + "theoretical_loss": 3.4236023668707354, + "tokens_seen": 2048196608 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1128475, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.619576930999756, + "objective/train/theoretical_loss": 3.4235932761551826, + "objective/train/tokens_used": 407121376, + "theoretical_loss": 3.4235932761551826, + "tokens_seen": 2048262144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858787878787879, + "loss": 2.5873, + "theoretical_loss": 3.4235932761551826, + "tokens_seen": 2048262144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038585858585858586, + "loss": 2.8021, + "theoretical_loss": 3.4235841858119302, + "tokens_seen": 2048327680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858383838383838, + "loss": 2.5011, + "theoretical_loss": 3.4235750958409517, + "tokens_seen": 2048393216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038581818181818183, + "loss": 2.7487, + "theoretical_loss": 3.4235660062422184, + "tokens_seen": 2048458752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857979797979798, + "loss": 2.5642, + "theoretical_loss": 3.4235569170157047, + "tokens_seen": 2048524288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857777777777778, + "loss": 2.6144, + "theoretical_loss": 3.423547828161383, + "tokens_seen": 2048589824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038575757575757576, + "loss": 2.6756, + "theoretical_loss": 3.4235387396792256, + "tokens_seen": 2048655360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857373737373737, + "loss": 2.501, + "theoretical_loss": 3.423529651569206, + "tokens_seen": 2048720896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857171717171717, + "loss": 2.4741, + "theoretical_loss": 3.423520563831296, + "tokens_seen": 2048786432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038569696969696974, + "loss": 2.6386, + "theoretical_loss": 3.42351147646547, + "tokens_seen": 2048851968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856767676767677, + "loss": 2.8198, + "theoretical_loss": 3.4235023894717003, + "tokens_seen": 2048917504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038565656565656565, + "loss": 2.8276, + "theoretical_loss": 3.423493302849959, + "tokens_seen": 2048983040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856363636363636, + "loss": 2.6402, + "theoretical_loss": 3.4234842166002197, + "tokens_seen": 2049048576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856161616161616, + "loss": 2.4989, + "theoretical_loss": 3.4234751307224554, + "tokens_seen": 2049114112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038559595959595963, + "loss": 2.5747, + "theoretical_loss": 3.4234660452166383, + "tokens_seen": 2049179648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855757575757576, + "loss": 2.3174, + "theoretical_loss": 3.423456960082742, + "tokens_seen": 2049245184 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038555555555555554, + "loss": 2.5597, + "theoretical_loss": 3.423447875320739, + "tokens_seen": 2049310720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855353535353535, + "loss": 2.396, + "theoretical_loss": 3.423438790930602, + "tokens_seen": 2049376256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038551515151515156, + "loss": 2.3726, + "theoretical_loss": 3.4234297069123043, + "tokens_seen": 2049441792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854949494949495, + "loss": 2.4606, + "theoretical_loss": 3.4234206232658186, + "tokens_seen": 2049507328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854747474747475, + "loss": 2.6207, + "theoretical_loss": 3.423411539991118, + "tokens_seen": 2049572864 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038545454545454544, + "loss": 2.3535, + "theoretical_loss": 3.4234024570881747, + "tokens_seen": 2049638400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854343434343434, + "loss": 2.5528, + "theoretical_loss": 3.4233933745569627, + "tokens_seen": 2049703936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038541414141414146, + "loss": 2.4746, + "theoretical_loss": 3.4233842923974542, + "tokens_seen": 2049769472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853939393939394, + "loss": 2.4524, + "theoretical_loss": 3.423375210609622, + "tokens_seen": 2049835008 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1129543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6075730323791504, + "objective/train/theoretical_loss": 3.423366129193439, + "objective/train/tokens_used": 408759776, + "theoretical_loss": 3.423366129193439, + "tokens_seen": 2049900544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038537373737373737, + "loss": 2.5068, + "theoretical_loss": 3.423366129193439, + "tokens_seen": 2049900544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038535353535353533, + "loss": 2.8476, + "theoretical_loss": 3.423357048148879, + "tokens_seen": 2049966080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853333333333334, + "loss": 2.5966, + "theoretical_loss": 3.4233479674759137, + "tokens_seen": 2050031616 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038531313131313135, + "loss": 2.5537, + "theoretical_loss": 3.4233388871745167, + "tokens_seen": 2050097152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852929292929293, + "loss": 2.5581, + "theoretical_loss": 3.4233298072446607, + "tokens_seen": 2050162688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038527272727272726, + "loss": 2.5214, + "theoretical_loss": 3.4233207276863187, + "tokens_seen": 2050228224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852525252525252, + "loss": 2.3995, + "theoretical_loss": 3.423311648499464, + "tokens_seen": 2050293760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852323232323233, + "loss": 2.682, + "theoretical_loss": 3.4233025696840684, + "tokens_seen": 2050359296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038521212121212124, + "loss": 2.5779, + "theoretical_loss": 3.4232934912401065, + "tokens_seen": 2050424832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851919191919192, + "loss": 2.7086, + "theoretical_loss": 3.4232844131675497, + "tokens_seen": 2050490368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038517171717171716, + "loss": 2.5114, + "theoretical_loss": 3.4232753354663714, + "tokens_seen": 2050555904 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038515151515151517, + "loss": 2.6217, + "theoretical_loss": 3.4232662581365454, + "tokens_seen": 2050621440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851313131313132, + "loss": 2.7702, + "theoretical_loss": 3.4232571811780437, + "tokens_seen": 2050686976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038511111111111114, + "loss": 2.4582, + "theoretical_loss": 3.423248104590839, + "tokens_seen": 2050752512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850909090909091, + "loss": 2.4804, + "theoretical_loss": 3.423239028374905, + "tokens_seen": 2050818048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038507070707070705, + "loss": 2.4453, + "theoretical_loss": 3.423229952530215, + "tokens_seen": 2050883584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038505050505050506, + "loss": 2.5674, + "theoretical_loss": 3.4232208770567403, + "tokens_seen": 2050949120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038503030303030307, + "loss": 2.5093, + "theoretical_loss": 3.423211801954455, + "tokens_seen": 2051014656 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038501010101010103, + "loss": 2.4003, + "theoretical_loss": 3.4232027272233325, + "tokens_seen": 2051080192 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384989898989899, + "loss": 2.4936, + "theoretical_loss": 3.423193652863345, + "tokens_seen": 2051145728 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384969696969697, + "loss": 2.6507, + "theoretical_loss": 3.4231845788744657, + "tokens_seen": 2051211264 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038494949494949495, + "loss": 2.6008, + "theoretical_loss": 3.4231755052566673, + "tokens_seen": 2051276800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038492929292929296, + "loss": 2.6672, + "theoretical_loss": 3.423166432009923, + "tokens_seen": 2051342336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849090909090909, + "loss": 2.4185, + "theoretical_loss": 3.4231573591342057, + "tokens_seen": 2051407872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848888888888889, + "loss": 2.4452, + "theoretical_loss": 3.4231482866294884, + "tokens_seen": 2051473408 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1129971, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5400750637054443, + "objective/train/theoretical_loss": 3.4231392144957447, + "objective/train/tokens_used": 410398176, + "theoretical_loss": 3.4231392144957447, + "tokens_seen": 2051538944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848686868686869, + "loss": 2.544, + "theoretical_loss": 3.4231392144957447, + "tokens_seen": 2051538944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038484848484848485, + "loss": 2.5103, + "theoretical_loss": 3.423130142732947, + "tokens_seen": 2051604480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038482828282828286, + "loss": 2.6622, + "theoretical_loss": 3.423121071341068, + "tokens_seen": 2051670016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848080808080808, + "loss": 2.6093, + "theoretical_loss": 3.4231120003200806, + "tokens_seen": 2051735552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038478787878787877, + "loss": 2.7409, + "theoretical_loss": 3.4231029296699584, + "tokens_seen": 2051801088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847676767676768, + "loss": 2.3119, + "theoretical_loss": 3.4230938593906743, + "tokens_seen": 2051866624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038474747474747474, + "loss": 2.9422, + "theoretical_loss": 3.423084789482201, + "tokens_seen": 2051932160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038472727272727275, + "loss": 2.5798, + "theoretical_loss": 3.423075719944512, + "tokens_seen": 2051997696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847070707070707, + "loss": 2.6104, + "theoretical_loss": 3.4230666507775798, + "tokens_seen": 2052063232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846868686868687, + "loss": 2.6882, + "theoretical_loss": 3.4230575819813778, + "tokens_seen": 2052128768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846666666666667, + "loss": 2.7611, + "theoretical_loss": 3.4230485135558784, + "tokens_seen": 2052194304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038464646464646463, + "loss": 2.6087, + "theoretical_loss": 3.423039445501055, + "tokens_seen": 2052259840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038462626262626264, + "loss": 2.5599, + "theoretical_loss": 3.4230303778168802, + "tokens_seen": 2052325376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846060606060606, + "loss": 2.5465, + "theoretical_loss": 3.423021310503328, + "tokens_seen": 2052390912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845858585858586, + "loss": 2.5225, + "theoretical_loss": 3.4230122435603705, + "tokens_seen": 2052456448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038456565656565657, + "loss": 2.5076, + "theoretical_loss": 3.423003176987981, + "tokens_seen": 2052521984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845454545454545, + "loss": 2.4484, + "theoretical_loss": 3.422994110786133, + "tokens_seen": 2052587520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038452525252525254, + "loss": 2.4406, + "theoretical_loss": 3.4229850449547987, + "tokens_seen": 2052653056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038450505050505055, + "loss": 2.5912, + "theoretical_loss": 3.422975979493952, + "tokens_seen": 2052718592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844848484848485, + "loss": 2.6731, + "theoretical_loss": 3.422966914403565, + "tokens_seen": 2052784128 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038446464646464646, + "loss": 2.5045, + "theoretical_loss": 3.422957849683611, + "tokens_seen": 2052849664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844444444444444, + "loss": 2.5084, + "theoretical_loss": 3.4229487853340634, + "tokens_seen": 2052915200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038442424242424243, + "loss": 2.4632, + "theoretical_loss": 3.4229397213548953, + "tokens_seen": 2052980736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038440404040404044, + "loss": 2.79, + "theoretical_loss": 3.422930657746079, + "tokens_seen": 2053046272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843838383838384, + "loss": 2.653, + "theoretical_loss": 3.4229215945075886, + "tokens_seen": 2053111808 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1131091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.304107666015625, + "objective/train/theoretical_loss": 3.4229125316393967, + "objective/train/tokens_used": 412036576, + "theoretical_loss": 3.4229125316393967, + "tokens_seen": 2053177344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038436363636363635, + "loss": 2.3587, + "theoretical_loss": 3.4229125316393967, + "tokens_seen": 2053177344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843434343434343, + "loss": 2.6467, + "theoretical_loss": 3.4229034691414757, + "tokens_seen": 2053242880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843232323232324, + "loss": 2.6997, + "theoretical_loss": 3.4228944070137994, + "tokens_seen": 2053308416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038430303030303033, + "loss": 2.6112, + "theoretical_loss": 3.422885345256341, + "tokens_seen": 2053373952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842828282828283, + "loss": 2.9658, + "theoretical_loss": 3.4228762838690727, + "tokens_seen": 2053439488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038426262626262625, + "loss": 2.5993, + "theoretical_loss": 3.422867222851968, + "tokens_seen": 2053505024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842424242424242, + "loss": 2.3219, + "theoretical_loss": 3.4228581622050007, + "tokens_seen": 2053570560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038422222222222227, + "loss": 2.2982, + "theoretical_loss": 3.4228491019281426, + "tokens_seen": 2053636096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842020202020202, + "loss": 2.5316, + "theoretical_loss": 3.422840042021368, + "tokens_seen": 2053701632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841818181818182, + "loss": 2.5537, + "theoretical_loss": 3.4228309824846495, + "tokens_seen": 2053767168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038416161616161614, + "loss": 2.6209, + "theoretical_loss": 3.422821923317959, + "tokens_seen": 2053832704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038414141414141415, + "loss": 2.3912, + "theoretical_loss": 3.4228128645212714, + "tokens_seen": 2053898240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038412121212121216, + "loss": 2.6452, + "theoretical_loss": 3.422803806094559, + "tokens_seen": 2053963776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841010101010101, + "loss": 2.6819, + "theoretical_loss": 3.422794748037795, + "tokens_seen": 2054029312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840808080808081, + "loss": 2.8093, + "theoretical_loss": 3.4227856903509517, + "tokens_seen": 2054094848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038406060606060603, + "loss": 2.5679, + "theoretical_loss": 3.4227766330340033, + "tokens_seen": 2054160384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038404040404040404, + "loss": 2.4692, + "theoretical_loss": 3.4227675760869225, + "tokens_seen": 2054225920 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038402020202020205, + "loss": 2.4967, + "theoretical_loss": 3.4227585195096824, + "tokens_seen": 2054291456 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384, + "loss": 2.5947, + "theoretical_loss": 3.4227494633022557, + "tokens_seen": 2054356992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038397979797979797, + "loss": 2.4263, + "theoretical_loss": 3.422740407464616, + "tokens_seen": 2054422528 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383959595959596, + "loss": 2.5193, + "theoretical_loss": 3.4227313519967364, + "tokens_seen": 2054488064 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038393939393939393, + "loss": 2.2884, + "theoretical_loss": 3.4227222968985904, + "tokens_seen": 2054553600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038391919191919195, + "loss": 2.5641, + "theoretical_loss": 3.4227132421701496, + "tokens_seen": 2054619136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838989898989899, + "loss": 2.3804, + "theoretical_loss": 3.422704187811389, + "tokens_seen": 2054684672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038387878787878786, + "loss": 2.5944, + "theoretical_loss": 3.4226951338222804, + "tokens_seen": 2054750208 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1131564, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9922235012054443, + "objective/train/theoretical_loss": 3.422686080202797, + "objective/train/tokens_used": 413674976, + "theoretical_loss": 3.422686080202797, + "tokens_seen": 2054815744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038385858585858587, + "loss": 2.3964, + "theoretical_loss": 3.422686080202797, + "tokens_seen": 2054815744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038383838383838383, + "loss": 2.4571, + "theoretical_loss": 3.422677026952913, + "tokens_seen": 2054881280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038381818181818184, + "loss": 2.6484, + "theoretical_loss": 3.4226679740726, + "tokens_seen": 2054946816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837979797979798, + "loss": 2.5657, + "theoretical_loss": 3.4226589215618324, + "tokens_seen": 2055012352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837777777777778, + "loss": 2.4732, + "theoretical_loss": 3.4226498694205825, + "tokens_seen": 2055077888 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038375757575757576, + "loss": 2.5445, + "theoretical_loss": 3.4226408176488246, + "tokens_seen": 2055143424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837373737373737, + "loss": 2.3885, + "theoretical_loss": 3.42263176624653, + "tokens_seen": 2055208960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038371717171717173, + "loss": 2.7094, + "theoretical_loss": 3.4226227152136737, + "tokens_seen": 2055274496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836969696969697, + "loss": 2.5966, + "theoretical_loss": 3.4226136645502274, + "tokens_seen": 2055340032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836767676767677, + "loss": 2.7365, + "theoretical_loss": 3.422604614256165, + "tokens_seen": 2055405568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038365656565656566, + "loss": 2.5426, + "theoretical_loss": 3.422595564331459, + "tokens_seen": 2055471104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836363636363636, + "loss": 2.4673, + "theoretical_loss": 3.422586514776084, + "tokens_seen": 2055536640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836161616161616, + "loss": 2.4799, + "theoretical_loss": 3.4225774655900114, + "tokens_seen": 2055602176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038359595959595963, + "loss": 2.804, + "theoretical_loss": 3.422568416773216, + "tokens_seen": 2055667712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835757575757576, + "loss": 2.7354, + "theoretical_loss": 3.422559368325669, + "tokens_seen": 2055733248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038355555555555555, + "loss": 2.6043, + "theoretical_loss": 3.4225503202473453, + "tokens_seen": 2055798784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038353535353535356, + "loss": 2.5097, + "theoretical_loss": 3.4225412725382176, + "tokens_seen": 2055864320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835151515151515, + "loss": 2.6064, + "theoretical_loss": 3.4225322251982586, + "tokens_seen": 2055929856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038349494949494953, + "loss": 2.5131, + "theoretical_loss": 3.4225231782274417, + "tokens_seen": 2055995392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834747474747475, + "loss": 2.4516, + "theoretical_loss": 3.42251413162574, + "tokens_seen": 2056060928 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038345454545454544, + "loss": 2.7433, + "theoretical_loss": 3.4225050853931274, + "tokens_seen": 2056126464 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038343434343434345, + "loss": 2.7684, + "theoretical_loss": 3.4224960395295763, + "tokens_seen": 2056192000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038341414141414146, + "loss": 2.5327, + "theoretical_loss": 3.4224869940350597, + "tokens_seen": 2056257536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833939393939394, + "loss": 2.4565, + "theoretical_loss": 3.4224779489095516, + "tokens_seen": 2056323072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833737373737374, + "loss": 2.6484, + "theoretical_loss": 3.4224689041530247, + "tokens_seen": 2056388608 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1132626, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2589941024780273, + "objective/train/theoretical_loss": 3.422459859765452, + "objective/train/tokens_used": 415313376, + "theoretical_loss": 3.422459859765452, + "tokens_seen": 2056454144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038335353535353533, + "loss": 2.3534, + "theoretical_loss": 3.422459859765452, + "tokens_seen": 2056454144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038333333333333334, + "loss": 2.7033, + "theoretical_loss": 3.422450815746807, + "tokens_seen": 2056519680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038331313131313136, + "loss": 2.8655, + "theoretical_loss": 3.422441772097063, + "tokens_seen": 2056585216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832929292929293, + "loss": 2.6004, + "theoretical_loss": 3.422432728816193, + "tokens_seen": 2056650752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038327272727272727, + "loss": 2.8118, + "theoretical_loss": 3.42242368590417, + "tokens_seen": 2056716288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832525252525252, + "loss": 2.6264, + "theoretical_loss": 3.422414643360967, + "tokens_seen": 2056781824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832323232323233, + "loss": 2.329, + "theoretical_loss": 3.4224056011865587, + "tokens_seen": 2056847360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038321212121212125, + "loss": 2.7078, + "theoretical_loss": 3.4223965593809167, + "tokens_seen": 2056912896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831919191919192, + "loss": 2.6651, + "theoretical_loss": 3.4223875179440144, + "tokens_seen": 2056978432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038317171717171716, + "loss": 2.6018, + "theoretical_loss": 3.4223784768758256, + "tokens_seen": 2057043968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831515151515151, + "loss": 2.7493, + "theoretical_loss": 3.4223694361763233, + "tokens_seen": 2057109504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831313131313132, + "loss": 2.7182, + "theoretical_loss": 3.422360395845481, + "tokens_seen": 2057175040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038311111111111114, + "loss": 2.656, + "theoretical_loss": 3.422351355883271, + "tokens_seen": 2057240576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830909090909091, + "loss": 2.6502, + "theoretical_loss": 3.422342316289668, + "tokens_seen": 2057306112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038307070707070706, + "loss": 2.4645, + "theoretical_loss": 3.4223332770646437, + "tokens_seen": 2057371648 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383050505050505, + "loss": 2.7262, + "theoretical_loss": 3.422324238208172, + "tokens_seen": 2057437184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830303030303031, + "loss": 2.5653, + "theoretical_loss": 3.422315199720227, + "tokens_seen": 2057502720 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038301010101010103, + "loss": 2.7167, + "theoretical_loss": 3.42230616160078, + "tokens_seen": 2057568256 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382989898989899, + "loss": 2.6084, + "theoretical_loss": 3.422297123849806, + "tokens_seen": 2057633792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038296969696969695, + "loss": 2.6191, + "theoretical_loss": 3.4222880864672773, + "tokens_seen": 2057699328 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038294949494949496, + "loss": 2.7294, + "theoretical_loss": 3.4222790494531674, + "tokens_seen": 2057764864 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038292929292929297, + "loss": 2.4276, + "theoretical_loss": 3.4222700128074495, + "tokens_seen": 2057830400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829090909090909, + "loss": 2.2369, + "theoretical_loss": 3.4222609765300973, + "tokens_seen": 2057895936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828888888888889, + "loss": 2.7929, + "theoretical_loss": 3.4222519406210834, + "tokens_seen": 2057961472 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038286868686868684, + "loss": 2.5678, + "theoretical_loss": 3.422242905080381, + "tokens_seen": 2058027008 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1133082, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.546114683151245, + "objective/train/theoretical_loss": 3.422233869907964, + "objective/train/tokens_used": 416951776, + "theoretical_loss": 3.422233869907964, + "tokens_seen": 2058092544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038284848484848485, + "loss": 2.5294, + "theoretical_loss": 3.422233869907964, + "tokens_seen": 2058092544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038282828282828286, + "loss": 2.6326, + "theoretical_loss": 3.4222248351038056, + "tokens_seen": 2058158080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828080808080808, + "loss": 2.7425, + "theoretical_loss": 3.4222158006678787, + "tokens_seen": 2058223616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827878787878788, + "loss": 2.7415, + "theoretical_loss": 3.4222067666001563, + "tokens_seen": 2058289152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827676767676768, + "loss": 2.4934, + "theoretical_loss": 3.4221977329006124, + "tokens_seen": 2058354688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038274747474747474, + "loss": 2.7173, + "theoretical_loss": 3.4221886995692197, + "tokens_seen": 2058420224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038272727272727276, + "loss": 2.5987, + "theoretical_loss": 3.4221796666059516, + "tokens_seen": 2058485760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827070707070707, + "loss": 2.8897, + "theoretical_loss": 3.422170634010782, + "tokens_seen": 2058551296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038268686868686867, + "loss": 2.9005, + "theoretical_loss": 3.4221616017836833, + "tokens_seen": 2058616832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826666666666667, + "loss": 2.8517, + "theoretical_loss": 3.422152569924629, + "tokens_seen": 2058682368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038264646464646464, + "loss": 2.697, + "theoretical_loss": 3.422143538433593, + "tokens_seen": 2058747904 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038262626262626265, + "loss": 2.5326, + "theoretical_loss": 3.422134507310548, + "tokens_seen": 2058813440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826060606060606, + "loss": 2.5776, + "theoretical_loss": 3.422125476555467, + "tokens_seen": 2058878976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825858585858586, + "loss": 2.4844, + "theoretical_loss": 3.4221164461683236, + "tokens_seen": 2058944512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038256565656565657, + "loss": 2.6396, + "theoretical_loss": 3.4221074161490916, + "tokens_seen": 2059010048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038254545454545453, + "loss": 2.9819, + "theoretical_loss": 3.422098386497744, + "tokens_seen": 2059075584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038252525252525254, + "loss": 2.5535, + "theoretical_loss": 3.422089357214254, + "tokens_seen": 2059141120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825050505050505, + "loss": 2.7637, + "theoretical_loss": 3.4220803282985948, + "tokens_seen": 2059206656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824848484848485, + "loss": 2.4363, + "theoretical_loss": 3.4220712997507396, + "tokens_seen": 2059272192 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038246464646464647, + "loss": 2.6999, + "theoretical_loss": 3.4220622715706623, + "tokens_seen": 2059337728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824444444444444, + "loss": 2.7858, + "theoretical_loss": 3.4220532437583353, + "tokens_seen": 2059403264 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038242424242424243, + "loss": 2.5095, + "theoretical_loss": 3.4220442163137332, + "tokens_seen": 2059468800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038240404040404044, + "loss": 2.5913, + "theoretical_loss": 3.422035189236828, + "tokens_seen": 2059534336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823838383838384, + "loss": 2.3923, + "theoretical_loss": 3.4220261625275943, + "tokens_seen": 2059599872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038236363636363636, + "loss": 2.629, + "theoretical_loss": 3.422017136186004, + "tokens_seen": 2059665408 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1134328, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9984428882598877, + "objective/train/theoretical_loss": 3.422008110212032, + "objective/train/tokens_used": 418590176, + "theoretical_loss": 3.422008110212032, + "tokens_seen": 2059730944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823434343434343, + "loss": 2.7817, + "theoretical_loss": 3.422008110212032, + "tokens_seen": 2059730944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823232323232323, + "loss": 2.604, + "theoretical_loss": 3.42199908460565, + "tokens_seen": 2059796480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038230303030303034, + "loss": 2.6303, + "theoretical_loss": 3.4219900593668324, + "tokens_seen": 2059862016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822828282828283, + "loss": 2.594, + "theoretical_loss": 3.421981034495553, + "tokens_seen": 2059927552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038226262626262625, + "loss": 2.6818, + "theoretical_loss": 3.421972009991783, + "tokens_seen": 2059993088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822424242424242, + "loss": 2.5402, + "theoretical_loss": 3.4219629858554983, + "tokens_seen": 2060058624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822222222222223, + "loss": 2.3345, + "theoretical_loss": 3.4219539620866706, + "tokens_seen": 2060124160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038220202020202023, + "loss": 2.7212, + "theoretical_loss": 3.421944938685274, + "tokens_seen": 2060189696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821818181818182, + "loss": 2.6767, + "theoretical_loss": 3.4219359156512814, + "tokens_seen": 2060255232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038216161616161614, + "loss": 2.4343, + "theoretical_loss": 3.4219268929846662, + "tokens_seen": 2060320768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821414141414141, + "loss": 2.4259, + "theoretical_loss": 3.4219178706854025, + "tokens_seen": 2060386304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038212121212121217, + "loss": 2.6288, + "theoretical_loss": 3.4219088487534624, + "tokens_seen": 2060451840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821010101010101, + "loss": 2.6574, + "theoretical_loss": 3.4218998271888204, + "tokens_seen": 2060517376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820808080808081, + "loss": 2.5343, + "theoretical_loss": 3.4218908059914495, + "tokens_seen": 2060582912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038206060606060604, + "loss": 2.6432, + "theoretical_loss": 3.4218817851613226, + "tokens_seen": 2060648448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038204040404040405, + "loss": 2.6204, + "theoretical_loss": 3.4218727646984135, + "tokens_seen": 2060713984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038202020202020206, + "loss": 2.6521, + "theoretical_loss": 3.4218637446026956, + "tokens_seen": 2060779520 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382, + "loss": 2.7458, + "theoretical_loss": 3.4218547248741427, + "tokens_seen": 2060845056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038197979797979797, + "loss": 2.8135, + "theoretical_loss": 3.4218457055127267, + "tokens_seen": 2060910592 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038195959595959593, + "loss": 2.5271, + "theoretical_loss": 3.4218366865184224, + "tokens_seen": 2060976128 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038193939393939394, + "loss": 2.7021, + "theoretical_loss": 3.421827667891203, + "tokens_seen": 2061041664 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038191919191919195, + "loss": 2.5729, + "theoretical_loss": 3.4218186496310414, + "tokens_seen": 2061107200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818989898989899, + "loss": 2.416, + "theoretical_loss": 3.4218096317379114, + "tokens_seen": 2061172736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038187878787878786, + "loss": 2.6395, + "theoretical_loss": 3.4218006142117856, + "tokens_seen": 2061238272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818585858585859, + "loss": 2.6007, + "theoretical_loss": 3.4217915970526387, + "tokens_seen": 2061303808 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1135184, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6123404502868652, + "objective/train/theoretical_loss": 3.421782580260443, + "objective/train/tokens_used": 420228576, + "theoretical_loss": 3.421782580260443, + "tokens_seen": 2061369344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818383838383839, + "loss": 2.6084, + "theoretical_loss": 3.421782580260443, + "tokens_seen": 2061369344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038181818181818184, + "loss": 2.6643, + "theoretical_loss": 3.4217735638351727, + "tokens_seen": 2061434880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817979797979798, + "loss": 2.6587, + "theoretical_loss": 3.4217645477768004, + "tokens_seen": 2061500416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038177777777777776, + "loss": 2.6285, + "theoretical_loss": 3.4217555320853004, + "tokens_seen": 2061565952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038175757575757577, + "loss": 2.5553, + "theoretical_loss": 3.4217465167606456, + "tokens_seen": 2061631488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817373737373738, + "loss": 2.6293, + "theoretical_loss": 3.4217375018028084, + "tokens_seen": 2061697024 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038171717171717174, + "loss": 2.6729, + "theoretical_loss": 3.421728487211764, + "tokens_seen": 2061762560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816969696969697, + "loss": 2.617, + "theoretical_loss": 3.4217194729874856, + "tokens_seen": 2061828096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816767676767677, + "loss": 2.7, + "theoretical_loss": 3.4217104591299456, + "tokens_seen": 2061893632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038165656565656566, + "loss": 2.7145, + "theoretical_loss": 3.4217014456391177, + "tokens_seen": 2061959168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038163636363636367, + "loss": 2.501, + "theoretical_loss": 3.421692432514976, + "tokens_seen": 2062024704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038161616161616163, + "loss": 2.4279, + "theoretical_loss": 3.421683419757493, + "tokens_seen": 2062090240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815959595959596, + "loss": 2.4535, + "theoretical_loss": 3.421674407366643, + "tokens_seen": 2062155776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815757575757576, + "loss": 2.5424, + "theoretical_loss": 3.4216653953423988, + "tokens_seen": 2062221312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038155555555555555, + "loss": 2.5376, + "theoretical_loss": 3.4216563836847342, + "tokens_seen": 2062286848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038153535353535357, + "loss": 2.6659, + "theoretical_loss": 3.4216473723936223, + "tokens_seen": 2062352384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815151515151515, + "loss": 2.5682, + "theoretical_loss": 3.421638361469037, + "tokens_seen": 2062417920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814949494949495, + "loss": 2.52, + "theoretical_loss": 3.421629350910951, + "tokens_seen": 2062483456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814747474747475, + "loss": 2.4654, + "theoretical_loss": 3.421620340719339, + "tokens_seen": 2062548992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038145454545454545, + "loss": 2.5279, + "theoretical_loss": 3.421611330894174, + "tokens_seen": 2062614528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038143434343434346, + "loss": 2.3815, + "theoretical_loss": 3.421602321435428, + "tokens_seen": 2062680064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814141414141414, + "loss": 2.8111, + "theoretical_loss": 3.4215933123430764, + "tokens_seen": 2062745600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813939393939394, + "loss": 2.4246, + "theoretical_loss": 3.4215843036170916, + "tokens_seen": 2062811136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813737373737374, + "loss": 2.5496, + "theoretical_loss": 3.421575295257447, + "tokens_seen": 2062876672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038135353535353534, + "loss": 2.5605, + "theoretical_loss": 3.421566287264117, + "tokens_seen": 2062942208 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1135835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9515092372894287, + "objective/train/theoretical_loss": 3.4215572796370743, + "objective/train/tokens_used": 421866976, + "theoretical_loss": 3.4215572796370743, + "tokens_seen": 2063007744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038133333333333335, + "loss": 2.7702, + "theoretical_loss": 3.4215572796370743, + "tokens_seen": 2063007744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813131313131313, + "loss": 2.7661, + "theoretical_loss": 3.4215482723762927, + "tokens_seen": 2063073280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812929292929293, + "loss": 2.5597, + "theoretical_loss": 3.421539265481745, + "tokens_seen": 2063138816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812727272727273, + "loss": 2.8065, + "theoretical_loss": 3.421530258953406, + "tokens_seen": 2063204352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038125252525252523, + "loss": 2.5136, + "theoretical_loss": 3.4215212527912477, + "tokens_seen": 2063269888 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038123232323232324, + "loss": 2.638, + "theoretical_loss": 3.4215122469952446, + "tokens_seen": 2063335424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038121212121212125, + "loss": 2.6101, + "theoretical_loss": 3.4215032415653694, + "tokens_seen": 2063400960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811919191919192, + "loss": 2.5277, + "theoretical_loss": 3.421494236501596, + "tokens_seen": 2063466496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038117171717171717, + "loss": 2.683, + "theoretical_loss": 3.4214852318038984, + "tokens_seen": 2063532032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811515151515151, + "loss": 2.6695, + "theoretical_loss": 3.421476227472249, + "tokens_seen": 2063597568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038113131313131314, + "loss": 2.6012, + "theoretical_loss": 3.421467223506623, + "tokens_seen": 2063663104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038111111111111115, + "loss": 2.6724, + "theoretical_loss": 3.421458219906992, + "tokens_seen": 2063728640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810909090909091, + "loss": 2.5402, + "theoretical_loss": 3.4214492166733304, + "tokens_seen": 2063794176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038107070707070706, + "loss": 2.6403, + "theoretical_loss": 3.421440213805612, + "tokens_seen": 2063859712 + }, + { + "epoch": 0.25, + "learning_rate": 0.000381050505050505, + "loss": 2.5712, + "theoretical_loss": 3.4214312113038092, + "tokens_seen": 2063925248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810303030303031, + "loss": 2.7039, + "theoretical_loss": 3.4214222091678965, + "tokens_seen": 2063990784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038101010101010104, + "loss": 2.5423, + "theoretical_loss": 3.421413207397847, + "tokens_seen": 2064056320 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380989898989899, + "loss": 2.7038, + "theoretical_loss": 3.4214042059936345, + "tokens_seen": 2064121856 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038096969696969695, + "loss": 2.7188, + "theoretical_loss": 3.4213952049552327, + "tokens_seen": 2064187392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809494949494949, + "loss": 2.5841, + "theoretical_loss": 3.421386204282615, + "tokens_seen": 2064252928 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380929292929293, + "loss": 2.9913, + "theoretical_loss": 3.4213772039757537, + "tokens_seen": 2064318464 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038090909090909093, + "loss": 2.5172, + "theoretical_loss": 3.421368204034624, + "tokens_seen": 2064384000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808888888888889, + "loss": 2.6884, + "theoretical_loss": 3.4213592044591987, + "tokens_seen": 2064449536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038086868686868685, + "loss": 2.5919, + "theoretical_loss": 3.421350205249451, + "tokens_seen": 2064515072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038084848484848486, + "loss": 2.545, + "theoretical_loss": 3.421341206405355, + "tokens_seen": 2064580608 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1136767, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7607131004333496, + "objective/train/theoretical_loss": 3.421332207926884, + "objective/train/tokens_used": 423505376, + "theoretical_loss": 3.421332207926884, + "tokens_seen": 2064646144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038082828282828287, + "loss": 2.6132, + "theoretical_loss": 3.421332207926884, + "tokens_seen": 2064646144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808080808080808, + "loss": 2.5181, + "theoretical_loss": 3.421323209814012, + "tokens_seen": 2064711680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807878787878788, + "loss": 2.7202, + "theoretical_loss": 3.4213142120667124, + "tokens_seen": 2064777216 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038076767676767674, + "loss": 2.6562, + "theoretical_loss": 3.421305214684958, + "tokens_seen": 2064842752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038074747474747475, + "loss": 2.6323, + "theoretical_loss": 3.421296217668723, + "tokens_seen": 2064908288 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038072727272727276, + "loss": 2.5255, + "theoretical_loss": 3.4212872210179803, + "tokens_seen": 2064973824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807070707070707, + "loss": 2.6225, + "theoretical_loss": 3.4212782247327045, + "tokens_seen": 2065039360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806868686868687, + "loss": 2.7296, + "theoretical_loss": 3.4212692288128683, + "tokens_seen": 2065104896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806666666666667, + "loss": 2.5503, + "theoretical_loss": 3.4212602332584456, + "tokens_seen": 2065170432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038064646464646464, + "loss": 2.7646, + "theoretical_loss": 3.4212512380694102, + "tokens_seen": 2065235968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038062626262626265, + "loss": 2.461, + "theoretical_loss": 3.421242243245735, + "tokens_seen": 2065301504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806060606060606, + "loss": 2.5117, + "theoretical_loss": 3.421233248787394, + "tokens_seen": 2065367040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038058585858585857, + "loss": 2.7507, + "theoretical_loss": 3.421224254694361, + "tokens_seen": 2065432576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805656565656566, + "loss": 2.4589, + "theoretical_loss": 3.4212152609666093, + "tokens_seen": 2065498112 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038054545454545454, + "loss": 2.7945, + "theoretical_loss": 3.421206267604112, + "tokens_seen": 2065563648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038052525252525255, + "loss": 2.7459, + "theoretical_loss": 3.4211972746068438, + "tokens_seen": 2065629184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805050505050505, + "loss": 2.5679, + "theoretical_loss": 3.421188281974777, + "tokens_seen": 2065694720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804848484848485, + "loss": 2.7287, + "theoretical_loss": 3.4211792897078865, + "tokens_seen": 2065760256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038046464646464647, + "loss": 2.8253, + "theoretical_loss": 3.4211702978061447, + "tokens_seen": 2065825792 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038044444444444443, + "loss": 2.938, + "theoretical_loss": 3.421161306269526, + "tokens_seen": 2065891328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038042424242424244, + "loss": 2.7181, + "theoretical_loss": 3.421152315098003, + "tokens_seen": 2065956864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804040404040404, + "loss": 2.5896, + "theoretical_loss": 3.4211433242915508, + "tokens_seen": 2066022400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803838383838384, + "loss": 2.5979, + "theoretical_loss": 3.4211343338501417, + "tokens_seen": 2066087936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038036363636363636, + "loss": 2.7055, + "theoretical_loss": 3.42112534377375, + "tokens_seen": 2066153472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803434343434343, + "loss": 2.8343, + "theoretical_loss": 3.421116354062349, + "tokens_seen": 2066219008 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1137905, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.325758457183838, + "objective/train/theoretical_loss": 3.4211073647159123, + "objective/train/tokens_used": 425143776, + "theoretical_loss": 3.4211073647159123, + "tokens_seen": 2066284544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038032323232323233, + "loss": 2.7325, + "theoretical_loss": 3.4211073647159123, + "tokens_seen": 2066284544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038030303030303034, + "loss": 2.5732, + "theoretical_loss": 3.421098375734414, + "tokens_seen": 2066350080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802828282828283, + "loss": 2.7549, + "theoretical_loss": 3.4210893871178265, + "tokens_seen": 2066415616 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038026262626262626, + "loss": 2.768, + "theoretical_loss": 3.421080398866125, + "tokens_seen": 2066481152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802424242424242, + "loss": 2.6855, + "theoretical_loss": 3.4210714109792817, + "tokens_seen": 2066546688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802222222222222, + "loss": 2.7331, + "theoretical_loss": 3.421062423457271, + "tokens_seen": 2066612224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038020202020202024, + "loss": 2.4095, + "theoretical_loss": 3.421053436300067, + "tokens_seen": 2066677760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801818181818182, + "loss": 2.695, + "theoretical_loss": 3.421044449507642, + "tokens_seen": 2066743296 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038016161616161615, + "loss": 2.7255, + "theoretical_loss": 3.4210354630799706, + "tokens_seen": 2066808832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038014141414141416, + "loss": 2.5248, + "theoretical_loss": 3.4210264770170262, + "tokens_seen": 2066874368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038012121212121217, + "loss": 2.6969, + "theoretical_loss": 3.421017491318782, + "tokens_seen": 2066939904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038010101010101013, + "loss": 2.613, + "theoretical_loss": 3.4210085059852124, + "tokens_seen": 2067005440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800808080808081, + "loss": 2.6488, + "theoretical_loss": 3.4209995210162907, + "tokens_seen": 2067070976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038006060606060604, + "loss": 2.6582, + "theoretical_loss": 3.42099053641199, + "tokens_seen": 2067136512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038004040404040405, + "loss": 2.4411, + "theoretical_loss": 3.420981552172285, + "tokens_seen": 2067202048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038002020202020206, + "loss": 2.6109, + "theoretical_loss": 3.420972568297149, + "tokens_seen": 2067267584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038, + "loss": 2.5581, + "theoretical_loss": 3.4209635847865547, + "tokens_seen": 2067333120 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379979797979798, + "loss": 2.7596, + "theoretical_loss": 3.420954601640477, + "tokens_seen": 2067398656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037995959595959593, + "loss": 2.564, + "theoretical_loss": 3.420945618858889, + "tokens_seen": 2067464192 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379939393939394, + "loss": 2.7477, + "theoretical_loss": 3.4209366364417644, + "tokens_seen": 2067529728 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037991919191919196, + "loss": 2.4975, + "theoretical_loss": 3.420927654389077, + "tokens_seen": 2067595264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798989898989899, + "loss": 2.7735, + "theoretical_loss": 3.4209186727008, + "tokens_seen": 2067660800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037987878787878787, + "loss": 2.5042, + "theoretical_loss": 3.4209096913769073, + "tokens_seen": 2067726336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037985858585858583, + "loss": 2.7599, + "theoretical_loss": 3.420900710417373, + "tokens_seen": 2067791872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798383838383839, + "loss": 2.7375, + "theoretical_loss": 3.4208917298221704, + "tokens_seen": 2067857408 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1138616, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.164562940597534, + "objective/train/theoretical_loss": 3.4208827495912733, + "objective/train/tokens_used": 426782176, + "theoretical_loss": 3.4208827495912733, + "tokens_seen": 2067922944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037981818181818185, + "loss": 2.6866, + "theoretical_loss": 3.4208827495912733, + "tokens_seen": 2067922944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797979797979798, + "loss": 2.5767, + "theoretical_loss": 3.420873769724655, + "tokens_seen": 2067988480 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037977777777777776, + "loss": 2.4466, + "theoretical_loss": 3.4208647902222893, + "tokens_seen": 2068054016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797575757575757, + "loss": 2.6539, + "theoretical_loss": 3.4208558110841505, + "tokens_seen": 2068119552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797373737373738, + "loss": 2.6817, + "theoretical_loss": 3.420846832310212, + "tokens_seen": 2068185088 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037971717171717174, + "loss": 2.6445, + "theoretical_loss": 3.420837853900447, + "tokens_seen": 2068250624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796969696969697, + "loss": 2.6812, + "theoretical_loss": 3.4208288758548293, + "tokens_seen": 2068316160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037967676767676766, + "loss": 2.5428, + "theoretical_loss": 3.420819898173333, + "tokens_seen": 2068381696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037965656565656567, + "loss": 2.6958, + "theoretical_loss": 3.420810920855932, + "tokens_seen": 2068447232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796363636363637, + "loss": 2.5625, + "theoretical_loss": 3.4208019439025987, + "tokens_seen": 2068512768 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037961616161616163, + "loss": 2.7598, + "theoretical_loss": 3.4207929673133086, + "tokens_seen": 2068578304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795959595959596, + "loss": 2.6718, + "theoretical_loss": 3.420783991088034, + "tokens_seen": 2068643840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037957575757575755, + "loss": 2.8103, + "theoretical_loss": 3.4207750152267495, + "tokens_seen": 2068709376 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037955555555555556, + "loss": 2.5705, + "theoretical_loss": 3.420766039729428, + "tokens_seen": 2068774912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037953535353535357, + "loss": 2.6372, + "theoretical_loss": 3.420757064596044, + "tokens_seen": 2068840448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037951515151515153, + "loss": 2.8598, + "theoretical_loss": 3.4207480898265707, + "tokens_seen": 2068905984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794949494949495, + "loss": 2.3985, + "theoretical_loss": 3.4207391154209823, + "tokens_seen": 2068971520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794747474747475, + "loss": 2.8237, + "theoretical_loss": 3.4207301413792512, + "tokens_seen": 2069037056 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037945454545454545, + "loss": 2.5604, + "theoretical_loss": 3.420721167701353, + "tokens_seen": 2069102592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037943434343434346, + "loss": 2.4821, + "theoretical_loss": 3.4207121943872605, + "tokens_seen": 2069168128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794141414141414, + "loss": 2.6009, + "theoretical_loss": 3.4207032214369475, + "tokens_seen": 2069233664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793939393939394, + "loss": 2.7229, + "theoretical_loss": 3.4206942488503875, + "tokens_seen": 2069299200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793737373737374, + "loss": 2.7748, + "theoretical_loss": 3.4206852766275544, + "tokens_seen": 2069364736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037935353535353535, + "loss": 2.6456, + "theoretical_loss": 3.420676304768422, + "tokens_seen": 2069430272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037933333333333336, + "loss": 2.8453, + "theoretical_loss": 3.420667333272964, + "tokens_seen": 2069495808 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1139859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9761197566986084, + "objective/train/theoretical_loss": 3.420658362141154, + "objective/train/tokens_used": 428420576, + "theoretical_loss": 3.420658362141154, + "tokens_seen": 2069561344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793131313131313, + "loss": 2.645, + "theoretical_loss": 3.420658362141154, + "tokens_seen": 2069561344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792929292929293, + "loss": 2.4449, + "theoretical_loss": 3.420649391372966, + "tokens_seen": 2069626880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792727272727273, + "loss": 2.6226, + "theoretical_loss": 3.4206404209683736, + "tokens_seen": 2069692416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037925252525252524, + "loss": 2.5629, + "theoretical_loss": 3.4206314509273508, + "tokens_seen": 2069757952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037923232323232325, + "loss": 2.7043, + "theoretical_loss": 3.4206224812498713, + "tokens_seen": 2069823488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792121212121212, + "loss": 2.7883, + "theoretical_loss": 3.4206135119359082, + "tokens_seen": 2069889024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791919191919192, + "loss": 2.4786, + "theoretical_loss": 3.420604542985436, + "tokens_seen": 2069954560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791717171717172, + "loss": 2.811, + "theoretical_loss": 3.420595574398429, + "tokens_seen": 2070020096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037915151515151513, + "loss": 2.5502, + "theoretical_loss": 3.420586606174859, + "tokens_seen": 2070085632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037913131313131314, + "loss": 2.7804, + "theoretical_loss": 3.4205776383147013, + "tokens_seen": 2070151168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037911111111111115, + "loss": 2.5411, + "theoretical_loss": 3.4205686708179295, + "tokens_seen": 2070216704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790909090909091, + "loss": 2.6516, + "theoretical_loss": 3.4205597036845177, + "tokens_seen": 2070282240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037907070707070707, + "loss": 2.5921, + "theoretical_loss": 3.4205507369144383, + "tokens_seen": 2070347776 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379050505050505, + "loss": 2.7129, + "theoretical_loss": 3.4205417705076666, + "tokens_seen": 2070413312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037903030303030303, + "loss": 2.4977, + "theoretical_loss": 3.420532804464175, + "tokens_seen": 2070478848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037901010101010105, + "loss": 2.6412, + "theoretical_loss": 3.4205238387839385, + "tokens_seen": 2070544384 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378989898989899, + "loss": 2.4797, + "theoretical_loss": 3.4205148734669306, + "tokens_seen": 2070609920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037896969696969696, + "loss": 2.4803, + "theoretical_loss": 3.4205059085131246, + "tokens_seen": 2070675456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789494949494949, + "loss": 2.5416, + "theoretical_loss": 3.420496943922495, + "tokens_seen": 2070740992 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378929292929293, + "loss": 2.7728, + "theoretical_loss": 3.4204879796950145, + "tokens_seen": 2070806528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037890909090909094, + "loss": 2.6949, + "theoretical_loss": 3.420479015830658, + "tokens_seen": 2070872064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788888888888889, + "loss": 2.6882, + "theoretical_loss": 3.4204700523293985, + "tokens_seen": 2070937600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037886868686868685, + "loss": 2.7507, + "theoretical_loss": 3.4204610891912104, + "tokens_seen": 2071003136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788484848484848, + "loss": 2.6074, + "theoretical_loss": 3.4204521264160674, + "tokens_seen": 2071068672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788282828282829, + "loss": 2.6161, + "theoretical_loss": 3.4204431640039434, + "tokens_seen": 2071134208 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1140861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.942692518234253, + "objective/train/theoretical_loss": 3.4204342019548113, + "objective/train/tokens_used": 430058976, + "theoretical_loss": 3.4204342019548113, + "tokens_seen": 2071199744 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037880808080808083, + "loss": 2.9214, + "theoretical_loss": 3.4204342019548113, + "tokens_seen": 2071199744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787878787878788, + "loss": 2.4544, + "theoretical_loss": 3.4204252402686466, + "tokens_seen": 2071265280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037876767676767674, + "loss": 2.8409, + "theoretical_loss": 3.4204162789454213, + "tokens_seen": 2071330816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037874747474747476, + "loss": 2.4468, + "theoretical_loss": 3.42040731798511, + "tokens_seen": 2071396352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037872727272727277, + "loss": 2.4653, + "theoretical_loss": 3.420398357387687, + "tokens_seen": 2071461888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787070707070707, + "loss": 2.7628, + "theoretical_loss": 3.4203893971531256, + "tokens_seen": 2071527424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786868686868687, + "loss": 2.4264, + "theoretical_loss": 3.4203804372814, + "tokens_seen": 2071592960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037866666666666664, + "loss": 2.6893, + "theoretical_loss": 3.420371477772483, + "tokens_seen": 2071658496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037864646464646465, + "loss": 2.8688, + "theoretical_loss": 3.4203625186263498, + "tokens_seen": 2071724032 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037862626262626266, + "loss": 2.7391, + "theoretical_loss": 3.4203535598429737, + "tokens_seen": 2071789568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786060606060606, + "loss": 2.783, + "theoretical_loss": 3.4203446014223284, + "tokens_seen": 2071855104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785858585858586, + "loss": 2.6383, + "theoretical_loss": 3.4203356433643877, + "tokens_seen": 2071920640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785656565656566, + "loss": 2.61, + "theoretical_loss": 3.4203266856691252, + "tokens_seen": 2071986176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037854545454545454, + "loss": 2.6615, + "theoretical_loss": 3.4203177283365154, + "tokens_seen": 2072051712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037852525252525255, + "loss": 2.6566, + "theoretical_loss": 3.420308771366532, + "tokens_seen": 2072117248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785050505050505, + "loss": 2.6085, + "theoretical_loss": 3.420299814759148, + "tokens_seen": 2072182784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037848484848484847, + "loss": 2.6634, + "theoretical_loss": 3.420290858514339, + "tokens_seen": 2072248320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784646464646465, + "loss": 2.6108, + "theoretical_loss": 3.4202819026320768, + "tokens_seen": 2072313856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784444444444445, + "loss": 2.7928, + "theoretical_loss": 3.4202729471123368, + "tokens_seen": 2072379392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037842424242424244, + "loss": 2.8207, + "theoretical_loss": 3.4202639919550926, + "tokens_seen": 2072444928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784040404040404, + "loss": 2.9374, + "theoretical_loss": 3.420255037160317, + "tokens_seen": 2072510464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783838383838384, + "loss": 2.6872, + "theoretical_loss": 3.420246082727985, + "tokens_seen": 2072576000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037836363636363637, + "loss": 2.6963, + "theoretical_loss": 3.42023712865807, + "tokens_seen": 2072641536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783434343434344, + "loss": 2.674, + "theoretical_loss": 3.4202281749505463, + "tokens_seen": 2072707072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037832323232323234, + "loss": 2.6523, + "theoretical_loss": 3.4202192216053873, + "tokens_seen": 2072772608 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1141535, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.150759696960449, + "objective/train/theoretical_loss": 3.420210268622567, + "objective/train/tokens_used": 431697376, + "theoretical_loss": 3.420210268622567, + "tokens_seen": 2072838144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783030303030303, + "loss": 2.7664, + "theoretical_loss": 3.420210268622567, + "tokens_seen": 2072838144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782828282828283, + "loss": 2.7415, + "theoretical_loss": 3.4202013160020597, + "tokens_seen": 2072903680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037826262626262626, + "loss": 2.9062, + "theoretical_loss": 3.4201923637438383, + "tokens_seen": 2072969216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782424242424243, + "loss": 2.8399, + "theoretical_loss": 3.4201834118478773, + "tokens_seen": 2073034752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037822222222222223, + "loss": 2.7105, + "theoretical_loss": 3.4201744603141515, + "tokens_seen": 2073100288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782020202020202, + "loss": 2.4727, + "theoretical_loss": 3.420165509142633, + "tokens_seen": 2073165824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781818181818182, + "loss": 2.3444, + "theoretical_loss": 3.420156558333297, + "tokens_seen": 2073231360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037816161616161615, + "loss": 2.5233, + "theoretical_loss": 3.4201476078861166, + "tokens_seen": 2073296896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037814141414141417, + "loss": 2.4147, + "theoretical_loss": 3.420138657801066, + "tokens_seen": 2073362432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781212121212121, + "loss": 2.7456, + "theoretical_loss": 3.42012970807812, + "tokens_seen": 2073427968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037810101010101013, + "loss": 2.935, + "theoretical_loss": 3.4201207587172506, + "tokens_seen": 2073493504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780808080808081, + "loss": 2.6897, + "theoretical_loss": 3.4201118097184335, + "tokens_seen": 2073559040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037806060606060605, + "loss": 2.645, + "theoretical_loss": 3.4201028610816415, + "tokens_seen": 2073624576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037804040404040406, + "loss": 2.6722, + "theoretical_loss": 3.4200939128068493, + "tokens_seen": 2073690112 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378020202020202, + "loss": 2.544, + "theoretical_loss": 3.4200849648940306, + "tokens_seen": 2073755648 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378, + "loss": 2.7328, + "theoretical_loss": 3.420076017343159, + "tokens_seen": 2073821184 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377979797979798, + "loss": 3.0095, + "theoretical_loss": 3.420067070154208, + "tokens_seen": 2073886720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037795959595959594, + "loss": 2.5682, + "theoretical_loss": 3.4200581233271525, + "tokens_seen": 2073952256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037793939393939395, + "loss": 2.576, + "theoretical_loss": 3.420049176861966, + "tokens_seen": 2074017792 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037791919191919196, + "loss": 2.7552, + "theoretical_loss": 3.420040230758622, + "tokens_seen": 2074083328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778989898989899, + "loss": 2.6943, + "theoretical_loss": 3.420031285017096, + "tokens_seen": 2074148864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778787878787879, + "loss": 2.6718, + "theoretical_loss": 3.42002233963736, + "tokens_seen": 2074214400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037785858585858583, + "loss": 2.6333, + "theoretical_loss": 3.4200133946193887, + "tokens_seen": 2074279936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037783838383838384, + "loss": 2.4596, + "theoretical_loss": 3.420004449963156, + "tokens_seen": 2074345472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037781818181818186, + "loss": 2.4134, + "theoretical_loss": 3.419995505668637, + "tokens_seen": 2074411008 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1142675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8896119594573975, + "objective/train/theoretical_loss": 3.4199865617358034, + "objective/train/tokens_used": 433335776, + "theoretical_loss": 3.4199865617358034, + "tokens_seen": 2074476544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777979797979798, + "loss": 2.7336, + "theoretical_loss": 3.4199865617358034, + "tokens_seen": 2074476544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037777777777777777, + "loss": 2.6664, + "theoretical_loss": 3.419977618164631, + "tokens_seen": 2074542080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777575757575757, + "loss": 2.5695, + "theoretical_loss": 3.4199686749550926, + "tokens_seen": 2074607616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777373737373738, + "loss": 2.7597, + "theoretical_loss": 3.419959732107163, + "tokens_seen": 2074673152 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037771717171717175, + "loss": 2.3906, + "theoretical_loss": 3.419950789620816, + "tokens_seen": 2074738688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776969696969697, + "loss": 2.5256, + "theoretical_loss": 3.419941847496025, + "tokens_seen": 2074804224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037767676767676766, + "loss": 2.6481, + "theoretical_loss": 3.419932905732764, + "tokens_seen": 2074869760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776565656565656, + "loss": 2.6393, + "theoretical_loss": 3.4199239643310078, + "tokens_seen": 2074935296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776363636363637, + "loss": 2.4312, + "theoretical_loss": 3.41991502329073, + "tokens_seen": 2075000832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037761616161616164, + "loss": 2.6489, + "theoretical_loss": 3.4199060826119037, + "tokens_seen": 2075066368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775959595959596, + "loss": 2.7394, + "theoretical_loss": 3.419897142294504, + "tokens_seen": 2075131904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037757575757575755, + "loss": 2.649, + "theoretical_loss": 3.4198882023385044, + "tokens_seen": 2075197440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037755555555555557, + "loss": 2.7409, + "theoretical_loss": 3.419879262743879, + "tokens_seen": 2075262976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775353535353536, + "loss": 2.7992, + "theoretical_loss": 3.419870323510602, + "tokens_seen": 2075328512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037751515151515153, + "loss": 2.7074, + "theoretical_loss": 3.4198613846386468, + "tokens_seen": 2075394048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774949494949495, + "loss": 2.4963, + "theoretical_loss": 3.4198524461279876, + "tokens_seen": 2075459584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037747474747474745, + "loss": 2.8299, + "theoretical_loss": 3.419843507978599, + "tokens_seen": 2075525120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037745454545454546, + "loss": 2.6015, + "theoretical_loss": 3.419834570190454, + "tokens_seen": 2075590656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037743434343434347, + "loss": 2.6814, + "theoretical_loss": 3.4198256327635272, + "tokens_seen": 2075656192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774141414141414, + "loss": 2.8352, + "theoretical_loss": 3.4198166956977927, + "tokens_seen": 2075721728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773939393939394, + "loss": 2.7546, + "theoretical_loss": 3.4198077589932234, + "tokens_seen": 2075787264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773737373737374, + "loss": 2.6698, + "theoretical_loss": 3.4197988226497955, + "tokens_seen": 2075852800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037735353535353535, + "loss": 2.7075, + "theoretical_loss": 3.4197898866674805, + "tokens_seen": 2075918336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037733333333333336, + "loss": 2.5608, + "theoretical_loss": 3.4197809510462545, + "tokens_seen": 2075983872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773131313131313, + "loss": 2.3638, + "theoretical_loss": 3.41977201578609, + "tokens_seen": 2076049408 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1143849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.351544141769409, + "objective/train/theoretical_loss": 3.4197630808869617, + "objective/train/tokens_used": 434974176, + "theoretical_loss": 3.4197630808869617, + "tokens_seen": 2076114944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772929292929293, + "loss": 2.6191, + "theoretical_loss": 3.4197630808869617, + "tokens_seen": 2076114944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772727272727273, + "loss": 2.8404, + "theoretical_loss": 3.4197541463488434, + "tokens_seen": 2076180480 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037725252525252524, + "loss": 2.7983, + "theoretical_loss": 3.41974521217171, + "tokens_seen": 2076246016 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037723232323232325, + "loss": 2.6373, + "theoretical_loss": 3.419736278355534, + "tokens_seen": 2076311552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772121212121212, + "loss": 2.6044, + "theoretical_loss": 3.4197273449002905, + "tokens_seen": 2076377088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771919191919192, + "loss": 2.3621, + "theoretical_loss": 3.4197184118059534, + "tokens_seen": 2076442624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771717171717172, + "loss": 2.8032, + "theoretical_loss": 3.419709479072496, + "tokens_seen": 2076508160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037715151515151514, + "loss": 2.7246, + "theoretical_loss": 3.419700546699893, + "tokens_seen": 2076573696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037713131313131315, + "loss": 2.773, + "theoretical_loss": 3.4196916146881184, + "tokens_seen": 2076639232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771111111111111, + "loss": 2.87, + "theoretical_loss": 3.4196826830371467, + "tokens_seen": 2076704768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770909090909091, + "loss": 2.6496, + "theoretical_loss": 3.419673751746951, + "tokens_seen": 2076770304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037707070707070707, + "loss": 2.5852, + "theoretical_loss": 3.4196648208175056, + "tokens_seen": 2076835840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037705050505050503, + "loss": 2.7811, + "theoretical_loss": 3.4196558902487846, + "tokens_seen": 2076901376 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037703030303030304, + "loss": 2.5733, + "theoretical_loss": 3.4196469600407626, + "tokens_seen": 2076966912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037701010101010105, + "loss": 2.5622, + "theoretical_loss": 3.419638030193413, + "tokens_seen": 2077032448 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376989898989899, + "loss": 2.6142, + "theoretical_loss": 3.4196291007067097, + "tokens_seen": 2077097984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037696969696969696, + "loss": 2.6837, + "theoretical_loss": 3.419620171580627, + "tokens_seen": 2077163520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769494949494949, + "loss": 2.5987, + "theoretical_loss": 3.4196112428151397, + "tokens_seen": 2077229056 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037692929292929293, + "loss": 2.7628, + "theoretical_loss": 3.419602314410221, + "tokens_seen": 2077294592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037690909090909094, + "loss": 2.5048, + "theoretical_loss": 3.419593386365845, + "tokens_seen": 2077360128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768888888888889, + "loss": 2.5455, + "theoretical_loss": 3.4195844586819857, + "tokens_seen": 2077425664 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037686868686868686, + "loss": 2.6995, + "theoretical_loss": 3.4195755313586176, + "tokens_seen": 2077491200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768484848484848, + "loss": 2.901, + "theoretical_loss": 3.419566604395715, + "tokens_seen": 2077556736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768282828282829, + "loss": 2.6293, + "theoretical_loss": 3.419557677793251, + "tokens_seen": 2077622272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037680808080808084, + "loss": 2.4327, + "theoretical_loss": 3.4195487515512006, + "tokens_seen": 2077687808 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1144478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4185636043548584, + "objective/train/theoretical_loss": 3.4195398256695375, + "objective/train/tokens_used": 436612576, + "theoretical_loss": 3.4195398256695375, + "tokens_seen": 2077753344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767878787878788, + "loss": 2.756, + "theoretical_loss": 3.4195398256695375, + "tokens_seen": 2077753344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037676767676767675, + "loss": 2.6662, + "theoretical_loss": 3.4195309001482355, + "tokens_seen": 2077818880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767474747474747, + "loss": 2.5164, + "theoretical_loss": 3.419521974987269, + "tokens_seen": 2077884416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037672727272727277, + "loss": 2.7099, + "theoretical_loss": 3.4195130501866124, + "tokens_seen": 2077949952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037670707070707073, + "loss": 2.5342, + "theoretical_loss": 3.4195041257462395, + "tokens_seen": 2078015488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766868686868687, + "loss": 2.5396, + "theoretical_loss": 3.419495201666124, + "tokens_seen": 2078081024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037666666666666664, + "loss": 2.5388, + "theoretical_loss": 3.4194862779462407, + "tokens_seen": 2078146560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766464646464647, + "loss": 2.6906, + "theoretical_loss": 3.4194773545865633, + "tokens_seen": 2078212096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037662626262626266, + "loss": 2.7033, + "theoretical_loss": 3.419468431587066, + "tokens_seen": 2078277632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766060606060606, + "loss": 2.6427, + "theoretical_loss": 3.419459508947723, + "tokens_seen": 2078343168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765858585858586, + "loss": 2.6233, + "theoretical_loss": 3.419450586668508, + "tokens_seen": 2078408704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037656565656565654, + "loss": 2.671, + "theoretical_loss": 3.419441664749395, + "tokens_seen": 2078474240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765454545454546, + "loss": 2.5917, + "theoretical_loss": 3.419432743190359, + "tokens_seen": 2078539776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037652525252525256, + "loss": 2.7356, + "theoretical_loss": 3.4194238219913737, + "tokens_seen": 2078605312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765050505050505, + "loss": 2.5018, + "theoretical_loss": 3.419414901152413, + "tokens_seen": 2078670848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037648484848484847, + "loss": 2.6064, + "theoretical_loss": 3.4194059806734507, + "tokens_seen": 2078736384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037646464646464643, + "loss": 2.8587, + "theoretical_loss": 3.419397060554462, + "tokens_seen": 2078801920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764444444444445, + "loss": 2.5207, + "theoretical_loss": 3.41938814079542, + "tokens_seen": 2078867456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037642424242424245, + "loss": 2.644, + "theoretical_loss": 3.4193792213962992, + "tokens_seen": 2078932992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764040404040404, + "loss": 2.4817, + "theoretical_loss": 3.4193703023570743, + "tokens_seen": 2078998528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037638383838383836, + "loss": 2.4036, + "theoretical_loss": 3.4193613836777184, + "tokens_seen": 2079064064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763636363636364, + "loss": 2.5825, + "theoretical_loss": 3.419352465358206, + "tokens_seen": 2079129600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763434343434344, + "loss": 2.6791, + "theoretical_loss": 3.419343547398512, + "tokens_seen": 2079195136 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037632323232323234, + "loss": 2.6596, + "theoretical_loss": 3.4193346297986094, + "tokens_seen": 2079260672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763030303030303, + "loss": 2.6833, + "theoretical_loss": 3.419325712558473, + "tokens_seen": 2079326208 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1145756, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6733858585357666, + "objective/train/theoretical_loss": 3.4193167956780766, + "objective/train/tokens_used": 438250976, + "theoretical_loss": 3.4193167956780766, + "tokens_seen": 2079391744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037628282828282826, + "loss": 2.4661, + "theoretical_loss": 3.4193167956780766, + "tokens_seen": 2079391744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037626262626262627, + "loss": 2.744, + "theoretical_loss": 3.4193078791573948, + "tokens_seen": 2079457280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762424242424243, + "loss": 2.611, + "theoretical_loss": 3.4192989629964012, + "tokens_seen": 2079522816 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037622222222222224, + "loss": 2.7055, + "theoretical_loss": 3.4192900471950707, + "tokens_seen": 2079588352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762020202020202, + "loss": 2.6647, + "theoretical_loss": 3.419281131753377, + "tokens_seen": 2079653888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761818181818182, + "loss": 2.732, + "theoretical_loss": 3.419272216671294, + "tokens_seen": 2079719424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037616161616161616, + "loss": 2.6474, + "theoretical_loss": 3.4192633019487966, + "tokens_seen": 2079784960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037614141414141417, + "loss": 2.6054, + "theoretical_loss": 3.4192543875858576, + "tokens_seen": 2079850496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037612121212121213, + "loss": 2.714, + "theoretical_loss": 3.419245473582453, + "tokens_seen": 2079916032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761010101010101, + "loss": 2.6577, + "theoretical_loss": 3.4192365599385557, + "tokens_seen": 2079981568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760808080808081, + "loss": 2.7123, + "theoretical_loss": 3.4192276466541403, + "tokens_seen": 2080047104 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037606060606060605, + "loss": 2.9202, + "theoretical_loss": 3.4192187337291804, + "tokens_seen": 2080112640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037604040404040406, + "loss": 2.7607, + "theoretical_loss": 3.4192098211636512, + "tokens_seen": 2080178176 + }, + { + "epoch": 0.26, + "learning_rate": 0.000376020202020202, + "loss": 2.6153, + "theoretical_loss": 3.419200908957526, + "tokens_seen": 2080243712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037600000000000003, + "loss": 2.6651, + "theoretical_loss": 3.41919199711078, + "tokens_seen": 2080309248 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375979797979798, + "loss": 2.7883, + "theoretical_loss": 3.419183085623386, + "tokens_seen": 2080374784 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037595959595959595, + "loss": 2.5957, + "theoretical_loss": 3.4191741744953195, + "tokens_seen": 2080440320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037593939393939396, + "loss": 2.4795, + "theoretical_loss": 3.419165263726554, + "tokens_seen": 2080505856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759191919191919, + "loss": 2.4591, + "theoretical_loss": 3.4191563533170637, + "tokens_seen": 2080571392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758989898989899, + "loss": 2.5381, + "theoretical_loss": 3.419147443266823, + "tokens_seen": 2080636928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758787878787879, + "loss": 2.76, + "theoretical_loss": 3.4191385335758055, + "tokens_seen": 2080702464 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037585858585858584, + "loss": 2.8718, + "theoretical_loss": 3.419129624243986, + "tokens_seen": 2080768000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037583838383838385, + "loss": 2.6215, + "theoretical_loss": 3.4191207152713394, + "tokens_seen": 2080833536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037581818181818186, + "loss": 2.7257, + "theoretical_loss": 3.4191118066578383, + "tokens_seen": 2080899072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757979797979798, + "loss": 2.7604, + "theoretical_loss": 3.419102898403459, + "tokens_seen": 2080964608 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1146429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.32552433013916, + "objective/train/theoretical_loss": 3.419093990508173, + "objective/train/tokens_used": 439889376, + "theoretical_loss": 3.419093990508173, + "tokens_seen": 2081030144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757777777777778, + "loss": 2.6839, + "theoretical_loss": 3.419093990508173, + "tokens_seen": 2081030144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037575757575757573, + "loss": 2.6432, + "theoretical_loss": 3.4190850829719563, + "tokens_seen": 2081095680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037573737373737374, + "loss": 2.557, + "theoretical_loss": 3.4190761757947836, + "tokens_seen": 2081161216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037571717171717175, + "loss": 2.5485, + "theoretical_loss": 3.4190672689766277, + "tokens_seen": 2081226752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756969696969697, + "loss": 2.7005, + "theoretical_loss": 3.4190583625174633, + "tokens_seen": 2081292288 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037567676767676767, + "loss": 2.6643, + "theoretical_loss": 3.419049456417265, + "tokens_seen": 2081357824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756565656565656, + "loss": 2.5289, + "theoretical_loss": 3.419040550676007, + "tokens_seen": 2081423360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756363636363637, + "loss": 2.8114, + "theoretical_loss": 3.419031645293663, + "tokens_seen": 2081488896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037561616161616165, + "loss": 2.7906, + "theoretical_loss": 3.4190227402702074, + "tokens_seen": 2081554432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755959595959596, + "loss": 2.9096, + "theoretical_loss": 3.419013835605615, + "tokens_seen": 2081619968 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037557575757575756, + "loss": 2.5331, + "theoretical_loss": 3.4190049312998596, + "tokens_seen": 2081685504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755555555555555, + "loss": 2.7058, + "theoretical_loss": 3.4189960273529154, + "tokens_seen": 2081751040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755353535353536, + "loss": 2.726, + "theoretical_loss": 3.4189871237647567, + "tokens_seen": 2081816576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037551515151515154, + "loss": 2.562, + "theoretical_loss": 3.4189782205353576, + "tokens_seen": 2081882112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754949494949495, + "loss": 2.5914, + "theoretical_loss": 3.4189693176646925, + "tokens_seen": 2081947648 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037547474747474745, + "loss": 2.7903, + "theoretical_loss": 3.4189604151527364, + "tokens_seen": 2082013184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037545454545454546, + "loss": 2.6528, + "theoretical_loss": 3.4189515129994623, + "tokens_seen": 2082078720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754343434343435, + "loss": 2.6828, + "theoretical_loss": 3.418942611204845, + "tokens_seen": 2082144256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037541414141414143, + "loss": 2.5347, + "theoretical_loss": 3.418933709768859, + "tokens_seen": 2082209792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753939393939394, + "loss": 2.6722, + "theoretical_loss": 3.418924808691478, + "tokens_seen": 2082275328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037537373737373735, + "loss": 2.4734, + "theoretical_loss": 3.4189159079726767, + "tokens_seen": 2082340864 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037535353535353536, + "loss": 2.6157, + "theoretical_loss": 3.4189070076124293, + "tokens_seen": 2082406400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037533333333333337, + "loss": 2.7297, + "theoretical_loss": 3.41889810761071, + "tokens_seen": 2082471936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753131313131313, + "loss": 2.6283, + "theoretical_loss": 3.418889207967493, + "tokens_seen": 2082537472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752929292929293, + "loss": 2.3652, + "theoretical_loss": 3.418880308682753, + "tokens_seen": 2082603008 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1147399, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0194616317749023, + "objective/train/theoretical_loss": 3.4188714097564636, + "objective/train/tokens_used": 441527776, + "theoretical_loss": 3.4188714097564636, + "tokens_seen": 2082668544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752727272727273, + "loss": 2.8635, + "theoretical_loss": 3.4188714097564636, + "tokens_seen": 2082668544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037525252525252525, + "loss": 2.7142, + "theoretical_loss": 3.4188625111886, + "tokens_seen": 2082734080 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037523232323232326, + "loss": 2.5603, + "theoretical_loss": 3.4188536129791354, + "tokens_seen": 2082799616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752121212121212, + "loss": 2.668, + "theoretical_loss": 3.418844715128045, + "tokens_seen": 2082865152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751919191919192, + "loss": 2.9046, + "theoretical_loss": 3.418835817635303, + "tokens_seen": 2082930688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751717171717172, + "loss": 2.5116, + "theoretical_loss": 3.4188269205008828, + "tokens_seen": 2082996224 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037515151515151514, + "loss": 2.5802, + "theoretical_loss": 3.41881802372476, + "tokens_seen": 2083061760 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037513131313131315, + "loss": 2.6405, + "theoretical_loss": 3.4188091273069072, + "tokens_seen": 2083127296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751111111111111, + "loss": 2.613, + "theoretical_loss": 3.4188002312473005, + "tokens_seen": 2083192832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750909090909091, + "loss": 2.6961, + "theoretical_loss": 3.4187913355459134, + "tokens_seen": 2083258368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750707070707071, + "loss": 2.5561, + "theoretical_loss": 3.41878244020272, + "tokens_seen": 2083323904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037505050505050503, + "loss": 2.6632, + "theoretical_loss": 3.418773545217695, + "tokens_seen": 2083389440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037503030303030305, + "loss": 3.0064, + "theoretical_loss": 3.418764650590812, + "tokens_seen": 2083454976 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375010101010101, + "loss": 2.7303, + "theoretical_loss": 3.4187557563220463, + "tokens_seen": 2083520512 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374989898989899, + "loss": 2.7951, + "theoretical_loss": 3.4187468624113717, + "tokens_seen": 2083586048 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037496969696969697, + "loss": 2.4562, + "theoretical_loss": 3.4187379688587627, + "tokens_seen": 2083651584 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374949494949495, + "loss": 2.6969, + "theoretical_loss": 3.4187290756641935, + "tokens_seen": 2083717120 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037492929292929294, + "loss": 2.3717, + "theoretical_loss": 3.4187201828276383, + "tokens_seen": 2083782656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749090909090909, + "loss": 2.6182, + "theoretical_loss": 3.418711290349072, + "tokens_seen": 2083848192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748888888888889, + "loss": 2.7406, + "theoretical_loss": 3.418702398228468, + "tokens_seen": 2083913728 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037486868686868686, + "loss": 2.6609, + "theoretical_loss": 3.418693506465801, + "tokens_seen": 2083979264 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748484848484849, + "loss": 2.6927, + "theoretical_loss": 3.4186846150610464, + "tokens_seen": 2084044800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037482828282828283, + "loss": 2.9106, + "theoretical_loss": 3.418675724014177, + "tokens_seen": 2084110336 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037480808080808084, + "loss": 2.7649, + "theoretical_loss": 3.418666833325168, + "tokens_seen": 2084175872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747878787878788, + "loss": 2.6435, + "theoretical_loss": 3.4186579429939927, + "tokens_seen": 2084241408 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1148024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3189051151275635, + "objective/train/theoretical_loss": 3.4186490530206273, + "objective/train/tokens_used": 443166176, + "theoretical_loss": 3.4186490530206273, + "tokens_seen": 2084306944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037476767676767676, + "loss": 2.4753, + "theoretical_loss": 3.4186490530206273, + "tokens_seen": 2084306944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037474747474747477, + "loss": 2.7101, + "theoretical_loss": 3.4186401634050445, + "tokens_seen": 2084372480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747272727272727, + "loss": 2.6549, + "theoretical_loss": 3.4186312741472196, + "tokens_seen": 2084438016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037470707070707073, + "loss": 2.6999, + "theoretical_loss": 3.4186223852471262, + "tokens_seen": 2084503552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746868686868687, + "loss": 2.6416, + "theoretical_loss": 3.4186134967047392, + "tokens_seen": 2084569088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037466666666666665, + "loss": 2.6206, + "theoretical_loss": 3.4186046085200332, + "tokens_seen": 2084634624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037464646464646466, + "loss": 2.6309, + "theoretical_loss": 3.418595720692982, + "tokens_seen": 2084700160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037462626262626267, + "loss": 2.7098, + "theoretical_loss": 3.41858683322356, + "tokens_seen": 2084765696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037460606060606063, + "loss": 2.3725, + "theoretical_loss": 3.418577946111742, + "tokens_seen": 2084831232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745858585858586, + "loss": 2.7087, + "theoretical_loss": 3.4185690593575018, + "tokens_seen": 2084896768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037456565656565654, + "loss": 2.7147, + "theoretical_loss": 3.418560172960814, + "tokens_seen": 2084962304 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037454545454545455, + "loss": 2.5222, + "theoretical_loss": 3.418551286921653, + "tokens_seen": 2085027840 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037452525252525256, + "loss": 2.4876, + "theoretical_loss": 3.4185424012399936, + "tokens_seen": 2085093376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745050505050505, + "loss": 2.6135, + "theoretical_loss": 3.4185335159158097, + "tokens_seen": 2085158912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744848484848485, + "loss": 2.5551, + "theoretical_loss": 3.4185246309490758, + "tokens_seen": 2085224448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037446464646464643, + "loss": 2.5469, + "theoretical_loss": 3.418515746339766, + "tokens_seen": 2085289984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744444444444445, + "loss": 2.536, + "theoretical_loss": 3.418506862087855, + "tokens_seen": 2085355520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037442424242424246, + "loss": 2.4808, + "theoretical_loss": 3.4184979781933174, + "tokens_seen": 2085421056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744040404040404, + "loss": 2.5116, + "theoretical_loss": 3.418489094656127, + "tokens_seen": 2085486592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037438383838383837, + "loss": 2.597, + "theoretical_loss": 3.418480211476259, + "tokens_seen": 2085552128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743636363636363, + "loss": 2.7988, + "theoretical_loss": 3.4184713286536867, + "tokens_seen": 2085617664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743434343434344, + "loss": 2.6326, + "theoretical_loss": 3.4184624461883857, + "tokens_seen": 2085683200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037432323232323235, + "loss": 2.5852, + "theoretical_loss": 3.41845356408033, + "tokens_seen": 2085748736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743030303030303, + "loss": 2.6487, + "theoretical_loss": 3.4184446823294934, + "tokens_seen": 2085814272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037428282828282826, + "loss": 2.6216, + "theoretical_loss": 3.418435800935851, + "tokens_seen": 2085879808 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1149004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6764867305755615, + "objective/train/theoretical_loss": 3.4184269198993773, + "objective/train/tokens_used": 444804576, + "theoretical_loss": 3.4184269198993773, + "tokens_seen": 2085945344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742626262626263, + "loss": 2.7363, + "theoretical_loss": 3.4184269198993773, + "tokens_seen": 2085945344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742424242424243, + "loss": 2.4787, + "theoretical_loss": 3.418418039220046, + "tokens_seen": 2086010880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037422222222222224, + "loss": 2.5255, + "theoretical_loss": 3.4184091588978314, + "tokens_seen": 2086076416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742020202020202, + "loss": 2.8832, + "theoretical_loss": 3.418400278932709, + "tokens_seen": 2086141952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037418181818181815, + "loss": 2.6207, + "theoretical_loss": 3.418391399324653, + "tokens_seen": 2086207488 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037416161616161617, + "loss": 2.5225, + "theoretical_loss": 3.418382520073637, + "tokens_seen": 2086273024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741414141414142, + "loss": 2.5866, + "theoretical_loss": 3.4183736411796364, + "tokens_seen": 2086338560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037412121212121213, + "loss": 2.7921, + "theoretical_loss": 3.4183647626426246, + "tokens_seen": 2086404096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741010101010101, + "loss": 2.6267, + "theoretical_loss": 3.4183558844625765, + "tokens_seen": 2086469632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740808080808081, + "loss": 2.736, + "theoretical_loss": 3.418347006639467, + "tokens_seen": 2086535168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037406060606060606, + "loss": 2.5893, + "theoretical_loss": 3.41833812917327, + "tokens_seen": 2086600704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037404040404040407, + "loss": 2.4624, + "theoretical_loss": 3.4183292520639603, + "tokens_seen": 2086666240 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374020202020202, + "loss": 2.7353, + "theoretical_loss": 3.418320375311512, + "tokens_seen": 2086731776 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374, + "loss": 2.7195, + "theoretical_loss": 3.4183114989159, + "tokens_seen": 2086797312 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373979797979798, + "loss": 2.7483, + "theoretical_loss": 3.4183026228770976, + "tokens_seen": 2086862848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037395959595959595, + "loss": 2.7238, + "theoretical_loss": 3.4182937471950807, + "tokens_seen": 2086928384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037393939393939396, + "loss": 2.6748, + "theoretical_loss": 3.4182848718698233, + "tokens_seen": 2086993920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739191919191919, + "loss": 2.4875, + "theoretical_loss": 3.4182759969012992, + "tokens_seen": 2087059456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037389898989898993, + "loss": 2.6688, + "theoretical_loss": 3.4182671222894836, + "tokens_seen": 2087124992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738787878787879, + "loss": 2.831, + "theoretical_loss": 3.418258248034351, + "tokens_seen": 2087190528 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037385858585858584, + "loss": 2.4438, + "theoretical_loss": 3.4182493741358755, + "tokens_seen": 2087256064 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037383838383838386, + "loss": 2.6071, + "theoretical_loss": 3.418240500594031, + "tokens_seen": 2087321600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738181818181818, + "loss": 2.7333, + "theoretical_loss": 3.4182316274087934, + "tokens_seen": 2087387136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737979797979798, + "loss": 2.7107, + "theoretical_loss": 3.4182227545801362, + "tokens_seen": 2087452672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737777777777778, + "loss": 2.491, + "theoretical_loss": 3.418213882108034, + "tokens_seen": 2087518208 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1149506, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7207868099212646, + "objective/train/theoretical_loss": 3.4182050099924615, + "objective/train/tokens_used": 446442976, + "theoretical_loss": 3.4182050099924615, + "tokens_seen": 2087583744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037375757575757574, + "loss": 2.615, + "theoretical_loss": 3.4182050099924615, + "tokens_seen": 2087583744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037373737373737375, + "loss": 2.6447, + "theoretical_loss": 3.4181961382333927, + "tokens_seen": 2087649280 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037371717171717176, + "loss": 2.7587, + "theoretical_loss": 3.4181872668308024, + "tokens_seen": 2087714816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736969696969697, + "loss": 2.8354, + "theoretical_loss": 3.4181783957846656, + "tokens_seen": 2087780352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037367676767676767, + "loss": 2.4997, + "theoretical_loss": 3.418169525094956, + "tokens_seen": 2087845888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037365656565656563, + "loss": 2.6017, + "theoretical_loss": 3.4181606547616488, + "tokens_seen": 2087911424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037363636363636364, + "loss": 2.7573, + "theoretical_loss": 3.4181517847847176, + "tokens_seen": 2087976960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037361616161616165, + "loss": 2.5815, + "theoretical_loss": 3.418142915164138, + "tokens_seen": 2088042496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735959595959596, + "loss": 2.8231, + "theoretical_loss": 3.4181340458998832, + "tokens_seen": 2088108032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037357575757575757, + "loss": 2.6708, + "theoretical_loss": 3.4181251769919285, + "tokens_seen": 2088173568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735555555555555, + "loss": 2.6358, + "theoretical_loss": 3.418116308440248, + "tokens_seen": 2088239104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735353535353536, + "loss": 2.6792, + "theoretical_loss": 3.4181074402448175, + "tokens_seen": 2088304640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037351515151515154, + "loss": 2.8498, + "theoretical_loss": 3.4180985724056097, + "tokens_seen": 2088370176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734949494949495, + "loss": 2.8895, + "theoretical_loss": 3.4180897049226004, + "tokens_seen": 2088435712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037347474747474746, + "loss": 2.4677, + "theoretical_loss": 3.418080837795763, + "tokens_seen": 2088501248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734545454545454, + "loss": 2.5268, + "theoretical_loss": 3.418071971025073, + "tokens_seen": 2088566784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734343434343435, + "loss": 2.797, + "theoretical_loss": 3.4180631046105048, + "tokens_seen": 2088632320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037341414141414144, + "loss": 2.8185, + "theoretical_loss": 3.4180542385520325, + "tokens_seen": 2088697856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733939393939394, + "loss": 2.8718, + "theoretical_loss": 3.4180453728496305, + "tokens_seen": 2088763392 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037337373737373735, + "loss": 2.769, + "theoretical_loss": 3.4180365075032744, + "tokens_seen": 2088828928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733535353535353, + "loss": 2.4134, + "theoretical_loss": 3.4180276425129374, + "tokens_seen": 2088894464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733333333333334, + "loss": 2.7009, + "theoretical_loss": 3.4180187778785953, + "tokens_seen": 2088960000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037331313131313133, + "loss": 2.6487, + "theoretical_loss": 3.418009913600221, + "tokens_seen": 2089025536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732929292929293, + "loss": 2.809, + "theoretical_loss": 3.4180010496777906, + "tokens_seen": 2089091072 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037327272727272724, + "loss": 2.6093, + "theoretical_loss": 3.417992186111278, + "tokens_seen": 2089156608 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1150478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.582503080368042, + "objective/train/theoretical_loss": 3.4179833229006578, + "objective/train/tokens_used": 448081376, + "theoretical_loss": 3.4179833229006578, + "tokens_seen": 2089222144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732525252525253, + "loss": 2.5808, + "theoretical_loss": 3.4179833229006578, + "tokens_seen": 2089222144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037323232323232327, + "loss": 2.5832, + "theoretical_loss": 3.417974460045904, + "tokens_seen": 2089287680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732121212121212, + "loss": 2.8097, + "theoretical_loss": 3.417965597546992, + "tokens_seen": 2089353216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731919191919192, + "loss": 2.8212, + "theoretical_loss": 3.417956735403896, + "tokens_seen": 2089418752 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037317171717171714, + "loss": 2.7588, + "theoretical_loss": 3.4179478736165914, + "tokens_seen": 2089484288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731515151515152, + "loss": 2.6283, + "theoretical_loss": 3.417939012185051, + "tokens_seen": 2089549824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037313131313131316, + "loss": 2.7115, + "theoretical_loss": 3.4179301511092506, + "tokens_seen": 2089615360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731111111111111, + "loss": 2.4287, + "theoretical_loss": 3.4179212903891645, + "tokens_seen": 2089680896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037309090909090907, + "loss": 2.5453, + "theoretical_loss": 3.417912430024767, + "tokens_seen": 2089746432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730707070707071, + "loss": 2.5528, + "theoretical_loss": 3.4179035700160334, + "tokens_seen": 2089811968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730505050505051, + "loss": 2.6557, + "theoretical_loss": 3.417894710362937, + "tokens_seen": 2089877504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037303030303030305, + "loss": 2.5877, + "theoretical_loss": 3.4178858510654537, + "tokens_seen": 2089943040 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373010101010101, + "loss": 2.4979, + "theoretical_loss": 3.4178769921235572, + "tokens_seen": 2090008576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037298989898989896, + "loss": 2.6663, + "theoretical_loss": 3.4178681335372225, + "tokens_seen": 2090074112 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372969696969697, + "loss": 2.735, + "theoretical_loss": 3.4178592753064243, + "tokens_seen": 2090139648 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372949494949495, + "loss": 2.4537, + "theoretical_loss": 3.417850417431137, + "tokens_seen": 2090205184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037292929292929294, + "loss": 2.5839, + "theoretical_loss": 3.417841559911335, + "tokens_seen": 2090270720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729090909090909, + "loss": 2.6842, + "theoretical_loss": 3.417832702746993, + "tokens_seen": 2090336256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728888888888889, + "loss": 2.6375, + "theoretical_loss": 3.4178238459380856, + "tokens_seen": 2090401792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037286868686868687, + "loss": 2.6688, + "theoretical_loss": 3.4178149894845875, + "tokens_seen": 2090467328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728484848484849, + "loss": 2.6437, + "theoretical_loss": 3.417806133386473, + "tokens_seen": 2090532864 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037282828282828284, + "loss": 2.6284, + "theoretical_loss": 3.417797277643717, + "tokens_seen": 2090598400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728080808080808, + "loss": 2.5581, + "theoretical_loss": 3.417788422256294, + "tokens_seen": 2090663936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727878787878788, + "loss": 2.6807, + "theoretical_loss": 3.417779567224179, + "tokens_seen": 2090729472 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037276767676767676, + "loss": 2.5903, + "theoretical_loss": 3.417770712547346, + "tokens_seen": 2090795008 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1151517, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.077552556991577, + "objective/train/theoretical_loss": 3.41776185822577, + "objective/train/tokens_used": 449719776, + "theoretical_loss": 3.41776185822577, + "tokens_seen": 2090860544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037274747474747477, + "loss": 2.8183, + "theoretical_loss": 3.41776185822577, + "tokens_seen": 2090860544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037272727272727273, + "loss": 2.757, + "theoretical_loss": 3.417753004259425, + "tokens_seen": 2090926080 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037270707070707074, + "loss": 2.7178, + "theoretical_loss": 3.4177441506482866, + "tokens_seen": 2090991616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726868686868687, + "loss": 2.8016, + "theoretical_loss": 3.417735297392328, + "tokens_seen": 2091057152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037266666666666665, + "loss": 2.603, + "theoretical_loss": 3.417726444491526, + "tokens_seen": 2091122688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037264646464646466, + "loss": 2.6375, + "theoretical_loss": 3.417717591945853, + "tokens_seen": 2091188224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726262626262626, + "loss": 2.7086, + "theoretical_loss": 3.4177087397552848, + "tokens_seen": 2091253760 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037260606060606063, + "loss": 2.4693, + "theoretical_loss": 3.4176998879197957, + "tokens_seen": 2091319296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725858585858586, + "loss": 2.766, + "theoretical_loss": 3.4176910364393605, + "tokens_seen": 2091384832 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037256565656565655, + "loss": 2.6232, + "theoretical_loss": 3.4176821853139536, + "tokens_seen": 2091450368 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037254545454545456, + "loss": 2.5957, + "theoretical_loss": 3.41767333454355, + "tokens_seen": 2091515904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037252525252525257, + "loss": 2.6096, + "theoretical_loss": 3.417664484128124, + "tokens_seen": 2091581440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725050505050505, + "loss": 2.8325, + "theoretical_loss": 3.4176556340676507, + "tokens_seen": 2091646976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724848484848485, + "loss": 2.8292, + "theoretical_loss": 3.4176467843621037, + "tokens_seen": 2091712512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037246464646464644, + "loss": 2.7331, + "theoretical_loss": 3.417637935011459, + "tokens_seen": 2091778048 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037244444444444445, + "loss": 2.6483, + "theoretical_loss": 3.4176290860156904, + "tokens_seen": 2091843584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037242424242424246, + "loss": 2.7649, + "theoretical_loss": 3.4176202373747726, + "tokens_seen": 2091909120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724040404040404, + "loss": 2.4776, + "theoretical_loss": 3.4176113890886803, + "tokens_seen": 2091974656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723838383838384, + "loss": 2.5944, + "theoretical_loss": 3.4176025411573887, + "tokens_seen": 2092040192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037236363636363633, + "loss": 2.5043, + "theoretical_loss": 3.417593693580872, + "tokens_seen": 2092105728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723434343434344, + "loss": 2.4376, + "theoretical_loss": 3.4175848463591043, + "tokens_seen": 2092171264 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037232323232323235, + "loss": 2.9385, + "theoretical_loss": 3.4175759994920614, + "tokens_seen": 2092236800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723030303030303, + "loss": 2.5103, + "theoretical_loss": 3.417567152979717, + "tokens_seen": 2092302336 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037228282828282827, + "loss": 2.6269, + "theoretical_loss": 3.4175583068220465, + "tokens_seen": 2092367872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722626262626262, + "loss": 2.7398, + "theoretical_loss": 3.417549461019024, + "tokens_seen": 2092433408 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1152178, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0097758769989014, + "objective/train/theoretical_loss": 3.4175406155706245, + "objective/train/tokens_used": 451358176, + "theoretical_loss": 3.4175406155706245, + "tokens_seen": 2092498944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722424242424243, + "loss": 2.4834, + "theoretical_loss": 3.4175406155706245, + "tokens_seen": 2092498944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037222222222222225, + "loss": 2.4869, + "theoretical_loss": 3.417531770476823, + "tokens_seen": 2092564480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722020202020202, + "loss": 2.7087, + "theoretical_loss": 3.4175229257375936, + "tokens_seen": 2092630016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037218181818181816, + "loss": 2.6086, + "theoretical_loss": 3.417514081352911, + "tokens_seen": 2092695552 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037216161616161617, + "loss": 2.666, + "theoretical_loss": 3.41750523732275, + "tokens_seen": 2092761088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721414141414142, + "loss": 2.7373, + "theoretical_loss": 3.4174963936470855, + "tokens_seen": 2092826624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037212121212121214, + "loss": 2.7108, + "theoretical_loss": 3.417487550325892, + "tokens_seen": 2092892160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721010101010101, + "loss": 2.6518, + "theoretical_loss": 3.417478707359144, + "tokens_seen": 2092957696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037208080808080805, + "loss": 2.5699, + "theoretical_loss": 3.417469864746817, + "tokens_seen": 2093023232 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037206060606060606, + "loss": 2.6461, + "theoretical_loss": 3.417461022488885, + "tokens_seen": 2093088768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720404040404041, + "loss": 2.4359, + "theoretical_loss": 3.417452180585322, + "tokens_seen": 2093154304 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037202020202020203, + "loss": 2.5586, + "theoretical_loss": 3.4174433390361045, + "tokens_seen": 2093219840 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372, + "loss": 2.4325, + "theoretical_loss": 3.4174344978412057, + "tokens_seen": 2093285376 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371979797979798, + "loss": 2.569, + "theoretical_loss": 3.417425657000601, + "tokens_seen": 2093350912 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037195959595959596, + "loss": 2.6863, + "theoretical_loss": 3.417416816514265, + "tokens_seen": 2093416448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037193939393939397, + "loss": 2.3295, + "theoretical_loss": 3.4174079763821723, + "tokens_seen": 2093481984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719191919191919, + "loss": 2.5572, + "theoretical_loss": 3.4173991366042973, + "tokens_seen": 2093547520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718989898989899, + "loss": 2.4566, + "theoretical_loss": 3.417390297180616, + "tokens_seen": 2093613056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718787878787879, + "loss": 2.4297, + "theoretical_loss": 3.4173814581111017, + "tokens_seen": 2093678592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037185858585858585, + "loss": 2.8283, + "theoretical_loss": 3.4173726193957297, + "tokens_seen": 2093744128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037183838383838386, + "loss": 2.6182, + "theoretical_loss": 3.4173637810344744, + "tokens_seen": 2093809664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718181818181818, + "loss": 2.4856, + "theoretical_loss": 3.417354943027311, + "tokens_seen": 2093875200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037179797979797983, + "loss": 2.6159, + "theoretical_loss": 3.417346105374214, + "tokens_seen": 2093940736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717777777777778, + "loss": 2.5668, + "theoretical_loss": 3.4173372680751584, + "tokens_seen": 2094006272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037175757575757574, + "loss": 2.6343, + "theoretical_loss": 3.4173284311301186, + "tokens_seen": 2094071808 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1153480, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.500739574432373, + "objective/train/theoretical_loss": 3.417319594539069, + "objective/train/tokens_used": 452996576, + "theoretical_loss": 3.417319594539069, + "tokens_seen": 2094137344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037173737373737375, + "loss": 2.6112, + "theoretical_loss": 3.417319594539069, + "tokens_seen": 2094137344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717171717171717, + "loss": 2.6866, + "theoretical_loss": 3.4173107583019853, + "tokens_seen": 2094202880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716969696969697, + "loss": 2.6229, + "theoretical_loss": 3.417301922418842, + "tokens_seen": 2094268416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716767676767677, + "loss": 2.606, + "theoretical_loss": 3.4172930868896128, + "tokens_seen": 2094333952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037165656565656564, + "loss": 2.5647, + "theoretical_loss": 3.417284251714274, + "tokens_seen": 2094399488 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037163636363636365, + "loss": 2.6784, + "theoretical_loss": 3.4172754168927986, + "tokens_seen": 2094465024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716161616161616, + "loss": 2.5724, + "theoretical_loss": 3.417266582425163, + "tokens_seen": 2094530560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715959595959596, + "loss": 2.3217, + "theoretical_loss": 3.417257748311341, + "tokens_seen": 2094596096 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037157575757575757, + "loss": 2.4304, + "theoretical_loss": 3.4172489145513083, + "tokens_seen": 2094661632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037155555555555553, + "loss": 2.2137, + "theoretical_loss": 3.4172400811450387, + "tokens_seen": 2094727168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037153535353535354, + "loss": 2.6197, + "theoretical_loss": 3.4172312480925067, + "tokens_seen": 2094792704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037151515151515155, + "loss": 2.603, + "theoretical_loss": 3.417222415393688, + "tokens_seen": 2094858240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714949494949495, + "loss": 2.5942, + "theoretical_loss": 3.4172135830485573, + "tokens_seen": 2094923776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037147474747474746, + "loss": 2.5866, + "theoretical_loss": 3.4172047510570893, + "tokens_seen": 2094989312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714545454545455, + "loss": 2.4817, + "theoretical_loss": 3.417195919419258, + "tokens_seen": 2095054848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037143434343434343, + "loss": 2.6287, + "theoretical_loss": 3.417187088135039, + "tokens_seen": 2095120384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037141414141414144, + "loss": 2.3498, + "theoretical_loss": 3.417178257204407, + "tokens_seen": 2095185920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713939393939394, + "loss": 2.454, + "theoretical_loss": 3.4171694266273365, + "tokens_seen": 2095251456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037137373737373736, + "loss": 2.5625, + "theoretical_loss": 3.4171605964038023, + "tokens_seen": 2095316992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037135353535353537, + "loss": 2.6528, + "theoretical_loss": 3.417151766533779, + "tokens_seen": 2095382528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713333333333334, + "loss": 2.5552, + "theoretical_loss": 3.4171429370172426, + "tokens_seen": 2095448064 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037131313131313134, + "loss": 2.4329, + "theoretical_loss": 3.4171341078541664, + "tokens_seen": 2095513600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712929292929293, + "loss": 2.6025, + "theoretical_loss": 3.4171252790445257, + "tokens_seen": 2095579136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037127272727272725, + "loss": 2.6012, + "theoretical_loss": 3.4171164505882956, + "tokens_seen": 2095644672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037125252525252526, + "loss": 2.6446, + "theoretical_loss": 3.4171076224854504, + "tokens_seen": 2095710208 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1154065, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8171300888061523, + "objective/train/theoretical_loss": 3.4170987947359652, + "objective/train/tokens_used": 454634976, + "theoretical_loss": 3.4170987947359652, + "tokens_seen": 2095775744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037123232323232327, + "loss": 2.7618, + "theoretical_loss": 3.4170987947359652, + "tokens_seen": 2095775744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037121212121212123, + "loss": 2.3248, + "theoretical_loss": 3.4170899673398147, + "tokens_seen": 2095841280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711919191919192, + "loss": 2.4875, + "theoretical_loss": 3.4170811402969745, + "tokens_seen": 2095906816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037117171717171714, + "loss": 2.416, + "theoretical_loss": 3.417072313607418, + "tokens_seen": 2095972352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711515151515152, + "loss": 2.5119, + "theoretical_loss": 3.417063487271121, + "tokens_seen": 2096037888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037113131313131316, + "loss": 2.7506, + "theoretical_loss": 3.4170546612880575, + "tokens_seen": 2096103424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711111111111111, + "loss": 2.6149, + "theoretical_loss": 3.417045835658204, + "tokens_seen": 2096168960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710909090909091, + "loss": 2.7236, + "theoretical_loss": 3.4170370103815335, + "tokens_seen": 2096234496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037107070707070703, + "loss": 2.5956, + "theoretical_loss": 3.417028185458021, + "tokens_seen": 2096300032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710505050505051, + "loss": 2.506, + "theoretical_loss": 3.4170193608876427, + "tokens_seen": 2096365568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037103030303030306, + "loss": 2.578, + "theoretical_loss": 3.4170105366703725, + "tokens_seen": 2096431104 + }, + { + "epoch": 0.27, + "learning_rate": 0.000371010101010101, + "loss": 2.5963, + "theoretical_loss": 3.4170017128061847, + "tokens_seen": 2096496640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037098989898989897, + "loss": 2.7297, + "theoretical_loss": 3.416992889295055, + "tokens_seen": 2096562176 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370969696969697, + "loss": 2.8348, + "theoretical_loss": 3.4169840661369584, + "tokens_seen": 2096627712 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370949494949495, + "loss": 2.5569, + "theoretical_loss": 3.4169752433318688, + "tokens_seen": 2096693248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037092929292929295, + "loss": 2.6674, + "theoretical_loss": 3.4169664208797617, + "tokens_seen": 2096758784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709090909090909, + "loss": 2.9471, + "theoretical_loss": 3.416957598780612, + "tokens_seen": 2096824320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037088888888888886, + "loss": 2.4803, + "theoretical_loss": 3.416948777034394, + "tokens_seen": 2096889856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708686868686869, + "loss": 2.9403, + "theoretical_loss": 3.416939955641083, + "tokens_seen": 2096955392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708484848484849, + "loss": 2.6036, + "theoretical_loss": 3.416931134600654, + "tokens_seen": 2097020928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037082828282828284, + "loss": 2.6213, + "theoretical_loss": 3.4169223139130813, + "tokens_seen": 2097086464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708080808080808, + "loss": 2.5021, + "theoretical_loss": 3.41691349357834, + "tokens_seen": 2097152000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707878787878788, + "loss": 2.5866, + "theoretical_loss": 3.416904673596406, + "tokens_seen": 2097217536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037076767676767677, + "loss": 2.7012, + "theoretical_loss": 3.416895853967252, + "tokens_seen": 2097283072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707474747474748, + "loss": 2.6695, + "theoretical_loss": 3.4168870346908546, + "tokens_seen": 2097348608 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1155055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3634607791900635, + "objective/train/theoretical_loss": 3.416878215767188, + "objective/train/tokens_used": 456273376, + "theoretical_loss": 3.416878215767188, + "tokens_seen": 2097414144 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037072727272727273, + "loss": 2.6518, + "theoretical_loss": 3.416878215767188, + "tokens_seen": 2097414144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707070707070707, + "loss": 2.7283, + "theoretical_loss": 3.416869397196227, + "tokens_seen": 2097479680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706868686868687, + "loss": 2.6649, + "theoretical_loss": 3.416860578977947, + "tokens_seen": 2097545216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037066666666666666, + "loss": 2.6906, + "theoretical_loss": 3.416851761112323, + "tokens_seen": 2097610752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037064646464646467, + "loss": 2.6143, + "theoretical_loss": 3.416842943599329, + "tokens_seen": 2097676288 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037062626262626263, + "loss": 2.3629, + "theoretical_loss": 3.41683412643894, + "tokens_seen": 2097741824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037060606060606064, + "loss": 2.8845, + "theoretical_loss": 3.4168253096311316, + "tokens_seen": 2097807360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705858585858586, + "loss": 2.7976, + "theoretical_loss": 3.4168164931758778, + "tokens_seen": 2097872896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037056565656565655, + "loss": 2.7928, + "theoretical_loss": 3.4168076770731544, + "tokens_seen": 2097938432 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037054545454545456, + "loss": 2.6009, + "theoretical_loss": 3.416798861322936, + "tokens_seen": 2098003968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705252525252525, + "loss": 2.7349, + "theoretical_loss": 3.416790045925197, + "tokens_seen": 2098069504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037050505050505053, + "loss": 2.6537, + "theoretical_loss": 3.416781230879913, + "tokens_seen": 2098135040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704848484848485, + "loss": 2.5315, + "theoretical_loss": 3.416772416187058, + "tokens_seen": 2098200576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037046464646464644, + "loss": 2.3986, + "theoretical_loss": 3.416763601846608, + "tokens_seen": 2098266112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037044444444444446, + "loss": 2.4969, + "theoretical_loss": 3.4167547878585376, + "tokens_seen": 2098331648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037042424242424247, + "loss": 2.6538, + "theoretical_loss": 3.416745974222821, + "tokens_seen": 2098397184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704040404040404, + "loss": 2.5273, + "theoretical_loss": 3.4167371609394337, + "tokens_seen": 2098462720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703838383838384, + "loss": 2.7185, + "theoretical_loss": 3.41672834800835, + "tokens_seen": 2098528256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037036363636363634, + "loss": 2.7387, + "theoretical_loss": 3.416719535429546, + "tokens_seen": 2098593792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037034343434343435, + "loss": 2.6902, + "theoretical_loss": 3.416710723202996, + "tokens_seen": 2098659328 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037032323232323236, + "loss": 2.6166, + "theoretical_loss": 3.4167019113286745, + "tokens_seen": 2098724864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703030303030303, + "loss": 2.2139, + "theoretical_loss": 3.4166930998065568, + "tokens_seen": 2098790400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702828282828283, + "loss": 2.6439, + "theoretical_loss": 3.4166842886366178, + "tokens_seen": 2098855936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037026262626262623, + "loss": 2.8034, + "theoretical_loss": 3.416675477818832, + "tokens_seen": 2098921472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702424242424243, + "loss": 2.5019, + "theoretical_loss": 3.4166666673531756, + "tokens_seen": 2098987008 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1156556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7254951000213623, + "objective/train/theoretical_loss": 3.4166578572396222, + "objective/train/tokens_used": 457911776, + "theoretical_loss": 3.4166578572396222, + "tokens_seen": 2099052544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037022222222222225, + "loss": 2.4085, + "theoretical_loss": 3.4166578572396222, + "tokens_seen": 2099052544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702020202020202, + "loss": 2.739, + "theoretical_loss": 3.4166490474781477, + "tokens_seen": 2099118080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037018181818181817, + "loss": 2.5515, + "theoretical_loss": 3.416640238068726, + "tokens_seen": 2099183616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701616161616161, + "loss": 2.6694, + "theoretical_loss": 3.416631429011333, + "tokens_seen": 2099249152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701414141414142, + "loss": 2.6967, + "theoretical_loss": 3.416622620305943, + "tokens_seen": 2099314688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037012121212121215, + "loss": 2.5425, + "theoretical_loss": 3.4166138119525313, + "tokens_seen": 2099380224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701010101010101, + "loss": 2.8095, + "theoretical_loss": 3.416605003951073, + "tokens_seen": 2099445760 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037008080808080806, + "loss": 2.5167, + "theoretical_loss": 3.4165961963015423, + "tokens_seen": 2099511296 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370060606060606, + "loss": 2.6135, + "theoretical_loss": 3.416587389003915, + "tokens_seen": 2099576832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700404040404041, + "loss": 2.635, + "theoretical_loss": 3.4165785820581656, + "tokens_seen": 2099642368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037002020202020204, + "loss": 2.6785, + "theoretical_loss": 3.416569775464269, + "tokens_seen": 2099707904 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037, + "loss": 2.7011, + "theoretical_loss": 3.4165609692222008, + "tokens_seen": 2099773440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036997979797979795, + "loss": 2.7845, + "theoretical_loss": 3.416552163331935, + "tokens_seen": 2099838976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036995959595959596, + "loss": 2.5003, + "theoretical_loss": 3.416543357793447, + "tokens_seen": 2099904512 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369939393939394, + "loss": 2.5218, + "theoretical_loss": 3.4165345526067123, + "tokens_seen": 2099970048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036991919191919193, + "loss": 2.5563, + "theoretical_loss": 3.416525747771705, + "tokens_seen": 2100035584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698989898989899, + "loss": 2.7174, + "theoretical_loss": 3.416516943288401, + "tokens_seen": 2100101120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036987878787878784, + "loss": 2.7711, + "theoretical_loss": 3.4165081391567744, + "tokens_seen": 2100166656 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036985858585858586, + "loss": 2.6654, + "theoretical_loss": 3.4164993353768005, + "tokens_seen": 2100232192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036983838383838387, + "loss": 2.6669, + "theoretical_loss": 3.4164905319484546, + "tokens_seen": 2100297728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698181818181818, + "loss": 2.6784, + "theoretical_loss": 3.4164817288717106, + "tokens_seen": 2100363264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697979797979798, + "loss": 2.4752, + "theoretical_loss": 3.416472926146545, + "tokens_seen": 2100428800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697777777777778, + "loss": 2.4664, + "theoretical_loss": 3.416464123772932, + "tokens_seen": 2100494336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697575757575758, + "loss": 2.6676, + "theoretical_loss": 3.4164553217508464, + "tokens_seen": 2100559872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036973737373737376, + "loss": 2.5018, + "theoretical_loss": 3.416446520080264, + "tokens_seen": 2100625408 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1157195, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8589794635772705, + "objective/train/theoretical_loss": 3.4164377187611588, + "objective/train/tokens_used": 459550176, + "theoretical_loss": 3.4164377187611588, + "tokens_seen": 2100690944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697171717171717, + "loss": 2.7354, + "theoretical_loss": 3.4164377187611588, + "tokens_seen": 2100690944 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036969696969696967, + "loss": 2.545, + "theoretical_loss": 3.4164289177935063, + "tokens_seen": 2100756480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696767676767677, + "loss": 2.6809, + "theoretical_loss": 3.4164201171772817, + "tokens_seen": 2100822016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696565656565657, + "loss": 2.7996, + "theoretical_loss": 3.4164113169124595, + "tokens_seen": 2100887552 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036963636363636365, + "loss": 2.4825, + "theoretical_loss": 3.416402516999015, + "tokens_seen": 2100953088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696161616161616, + "loss": 2.5685, + "theoretical_loss": 3.4163937174369234, + "tokens_seen": 2101018624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695959595959596, + "loss": 2.7909, + "theoretical_loss": 3.416384918226159, + "tokens_seen": 2101084160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695757575757576, + "loss": 2.41, + "theoretical_loss": 3.4163761193666975, + "tokens_seen": 2101149696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695555555555556, + "loss": 2.5918, + "theoretical_loss": 3.416367320858514, + "tokens_seen": 2101215232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036953535353535354, + "loss": 2.5184, + "theoretical_loss": 3.416358522701583, + "tokens_seen": 2101280768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695151515151515, + "loss": 2.5719, + "theoretical_loss": 3.4163497248958805, + "tokens_seen": 2101346304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694949494949495, + "loss": 2.7546, + "theoretical_loss": 3.4163409274413796, + "tokens_seen": 2101411840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036947474747474747, + "loss": 2.5471, + "theoretical_loss": 3.4163321303380574, + "tokens_seen": 2101477376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694545454545455, + "loss": 2.6367, + "theoretical_loss": 3.416323333585888, + "tokens_seen": 2101542912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036943434343434344, + "loss": 2.7109, + "theoretical_loss": 3.416314537184846, + "tokens_seen": 2101608448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036941414141414145, + "loss": 2.8343, + "theoretical_loss": 3.416305741134907, + "tokens_seen": 2101673984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693939393939394, + "loss": 2.5901, + "theoretical_loss": 3.4162969454360463, + "tokens_seen": 2101739520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036937373737373736, + "loss": 2.6116, + "theoretical_loss": 3.4162881500882385, + "tokens_seen": 2101805056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693535353535354, + "loss": 2.5429, + "theoretical_loss": 3.4162793550914587, + "tokens_seen": 2101870592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036933333333333333, + "loss": 2.7821, + "theoretical_loss": 3.416270560445682, + "tokens_seen": 2101936128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036931313131313134, + "loss": 2.7114, + "theoretical_loss": 3.4162617661508836, + "tokens_seen": 2102001664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692929292929293, + "loss": 2.6573, + "theoretical_loss": 3.4162529722070385, + "tokens_seen": 2102067200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036927272727272725, + "loss": 2.5542, + "theoretical_loss": 3.416244178614121, + "tokens_seen": 2102132736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036925252525252527, + "loss": 2.4011, + "theoretical_loss": 3.416235385372107, + "tokens_seen": 2102198272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692323232323233, + "loss": 2.4574, + "theoretical_loss": 3.416226592480972, + "tokens_seen": 2102263808 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1158634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4343600273132324, + "objective/train/theoretical_loss": 3.41621779994069, + "objective/train/tokens_used": 461188576, + "theoretical_loss": 3.41621779994069, + "tokens_seen": 2102329344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036921212121212123, + "loss": 2.7046, + "theoretical_loss": 3.41621779994069, + "tokens_seen": 2102329344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691919191919192, + "loss": 2.8016, + "theoretical_loss": 3.4162090077512364, + "tokens_seen": 2102394880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036917171717171715, + "loss": 2.6305, + "theoretical_loss": 3.4162002159125864, + "tokens_seen": 2102460416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036915151515151516, + "loss": 2.6215, + "theoretical_loss": 3.416191424424715, + "tokens_seen": 2102525952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036913131313131317, + "loss": 2.7357, + "theoretical_loss": 3.4161826332875975, + "tokens_seen": 2102591488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691111111111111, + "loss": 2.4573, + "theoretical_loss": 3.4161738425012085, + "tokens_seen": 2102657024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690909090909091, + "loss": 2.7079, + "theoretical_loss": 3.4161650520655233, + "tokens_seen": 2102722560 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036907070707070704, + "loss": 2.522, + "theoretical_loss": 3.4161562619805173, + "tokens_seen": 2102788096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690505050505051, + "loss": 2.8353, + "theoretical_loss": 3.4161474722461653, + "tokens_seen": 2102853632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036903030303030306, + "loss": 2.5757, + "theoretical_loss": 3.4161386828624423, + "tokens_seen": 2102919168 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369010101010101, + "loss": 2.6205, + "theoretical_loss": 3.416129893829323, + "tokens_seen": 2102984704 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368989898989899, + "loss": 2.631, + "theoretical_loss": 3.4161211051467837, + "tokens_seen": 2103050240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036896969696969693, + "loss": 2.4927, + "theoretical_loss": 3.4161123168147984, + "tokens_seen": 2103115776 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368949494949495, + "loss": 2.7321, + "theoretical_loss": 3.4161035288333426, + "tokens_seen": 2103181312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036892929292929295, + "loss": 2.5872, + "theoretical_loss": 3.416094741202391, + "tokens_seen": 2103246848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689090909090909, + "loss": 2.9086, + "theoretical_loss": 3.41608595392192, + "tokens_seen": 2103312384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036888888888888887, + "loss": 2.5979, + "theoretical_loss": 3.416077166991903, + "tokens_seen": 2103377920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688686868686869, + "loss": 2.522, + "theoretical_loss": 3.416068380412316, + "tokens_seen": 2103443456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688484848484849, + "loss": 2.62, + "theoretical_loss": 3.416059594183134, + "tokens_seen": 2103508992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036882828282828285, + "loss": 2.7018, + "theoretical_loss": 3.416050808304332, + "tokens_seen": 2103574528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688080808080808, + "loss": 2.4874, + "theoretical_loss": 3.416042022775885, + "tokens_seen": 2103640064 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036878787878787876, + "loss": 2.7253, + "theoretical_loss": 3.4160332375977687, + "tokens_seen": 2103705600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036876767676767677, + "loss": 2.6079, + "theoretical_loss": 3.4160244527699577, + "tokens_seen": 2103771136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687474747474748, + "loss": 2.808, + "theoretical_loss": 3.4160156682924274, + "tokens_seen": 2103836672 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036872727272727274, + "loss": 2.5926, + "theoretical_loss": 3.416006884165152, + "tokens_seen": 2103902208 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1159301, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5771374702453613, + "objective/train/theoretical_loss": 3.4159981003881086, + "objective/train/tokens_used": 462826976, + "theoretical_loss": 3.4159981003881086, + "tokens_seen": 2103967744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687070707070707, + "loss": 2.724, + "theoretical_loss": 3.4159981003881086, + "tokens_seen": 2103967744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686868686868687, + "loss": 2.6223, + "theoretical_loss": 3.4159893169612703, + "tokens_seen": 2104033280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036866666666666667, + "loss": 2.5754, + "theoretical_loss": 3.415980533884613, + "tokens_seen": 2104098816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686464646464647, + "loss": 2.6732, + "theoretical_loss": 3.4159717511581116, + "tokens_seen": 2104164352 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036862626262626263, + "loss": 2.7595, + "theoretical_loss": 3.4159629687817423, + "tokens_seen": 2104229888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686060606060606, + "loss": 2.4892, + "theoretical_loss": 3.415954186755479, + "tokens_seen": 2104295424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685858585858586, + "loss": 2.6573, + "theoretical_loss": 3.415945405079297, + "tokens_seen": 2104360960 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036856565656565656, + "loss": 2.7806, + "theoretical_loss": 3.4159366237531725, + "tokens_seen": 2104426496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036854545454545457, + "loss": 2.5188, + "theoretical_loss": 3.415927842777079, + "tokens_seen": 2104492032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685252525252525, + "loss": 2.4961, + "theoretical_loss": 3.4159190621509934, + "tokens_seen": 2104557568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036850505050505054, + "loss": 2.5838, + "theoretical_loss": 3.4159102818748894, + "tokens_seen": 2104623104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684848484848485, + "loss": 2.5448, + "theoretical_loss": 3.415901501948743, + "tokens_seen": 2104688640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036846464646464645, + "loss": 2.6647, + "theoretical_loss": 3.415892722372529, + "tokens_seen": 2104754176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036844444444444446, + "loss": 2.6084, + "theoretical_loss": 3.4158839431462225, + "tokens_seen": 2104819712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684242424242424, + "loss": 2.6192, + "theoretical_loss": 3.415875164269799, + "tokens_seen": 2104885248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036840404040404043, + "loss": 2.6547, + "theoretical_loss": 3.415866385743233, + "tokens_seen": 2104950784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683838383838384, + "loss": 2.4461, + "theoretical_loss": 3.4158576075665006, + "tokens_seen": 2105016320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036836363636363634, + "loss": 2.4437, + "theoretical_loss": 3.415848829739576, + "tokens_seen": 2105081856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036834343434343435, + "loss": 2.6591, + "theoretical_loss": 3.4158400522624355, + "tokens_seen": 2105147392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683232323232323, + "loss": 2.7838, + "theoretical_loss": 3.4158312751350532, + "tokens_seen": 2105212928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683030303030303, + "loss": 2.541, + "theoretical_loss": 3.415822498357405, + "tokens_seen": 2105278464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682828282828283, + "loss": 2.6542, + "theoretical_loss": 3.4158137219294655, + "tokens_seen": 2105344000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036826262626262624, + "loss": 2.4815, + "theoretical_loss": 3.4158049458512103, + "tokens_seen": 2105409536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036824242424242425, + "loss": 2.7231, + "theoretical_loss": 3.415796170122615, + "tokens_seen": 2105475072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036822222222222226, + "loss": 2.5703, + "theoretical_loss": 3.4157873947436537, + "tokens_seen": 2105540608 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1160284, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0382537841796875, + "objective/train/theoretical_loss": 3.415778619714302, + "objective/train/tokens_used": 464465376, + "theoretical_loss": 3.415778619714302, + "tokens_seen": 2105606144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682020202020202, + "loss": 2.6972, + "theoretical_loss": 3.415778619714302, + "tokens_seen": 2105606144 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036818181818181817, + "loss": 2.4719, + "theoretical_loss": 3.4157698450345353, + "tokens_seen": 2105671680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036816161616161613, + "loss": 2.6513, + "theoretical_loss": 3.4157610707043284, + "tokens_seen": 2105737216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036814141414141414, + "loss": 2.6362, + "theoretical_loss": 3.4157522967236575, + "tokens_seen": 2105802752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036812121212121215, + "loss": 2.4002, + "theoretical_loss": 3.415743523092497, + "tokens_seen": 2105868288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681010101010101, + "loss": 2.7257, + "theoretical_loss": 3.415734749810822, + "tokens_seen": 2105933824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036808080808080806, + "loss": 2.4798, + "theoretical_loss": 3.4157259768786075, + "tokens_seen": 2105999360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680606060606061, + "loss": 2.3792, + "theoretical_loss": 3.41571720429583, + "tokens_seen": 2106064896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680404040404041, + "loss": 2.7184, + "theoretical_loss": 3.415708432062463, + "tokens_seen": 2106130432 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036802020202020204, + "loss": 2.6301, + "theoretical_loss": 3.415699660178483, + "tokens_seen": 2106195968 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368, + "loss": 2.629, + "theoretical_loss": 3.415690888643865, + "tokens_seen": 2106261504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036797979797979796, + "loss": 2.8607, + "theoretical_loss": 3.415682117458583, + "tokens_seen": 2106327040 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036795959595959597, + "loss": 2.4124, + "theoretical_loss": 3.415673346622614, + "tokens_seen": 2106392576 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367939393939394, + "loss": 2.5606, + "theoretical_loss": 3.4156645761359323, + "tokens_seen": 2106458112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036791919191919194, + "loss": 2.471, + "theoretical_loss": 3.415655805998513, + "tokens_seen": 2106523648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678989898989899, + "loss": 2.4754, + "theoretical_loss": 3.4156470362103315, + "tokens_seen": 2106589184 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036787878787878785, + "loss": 2.6735, + "theoretical_loss": 3.415638266771363, + "tokens_seen": 2106654720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678585858585859, + "loss": 2.4972, + "theoretical_loss": 3.415629497681583, + "tokens_seen": 2106720256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036783838383838387, + "loss": 2.6408, + "theoretical_loss": 3.415620728940967, + "tokens_seen": 2106785792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036781818181818183, + "loss": 2.4854, + "theoretical_loss": 3.4156119605494895, + "tokens_seen": 2106851328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677979797979798, + "loss": 2.6254, + "theoretical_loss": 3.4156031925071257, + "tokens_seen": 2106916864 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036777777777777774, + "loss": 2.3775, + "theoretical_loss": 3.4155944248138512, + "tokens_seen": 2106982400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677575757575758, + "loss": 2.688, + "theoretical_loss": 3.4155856574696415, + "tokens_seen": 2107047936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036773737373737376, + "loss": 2.6889, + "theoretical_loss": 3.4155768904744717, + "tokens_seen": 2107113472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677171717171717, + "loss": 2.2961, + "theoretical_loss": 3.415568123828317, + "tokens_seen": 2107179008 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1161731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1398634910583496, + "objective/train/theoretical_loss": 3.415559357531152, + "objective/train/tokens_used": 466103776, + "theoretical_loss": 3.415559357531152, + "tokens_seen": 2107244544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676969696969697, + "loss": 2.433, + "theoretical_loss": 3.415559357531152, + "tokens_seen": 2107244544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676767676767677, + "loss": 2.4995, + "theoretical_loss": 3.4155505915829525, + "tokens_seen": 2107310080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676565656565657, + "loss": 2.6416, + "theoretical_loss": 3.415541825983694, + "tokens_seen": 2107375616 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036763636363636366, + "loss": 2.4995, + "theoretical_loss": 3.4155330607333516, + "tokens_seen": 2107441152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676161616161616, + "loss": 2.7959, + "theoretical_loss": 3.4155242958319003, + "tokens_seen": 2107506688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036759595959595957, + "loss": 2.4846, + "theoretical_loss": 3.4155155312793157, + "tokens_seen": 2107572224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675757575757576, + "loss": 2.7947, + "theoretical_loss": 3.415506767075573, + "tokens_seen": 2107637760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675555555555556, + "loss": 2.6484, + "theoretical_loss": 3.4154980032206472, + "tokens_seen": 2107703296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036753535353535355, + "loss": 2.5693, + "theoretical_loss": 3.415489239714514, + "tokens_seen": 2107768832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675151515151515, + "loss": 2.754, + "theoretical_loss": 3.415480476557148, + "tokens_seen": 2107834368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674949494949495, + "loss": 2.6904, + "theoretical_loss": 3.4154717137485253, + "tokens_seen": 2107899904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674747474747475, + "loss": 2.6066, + "theoretical_loss": 3.4154629512886205, + "tokens_seen": 2107965440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674545454545455, + "loss": 2.6974, + "theoretical_loss": 3.4154541891774093, + "tokens_seen": 2108030976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036743434343434344, + "loss": 2.5926, + "theoretical_loss": 3.415445427414867, + "tokens_seen": 2108096512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674141414141414, + "loss": 2.6391, + "theoretical_loss": 3.4154366660009687, + "tokens_seen": 2108162048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673939393939394, + "loss": 2.5497, + "theoretical_loss": 3.4154279049356897, + "tokens_seen": 2108227584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036737373737373737, + "loss": 2.6752, + "theoretical_loss": 3.4154191442190056, + "tokens_seen": 2108293120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673535353535354, + "loss": 2.644, + "theoretical_loss": 3.4154103838508907, + "tokens_seen": 2108358656 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036733333333333334, + "loss": 2.6448, + "theoretical_loss": 3.415401623831322, + "tokens_seen": 2108424192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036731313131313135, + "loss": 2.5945, + "theoretical_loss": 3.415392864160273, + "tokens_seen": 2108489728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672929292929293, + "loss": 2.6826, + "theoretical_loss": 3.41538410483772, + "tokens_seen": 2108555264 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036727272727272726, + "loss": 2.9419, + "theoretical_loss": 3.4153753458636382, + "tokens_seen": 2108620800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036725252525252527, + "loss": 2.4951, + "theoretical_loss": 3.415366587238003, + "tokens_seen": 2108686336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036723232323232323, + "loss": 2.6084, + "theoretical_loss": 3.415357828960789, + "tokens_seen": 2108751872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036721212121212124, + "loss": 2.5701, + "theoretical_loss": 3.4153490710319723, + "tokens_seen": 2108817408 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1162303, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.828242063522339, + "objective/train/theoretical_loss": 3.4153403134515283, + "objective/train/tokens_used": 467742176, + "theoretical_loss": 3.4153403134515283, + "tokens_seen": 2108882944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671919191919192, + "loss": 2.588, + "theoretical_loss": 3.4153403134515283, + "tokens_seen": 2108882944 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036717171717171715, + "loss": 2.5231, + "theoretical_loss": 3.4153315562194315, + "tokens_seen": 2108948480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036715151515151516, + "loss": 2.8862, + "theoretical_loss": 3.415322799335658, + "tokens_seen": 2109014016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671313131313132, + "loss": 2.7052, + "theoretical_loss": 3.4153140428001825, + "tokens_seen": 2109079552 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036711111111111113, + "loss": 2.4027, + "theoretical_loss": 3.4153052866129814, + "tokens_seen": 2109145088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670909090909091, + "loss": 2.4914, + "theoretical_loss": 3.4152965307740284, + "tokens_seen": 2109210624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036707070707070705, + "loss": 2.8417, + "theoretical_loss": 3.4152877752833, + "tokens_seen": 2109276160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036705050505050506, + "loss": 2.6103, + "theoretical_loss": 3.4152790201407717, + "tokens_seen": 2109341696 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036703030303030307, + "loss": 2.529, + "theoretical_loss": 3.4152702653464178, + "tokens_seen": 2109407232 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367010101010101, + "loss": 2.6543, + "theoretical_loss": 3.4152615109002142, + "tokens_seen": 2109472768 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366989898989899, + "loss": 2.958, + "theoretical_loss": 3.4152527568021362, + "tokens_seen": 2109538304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036696969696969694, + "loss": 2.6509, + "theoretical_loss": 3.41524400305216, + "tokens_seen": 2109603840 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366949494949495, + "loss": 2.5489, + "theoretical_loss": 3.4152352496502596, + "tokens_seen": 2109669376 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036692929292929296, + "loss": 2.6989, + "theoretical_loss": 3.4152264965964108, + "tokens_seen": 2109734912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669090909090909, + "loss": 2.483, + "theoretical_loss": 3.4152177438905893, + "tokens_seen": 2109800448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668888888888889, + "loss": 2.5866, + "theoretical_loss": 3.4152089915327695, + "tokens_seen": 2109865984 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036686868686868683, + "loss": 2.5572, + "theoretical_loss": 3.415200239522928, + "tokens_seen": 2109931520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668484848484849, + "loss": 2.5575, + "theoretical_loss": 3.4151914878610397, + "tokens_seen": 2109997056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036682828282828285, + "loss": 2.4725, + "theoretical_loss": 3.4151827365470795, + "tokens_seen": 2110062592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668080808080808, + "loss": 2.7051, + "theoretical_loss": 3.4151739855810233, + "tokens_seen": 2110128128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036678787878787877, + "loss": 2.5453, + "theoretical_loss": 3.4151652349628465, + "tokens_seen": 2110193664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667676767676767, + "loss": 2.5881, + "theoretical_loss": 3.415156484692524, + "tokens_seen": 2110259200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667474747474748, + "loss": 2.6439, + "theoretical_loss": 3.415147734770031, + "tokens_seen": 2110324736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036672727272727275, + "loss": 2.6104, + "theoretical_loss": 3.4151389851953438, + "tokens_seen": 2110390272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667070707070707, + "loss": 2.6512, + "theoretical_loss": 3.4151302359684372, + "tokens_seen": 2110455808 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1163684, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.875441312789917, + "objective/train/theoretical_loss": 3.4151214870892868, + "objective/train/tokens_used": 469380576, + "theoretical_loss": 3.4151214870892868, + "tokens_seen": 2110521344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036668686868686866, + "loss": 2.6088, + "theoretical_loss": 3.4151214870892868, + "tokens_seen": 2110521344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036666666666666667, + "loss": 2.6473, + "theoretical_loss": 3.4151127385578675, + "tokens_seen": 2110586880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666464646464647, + "loss": 2.6037, + "theoretical_loss": 3.4151039903741554, + "tokens_seen": 2110652416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036662626262626264, + "loss": 2.585, + "theoretical_loss": 3.415095242538125, + "tokens_seen": 2110717952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666060606060606, + "loss": 2.4103, + "theoretical_loss": 3.4150864950497524, + "tokens_seen": 2110783488 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036658585858585855, + "loss": 2.4533, + "theoretical_loss": 3.415077747909013, + "tokens_seen": 2110849024 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036656565656565656, + "loss": 2.5017, + "theoretical_loss": 3.4150690011158815, + "tokens_seen": 2110914560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665454545454546, + "loss": 2.5898, + "theoretical_loss": 3.415060254670334, + "tokens_seen": 2110980096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036652525252525253, + "loss": 2.7279, + "theoretical_loss": 3.415051508572346, + "tokens_seen": 2111045632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665050505050505, + "loss": 2.6679, + "theoretical_loss": 3.415042762821892, + "tokens_seen": 2111111168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664848484848485, + "loss": 2.4494, + "theoretical_loss": 3.4150340174189484, + "tokens_seen": 2111176704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036646464646464646, + "loss": 2.3438, + "theoretical_loss": 3.41502527236349, + "tokens_seen": 2111242240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036644444444444447, + "loss": 2.6134, + "theoretical_loss": 3.4150165276554922, + "tokens_seen": 2111307776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664242424242424, + "loss": 2.7989, + "theoretical_loss": 3.4150077832949304, + "tokens_seen": 2111373312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664040404040404, + "loss": 2.7628, + "theoretical_loss": 3.4149990392817804, + "tokens_seen": 2111438848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663838383838384, + "loss": 2.7186, + "theoretical_loss": 3.4149902956160174, + "tokens_seen": 2111504384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663636363636364, + "loss": 2.6454, + "theoretical_loss": 3.414981552297617, + "tokens_seen": 2111569920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036634343434343436, + "loss": 2.7228, + "theoretical_loss": 3.4149728093265543, + "tokens_seen": 2111635456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663232323232323, + "loss": 2.5719, + "theoretical_loss": 3.414964066702805, + "tokens_seen": 2111700992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036630303030303033, + "loss": 2.6716, + "theoretical_loss": 3.4149553244263444, + "tokens_seen": 2111766528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662828282828283, + "loss": 2.6835, + "theoretical_loss": 3.414946582497148, + "tokens_seen": 2111832064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662626262626263, + "loss": 2.7341, + "theoretical_loss": 3.4149378409151905, + "tokens_seen": 2111897600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036624242424242425, + "loss": 2.7706, + "theoretical_loss": 3.4149290996804487, + "tokens_seen": 2111963136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662222222222222, + "loss": 2.8759, + "theoretical_loss": 3.414920358792897, + "tokens_seen": 2112028672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662020202020202, + "loss": 2.6096, + "theoretical_loss": 3.414911618252511, + "tokens_seen": 2112094208 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1164108, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.35992169380188, + "objective/train/theoretical_loss": 3.4149028780592667, + "objective/train/tokens_used": 471018976, + "theoretical_loss": 3.4149028780592667, + "tokens_seen": 2112159744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661818181818182, + "loss": 2.5137, + "theoretical_loss": 3.4149028780592667, + "tokens_seen": 2112159744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661616161616162, + "loss": 2.7378, + "theoretical_loss": 3.4148941382131386, + "tokens_seen": 2112225280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036614141414141415, + "loss": 2.695, + "theoretical_loss": 3.414885398714103, + "tokens_seen": 2112290816 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036612121212121216, + "loss": 2.3199, + "theoretical_loss": 3.414876659562135, + "tokens_seen": 2112356352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661010101010101, + "loss": 2.6186, + "theoretical_loss": 3.41486792075721, + "tokens_seen": 2112421888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036608080808080807, + "loss": 2.558, + "theoretical_loss": 3.4148591822993035, + "tokens_seen": 2112487424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660606060606061, + "loss": 2.4394, + "theoretical_loss": 3.4148504441883913, + "tokens_seen": 2112552960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036604040404040404, + "loss": 2.7204, + "theoretical_loss": 3.4148417064244487, + "tokens_seen": 2112618496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036602020202020205, + "loss": 2.5691, + "theoretical_loss": 3.414832969007451, + "tokens_seen": 2112684032 + }, + { + "epoch": 0.28, + "learning_rate": 0.000366, + "loss": 2.4832, + "theoretical_loss": 3.4148242319373727, + "tokens_seen": 2112749568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036597979797979796, + "loss": 2.6831, + "theoretical_loss": 3.414815495214191, + "tokens_seen": 2112815104 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365959595959596, + "loss": 2.7641, + "theoretical_loss": 3.41480675883788, + "tokens_seen": 2112880640 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365939393939394, + "loss": 2.7619, + "theoretical_loss": 3.4147980228084167, + "tokens_seen": 2112946176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036591919191919194, + "loss": 2.6204, + "theoretical_loss": 3.4147892871257746, + "tokens_seen": 2113011712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658989898989899, + "loss": 2.7348, + "theoretical_loss": 3.414780551789931, + "tokens_seen": 2113077248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036587878787878786, + "loss": 2.4798, + "theoretical_loss": 3.4147718168008607, + "tokens_seen": 2113142784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036585858585858587, + "loss": 2.4059, + "theoretical_loss": 3.4147630821585384, + "tokens_seen": 2113208320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658383838383839, + "loss": 2.6539, + "theoretical_loss": 3.4147543478629405, + "tokens_seen": 2113273856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036581818181818183, + "loss": 2.4668, + "theoretical_loss": 3.414745613914042, + "tokens_seen": 2113339392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657979797979798, + "loss": 2.6112, + "theoretical_loss": 3.4147368803118194, + "tokens_seen": 2113404928 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036577777777777775, + "loss": 2.7575, + "theoretical_loss": 3.4147281470562465, + "tokens_seen": 2113470464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657575757575758, + "loss": 2.5115, + "theoretical_loss": 3.4147194141473003, + "tokens_seen": 2113536000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036573737373737377, + "loss": 2.756, + "theoretical_loss": 3.414710681584956, + "tokens_seen": 2113601536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036571717171717173, + "loss": 2.88, + "theoretical_loss": 3.4147019493691877, + "tokens_seen": 2113667072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656969696969697, + "loss": 2.5343, + "theoretical_loss": 3.414693217499973, + "tokens_seen": 2113732608 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1165368, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.723964214324951, + "objective/train/theoretical_loss": 3.414684485977286, + "objective/train/tokens_used": 472657376, + "theoretical_loss": 3.414684485977286, + "tokens_seen": 2113798144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036567676767676764, + "loss": 2.5892, + "theoretical_loss": 3.414684485977286, + "tokens_seen": 2113798144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656565656565657, + "loss": 2.8461, + "theoretical_loss": 3.4146757548011024, + "tokens_seen": 2113863680 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036563636363636366, + "loss": 2.599, + "theoretical_loss": 3.4146670239713983, + "tokens_seen": 2113929216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656161616161616, + "loss": 2.7025, + "theoretical_loss": 3.4146582934881486, + "tokens_seen": 2113994752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655959595959596, + "loss": 2.6218, + "theoretical_loss": 3.414649563351329, + "tokens_seen": 2114060288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655757575757576, + "loss": 2.7486, + "theoretical_loss": 3.4146408335609157, + "tokens_seen": 2114125824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655555555555556, + "loss": 2.6214, + "theoretical_loss": 3.414632104116883, + "tokens_seen": 2114191360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036553535353535356, + "loss": 2.576, + "theoretical_loss": 3.4146233750192065, + "tokens_seen": 2114256896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655151515151515, + "loss": 2.5162, + "theoretical_loss": 3.4146146462678626, + "tokens_seen": 2114322432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036549494949494947, + "loss": 2.4917, + "theoretical_loss": 3.4146059178628265, + "tokens_seen": 2114387968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654747474747475, + "loss": 2.6346, + "theoretical_loss": 3.414597189804074, + "tokens_seen": 2114453504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654545454545455, + "loss": 2.5658, + "theoretical_loss": 3.4145884620915803, + "tokens_seen": 2114519040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036543434343434345, + "loss": 2.7962, + "theoretical_loss": 3.4145797347253204, + "tokens_seen": 2114584576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654141414141414, + "loss": 2.6246, + "theoretical_loss": 3.4145710077052707, + "tokens_seen": 2114650112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653939393939394, + "loss": 2.6092, + "theoretical_loss": 3.4145622810314062, + "tokens_seen": 2114715648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653737373737374, + "loss": 2.584, + "theoretical_loss": 3.4145535547037023, + "tokens_seen": 2114781184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653535353535354, + "loss": 2.6047, + "theoretical_loss": 3.4145448287221356, + "tokens_seen": 2114846720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036533333333333334, + "loss": 2.5933, + "theoretical_loss": 3.4145361030866805, + "tokens_seen": 2114912256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653131313131313, + "loss": 2.4416, + "theoretical_loss": 3.414527377797313, + "tokens_seen": 2114977792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652929292929293, + "loss": 2.6182, + "theoretical_loss": 3.414518652854009, + "tokens_seen": 2115043328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036527272727272727, + "loss": 2.7091, + "theoretical_loss": 3.4145099282567433, + "tokens_seen": 2115108864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652525252525253, + "loss": 2.5143, + "theoretical_loss": 3.414501204005492, + "tokens_seen": 2115174400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036523232323232323, + "loss": 2.54, + "theoretical_loss": 3.41449248010023, + "tokens_seen": 2115239936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036521212121212124, + "loss": 2.72, + "theoretical_loss": 3.4144837565409336, + "tokens_seen": 2115305472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651919191919192, + "loss": 2.6686, + "theoretical_loss": 3.4144750333275784, + "tokens_seen": 2115371008 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1165914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.618112087249756, + "objective/train/theoretical_loss": 3.4144663104601394, + "objective/train/tokens_used": 474295776, + "theoretical_loss": 3.4144663104601394, + "tokens_seen": 2115436544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036517171717171716, + "loss": 2.641, + "theoretical_loss": 3.4144663104601394, + "tokens_seen": 2115436544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036515151515151517, + "loss": 2.5092, + "theoretical_loss": 3.4144575879385926, + "tokens_seen": 2115502080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651313131313131, + "loss": 2.537, + "theoretical_loss": 3.414448865762913, + "tokens_seen": 2115567616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036511111111111114, + "loss": 2.8961, + "theoretical_loss": 3.414440143933077, + "tokens_seen": 2115633152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650909090909091, + "loss": 2.5186, + "theoretical_loss": 3.4144314224490597, + "tokens_seen": 2115698688 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036507070707070705, + "loss": 2.5034, + "theoretical_loss": 3.414422701310837, + "tokens_seen": 2115764224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036505050505050506, + "loss": 2.5035, + "theoretical_loss": 3.4144139805183835, + "tokens_seen": 2115829760 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365030303030303, + "loss": 2.7573, + "theoretical_loss": 3.4144052600716757, + "tokens_seen": 2115895296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036501010101010103, + "loss": 2.6071, + "theoretical_loss": 3.4143965399706895, + "tokens_seen": 2115960832 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364989898989899, + "loss": 2.6751, + "theoretical_loss": 3.414387820215399, + "tokens_seen": 2116026368 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036496969696969694, + "loss": 2.771, + "theoretical_loss": 3.4143791008057818, + "tokens_seen": 2116091904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036494949494949496, + "loss": 2.4936, + "theoretical_loss": 3.4143703817418123, + "tokens_seen": 2116157440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036492929292929297, + "loss": 2.551, + "theoretical_loss": 3.4143616630234654, + "tokens_seen": 2116222976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649090909090909, + "loss": 2.6326, + "theoretical_loss": 3.4143529446507186, + "tokens_seen": 2116288512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648888888888889, + "loss": 2.6491, + "theoretical_loss": 3.4143442266235455, + "tokens_seen": 2116354048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036486868686868684, + "loss": 2.6087, + "theoretical_loss": 3.4143355089419227, + "tokens_seen": 2116419584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036484848484848485, + "loss": 2.692, + "theoretical_loss": 3.414326791605826, + "tokens_seen": 2116485120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036482828282828286, + "loss": 2.2571, + "theoretical_loss": 3.414318074615231, + "tokens_seen": 2116550656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648080808080808, + "loss": 2.6154, + "theoretical_loss": 3.414309357970113, + "tokens_seen": 2116616192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036478787878787877, + "loss": 2.5167, + "theoretical_loss": 3.4143006416704473, + "tokens_seen": 2116681728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036476767676767673, + "loss": 2.6605, + "theoretical_loss": 3.41429192571621, + "tokens_seen": 2116747264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647474747474748, + "loss": 2.6692, + "theoretical_loss": 3.414283210107377, + "tokens_seen": 2116812800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036472727272727275, + "loss": 2.573, + "theoretical_loss": 3.414274494843923, + "tokens_seen": 2116878336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647070707070707, + "loss": 2.8211, + "theoretical_loss": 3.4142657799258243, + "tokens_seen": 2116943872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036468686868686867, + "loss": 2.7455, + "theoretical_loss": 3.4142570653530564, + "tokens_seen": 2117009408 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1167104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4354066848754883, + "objective/train/theoretical_loss": 3.414248351125595, + "objective/train/tokens_used": 475934176, + "theoretical_loss": 3.414248351125595, + "tokens_seen": 2117074944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646666666666666, + "loss": 2.5663, + "theoretical_loss": 3.414248351125595, + "tokens_seen": 2117074944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646464646464647, + "loss": 2.4757, + "theoretical_loss": 3.414239637243415, + "tokens_seen": 2117140480 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036462626262626264, + "loss": 2.6941, + "theoretical_loss": 3.4142309237064934, + "tokens_seen": 2117206016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646060606060606, + "loss": 2.6617, + "theoretical_loss": 3.414222210514805, + "tokens_seen": 2117271552 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036458585858585856, + "loss": 2.5227, + "theoretical_loss": 3.4142134976683254, + "tokens_seen": 2117337088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645656565656566, + "loss": 2.5767, + "theoretical_loss": 3.41420478516703, + "tokens_seen": 2117402624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645454545454546, + "loss": 2.6952, + "theoretical_loss": 3.414196073010895, + "tokens_seen": 2117468160 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036452525252525254, + "loss": 2.3163, + "theoretical_loss": 3.414187361199896, + "tokens_seen": 2117533696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645050505050505, + "loss": 2.5076, + "theoretical_loss": 3.414178649734008, + "tokens_seen": 2117599232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036448484848484845, + "loss": 2.5644, + "theoretical_loss": 3.414169938613208, + "tokens_seen": 2117664768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644646464646465, + "loss": 2.6019, + "theoretical_loss": 3.41416122783747, + "tokens_seen": 2117730304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036444444444444447, + "loss": 2.6005, + "theoretical_loss": 3.4141525174067704, + "tokens_seen": 2117795840 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036442424242424243, + "loss": 2.6914, + "theoretical_loss": 3.4141438073210857, + "tokens_seen": 2117861376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644040404040404, + "loss": 2.4907, + "theoretical_loss": 3.41413509758039, + "tokens_seen": 2117926912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643838383838384, + "loss": 2.7162, + "theoretical_loss": 3.41412638818466, + "tokens_seen": 2117992448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643636363636364, + "loss": 2.4166, + "theoretical_loss": 3.414117679133871, + "tokens_seen": 2118057984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036434343434343437, + "loss": 2.7053, + "theoretical_loss": 3.414108970427999, + "tokens_seen": 2118123520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643232323232323, + "loss": 2.5607, + "theoretical_loss": 3.414100262067019, + "tokens_seen": 2118189056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643030303030303, + "loss": 2.5588, + "theoretical_loss": 3.4140915540509074, + "tokens_seen": 2118254592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642828282828283, + "loss": 2.6071, + "theoretical_loss": 3.4140828463796398, + "tokens_seen": 2118320128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642626262626263, + "loss": 2.4847, + "theoretical_loss": 3.4140741390531915, + "tokens_seen": 2118385664 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036424242424242426, + "loss": 2.636, + "theoretical_loss": 3.4140654320715376, + "tokens_seen": 2118451200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642222222222222, + "loss": 2.6212, + "theoretical_loss": 3.414056725434655, + "tokens_seen": 2118516736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642020202020202, + "loss": 2.7221, + "theoretical_loss": 3.414048019142519, + "tokens_seen": 2118582272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641818181818182, + "loss": 2.551, + "theoretical_loss": 3.4140393131951052, + "tokens_seen": 2118647808 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1168338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4486634731292725, + "objective/train/theoretical_loss": 3.414030607592389, + "objective/train/tokens_used": 477572576, + "theoretical_loss": 3.414030607592389, + "tokens_seen": 2118713344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641616161616162, + "loss": 2.5584, + "theoretical_loss": 3.414030607592389, + "tokens_seen": 2118713344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036414141414141415, + "loss": 2.6926, + "theoretical_loss": 3.4140219023343468, + "tokens_seen": 2118778880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641212121212121, + "loss": 2.5152, + "theoretical_loss": 3.414013197420953, + "tokens_seen": 2118844416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641010101010101, + "loss": 2.7168, + "theoretical_loss": 3.414004492852185, + "tokens_seen": 2118909952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640808080808081, + "loss": 2.3342, + "theoretical_loss": 3.413995788628017, + "tokens_seen": 2118975488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640606060606061, + "loss": 2.6007, + "theoretical_loss": 3.4139870847484257, + "tokens_seen": 2119041024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036404040404040404, + "loss": 2.5816, + "theoretical_loss": 3.4139783812133864, + "tokens_seen": 2119106560 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036402020202020205, + "loss": 2.4268, + "theoretical_loss": 3.4139696780228745, + "tokens_seen": 2119172096 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364, + "loss": 2.7488, + "theoretical_loss": 3.4139609751768663, + "tokens_seen": 2119237632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036397979797979797, + "loss": 2.7172, + "theoretical_loss": 3.413952272675337, + "tokens_seen": 2119303168 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363959595959596, + "loss": 2.4971, + "theoretical_loss": 3.413943570518263, + "tokens_seen": 2119368704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036393939393939394, + "loss": 2.6373, + "theoretical_loss": 3.4139348687056192, + "tokens_seen": 2119434240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036391919191919195, + "loss": 2.6311, + "theoretical_loss": 3.413926167237382, + "tokens_seen": 2119499776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638989898989899, + "loss": 2.621, + "theoretical_loss": 3.4139174661135265, + "tokens_seen": 2119565312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036387878787878786, + "loss": 2.439, + "theoretical_loss": 3.413908765334029, + "tokens_seen": 2119630848 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036385858585858587, + "loss": 2.5993, + "theoretical_loss": 3.413900064898865, + "tokens_seen": 2119696384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638383838383839, + "loss": 2.5193, + "theoretical_loss": 3.41389136480801, + "tokens_seen": 2119761920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036381818181818184, + "loss": 2.5325, + "theoretical_loss": 3.4138826650614402, + "tokens_seen": 2119827456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637979797979798, + "loss": 2.5291, + "theoretical_loss": 3.413873965659131, + "tokens_seen": 2119892992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036377777777777775, + "loss": 2.6428, + "theoretical_loss": 3.413865266601058, + "tokens_seen": 2119958528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036375757575757576, + "loss": 2.7483, + "theoretical_loss": 3.4138565678871973, + "tokens_seen": 2120024064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637373737373738, + "loss": 2.5676, + "theoretical_loss": 3.4138478695175243, + "tokens_seen": 2120089600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036371717171717173, + "loss": 2.5161, + "theoretical_loss": 3.4138391714920147, + "tokens_seen": 2120155136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636969696969697, + "loss": 2.5857, + "theoretical_loss": 3.413830473810645, + "tokens_seen": 2120220672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036367676767676765, + "loss": 2.5532, + "theoretical_loss": 3.41382177647339, + "tokens_seen": 2120286208 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1168872, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.698695182800293, + "objective/train/theoretical_loss": 3.413813079480226, + "objective/train/tokens_used": 479210976, + "theoretical_loss": 3.413813079480226, + "tokens_seen": 2120351744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636565656565657, + "loss": 2.5234, + "theoretical_loss": 3.413813079480226, + "tokens_seen": 2120351744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036363636363636367, + "loss": 2.4915, + "theoretical_loss": 3.413804382831129, + "tokens_seen": 2120417280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636161616161616, + "loss": 2.7313, + "theoretical_loss": 3.4137956865260737, + "tokens_seen": 2120482816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635959595959596, + "loss": 2.7314, + "theoretical_loss": 3.413786990565037, + "tokens_seen": 2120548352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036357575757575754, + "loss": 2.8578, + "theoretical_loss": 3.413778294947994, + "tokens_seen": 2120613888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635555555555556, + "loss": 2.6198, + "theoretical_loss": 3.4137695996749207, + "tokens_seen": 2120679424 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036353535353535356, + "loss": 2.5495, + "theoretical_loss": 3.413760904745793, + "tokens_seen": 2120744960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635151515151515, + "loss": 2.5978, + "theoretical_loss": 3.413752210160586, + "tokens_seen": 2120810496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634949494949495, + "loss": 2.6271, + "theoretical_loss": 3.413743515919276, + "tokens_seen": 2120876032 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036347474747474743, + "loss": 2.5787, + "theoretical_loss": 3.413734822021839, + "tokens_seen": 2120941568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634545454545455, + "loss": 2.6924, + "theoretical_loss": 3.4137261284682503, + "tokens_seen": 2121007104 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036343434343434345, + "loss": 2.723, + "theoretical_loss": 3.413717435258486, + "tokens_seen": 2121072640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634141414141414, + "loss": 2.3627, + "theoretical_loss": 3.4137087423925214, + "tokens_seen": 2121138176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036339393939393937, + "loss": 2.3717, + "theoretical_loss": 3.413700049870333, + "tokens_seen": 2121203712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633737373737374, + "loss": 2.6768, + "theoretical_loss": 3.413691357691896, + "tokens_seen": 2121269248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633535353535354, + "loss": 2.5494, + "theoretical_loss": 3.413682665857187, + "tokens_seen": 2121334784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036333333333333335, + "loss": 2.5297, + "theoretical_loss": 3.4136739743661804, + "tokens_seen": 2121400320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633131313131313, + "loss": 2.4224, + "theoretical_loss": 3.4136652832188528, + "tokens_seen": 2121465856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036329292929292926, + "loss": 2.5031, + "theoretical_loss": 3.4136565924151805, + "tokens_seen": 2121531392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036327272727272727, + "loss": 2.5477, + "theoretical_loss": 3.4136479019551382, + "tokens_seen": 2121596928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632525252525253, + "loss": 2.661, + "theoretical_loss": 3.413639211838703, + "tokens_seen": 2121662464 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036323232323232324, + "loss": 2.8096, + "theoretical_loss": 3.4136305220658496, + "tokens_seen": 2121728000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632121212121212, + "loss": 2.763, + "theoretical_loss": 3.413621832636554, + "tokens_seen": 2121793536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631919191919192, + "loss": 2.5383, + "theoretical_loss": 3.4136131435507924, + "tokens_seen": 2121859072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036317171717171716, + "loss": 2.386, + "theoretical_loss": 3.4136044548085405, + "tokens_seen": 2121924608 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1169826, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2446186542510986, + "objective/train/theoretical_loss": 3.413595766409774, + "objective/train/tokens_used": 480849376, + "theoretical_loss": 3.413595766409774, + "tokens_seen": 2121990144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631515151515152, + "loss": 2.4437, + "theoretical_loss": 3.413595766409774, + "tokens_seen": 2121990144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036313131313131313, + "loss": 2.7176, + "theoretical_loss": 3.413587078354468, + "tokens_seen": 2122055680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631111111111111, + "loss": 2.379, + "theoretical_loss": 3.4135783906425994, + "tokens_seen": 2122121216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630909090909091, + "loss": 2.5955, + "theoretical_loss": 3.4135697032741437, + "tokens_seen": 2122186752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036307070707070706, + "loss": 2.6066, + "theoretical_loss": 3.4135610162490773, + "tokens_seen": 2122252288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036305050505050507, + "loss": 2.5977, + "theoretical_loss": 3.4135523295673744, + "tokens_seen": 2122317824 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363030303030303, + "loss": 2.5323, + "theoretical_loss": 3.4135436432290125, + "tokens_seen": 2122383360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036301010101010104, + "loss": 2.557, + "theoretical_loss": 3.413534957233966, + "tokens_seen": 2122448896 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362989898989899, + "loss": 2.6608, + "theoretical_loss": 3.4135262715822123, + "tokens_seen": 2122514432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036296969696969695, + "loss": 2.428, + "theoretical_loss": 3.413517586273726, + "tokens_seen": 2122579968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036294949494949496, + "loss": 2.7387, + "theoretical_loss": 3.4135089013084836, + "tokens_seen": 2122645504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629292929292929, + "loss": 2.6331, + "theoretical_loss": 3.4135002166864608, + "tokens_seen": 2122711040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036290909090909093, + "loss": 2.6378, + "theoretical_loss": 3.413491532407633, + "tokens_seen": 2122776576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628888888888889, + "loss": 2.6081, + "theoretical_loss": 3.4134828484719764, + "tokens_seen": 2122842112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628686868686869, + "loss": 2.6682, + "theoretical_loss": 3.4134741648794664, + "tokens_seen": 2122907648 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036284848484848485, + "loss": 2.5269, + "theoretical_loss": 3.4134654816300802, + "tokens_seen": 2122973184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036282828282828286, + "loss": 2.6664, + "theoretical_loss": 3.413456798723792, + "tokens_seen": 2123038720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628080808080808, + "loss": 2.7007, + "theoretical_loss": 3.4134481161605787, + "tokens_seen": 2123104256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627878787878788, + "loss": 2.5572, + "theoretical_loss": 3.4134394339404155, + "tokens_seen": 2123169792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627676767676768, + "loss": 2.7016, + "theoretical_loss": 3.4134307520632787, + "tokens_seen": 2123235328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036274747474747475, + "loss": 2.5064, + "theoretical_loss": 3.413422070529144, + "tokens_seen": 2123300864 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036272727272727276, + "loss": 2.6765, + "theoretical_loss": 3.413413389337988, + "tokens_seen": 2123366400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627070707070707, + "loss": 2.5349, + "theoretical_loss": 3.413404708489785, + "tokens_seen": 2123431936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036268686868686867, + "loss": 2.7782, + "theoretical_loss": 3.4133960279845117, + "tokens_seen": 2123497472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626666666666667, + "loss": 2.6642, + "theoretical_loss": 3.4133873478221446, + "tokens_seen": 2123563008 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1170397, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.436032295227051, + "objective/train/theoretical_loss": 3.4133786680026583, + "objective/train/tokens_used": 482487776, + "theoretical_loss": 3.4133786680026583, + "tokens_seen": 2123628544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626464646464647, + "loss": 2.5855, + "theoretical_loss": 3.4133786680026583, + "tokens_seen": 2123628544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036262626262626265, + "loss": 2.5702, + "theoretical_loss": 3.41336998852603, + "tokens_seen": 2123694080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626060606060606, + "loss": 2.4347, + "theoretical_loss": 3.4133613093922346, + "tokens_seen": 2123759616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036258585858585856, + "loss": 2.8192, + "theoretical_loss": 3.4133526306012483, + "tokens_seen": 2123825152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625656565656566, + "loss": 2.6402, + "theoretical_loss": 3.413343952153047, + "tokens_seen": 2123890688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625454545454546, + "loss": 2.6956, + "theoretical_loss": 3.4133352740476064, + "tokens_seen": 2123956224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036252525252525254, + "loss": 2.2476, + "theoretical_loss": 3.413326596284903, + "tokens_seen": 2124021760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625050505050505, + "loss": 2.7057, + "theoretical_loss": 3.413317918864912, + "tokens_seen": 2124087296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036248484848484846, + "loss": 2.4141, + "theoretical_loss": 3.4133092417876094, + "tokens_seen": 2124152832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624646464646465, + "loss": 2.5951, + "theoretical_loss": 3.4133005650529715, + "tokens_seen": 2124218368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624444444444445, + "loss": 2.6807, + "theoretical_loss": 3.4132918886609733, + "tokens_seen": 2124283904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036242424242424244, + "loss": 2.5597, + "theoretical_loss": 3.413283212611592, + "tokens_seen": 2124349440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624040404040404, + "loss": 2.4826, + "theoretical_loss": 3.4132745369048028, + "tokens_seen": 2124414976 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036238383838383835, + "loss": 2.698, + "theoretical_loss": 3.413265861540582, + "tokens_seen": 2124480512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623636363636364, + "loss": 2.5928, + "theoretical_loss": 3.413257186518904, + "tokens_seen": 2124546048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036234343434343437, + "loss": 2.7756, + "theoretical_loss": 3.413248511839747, + "tokens_seen": 2124611584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036232323232323233, + "loss": 2.6898, + "theoretical_loss": 3.413239837503085, + "tokens_seen": 2124677120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623030303030303, + "loss": 2.4804, + "theoretical_loss": 3.413231163508895, + "tokens_seen": 2124742656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622828282828283, + "loss": 2.5586, + "theoretical_loss": 3.4132224898571524, + "tokens_seen": 2124808192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622626262626263, + "loss": 2.4899, + "theoretical_loss": 3.4132138165478336, + "tokens_seen": 2124873728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036224242424242426, + "loss": 2.7261, + "theoretical_loss": 3.413205143580914, + "tokens_seen": 2124939264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622222222222222, + "loss": 2.483, + "theoretical_loss": 3.41319647095637, + "tokens_seen": 2125004800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622020202020202, + "loss": 2.6043, + "theoretical_loss": 3.413187798674177, + "tokens_seen": 2125070336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621818181818182, + "loss": 2.6508, + "theoretical_loss": 3.4131791267343115, + "tokens_seen": 2125135872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621616161616162, + "loss": 2.4916, + "theoretical_loss": 3.4131704551367488, + "tokens_seen": 2125201408 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1171543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5023303031921387, + "objective/train/theoretical_loss": 3.413161783881465, + "objective/train/tokens_used": 484126176, + "theoretical_loss": 3.413161783881465, + "tokens_seen": 2125266944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036214141414141416, + "loss": 2.5373, + "theoretical_loss": 3.413161783881465, + "tokens_seen": 2125266944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621212121212121, + "loss": 2.7232, + "theoretical_loss": 3.4131531129684367, + "tokens_seen": 2125332480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621010101010101, + "loss": 2.3724, + "theoretical_loss": 3.4131444423976394, + "tokens_seen": 2125398016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620808080808081, + "loss": 2.594, + "theoretical_loss": 3.4131357721690487, + "tokens_seen": 2125463552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620606060606061, + "loss": 2.4723, + "theoretical_loss": 3.4131271022826413, + "tokens_seen": 2125529088 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036204040404040405, + "loss": 2.6411, + "theoretical_loss": 3.4131184327383917, + "tokens_seen": 2125594624 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362020202020202, + "loss": 2.4801, + "theoretical_loss": 3.4131097635362777, + "tokens_seen": 2125660160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362, + "loss": 2.819, + "theoretical_loss": 3.413101094676274, + "tokens_seen": 2125725696 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361979797979798, + "loss": 2.5704, + "theoretical_loss": 3.4130924261583573, + "tokens_seen": 2125791232 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361959595959596, + "loss": 2.5092, + "theoretical_loss": 3.413083757982503, + "tokens_seen": 2125856768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036193939393939394, + "loss": 2.5521, + "theoretical_loss": 3.413075090148687, + "tokens_seen": 2125922304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619191919191919, + "loss": 2.7827, + "theoretical_loss": 3.413066422656885, + "tokens_seen": 2125987840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618989898989899, + "loss": 2.3831, + "theoretical_loss": 3.413057755507075, + "tokens_seen": 2126053376 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036187878787878787, + "loss": 2.6201, + "theoretical_loss": 3.41304908869923, + "tokens_seen": 2126118912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618585858585859, + "loss": 2.576, + "theoretical_loss": 3.413040422233328, + "tokens_seen": 2126184448 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036183838383838383, + "loss": 2.7195, + "theoretical_loss": 3.413031756109344, + "tokens_seen": 2126249984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036181818181818185, + "loss": 2.4798, + "theoretical_loss": 3.413023090327255, + "tokens_seen": 2126315520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617979797979798, + "loss": 2.6043, + "theoretical_loss": 3.413014424887036, + "tokens_seen": 2126381056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036177777777777776, + "loss": 2.5492, + "theoretical_loss": 3.413005759788663, + "tokens_seen": 2126446592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036175757575757577, + "loss": 2.4745, + "theoretical_loss": 3.4129970950321127, + "tokens_seen": 2126512128 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036173737373737373, + "loss": 2.5508, + "theoretical_loss": 3.412988430617361, + "tokens_seen": 2126577664 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036171717171717174, + "loss": 2.4879, + "theoretical_loss": 3.4129797665443826, + "tokens_seen": 2126643200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616969696969697, + "loss": 2.5086, + "theoretical_loss": 3.4129711028131546, + "tokens_seen": 2126708736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036167676767676765, + "loss": 2.6245, + "theoretical_loss": 3.4129624394236533, + "tokens_seen": 2126774272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036165656565656566, + "loss": 2.6815, + "theoretical_loss": 3.4129537763758537, + "tokens_seen": 2126839808 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1172068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6468193531036377, + "objective/train/theoretical_loss": 3.412945113669733, + "objective/train/tokens_used": 485764576, + "theoretical_loss": 3.412945113669733, + "tokens_seen": 2126905344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616363636363637, + "loss": 2.75, + "theoretical_loss": 3.412945113669733, + "tokens_seen": 2126905344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036161616161616163, + "loss": 2.6632, + "theoretical_loss": 3.4129364513052654, + "tokens_seen": 2126970880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615959595959596, + "loss": 2.6412, + "theoretical_loss": 3.4129277892824286, + "tokens_seen": 2127036416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036157575757575754, + "loss": 2.6567, + "theoretical_loss": 3.4129191276011985, + "tokens_seen": 2127101952 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036155555555555556, + "loss": 2.5087, + "theoretical_loss": 3.41291046626155, + "tokens_seen": 2127167488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036153535353535357, + "loss": 2.4819, + "theoretical_loss": 3.4129018052634597, + "tokens_seen": 2127233024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615151515151515, + "loss": 2.6294, + "theoretical_loss": 3.4128931446069037, + "tokens_seen": 2127298560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614949494949495, + "loss": 2.5302, + "theoretical_loss": 3.412884484291858, + "tokens_seen": 2127364096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036147474747474744, + "loss": 2.6261, + "theoretical_loss": 3.4128758243182986, + "tokens_seen": 2127429632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614545454545455, + "loss": 2.4306, + "theoretical_loss": 3.412867164686201, + "tokens_seen": 2127495168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036143434343434346, + "loss": 2.6665, + "theoretical_loss": 3.4128585053955423, + "tokens_seen": 2127560704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614141414141414, + "loss": 2.6082, + "theoretical_loss": 3.4128498464462975, + "tokens_seen": 2127626240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613939393939394, + "loss": 2.5125, + "theoretical_loss": 3.412841187838443, + "tokens_seen": 2127691776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036137373737373733, + "loss": 2.8392, + "theoretical_loss": 3.412832529571955, + "tokens_seen": 2127757312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613535353535354, + "loss": 2.4665, + "theoretical_loss": 3.4128238716468093, + "tokens_seen": 2127822848 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036133333333333335, + "loss": 2.4348, + "theoretical_loss": 3.4128152140629817, + "tokens_seen": 2127888384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613131313131313, + "loss": 2.5413, + "theoretical_loss": 3.412806556820449, + "tokens_seen": 2127953920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036129292929292927, + "loss": 2.6776, + "theoretical_loss": 3.4127978999191866, + "tokens_seen": 2128019456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612727272727273, + "loss": 2.6288, + "theoretical_loss": 3.412789243359171, + "tokens_seen": 2128084992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612525252525253, + "loss": 2.6286, + "theoretical_loss": 3.412780587140378, + "tokens_seen": 2128150528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036123232323232324, + "loss": 2.4641, + "theoretical_loss": 3.4127719312627827, + "tokens_seen": 2128216064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612121212121212, + "loss": 2.5957, + "theoretical_loss": 3.4127632757263626, + "tokens_seen": 2128281600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036119191919191916, + "loss": 2.7097, + "theoretical_loss": 3.4127546205310932, + "tokens_seen": 2128347136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611717171717172, + "loss": 2.5988, + "theoretical_loss": 3.4127459656769505, + "tokens_seen": 2128412672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611515151515152, + "loss": 2.4821, + "theoretical_loss": 3.4127373111639105, + "tokens_seen": 2128478208 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1173260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5667147636413574, + "objective/train/theoretical_loss": 3.4127286569919493, + "objective/train/tokens_used": 487402976, + "theoretical_loss": 3.4127286569919493, + "tokens_seen": 2128543744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036113131313131314, + "loss": 2.6403, + "theoretical_loss": 3.4127286569919493, + "tokens_seen": 2128543744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611111111111111, + "loss": 2.5117, + "theoretical_loss": 3.412720003161043, + "tokens_seen": 2128609280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610909090909091, + "loss": 2.5641, + "theoretical_loss": 3.412711349671168, + "tokens_seen": 2128674816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610707070707071, + "loss": 2.5267, + "theoretical_loss": 3.4127026965222997, + "tokens_seen": 2128740352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610505050505051, + "loss": 2.7131, + "theoretical_loss": 3.4126940437144144, + "tokens_seen": 2128805888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036103030303030303, + "loss": 2.558, + "theoretical_loss": 3.4126853912474884, + "tokens_seen": 2128871424 + }, + { + "epoch": 0.29, + "learning_rate": 0.000361010101010101, + "loss": 2.5396, + "theoretical_loss": 3.4126767391214976, + "tokens_seen": 2128936960 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360989898989899, + "loss": 2.6273, + "theoretical_loss": 3.412668087336418, + "tokens_seen": 2129002496 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360969696969697, + "loss": 2.7662, + "theoretical_loss": 3.412659435892226, + "tokens_seen": 2129068032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036094949494949497, + "loss": 2.6133, + "theoretical_loss": 3.412650784788897, + "tokens_seen": 2129133568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609292929292929, + "loss": 2.8016, + "theoretical_loss": 3.412642134026408, + "tokens_seen": 2129199104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036090909090909093, + "loss": 2.5226, + "theoretical_loss": 3.4126334836047345, + "tokens_seen": 2129264640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608888888888889, + "loss": 2.4984, + "theoretical_loss": 3.4126248335238527, + "tokens_seen": 2129330176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608686868686869, + "loss": 2.6015, + "theoretical_loss": 3.4126161837837383, + "tokens_seen": 2129395712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036084848484848486, + "loss": 2.6194, + "theoretical_loss": 3.412607534384368, + "tokens_seen": 2129461248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608282828282828, + "loss": 2.5166, + "theoretical_loss": 3.4125988853257176, + "tokens_seen": 2129526784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608080808080808, + "loss": 2.4291, + "theoretical_loss": 3.4125902366077634, + "tokens_seen": 2129592320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607878787878788, + "loss": 2.5511, + "theoretical_loss": 3.4125815882304806, + "tokens_seen": 2129657856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607676767676768, + "loss": 2.5663, + "theoretical_loss": 3.412572940193847, + "tokens_seen": 2129723392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036074747474747475, + "loss": 2.7557, + "theoretical_loss": 3.412564292497837, + "tokens_seen": 2129788928 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036072727272727276, + "loss": 2.5695, + "theoretical_loss": 3.412555645142428, + "tokens_seen": 2129854464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607070707070707, + "loss": 2.5838, + "theoretical_loss": 3.4125469981275955, + "tokens_seen": 2129920000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606868686868687, + "loss": 2.6765, + "theoretical_loss": 3.4125383514533154, + "tokens_seen": 2129985536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606666666666667, + "loss": 2.641, + "theoretical_loss": 3.412529705119564, + "tokens_seen": 2130051072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036064646464646464, + "loss": 2.5517, + "theoretical_loss": 3.4125210591263175, + "tokens_seen": 2130116608 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1174166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.371098041534424, + "objective/train/theoretical_loss": 3.4125124134735523, + "objective/train/tokens_used": 489041376, + "theoretical_loss": 3.4125124134735523, + "tokens_seen": 2130182144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036062626262626266, + "loss": 2.5198, + "theoretical_loss": 3.4125124134735523, + "tokens_seen": 2130182144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606060606060606, + "loss": 2.5662, + "theoretical_loss": 3.412503768161244, + "tokens_seen": 2130247680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036058585858585857, + "loss": 2.6296, + "theoretical_loss": 3.4124951231893688, + "tokens_seen": 2130313216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605656565656566, + "loss": 2.5507, + "theoretical_loss": 3.412486478557903, + "tokens_seen": 2130378752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605454545454546, + "loss": 2.799, + "theoretical_loss": 3.4124778342668227, + "tokens_seen": 2130444288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036052525252525255, + "loss": 2.7239, + "theoretical_loss": 3.412469190316104, + "tokens_seen": 2130509824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605050505050505, + "loss": 2.7608, + "theoretical_loss": 3.412460546705723, + "tokens_seen": 2130575360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036048484848484846, + "loss": 2.6064, + "theoretical_loss": 3.412451903435656, + "tokens_seen": 2130640896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036046464646464647, + "loss": 2.5552, + "theoretical_loss": 3.4124432605058788, + "tokens_seen": 2130706432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604444444444445, + "loss": 2.9222, + "theoretical_loss": 3.412434617916368, + "tokens_seen": 2130771968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036042424242424244, + "loss": 2.6447, + "theoretical_loss": 3.412425975667099, + "tokens_seen": 2130837504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604040404040404, + "loss": 2.6339, + "theoretical_loss": 3.4124173337580492, + "tokens_seen": 2130903040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036038383838383835, + "loss": 2.6131, + "theoretical_loss": 3.412408692189193, + "tokens_seen": 2130968576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603636363636364, + "loss": 2.716, + "theoretical_loss": 3.412400050960508, + "tokens_seen": 2131034112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603434343434344, + "loss": 2.7042, + "theoretical_loss": 3.4123914100719697, + "tokens_seen": 2131099648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036032323232323233, + "loss": 2.5032, + "theoretical_loss": 3.4123827695235542, + "tokens_seen": 2131165184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603030303030303, + "loss": 2.6595, + "theoretical_loss": 3.4123741293152383, + "tokens_seen": 2131230720 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036028282828282825, + "loss": 2.715, + "theoretical_loss": 3.412365489446998, + "tokens_seen": 2131296256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602626262626263, + "loss": 2.547, + "theoretical_loss": 3.4123568499188086, + "tokens_seen": 2131361792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036024242424242427, + "loss": 2.7751, + "theoretical_loss": 3.412348210730647, + "tokens_seen": 2131427328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602222222222222, + "loss": 2.4854, + "theoretical_loss": 3.412339571882489, + "tokens_seen": 2131492864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602020202020202, + "loss": 2.5917, + "theoretical_loss": 3.412330933374311, + "tokens_seen": 2131558400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036018181818181814, + "loss": 2.6232, + "theoretical_loss": 3.4123222952060894, + "tokens_seen": 2131623936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601616161616162, + "loss": 2.6344, + "theoretical_loss": 3.4123136573777995, + "tokens_seen": 2131689472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036014141414141416, + "loss": 2.6682, + "theoretical_loss": 3.412305019889419, + "tokens_seen": 2131755008 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1174844, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.656928300857544, + "objective/train/theoretical_loss": 3.412296382740922, + "objective/train/tokens_used": 490679776, + "theoretical_loss": 3.412296382740922, + "tokens_seen": 2131820544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601212121212121, + "loss": 2.5572, + "theoretical_loss": 3.412296382740922, + "tokens_seen": 2131820544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601010101010101, + "loss": 2.5109, + "theoretical_loss": 3.4122877459322867, + "tokens_seen": 2131886080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600808080808081, + "loss": 2.5999, + "theoretical_loss": 3.412279109463488, + "tokens_seen": 2131951616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600606060606061, + "loss": 2.4027, + "theoretical_loss": 3.4122704733345026, + "tokens_seen": 2132017152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036004040404040405, + "loss": 2.482, + "theoretical_loss": 3.412261837545307, + "tokens_seen": 2132082688 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360020202020202, + "loss": 2.4131, + "theoretical_loss": 3.4122532020958762, + "tokens_seen": 2132148224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035999999999999997, + "loss": 2.6003, + "theoretical_loss": 3.412244566986187, + "tokens_seen": 2132213760 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359979797979798, + "loss": 2.3046, + "theoretical_loss": 3.4122359322162166, + "tokens_seen": 2132279296 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359959595959596, + "loss": 2.5533, + "theoretical_loss": 3.4122272977859396, + "tokens_seen": 2132344832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035993939393939395, + "loss": 2.5854, + "theoretical_loss": 3.4122186636953336, + "tokens_seen": 2132410368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599191919191919, + "loss": 2.8106, + "theoretical_loss": 3.412210029944374, + "tokens_seen": 2132475904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598989898989899, + "loss": 2.5694, + "theoretical_loss": 3.4122013965330367, + "tokens_seen": 2132541440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035987878787878787, + "loss": 2.6185, + "theoretical_loss": 3.4121927634612987, + "tokens_seen": 2132606976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598585858585859, + "loss": 2.4607, + "theoretical_loss": 3.4121841307291354, + "tokens_seen": 2132672512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035983838383838384, + "loss": 2.7415, + "theoretical_loss": 3.4121754983365236, + "tokens_seen": 2132738048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598181818181818, + "loss": 2.5386, + "theoretical_loss": 3.4121668662834397, + "tokens_seen": 2132803584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597979797979798, + "loss": 2.5452, + "theoretical_loss": 3.4121582345698593, + "tokens_seen": 2132869120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035977777777777776, + "loss": 2.4101, + "theoretical_loss": 3.4121496031957586, + "tokens_seen": 2132934656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597575757575758, + "loss": 2.7979, + "theoretical_loss": 3.4121409721611147, + "tokens_seen": 2133000192 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035973737373737373, + "loss": 2.6733, + "theoretical_loss": 3.412132341465903, + "tokens_seen": 2133065728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035971717171717174, + "loss": 2.5141, + "theoretical_loss": 3.4121237111101, + "tokens_seen": 2133131264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596969696969697, + "loss": 2.503, + "theoretical_loss": 3.4121150810936816, + "tokens_seen": 2133196800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035967676767676766, + "loss": 2.5298, + "theoretical_loss": 3.4121064514166246, + "tokens_seen": 2133262336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035965656565656567, + "loss": 2.6807, + "theoretical_loss": 3.4120978220789047, + "tokens_seen": 2133327872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596363636363636, + "loss": 2.5674, + "theoretical_loss": 3.412089193080498, + "tokens_seen": 2133393408 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1175925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5075652599334717, + "objective/train/theoretical_loss": 3.412080564421382, + "objective/train/tokens_used": 492318176, + "theoretical_loss": 3.412080564421382, + "tokens_seen": 2133458944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035961616161616164, + "loss": 2.6865, + "theoretical_loss": 3.412080564421382, + "tokens_seen": 2133458944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595959595959596, + "loss": 2.822, + "theoretical_loss": 3.4120719361015315, + "tokens_seen": 2133524480 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035957575757575755, + "loss": 2.541, + "theoretical_loss": 3.4120633081209233, + "tokens_seen": 2133590016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035955555555555556, + "loss": 2.6752, + "theoretical_loss": 3.4120546804795335, + "tokens_seen": 2133655552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035953535353535357, + "loss": 2.6471, + "theoretical_loss": 3.412046053177339, + "tokens_seen": 2133721088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035951515151515153, + "loss": 2.6086, + "theoretical_loss": 3.412037426214315, + "tokens_seen": 2133786624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594949494949495, + "loss": 2.5331, + "theoretical_loss": 3.4120287995904386, + "tokens_seen": 2133852160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594747474747475, + "loss": 2.739, + "theoretical_loss": 3.4120201733056854, + "tokens_seen": 2133917696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035945454545454545, + "loss": 2.6104, + "theoretical_loss": 3.412011547360032, + "tokens_seen": 2133983232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035943434343434347, + "loss": 2.6294, + "theoretical_loss": 3.412002921753455, + "tokens_seen": 2134048768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594141414141414, + "loss": 2.8417, + "theoretical_loss": 3.41199429648593, + "tokens_seen": 2134114304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593939393939394, + "loss": 2.5628, + "theoretical_loss": 3.4119856715574333, + "tokens_seen": 2134179840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593737373737374, + "loss": 2.6903, + "theoretical_loss": 3.411977046967942, + "tokens_seen": 2134245376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593535353535354, + "loss": 2.7785, + "theoretical_loss": 3.411968422717431, + "tokens_seen": 2134310912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035933333333333336, + "loss": 2.5786, + "theoretical_loss": 3.411959798805878, + "tokens_seen": 2134376448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593131313131313, + "loss": 2.5277, + "theoretical_loss": 3.4119511752332583, + "tokens_seen": 2134441984 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035929292929292927, + "loss": 2.7237, + "theoretical_loss": 3.4119425519995485, + "tokens_seen": 2134507520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592727272727273, + "loss": 2.6926, + "theoretical_loss": 3.411933929104725, + "tokens_seen": 2134573056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592525252525253, + "loss": 2.6479, + "theoretical_loss": 3.411925306548764, + "tokens_seen": 2134638592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035923232323232325, + "loss": 2.6249, + "theoretical_loss": 3.4119166843316413, + "tokens_seen": 2134704128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592121212121212, + "loss": 2.6359, + "theoretical_loss": 3.4119080624533336, + "tokens_seen": 2134769664 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035919191919191916, + "loss": 2.7272, + "theoretical_loss": 3.411899440913818, + "tokens_seen": 2134835200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035917171717171723, + "loss": 2.7252, + "theoretical_loss": 3.4118908197130695, + "tokens_seen": 2134900736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591515151515152, + "loss": 2.7684, + "theoretical_loss": 3.411882198851065, + "tokens_seen": 2134966272 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035913131313131314, + "loss": 2.5349, + "theoretical_loss": 3.4118735783277803, + "tokens_seen": 2135031808 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1176552, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6072494983673096, + "objective/train/theoretical_loss": 3.411864958143192, + "objective/train/tokens_used": 493956576, + "theoretical_loss": 3.411864958143192, + "tokens_seen": 2135097344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591111111111111, + "loss": 2.4361, + "theoretical_loss": 3.411864958143192, + "tokens_seen": 2135097344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035909090909090906, + "loss": 2.639, + "theoretical_loss": 3.4118563382972766, + "tokens_seen": 2135162880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590707070707071, + "loss": 2.608, + "theoretical_loss": 3.4118477187900105, + "tokens_seen": 2135228416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590505050505051, + "loss": 2.6096, + "theoretical_loss": 3.4118390996213694, + "tokens_seen": 2135293952 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035903030303030304, + "loss": 2.6053, + "theoretical_loss": 3.41183048079133, + "tokens_seen": 2135359488 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359010101010101, + "loss": 2.5846, + "theoretical_loss": 3.4118218622998686, + "tokens_seen": 2135425024 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358989898989899, + "loss": 2.4498, + "theoretical_loss": 3.4118132441469617, + "tokens_seen": 2135490560 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358969696969697, + "loss": 2.7093, + "theoretical_loss": 3.411804626332585, + "tokens_seen": 2135556096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035894949494949497, + "loss": 2.5592, + "theoretical_loss": 3.4117960088567156, + "tokens_seen": 2135621632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035892929292929293, + "loss": 2.6863, + "theoretical_loss": 3.4117873917193293, + "tokens_seen": 2135687168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589090909090909, + "loss": 2.6119, + "theoretical_loss": 3.4117787749204025, + "tokens_seen": 2135752704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588888888888889, + "loss": 2.8139, + "theoretical_loss": 3.4117701584599116, + "tokens_seen": 2135818240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588686868686869, + "loss": 2.6701, + "theoretical_loss": 3.4117615423378327, + "tokens_seen": 2135883776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035884848484848486, + "loss": 2.6039, + "theoretical_loss": 3.4117529265541426, + "tokens_seen": 2135949312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588282828282828, + "loss": 2.5645, + "theoretical_loss": 3.4117443111088175, + "tokens_seen": 2136014848 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035880808080808083, + "loss": 2.7743, + "theoretical_loss": 3.411735696001833, + "tokens_seen": 2136080384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587878787878788, + "loss": 2.654, + "theoretical_loss": 3.411727081233166, + "tokens_seen": 2136145920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587676767676768, + "loss": 2.6771, + "theoretical_loss": 3.4117184668027933, + "tokens_seen": 2136211456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035874747474747476, + "loss": 2.7149, + "theoretical_loss": 3.41170985271069, + "tokens_seen": 2136276992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587272727272727, + "loss": 2.7189, + "theoretical_loss": 3.4117012389568337, + "tokens_seen": 2136342528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587070707070707, + "loss": 2.5876, + "theoretical_loss": 3.4116926255412006, + "tokens_seen": 2136408064 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586868686868687, + "loss": 2.6411, + "theoretical_loss": 3.411684012463766, + "tokens_seen": 2136473600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586666666666667, + "loss": 2.5305, + "theoretical_loss": 3.4116753997245075, + "tokens_seen": 2136539136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035864646464646465, + "loss": 2.5586, + "theoretical_loss": 3.4116667873234006, + "tokens_seen": 2136604672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586262626262626, + "loss": 2.6443, + "theoretical_loss": 3.4116581752604223, + "tokens_seen": 2136670208 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1177415, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7517342567443848, + "objective/train/theoretical_loss": 3.4116495635355486, + "objective/train/tokens_used": 495594976, + "theoretical_loss": 3.4116495635355486, + "tokens_seen": 2136735744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586060606060606, + "loss": 2.6526, + "theoretical_loss": 3.4116495635355486, + "tokens_seen": 2136735744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585858585858586, + "loss": 2.8409, + "theoretical_loss": 3.411640952148755, + "tokens_seen": 2136801280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585656565656566, + "loss": 2.6958, + "theoretical_loss": 3.4116323411000193, + "tokens_seen": 2136866816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035854545454545454, + "loss": 2.6911, + "theoretical_loss": 3.4116237303893175, + "tokens_seen": 2136932352 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035852525252525255, + "loss": 2.5204, + "theoretical_loss": 3.4116151200166254, + "tokens_seen": 2136997888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585050505050505, + "loss": 2.4892, + "theoretical_loss": 3.4116065099819197, + "tokens_seen": 2137063424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035848484848484847, + "loss": 2.7625, + "theoretical_loss": 3.4115979002851775, + "tokens_seen": 2137128960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584646464646465, + "loss": 2.748, + "theoretical_loss": 3.411589290926374, + "tokens_seen": 2137194496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035844444444444444, + "loss": 2.8012, + "theoretical_loss": 3.4115806819054857, + "tokens_seen": 2137260032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035842424242424245, + "loss": 2.4286, + "theoretical_loss": 3.411572073222489, + "tokens_seen": 2137325568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584040404040404, + "loss": 2.7613, + "theoretical_loss": 3.411563464877361, + "tokens_seen": 2137391104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035838383838383836, + "loss": 2.521, + "theoretical_loss": 3.4115548568700778, + "tokens_seen": 2137456640 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035836363636363637, + "loss": 2.5064, + "theoretical_loss": 3.4115462492006157, + "tokens_seen": 2137522176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583434343434344, + "loss": 2.687, + "theoretical_loss": 3.411537641868951, + "tokens_seen": 2137587712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035832323232323234, + "loss": 2.6783, + "theoretical_loss": 3.41152903487506, + "tokens_seen": 2137653248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583030303030303, + "loss": 2.6258, + "theoretical_loss": 3.411520428218919, + "tokens_seen": 2137718784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035828282828282825, + "loss": 2.4896, + "theoretical_loss": 3.411511821900505, + "tokens_seen": 2137784320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035826262626262626, + "loss": 2.8056, + "theoretical_loss": 3.4115032159197938, + "tokens_seen": 2137849856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582424242424243, + "loss": 2.712, + "theoretical_loss": 3.411494610276762, + "tokens_seen": 2137915392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035822222222222223, + "loss": 2.7067, + "theoretical_loss": 3.4114860049713855, + "tokens_seen": 2137980928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582020202020202, + "loss": 2.8186, + "theoretical_loss": 3.4114774000036414, + "tokens_seen": 2138046464 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035818181818181815, + "loss": 2.7648, + "theoretical_loss": 3.4114687953735063, + "tokens_seen": 2138112000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581616161616162, + "loss": 2.7007, + "theoretical_loss": 3.411460191080956, + "tokens_seen": 2138177536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035814141414141417, + "loss": 2.6543, + "theoretical_loss": 3.411451587125967, + "tokens_seen": 2138243072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581212121212121, + "loss": 2.6273, + "theoretical_loss": 3.4114429835085156, + "tokens_seen": 2138308608 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1177948, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.514267921447754, + "objective/train/theoretical_loss": 3.411434380228579, + "objective/train/tokens_used": 497233376, + "theoretical_loss": 3.411434380228579, + "tokens_seen": 2138374144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581010101010101, + "loss": 2.5898, + "theoretical_loss": 3.411434380228579, + "tokens_seen": 2138374144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035808080808080804, + "loss": 2.7825, + "theoretical_loss": 3.4114257772861327, + "tokens_seen": 2138439680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580606060606061, + "loss": 2.7285, + "theoretical_loss": 3.4114171746811537, + "tokens_seen": 2138505216 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035804040404040406, + "loss": 2.5128, + "theoretical_loss": 3.411408572413618, + "tokens_seen": 2138570752 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358020202020202, + "loss": 2.5706, + "theoretical_loss": 3.4113999704835023, + "tokens_seen": 2138636288 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358, + "loss": 2.8487, + "theoretical_loss": 3.411391368890783, + "tokens_seen": 2138701824 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357979797979798, + "loss": 2.7602, + "theoretical_loss": 3.4113827676354362, + "tokens_seen": 2138767360 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357959595959596, + "loss": 2.5372, + "theoretical_loss": 3.4113741667174384, + "tokens_seen": 2138832896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035793939393939395, + "loss": 2.7173, + "theoretical_loss": 3.411365566136767, + "tokens_seen": 2138898432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579191919191919, + "loss": 2.5033, + "theoretical_loss": 3.411356965893397, + "tokens_seen": 2138963968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035789898989898987, + "loss": 2.7777, + "theoretical_loss": 3.4113483659873056, + "tokens_seen": 2139029504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578787878787879, + "loss": 2.8195, + "theoretical_loss": 3.4113397664184695, + "tokens_seen": 2139095040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578585858585859, + "loss": 2.4594, + "theoretical_loss": 3.4113311671868645, + "tokens_seen": 2139160576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035783838383838385, + "loss": 2.7956, + "theoretical_loss": 3.4113225682924675, + "tokens_seen": 2139226112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578181818181818, + "loss": 2.8989, + "theoretical_loss": 3.4113139697352546, + "tokens_seen": 2139291648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577979797979798, + "loss": 2.5859, + "theoretical_loss": 3.411305371515202, + "tokens_seen": 2139357184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035777777777777777, + "loss": 2.6077, + "theoretical_loss": 3.4112967736322872, + "tokens_seen": 2139422720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577575757575758, + "loss": 2.5769, + "theoretical_loss": 3.411288176086486, + "tokens_seen": 2139488256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035773737373737374, + "loss": 2.6175, + "theoretical_loss": 3.4112795788777746, + "tokens_seen": 2139553792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577171717171717, + "loss": 2.4836, + "theoretical_loss": 3.41127098200613, + "tokens_seen": 2139619328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576969696969697, + "loss": 2.6395, + "theoretical_loss": 3.4112623854715283, + "tokens_seen": 2139684864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576767676767677, + "loss": 2.7262, + "theoretical_loss": 3.411253789273946, + "tokens_seen": 2139750400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576565656565657, + "loss": 2.6886, + "theoretical_loss": 3.41124519341336, + "tokens_seen": 2139815936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035763636363636363, + "loss": 2.6317, + "theoretical_loss": 3.4112365978897463, + "tokens_seen": 2139881472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035761616161616164, + "loss": 2.5842, + "theoretical_loss": 3.411228002703081, + "tokens_seen": 2139947008 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1178547, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.098909378051758, + "objective/train/theoretical_loss": 3.4112194078533413, + "objective/train/tokens_used": 498871776, + "theoretical_loss": 3.4112194078533413, + "tokens_seen": 2140012544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575959595959596, + "loss": 2.8142, + "theoretical_loss": 3.4112194078533413, + "tokens_seen": 2140012544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575757575757576, + "loss": 2.6792, + "theoretical_loss": 3.4112108133405035, + "tokens_seen": 2140078080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035755555555555557, + "loss": 2.9019, + "theoretical_loss": 3.4112022191645437, + "tokens_seen": 2140143616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575353535353535, + "loss": 2.6421, + "theoretical_loss": 3.4111936253254393, + "tokens_seen": 2140209152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035751515151515153, + "loss": 2.5492, + "theoretical_loss": 3.4111850318231656, + "tokens_seen": 2140274688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574949494949495, + "loss": 2.6782, + "theoretical_loss": 3.4111764386576997, + "tokens_seen": 2140340224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574747474747475, + "loss": 2.6779, + "theoretical_loss": 3.411167845829018, + "tokens_seen": 2140405760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035745454545454546, + "loss": 2.6727, + "theoretical_loss": 3.4111592533370976, + "tokens_seen": 2140471296 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035743434343434347, + "loss": 2.4204, + "theoretical_loss": 3.411150661181914, + "tokens_seen": 2140536832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035741414141414143, + "loss": 2.8483, + "theoretical_loss": 3.4111420693634438, + "tokens_seen": 2140602368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573939393939394, + "loss": 2.5644, + "theoretical_loss": 3.4111334778816644, + "tokens_seen": 2140667904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573737373737374, + "loss": 2.618, + "theoretical_loss": 3.411124886736551, + "tokens_seen": 2140733440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035735353535353535, + "loss": 2.5871, + "theoretical_loss": 3.4111162959280814, + "tokens_seen": 2140798976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035733333333333336, + "loss": 2.865, + "theoretical_loss": 3.4111077054562315, + "tokens_seen": 2140864512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573131313131313, + "loss": 2.8388, + "theoretical_loss": 3.4110991153209778, + "tokens_seen": 2140930048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572929292929293, + "loss": 2.8237, + "theoretical_loss": 3.4110905255222965, + "tokens_seen": 2140995584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572727272727273, + "loss": 2.5144, + "theoretical_loss": 3.4110819360601647, + "tokens_seen": 2141061120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572525252525253, + "loss": 2.5991, + "theoretical_loss": 3.4110733469345584, + "tokens_seen": 2141126656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035723232323232326, + "loss": 2.4743, + "theoretical_loss": 3.4110647581454545, + "tokens_seen": 2141192192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572121212121212, + "loss": 2.6059, + "theoretical_loss": 3.4110561696928294, + "tokens_seen": 2141257728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035719191919191917, + "loss": 2.6451, + "theoretical_loss": 3.4110475815766597, + "tokens_seen": 2141323264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571717171717172, + "loss": 2.6165, + "theoretical_loss": 3.4110389937969217, + "tokens_seen": 2141388800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571515151515152, + "loss": 2.5672, + "theoretical_loss": 3.411030406353592, + "tokens_seen": 2141454336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035713131313131315, + "loss": 2.6528, + "theoretical_loss": 3.4110218192466473, + "tokens_seen": 2141519872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571111111111111, + "loss": 2.8488, + "theoretical_loss": 3.411013232476064, + "tokens_seen": 2141585408 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1179742, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.639477014541626, + "objective/train/theoretical_loss": 3.4110046460418184, + "objective/train/tokens_used": 500510176, + "theoretical_loss": 3.4110046460418184, + "tokens_seen": 2141650944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035709090909090906, + "loss": 2.6017, + "theoretical_loss": 3.4110046460418184, + "tokens_seen": 2141650944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035707070707070713, + "loss": 2.7004, + "theoretical_loss": 3.410996059943887, + "tokens_seen": 2141716480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570505050505051, + "loss": 2.6781, + "theoretical_loss": 3.4109874741822477, + "tokens_seen": 2141782016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035703030303030304, + "loss": 2.8444, + "theoretical_loss": 3.4109788887568753, + "tokens_seen": 2141847552 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357010101010101, + "loss": 2.8455, + "theoretical_loss": 3.410970303667747, + "tokens_seen": 2141913088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035698989898989896, + "loss": 2.8306, + "theoretical_loss": 3.410961718914839, + "tokens_seen": 2141978624 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356969696969697, + "loss": 2.7306, + "theoretical_loss": 3.4109531344981283, + "tokens_seen": 2142044160 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356949494949495, + "loss": 2.6625, + "theoretical_loss": 3.4109445504175917, + "tokens_seen": 2142109696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035692929292929293, + "loss": 2.5678, + "theoretical_loss": 3.4109359666732053, + "tokens_seen": 2142175232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569090909090909, + "loss": 2.6553, + "theoretical_loss": 3.4109273832649456, + "tokens_seen": 2142240768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035688888888888885, + "loss": 2.6125, + "theoretical_loss": 3.410918800192789, + "tokens_seen": 2142306304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568686868686869, + "loss": 2.6823, + "theoretical_loss": 3.4109102174567125, + "tokens_seen": 2142371840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035684848484848487, + "loss": 2.6649, + "theoretical_loss": 3.4109016350566934, + "tokens_seen": 2142437376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568282828282828, + "loss": 2.7469, + "theoretical_loss": 3.410893052992706, + "tokens_seen": 2142502912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568080808080808, + "loss": 2.5226, + "theoretical_loss": 3.410884471264729, + "tokens_seen": 2142568448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567878787878788, + "loss": 2.3986, + "theoretical_loss": 3.410875889872738, + "tokens_seen": 2142633984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567676767676768, + "loss": 2.6273, + "theoretical_loss": 3.41086730881671, + "tokens_seen": 2142699520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035674747474747476, + "loss": 2.5577, + "theoretical_loss": 3.4108587280966214, + "tokens_seen": 2142765056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567272727272727, + "loss": 2.5092, + "theoretical_loss": 3.4108501477124484, + "tokens_seen": 2142830592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567070707070707, + "loss": 2.4898, + "theoretical_loss": 3.4108415676641677, + "tokens_seen": 2142896128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566868686868687, + "loss": 2.7255, + "theoretical_loss": 3.4108329879517565, + "tokens_seen": 2142961664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566666666666667, + "loss": 2.6781, + "theoretical_loss": 3.4108244085751904, + "tokens_seen": 2143027200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035664646464646466, + "loss": 2.6827, + "theoretical_loss": 3.4108158295344473, + "tokens_seen": 2143092736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566262626262626, + "loss": 2.7293, + "theoretical_loss": 3.4108072508295026, + "tokens_seen": 2143158272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566060606060606, + "loss": 2.4671, + "theoretical_loss": 3.4107986724603334, + "tokens_seen": 2143223808 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1180436, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.763584613800049, + "objective/train/theoretical_loss": 3.410790094426916, + "objective/train/tokens_used": 502148576, + "theoretical_loss": 3.410790094426916, + "tokens_seen": 2143289344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565858585858586, + "loss": 2.6251, + "theoretical_loss": 3.410790094426916, + "tokens_seen": 2143289344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565656565656566, + "loss": 2.7291, + "theoretical_loss": 3.4107815167292275, + "tokens_seen": 2143354880 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035654545454545455, + "loss": 2.4953, + "theoretical_loss": 3.410772939367244, + "tokens_seen": 2143420416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565252525252525, + "loss": 2.654, + "theoretical_loss": 3.4107643623409425, + "tokens_seen": 2143485952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565050505050505, + "loss": 2.6565, + "theoretical_loss": 3.410755785650299, + "tokens_seen": 2143551488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564848484848485, + "loss": 2.6607, + "theoretical_loss": 3.410747209295291, + "tokens_seen": 2143617024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564646464646465, + "loss": 2.7669, + "theoretical_loss": 3.410738633275894, + "tokens_seen": 2143682560 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035644444444444444, + "loss": 2.7056, + "theoretical_loss": 3.4107300575920854, + "tokens_seen": 2143748096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035642424242424245, + "loss": 2.8577, + "theoretical_loss": 3.410721482243842, + "tokens_seen": 2143813632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564040404040404, + "loss": 2.6011, + "theoretical_loss": 3.4107129072311393, + "tokens_seen": 2143879168 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035638383838383837, + "loss": 2.4695, + "theoretical_loss": 3.410704332553955, + "tokens_seen": 2143944704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563636363636364, + "loss": 2.62, + "theoretical_loss": 3.410695758212266, + "tokens_seen": 2144010240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035634343434343433, + "loss": 2.5418, + "theoretical_loss": 3.410687184206047, + "tokens_seen": 2144075776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035632323232323234, + "loss": 2.6616, + "theoretical_loss": 3.4106786105352773, + "tokens_seen": 2144141312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563030303030303, + "loss": 2.6573, + "theoretical_loss": 3.410670037199931, + "tokens_seen": 2144206848 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035628282828282826, + "loss": 2.7227, + "theoretical_loss": 3.410661464199986, + "tokens_seen": 2144272384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035626262626262627, + "loss": 2.9286, + "theoretical_loss": 3.4106528915354195, + "tokens_seen": 2144337920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562424242424243, + "loss": 2.7204, + "theoretical_loss": 3.410644319206207, + "tokens_seen": 2144403456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035622222222222224, + "loss": 2.7795, + "theoretical_loss": 3.410635747212325, + "tokens_seen": 2144468992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562020202020202, + "loss": 2.6493, + "theoretical_loss": 3.4106271755537514, + "tokens_seen": 2144534528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035618181818181815, + "loss": 2.7353, + "theoretical_loss": 3.4106186042304616, + "tokens_seen": 2144600064 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035616161616161616, + "loss": 2.6767, + "theoretical_loss": 3.410610033242433, + "tokens_seen": 2144665600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561414141414142, + "loss": 2.6527, + "theoretical_loss": 3.410601462589642, + "tokens_seen": 2144731136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035612121212121213, + "loss": 2.5647, + "theoretical_loss": 3.4105928922720654, + "tokens_seen": 2144796672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561010101010101, + "loss": 2.6571, + "theoretical_loss": 3.4105843222896794, + "tokens_seen": 2144862208 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1181645, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3639755249023438, + "objective/train/theoretical_loss": 3.410575752642461, + "objective/train/tokens_used": 503786976, + "theoretical_loss": 3.410575752642461, + "tokens_seen": 2144927744 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035608080808080804, + "loss": 2.7949, + "theoretical_loss": 3.410575752642461, + "tokens_seen": 2144927744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560606060606061, + "loss": 2.5403, + "theoretical_loss": 3.4105671833303868, + "tokens_seen": 2144993280 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035604040404040407, + "loss": 2.71, + "theoretical_loss": 3.4105586143534334, + "tokens_seen": 2145058816 + }, + { + "epoch": 0.3, + "learning_rate": 0.000356020202020202, + "loss": 2.5801, + "theoretical_loss": 3.4105500457115774, + "tokens_seen": 2145124352 + }, + { + "epoch": 0.3, + "learning_rate": 0.000356, + "loss": 2.5069, + "theoretical_loss": 3.410541477404796, + "tokens_seen": 2145189888 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355979797979798, + "loss": 2.815, + "theoretical_loss": 3.4105329094330648, + "tokens_seen": 2145255424 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355959595959596, + "loss": 2.5818, + "theoretical_loss": 3.4105243417963615, + "tokens_seen": 2145320960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035593939393939396, + "loss": 2.4357, + "theoretical_loss": 3.4105157744946624, + "tokens_seen": 2145386496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559191919191919, + "loss": 2.6195, + "theoretical_loss": 3.4105072075279432, + "tokens_seen": 2145452032 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035589898989898987, + "loss": 2.7188, + "theoretical_loss": 3.410498640896183, + "tokens_seen": 2145517568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035587878787878794, + "loss": 2.6192, + "theoretical_loss": 3.410490074599356, + "tokens_seen": 2145583104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558585858585859, + "loss": 2.4677, + "theoretical_loss": 3.4104815086374396, + "tokens_seen": 2145648640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035583838383838385, + "loss": 2.7138, + "theoretical_loss": 3.4104729430104115, + "tokens_seen": 2145714176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558181818181818, + "loss": 2.8676, + "theoretical_loss": 3.410464377718247, + "tokens_seen": 2145779712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035579797979797977, + "loss": 2.7443, + "theoretical_loss": 3.4104558127609232, + "tokens_seen": 2145845248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035577777777777783, + "loss": 2.9617, + "theoretical_loss": 3.4104472481384174, + "tokens_seen": 2145910784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557575757575758, + "loss": 2.5756, + "theoretical_loss": 3.410438683850706, + "tokens_seen": 2145976320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035573737373737374, + "loss": 2.8851, + "theoretical_loss": 3.4104301198977653, + "tokens_seen": 2146041856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557171717171717, + "loss": 2.8399, + "theoretical_loss": 3.410421556279572, + "tokens_seen": 2146107392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556969696969697, + "loss": 2.5007, + "theoretical_loss": 3.410412992996103, + "tokens_seen": 2146172928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556767676767677, + "loss": 2.5867, + "theoretical_loss": 3.4104044300473353, + "tokens_seen": 2146238464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556565656565657, + "loss": 2.6169, + "theoretical_loss": 3.410395867433245, + "tokens_seen": 2146304000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035563636363636364, + "loss": 2.5714, + "theoretical_loss": 3.4103873051538094, + "tokens_seen": 2146369536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556161616161616, + "loss": 2.6238, + "theoretical_loss": 3.410378743209005, + "tokens_seen": 2146435072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555959595959596, + "loss": 2.685, + "theoretical_loss": 3.4103701815988083, + "tokens_seen": 2146500608 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1182191, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.627549886703491, + "objective/train/theoretical_loss": 3.4103616203231955, + "objective/train/tokens_used": 505425376, + "theoretical_loss": 3.4103616203231955, + "tokens_seen": 2146566144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555757575757576, + "loss": 2.7703, + "theoretical_loss": 3.4103616203231955, + "tokens_seen": 2146566144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035555555555555557, + "loss": 2.5253, + "theoretical_loss": 3.4103530593821447, + "tokens_seen": 2146631680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035553535353535353, + "loss": 2.9456, + "theoretical_loss": 3.4103444987756317, + "tokens_seen": 2146697216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035551515151515154, + "loss": 2.5637, + "theoretical_loss": 3.4103359385036334, + "tokens_seen": 2146762752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554949494949495, + "loss": 2.5552, + "theoretical_loss": 3.4103273785661266, + "tokens_seen": 2146828288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554747474747475, + "loss": 2.7312, + "theoretical_loss": 3.4103188189630878, + "tokens_seen": 2146893824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035545454545454547, + "loss": 2.7148, + "theoretical_loss": 3.4103102596944934, + "tokens_seen": 2146959360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554343434343434, + "loss": 2.9333, + "theoretical_loss": 3.4103017007603205, + "tokens_seen": 2147024896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035541414141414143, + "loss": 2.5406, + "theoretical_loss": 3.4102931421605467, + "tokens_seen": 2147090432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553939393939394, + "loss": 2.8978, + "theoretical_loss": 3.410284583895147, + "tokens_seen": 2147155968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553737373737374, + "loss": 2.7002, + "theoretical_loss": 3.4102760259641, + "tokens_seen": 2147221504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035535353535353536, + "loss": 2.7219, + "theoretical_loss": 3.4102674683673806, + "tokens_seen": 2147287040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553333333333333, + "loss": 2.5009, + "theoretical_loss": 3.4102589111049664, + "tokens_seen": 2147352576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553131313131313, + "loss": 2.491, + "theoretical_loss": 3.4102503541768345, + "tokens_seen": 2147418112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552929292929293, + "loss": 2.7664, + "theoretical_loss": 3.4102417975829615, + "tokens_seen": 2147483648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552727272727273, + "loss": 2.6139, + "theoretical_loss": 3.4102332413233234, + "tokens_seen": 2147549184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035525252525252525, + "loss": 2.7421, + "theoretical_loss": 3.410224685397898, + "tokens_seen": 2147614720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035523232323232326, + "loss": 2.6755, + "theoretical_loss": 3.4102161298066607, + "tokens_seen": 2147680256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552121212121212, + "loss": 2.634, + "theoretical_loss": 3.41020757454959, + "tokens_seen": 2147745792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551919191919192, + "loss": 2.6799, + "theoretical_loss": 3.410199019626661, + "tokens_seen": 2147811328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551717171717172, + "loss": 2.7408, + "theoretical_loss": 3.410190465037852, + "tokens_seen": 2147876864 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035515151515151514, + "loss": 2.5752, + "theoretical_loss": 3.410181910783138, + "tokens_seen": 2147942400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035513131313131315, + "loss": 2.582, + "theoretical_loss": 3.4101733568624972, + "tokens_seen": 2148007936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551111111111111, + "loss": 2.6301, + "theoretical_loss": 3.4101648032759053, + "tokens_seen": 2148073472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035509090909090907, + "loss": 2.8678, + "theoretical_loss": 3.4101562500233404, + "tokens_seen": 2148139008 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1183255, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5413014888763428, + "objective/train/theoretical_loss": 3.4101476971047777, + "objective/train/tokens_used": 507063776, + "theoretical_loss": 3.4101476971047777, + "tokens_seen": 2148204544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550707070707071, + "loss": 2.5406, + "theoretical_loss": 3.4101476971047777, + "tokens_seen": 2148204544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550505050505051, + "loss": 2.511, + "theoretical_loss": 3.4101391445201954, + "tokens_seen": 2148270080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035503030303030305, + "loss": 2.6562, + "theoretical_loss": 3.410130592269569, + "tokens_seen": 2148335616 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355010101010101, + "loss": 2.7914, + "theoretical_loss": 3.410122040352877, + "tokens_seen": 2148401152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035498989898989896, + "loss": 2.6844, + "theoretical_loss": 3.4101134887700937, + "tokens_seen": 2148466688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035496969696969697, + "loss": 2.5422, + "theoretical_loss": 3.410104937521198, + "tokens_seen": 2148532224 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354949494949495, + "loss": 2.7876, + "theoretical_loss": 3.410096386606166, + "tokens_seen": 2148597760 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035492929292929294, + "loss": 2.7034, + "theoretical_loss": 3.4100878360249736, + "tokens_seen": 2148663296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549090909090909, + "loss": 2.4995, + "theoretical_loss": 3.4100792857775994, + "tokens_seen": 2148728832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035488888888888885, + "loss": 2.6483, + "theoretical_loss": 3.4100707358640188, + "tokens_seen": 2148794368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548686868686869, + "loss": 2.775, + "theoretical_loss": 3.4100621862842084, + "tokens_seen": 2148859904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548484848484849, + "loss": 2.5338, + "theoretical_loss": 3.4100536370381462, + "tokens_seen": 2148925440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035482828282828283, + "loss": 2.9031, + "theoretical_loss": 3.410045088125808, + "tokens_seen": 2148990976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548080808080808, + "loss": 2.5603, + "theoretical_loss": 3.4100365395471712, + "tokens_seen": 2149056512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035478787878787875, + "loss": 2.5336, + "theoretical_loss": 3.4100279913022122, + "tokens_seen": 2149122048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547676767676768, + "loss": 2.7277, + "theoretical_loss": 3.410019443390908, + "tokens_seen": 2149187584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035474747474747477, + "loss": 2.7138, + "theoretical_loss": 3.410010895813236, + "tokens_seen": 2149253120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547272727272727, + "loss": 2.5253, + "theoretical_loss": 3.4100023485691713, + "tokens_seen": 2149318656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547070707070707, + "loss": 2.7073, + "theoretical_loss": 3.4099938016586924, + "tokens_seen": 2149384192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546868686868687, + "loss": 2.5695, + "theoretical_loss": 3.409985255081775, + "tokens_seen": 2149449728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546666666666667, + "loss": 2.5718, + "theoretical_loss": 3.4099767088383968, + "tokens_seen": 2149515264 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035464646464646466, + "loss": 2.6058, + "theoretical_loss": 3.409968162928534, + "tokens_seen": 2149580800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546262626262626, + "loss": 2.6631, + "theoretical_loss": 3.409959617352164, + "tokens_seen": 2149646336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546060606060606, + "loss": 2.7089, + "theoretical_loss": 3.4099510721092625, + "tokens_seen": 2149711872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545858585858586, + "loss": 2.356, + "theoretical_loss": 3.4099425271998074, + "tokens_seen": 2149777408 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1183881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.526425361633301, + "objective/train/theoretical_loss": 3.4099339826237753, + "objective/train/tokens_used": 508702176, + "theoretical_loss": 3.4099339826237753, + "tokens_seen": 2149842944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545656565656566, + "loss": 2.6612, + "theoretical_loss": 3.4099339826237753, + "tokens_seen": 2149842944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035454545454545455, + "loss": 2.6441, + "theoretical_loss": 3.409925438381143, + "tokens_seen": 2149908480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545252525252525, + "loss": 2.4833, + "theoretical_loss": 3.4099168944718867, + "tokens_seen": 2149974016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545050505050505, + "loss": 2.6153, + "theoretical_loss": 3.409908350895984, + "tokens_seen": 2150039552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544848484848485, + "loss": 2.5933, + "theoretical_loss": 3.409899807653412, + "tokens_seen": 2150105088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544646464646465, + "loss": 2.4394, + "theoretical_loss": 3.4098912647441466, + "tokens_seen": 2150170624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035444444444444445, + "loss": 2.4475, + "theoretical_loss": 3.409882722168165, + "tokens_seen": 2150236160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544242424242424, + "loss": 2.7372, + "theoretical_loss": 3.4098741799254446, + "tokens_seen": 2150301696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544040404040404, + "loss": 2.642, + "theoretical_loss": 3.409865638015961, + "tokens_seen": 2150367232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035438383838383837, + "loss": 2.6953, + "theoretical_loss": 3.4098570964396924, + "tokens_seen": 2150432768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543636363636364, + "loss": 2.8193, + "theoretical_loss": 3.4098485551966147, + "tokens_seen": 2150498304 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035434343434343434, + "loss": 2.6868, + "theoretical_loss": 3.409840014286705, + "tokens_seen": 2150563840 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035432323232323235, + "loss": 2.6162, + "theoretical_loss": 3.409831473709941, + "tokens_seen": 2150629376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543030303030303, + "loss": 2.8, + "theoretical_loss": 3.409822933466298, + "tokens_seen": 2150694912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542828282828283, + "loss": 2.6104, + "theoretical_loss": 3.409814393555754, + "tokens_seen": 2150760448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542626262626263, + "loss": 2.7018, + "theoretical_loss": 3.4098058539782854, + "tokens_seen": 2150825984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035424242424242423, + "loss": 2.5429, + "theoretical_loss": 3.4097973147338694, + "tokens_seen": 2150891520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035422222222222224, + "loss": 2.4372, + "theoretical_loss": 3.4097887758224825, + "tokens_seen": 2150957056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542020202020202, + "loss": 2.6098, + "theoretical_loss": 3.4097802372441013, + "tokens_seen": 2151022592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541818181818182, + "loss": 2.6754, + "theoretical_loss": 3.4097716989987035, + "tokens_seen": 2151088128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035416161616161617, + "loss": 2.4958, + "theoretical_loss": 3.409763161086266, + "tokens_seen": 2151153664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541414141414142, + "loss": 2.6487, + "theoretical_loss": 3.4097546235067644, + "tokens_seen": 2151219200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035412121212121214, + "loss": 2.8716, + "theoretical_loss": 3.409746086260177, + "tokens_seen": 2151284736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541010101010101, + "loss": 2.6091, + "theoretical_loss": 3.409737549346479, + "tokens_seen": 2151350272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540808080808081, + "loss": 2.71, + "theoretical_loss": 3.4097290127656494, + "tokens_seen": 2151415808 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1184815, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6544196605682373, + "objective/train/theoretical_loss": 3.409720476517664, + "objective/train/tokens_used": 510340576, + "theoretical_loss": 3.409720476517664, + "tokens_seen": 2151481344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035406060606060606, + "loss": 2.6166, + "theoretical_loss": 3.409720476517664, + "tokens_seen": 2151481344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035404040404040407, + "loss": 2.7207, + "theoretical_loss": 3.4097119406024996, + "tokens_seen": 2151546880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035402020202020203, + "loss": 2.5904, + "theoretical_loss": 3.4097034050201334, + "tokens_seen": 2151612416 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354, + "loss": 2.6907, + "theoretical_loss": 3.4096948697705414, + "tokens_seen": 2151677952 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353979797979798, + "loss": 2.6082, + "theoretical_loss": 3.4096863348537014, + "tokens_seen": 2151743488 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353959595959596, + "loss": 2.7023, + "theoretical_loss": 3.4096778002695904, + "tokens_seen": 2151809024 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035393939393939396, + "loss": 2.6989, + "theoretical_loss": 3.409669266018185, + "tokens_seen": 2151874560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539191919191919, + "loss": 2.5781, + "theoretical_loss": 3.4096607320994616, + "tokens_seen": 2151940096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538989898989899, + "loss": 2.4732, + "theoretical_loss": 3.409652198513398, + "tokens_seen": 2152005632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538787878787879, + "loss": 2.632, + "theoretical_loss": 3.4096436652599706, + "tokens_seen": 2152071168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538585858585859, + "loss": 2.7633, + "theoretical_loss": 3.4096351323391563, + "tokens_seen": 2152136704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035383838383838386, + "loss": 2.5427, + "theoretical_loss": 3.409626599750932, + "tokens_seen": 2152202240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538181818181818, + "loss": 2.627, + "theoretical_loss": 3.4096180674952747, + "tokens_seen": 2152267776 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035379797979797977, + "loss": 2.4213, + "theoretical_loss": 3.4096095355721614, + "tokens_seen": 2152333312 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035377777777777784, + "loss": 2.4559, + "theoretical_loss": 3.409601003981569, + "tokens_seen": 2152398848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537575757575758, + "loss": 2.4325, + "theoretical_loss": 3.409592472723474, + "tokens_seen": 2152464384 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035373737373737375, + "loss": 2.6978, + "theoretical_loss": 3.4095839417978544, + "tokens_seen": 2152529920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537171717171717, + "loss": 2.6345, + "theoretical_loss": 3.4095754112046857, + "tokens_seen": 2152595456 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035369696969696966, + "loss": 2.704, + "theoretical_loss": 3.409566880943945, + "tokens_seen": 2152660992 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035367676767676773, + "loss": 2.4888, + "theoretical_loss": 3.409558351015611, + "tokens_seen": 2152726528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536565656565657, + "loss": 2.4636, + "theoretical_loss": 3.4095498214196587, + "tokens_seen": 2152792064 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035363636363636364, + "loss": 2.5985, + "theoretical_loss": 3.4095412921560655, + "tokens_seen": 2152857600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536161616161616, + "loss": 2.547, + "theoretical_loss": 3.409532763224809, + "tokens_seen": 2152923136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035359595959595956, + "loss": 2.6535, + "theoretical_loss": 3.4095242346258656, + "tokens_seen": 2152988672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535757575757576, + "loss": 2.6403, + "theoretical_loss": 3.409515706359212, + "tokens_seen": 2153054208 + }, + { + "debugging/Self-BLEU-5": 0.39580322091234643, + "debugging/distinct-1-grams": 0.797421315948946, + "debugging/distinct-2-grams": 0.9612328752077172, + "debugging/entropy-1-grams": 5.484895349701254, + "debugging/entropy-2-grams": 6.169724221975992, + "debugging/length": 483.25, + "debugging/num_segments": 8, + "epoch": 0.3, + "objective/train/docs_used": 1185854, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8254241943359375, + "objective/train/theoretical_loss": 3.4095071784248256, + "objective/train/tokens_used": 511978976, + "theoretical_loss": 3.4095071784248256, + "tokens_seen": 2153119744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535555555555556, + "loss": 2.5479, + "theoretical_loss": 3.4095071784248256, + "tokens_seen": 2153119744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035353535353535354, + "loss": 2.6056, + "theoretical_loss": 3.409498650822683, + "tokens_seen": 2153185280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535151515151515, + "loss": 2.6974, + "theoretical_loss": 3.4094901235527613, + "tokens_seen": 2153250816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534949494949495, + "loss": 2.3753, + "theoretical_loss": 3.409481596615038, + "tokens_seen": 2153316352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534747474747475, + "loss": 2.5776, + "theoretical_loss": 3.4094730700094886, + "tokens_seen": 2153381888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035345454545454547, + "loss": 2.5597, + "theoretical_loss": 3.4094645437360915, + "tokens_seen": 2153447424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035343434343434343, + "loss": 2.773, + "theoretical_loss": 3.409456017794823, + "tokens_seen": 2153512960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534141414141414, + "loss": 2.6986, + "theoretical_loss": 3.4094474921856603, + "tokens_seen": 2153578496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533939393939394, + "loss": 2.6894, + "theoretical_loss": 3.40943896690858, + "tokens_seen": 2153644032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533737373737374, + "loss": 2.916, + "theoretical_loss": 3.4094304419635595, + "tokens_seen": 2153709568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035335353535353536, + "loss": 2.6452, + "theoretical_loss": 3.409421917350575, + "tokens_seen": 2153775104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533333333333333, + "loss": 2.6145, + "theoretical_loss": 3.409413393069605, + "tokens_seen": 2153840640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035331313131313133, + "loss": 2.8686, + "theoretical_loss": 3.4094048691206247, + "tokens_seen": 2153906176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532929292929293, + "loss": 2.8178, + "theoretical_loss": 3.4093963455036125, + "tokens_seen": 2153971712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532727272727273, + "loss": 2.6879, + "theoretical_loss": 3.4093878222185445, + "tokens_seen": 2154037248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035325252525252526, + "loss": 2.623, + "theoretical_loss": 3.4093792992653977, + "tokens_seen": 2154102784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532323232323232, + "loss": 2.7944, + "theoretical_loss": 3.409370776644149, + "tokens_seen": 2154168320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532121212121212, + "loss": 2.7522, + "theoretical_loss": 3.409362254354776, + "tokens_seen": 2154233856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531919191919192, + "loss": 2.5473, + "theoretical_loss": 3.409353732397255, + "tokens_seen": 2154299392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531717171717172, + "loss": 2.6862, + "theoretical_loss": 3.409345210771564, + "tokens_seen": 2154364928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035315151515151515, + "loss": 2.6817, + "theoretical_loss": 3.409336689477679, + "tokens_seen": 2154430464 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035313131313131316, + "loss": 2.5026, + "theoretical_loss": 3.409328168515577, + "tokens_seen": 2154496000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531111111111111, + "loss": 2.7158, + "theoretical_loss": 3.4093196478852352, + "tokens_seen": 2154561536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530909090909091, + "loss": 2.5761, + "theoretical_loss": 3.4093111275866312, + "tokens_seen": 2154627072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530707070707071, + "loss": 2.7904, + "theoretical_loss": 3.4093026076197415, + "tokens_seen": 2154692608 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1186317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.778775691986084, + "objective/train/theoretical_loss": 3.4092940879845424, + "objective/train/tokens_used": 513617376, + "theoretical_loss": 3.4092940879845424, + "tokens_seen": 2154758144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035305050505050504, + "loss": 2.5932, + "theoretical_loss": 3.4092940879845424, + "tokens_seen": 2154758144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035303030303030305, + "loss": 2.6707, + "theoretical_loss": 3.409285568681012, + "tokens_seen": 2154823680 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353010101010101, + "loss": 2.5527, + "theoretical_loss": 3.409277049709127, + "tokens_seen": 2154889216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035298989898989897, + "loss": 2.5178, + "theoretical_loss": 3.409268531068864, + "tokens_seen": 2154954752 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352969696969697, + "loss": 2.6367, + "theoretical_loss": 3.4092600127602006, + "tokens_seen": 2155020288 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352949494949495, + "loss": 2.5147, + "theoretical_loss": 3.409251494783113, + "tokens_seen": 2155085824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035292929292929295, + "loss": 2.6731, + "theoretical_loss": 3.4092429771375787, + "tokens_seen": 2155151360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529090909090909, + "loss": 2.5244, + "theoretical_loss": 3.4092344598235753, + "tokens_seen": 2155216896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035288888888888886, + "loss": 2.6141, + "theoretical_loss": 3.4092259428410783, + "tokens_seen": 2155282432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035286868686868687, + "loss": 2.6846, + "theoretical_loss": 3.4092174261900663, + "tokens_seen": 2155347968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528484848484849, + "loss": 2.5751, + "theoretical_loss": 3.409208909870516, + "tokens_seen": 2155413504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035282828282828284, + "loss": 2.625, + "theoretical_loss": 3.4092003938824034, + "tokens_seen": 2155479040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528080808080808, + "loss": 2.5676, + "theoretical_loss": 3.4091918782257062, + "tokens_seen": 2155544576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035278787878787875, + "loss": 2.5603, + "theoretical_loss": 3.409183362900402, + "tokens_seen": 2155610112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527676767676768, + "loss": 2.6779, + "theoretical_loss": 3.409174847906467, + "tokens_seen": 2155675648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527474747474748, + "loss": 2.597, + "theoretical_loss": 3.409166333243878, + "tokens_seen": 2155741184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035272727272727273, + "loss": 2.6806, + "theoretical_loss": 3.409157818912613, + "tokens_seen": 2155806720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527070707070707, + "loss": 2.6874, + "theoretical_loss": 3.4091493049126487, + "tokens_seen": 2155872256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035268686868686864, + "loss": 2.6295, + "theoretical_loss": 3.4091407912439617, + "tokens_seen": 2155937792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526666666666667, + "loss": 2.5543, + "theoretical_loss": 3.4091322779065294, + "tokens_seen": 2156003328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035264646464646467, + "loss": 2.5731, + "theoretical_loss": 3.409123764900329, + "tokens_seen": 2156068864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526262626262626, + "loss": 2.6069, + "theoretical_loss": 3.4091152522253374, + "tokens_seen": 2156134400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526060606060606, + "loss": 2.6663, + "theoretical_loss": 3.409106739881531, + "tokens_seen": 2156199936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035258585858585865, + "loss": 2.4908, + "theoretical_loss": 3.4090982278688875, + "tokens_seen": 2156265472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525656565656566, + "loss": 2.5468, + "theoretical_loss": 3.4090897161873848, + "tokens_seen": 2156331008 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1187470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.505795955657959, + "objective/train/theoretical_loss": 3.409081204836998, + "objective/train/tokens_used": 515255776, + "theoretical_loss": 3.409081204836998, + "tokens_seen": 2156396544 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035254545454545456, + "loss": 2.6541, + "theoretical_loss": 3.409081204836998, + "tokens_seen": 2156396544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525252525252525, + "loss": 2.4961, + "theoretical_loss": 3.4090726938177056, + "tokens_seen": 2156462080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525050505050505, + "loss": 2.595, + "theoretical_loss": 3.409064183129484, + "tokens_seen": 2156527616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035248484848484854, + "loss": 2.6051, + "theoretical_loss": 3.409055672772311, + "tokens_seen": 2156593152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524646464646465, + "loss": 2.6744, + "theoretical_loss": 3.4090471627461625, + "tokens_seen": 2156658688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035244444444444445, + "loss": 2.4898, + "theoretical_loss": 3.409038653051017, + "tokens_seen": 2156724224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524242424242424, + "loss": 2.6977, + "theoretical_loss": 3.4090301436868504, + "tokens_seen": 2156789760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524040404040404, + "loss": 2.5083, + "theoretical_loss": 3.40902163465364, + "tokens_seen": 2156855296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035238383838383843, + "loss": 2.6723, + "theoretical_loss": 3.4090131259513634, + "tokens_seen": 2156920832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523636363636364, + "loss": 2.6959, + "theoretical_loss": 3.4090046175799973, + "tokens_seen": 2156986368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035234343434343434, + "loss": 2.4903, + "theoretical_loss": 3.408996109539519, + "tokens_seen": 2157051904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523232323232323, + "loss": 2.4713, + "theoretical_loss": 3.408987601829905, + "tokens_seen": 2157117440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523030303030303, + "loss": 2.7848, + "theoretical_loss": 3.4089790944511327, + "tokens_seen": 2157182976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522828282828283, + "loss": 2.8257, + "theoretical_loss": 3.4089705874031795, + "tokens_seen": 2157248512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522626262626263, + "loss": 2.4483, + "theoretical_loss": 3.408962080686022, + "tokens_seen": 2157314048 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035224242424242424, + "loss": 2.8532, + "theoretical_loss": 3.4089535742996375, + "tokens_seen": 2157379584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035222222222222225, + "loss": 2.693, + "theoretical_loss": 3.4089450682440035, + "tokens_seen": 2157445120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522020202020202, + "loss": 2.6485, + "theoretical_loss": 3.408936562519096, + "tokens_seen": 2157510656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521818181818182, + "loss": 2.6114, + "theoretical_loss": 3.408928057124893, + "tokens_seen": 2157576192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521616161616162, + "loss": 2.567, + "theoretical_loss": 3.4089195520613718, + "tokens_seen": 2157641728 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035214141414141413, + "loss": 2.6924, + "theoretical_loss": 3.408911047328509, + "tokens_seen": 2157707264 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035212121212121214, + "loss": 2.4765, + "theoretical_loss": 3.4089025429262816, + "tokens_seen": 2157772800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521010101010101, + "loss": 2.6886, + "theoretical_loss": 3.408894038854667, + "tokens_seen": 2157838336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520808080808081, + "loss": 2.5828, + "theoretical_loss": 3.4088855351136425, + "tokens_seen": 2157903872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035206060606060607, + "loss": 2.507, + "theoretical_loss": 3.4088770317031845, + "tokens_seen": 2157969408 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1188127, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4245564937591553, + "objective/train/theoretical_loss": 3.4088685286232705, + "objective/train/tokens_used": 516894176, + "theoretical_loss": 3.4088685286232705, + "tokens_seen": 2158034944 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352040404040404, + "loss": 2.5853, + "theoretical_loss": 3.4088685286232705, + "tokens_seen": 2158034944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035202020202020203, + "loss": 2.6205, + "theoretical_loss": 3.408860025873878, + "tokens_seen": 2158100480 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352, + "loss": 2.6775, + "theoretical_loss": 3.4088515234549837, + "tokens_seen": 2158166016 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351979797979798, + "loss": 2.6753, + "theoretical_loss": 3.4088430213665646, + "tokens_seen": 2158231552 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035195959595959596, + "loss": 2.7198, + "theoretical_loss": 3.408834519608598, + "tokens_seen": 2158297088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035193939393939397, + "loss": 2.6947, + "theoretical_loss": 3.408826018181061, + "tokens_seen": 2158362624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519191919191919, + "loss": 2.816, + "theoretical_loss": 3.408817517083931, + "tokens_seen": 2158428160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518989898989899, + "loss": 2.4772, + "theoretical_loss": 3.4088090163171847, + "tokens_seen": 2158493696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518787878787879, + "loss": 2.6107, + "theoretical_loss": 3.4088005158807992, + "tokens_seen": 2158559232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035185858585858585, + "loss": 2.6709, + "theoretical_loss": 3.408792015774752, + "tokens_seen": 2158624768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035183838383838386, + "loss": 2.3833, + "theoretical_loss": 3.40878351599902, + "tokens_seen": 2158690304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518181818181818, + "loss": 2.6363, + "theoretical_loss": 3.408775016553581, + "tokens_seen": 2158755840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517979797979798, + "loss": 2.389, + "theoretical_loss": 3.408766517438411, + "tokens_seen": 2158821376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517777777777778, + "loss": 2.5093, + "theoretical_loss": 3.408758018653488, + "tokens_seen": 2158886912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517575757575758, + "loss": 2.6983, + "theoretical_loss": 3.4087495201987883, + "tokens_seen": 2158952448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035173737373737376, + "loss": 2.3098, + "theoretical_loss": 3.40874102207429, + "tokens_seen": 2159017984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517171717171717, + "loss": 2.6428, + "theoretical_loss": 3.4087325242799693, + "tokens_seen": 2159083520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035169696969696967, + "loss": 2.5138, + "theoretical_loss": 3.4087240268158046, + "tokens_seen": 2159149056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516767676767677, + "loss": 2.6145, + "theoretical_loss": 3.4087155296817717, + "tokens_seen": 2159214592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516565656565657, + "loss": 2.631, + "theoretical_loss": 3.4087070328778486, + "tokens_seen": 2159280128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035163636363636365, + "loss": 2.6058, + "theoretical_loss": 3.4086985364040125, + "tokens_seen": 2159345664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516161616161616, + "loss": 2.3295, + "theoretical_loss": 3.40869004026024, + "tokens_seen": 2159411200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035159595959595956, + "loss": 2.6824, + "theoretical_loss": 3.408681544446509, + "tokens_seen": 2159476736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515757575757576, + "loss": 2.4669, + "theoretical_loss": 3.4086730489627954, + "tokens_seen": 2159542272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515555555555556, + "loss": 2.8928, + "theoretical_loss": 3.4086645538090776, + "tokens_seen": 2159607808 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1189561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.801823139190674, + "objective/train/theoretical_loss": 3.4086560589853327, + "objective/train/tokens_used": 518532576, + "theoretical_loss": 3.4086560589853327, + "tokens_seen": 2159673344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035153535353535354, + "loss": 2.7104, + "theoretical_loss": 3.4086560589853327, + "tokens_seen": 2159673344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515151515151515, + "loss": 2.534, + "theoretical_loss": 3.4086475644915373, + "tokens_seen": 2159738880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035149494949494945, + "loss": 2.6664, + "theoretical_loss": 3.4086390703276686, + "tokens_seen": 2159804416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514747474747475, + "loss": 2.5441, + "theoretical_loss": 3.408630576493704, + "tokens_seen": 2159869952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514545454545455, + "loss": 2.495, + "theoretical_loss": 3.4086220829896208, + "tokens_seen": 2159935488 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035143434343434343, + "loss": 2.7479, + "theoretical_loss": 3.408613589815396, + "tokens_seen": 2160001024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514141414141414, + "loss": 2.7925, + "theoretical_loss": 3.408605096971007, + "tokens_seen": 2160066560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513939393939394, + "loss": 2.6846, + "theoretical_loss": 3.4085966044564304, + "tokens_seen": 2160132096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513737373737374, + "loss": 2.3807, + "theoretical_loss": 3.408588112271644, + "tokens_seen": 2160197632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035135353535353537, + "loss": 2.8208, + "theoretical_loss": 3.4085796204166243, + "tokens_seen": 2160263168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513333333333333, + "loss": 2.5076, + "theoretical_loss": 3.4085711288913494, + "tokens_seen": 2160328704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513131313131313, + "loss": 2.5498, + "theoretical_loss": 3.4085626376957965, + "tokens_seen": 2160394240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512929292929293, + "loss": 2.5672, + "theoretical_loss": 3.4085541468299416, + "tokens_seen": 2160459776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512727272727273, + "loss": 2.2968, + "theoretical_loss": 3.408545656293763, + "tokens_seen": 2160525312 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035125252525252526, + "loss": 2.6389, + "theoretical_loss": 3.4085371660872377, + "tokens_seen": 2160590848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512323232323232, + "loss": 2.4206, + "theoretical_loss": 3.4085286762103424, + "tokens_seen": 2160656384 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035121212121212123, + "loss": 2.7318, + "theoretical_loss": 3.4085201866630546, + "tokens_seen": 2160721920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511919191919192, + "loss": 2.6289, + "theoretical_loss": 3.408511697445352, + "tokens_seen": 2160787456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511717171717172, + "loss": 2.6766, + "theoretical_loss": 3.408503208557211, + "tokens_seen": 2160852992 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035115151515151515, + "loss": 2.5316, + "theoretical_loss": 3.4084947199986093, + "tokens_seen": 2160918528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511313131313131, + "loss": 2.7037, + "theoretical_loss": 3.408486231769524, + "tokens_seen": 2160984064 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511111111111111, + "loss": 2.6177, + "theoretical_loss": 3.4084777438699323, + "tokens_seen": 2161049600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510909090909091, + "loss": 2.6657, + "theoretical_loss": 3.408469256299812, + "tokens_seen": 2161115136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510707070707071, + "loss": 2.5146, + "theoretical_loss": 3.408460769059139, + "tokens_seen": 2161180672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035105050505050505, + "loss": 2.7131, + "theoretical_loss": 3.4084522821478913, + "tokens_seen": 2161246208 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1190021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.461451530456543, + "objective/train/theoretical_loss": 3.408443795566046, + "objective/train/tokens_used": 520170976, + "theoretical_loss": 3.408443795566046, + "tokens_seen": 2161311744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035103030303030306, + "loss": 2.5318, + "theoretical_loss": 3.408443795566046, + "tokens_seen": 2161311744 + }, + { + "epoch": 0.31, + "learning_rate": 0.000351010101010101, + "loss": 2.7198, + "theoretical_loss": 3.408435309313581, + "tokens_seen": 2161377280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035098989898989897, + "loss": 2.6733, + "theoretical_loss": 3.4084268233904726, + "tokens_seen": 2161442816 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350969696969697, + "loss": 2.5547, + "theoretical_loss": 3.4084183377966983, + "tokens_seen": 2161508352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035094949494949494, + "loss": 2.6012, + "theoretical_loss": 3.4084098525322357, + "tokens_seen": 2161573888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035092929292929295, + "loss": 2.7147, + "theoretical_loss": 3.4084013675970617, + "tokens_seen": 2161639424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509090909090909, + "loss": 2.6112, + "theoretical_loss": 3.408392882991153, + "tokens_seen": 2161704960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035088888888888886, + "loss": 2.3176, + "theoretical_loss": 3.4083843987144884, + "tokens_seen": 2161770496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508686868686869, + "loss": 2.7915, + "theoretical_loss": 3.408375914767044, + "tokens_seen": 2161836032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508484848484849, + "loss": 2.6781, + "theoretical_loss": 3.408367431148797, + "tokens_seen": 2161901568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035082828282828284, + "loss": 2.7152, + "theoretical_loss": 3.4083589478597247, + "tokens_seen": 2161967104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508080808080808, + "loss": 2.5664, + "theoretical_loss": 3.4083504648998044, + "tokens_seen": 2162032640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507878787878788, + "loss": 2.5902, + "theoretical_loss": 3.4083419822690137, + "tokens_seen": 2162098176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035076767676767677, + "loss": 2.5746, + "theoretical_loss": 3.4083334999673296, + "tokens_seen": 2162163712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507474747474748, + "loss": 2.7214, + "theoretical_loss": 3.4083250179947293, + "tokens_seen": 2162229248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035072727272727274, + "loss": 2.4374, + "theoretical_loss": 3.4083165363511903, + "tokens_seen": 2162294784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507070707070707, + "loss": 2.5229, + "theoretical_loss": 3.4083080550366898, + "tokens_seen": 2162360320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506868686868687, + "loss": 2.5961, + "theoretical_loss": 3.4082995740512048, + "tokens_seen": 2162425856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506666666666667, + "loss": 2.6604, + "theoretical_loss": 3.4082910933947126, + "tokens_seen": 2162491392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035064646464646467, + "loss": 2.4124, + "theoretical_loss": 3.408282613067191, + "tokens_seen": 2162556928 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035062626262626263, + "loss": 2.4736, + "theoretical_loss": 3.408274133068617, + "tokens_seen": 2162622464 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506060606060606, + "loss": 2.6433, + "theoretical_loss": 3.408265653398967, + "tokens_seen": 2162688000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505858585858586, + "loss": 2.5658, + "theoretical_loss": 3.4082571740582197, + "tokens_seen": 2162753536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505656565656566, + "loss": 2.3968, + "theoretical_loss": 3.4082486950463515, + "tokens_seen": 2162819072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035054545454545457, + "loss": 2.6287, + "theoretical_loss": 3.4082402163633394, + "tokens_seen": 2162884608 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1191353, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.469416618347168, + "objective/train/theoretical_loss": 3.4082317380091616, + "objective/train/tokens_used": 521809376, + "theoretical_loss": 3.4082317380091616, + "tokens_seen": 2162950144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505252525252525, + "loss": 2.6831, + "theoretical_loss": 3.4082317380091616, + "tokens_seen": 2162950144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505050505050505, + "loss": 2.5446, + "theoretical_loss": 3.408223259983795, + "tokens_seen": 2163015680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035048484848484854, + "loss": 2.7503, + "theoretical_loss": 3.408214782287217, + "tokens_seen": 2163081216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504646464646465, + "loss": 2.8492, + "theoretical_loss": 3.4082063049194042, + "tokens_seen": 2163146752 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035044444444444446, + "loss": 2.6935, + "theoretical_loss": 3.4081978278803344, + "tokens_seen": 2163212288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504242424242424, + "loss": 2.659, + "theoretical_loss": 3.4081893511699852, + "tokens_seen": 2163277824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035040404040404037, + "loss": 2.3811, + "theoretical_loss": 3.408180874788334, + "tokens_seen": 2163343360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035038383838383844, + "loss": 2.7312, + "theoretical_loss": 3.408172398735357, + "tokens_seen": 2163408896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503636363636364, + "loss": 2.7423, + "theoretical_loss": 3.4081639230110325, + "tokens_seen": 2163474432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035034343434343435, + "loss": 2.7973, + "theoretical_loss": 3.4081554476153375, + "tokens_seen": 2163539968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503232323232323, + "loss": 2.6042, + "theoretical_loss": 3.408146972548249, + "tokens_seen": 2163605504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035030303030303026, + "loss": 2.4051, + "theoretical_loss": 3.4081384978097446, + "tokens_seen": 2163671040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035028282828282833, + "loss": 2.5392, + "theoretical_loss": 3.408130023399802, + "tokens_seen": 2163736576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502626262626263, + "loss": 2.5968, + "theoretical_loss": 3.408121549318398, + "tokens_seen": 2163802112 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035024242424242424, + "loss": 2.5486, + "theoretical_loss": 3.40811307556551, + "tokens_seen": 2163867648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502222222222222, + "loss": 2.7453, + "theoretical_loss": 3.4081046021411154, + "tokens_seen": 2163933184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502020202020202, + "loss": 2.6839, + "theoretical_loss": 3.4080961290451914, + "tokens_seen": 2163998720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501818181818182, + "loss": 2.6244, + "theoretical_loss": 3.4080876562777154, + "tokens_seen": 2164064256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501616161616162, + "loss": 2.4852, + "theoretical_loss": 3.4080791838386646, + "tokens_seen": 2164129792 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035014141414141414, + "loss": 2.5086, + "theoretical_loss": 3.4080707117280165, + "tokens_seen": 2164195328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501212121212121, + "loss": 2.4746, + "theoretical_loss": 3.4080622399457483, + "tokens_seen": 2164260864 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501010101010101, + "loss": 2.529, + "theoretical_loss": 3.408053768491838, + "tokens_seen": 2164326400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500808080808081, + "loss": 2.2181, + "theoretical_loss": 3.4080452973662614, + "tokens_seen": 2164391936 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035006060606060607, + "loss": 2.7326, + "theoretical_loss": 3.4080368265689973, + "tokens_seen": 2164457472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035004040404040403, + "loss": 2.4137, + "theoretical_loss": 3.408028356100022, + "tokens_seen": 2164523008 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1192084, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4357054233551025, + "objective/train/theoretical_loss": 3.4080198859593143, + "objective/train/tokens_used": 523447776, + "theoretical_loss": 3.4080198859593143, + "tokens_seen": 2164588544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035002020202020204, + "loss": 2.5545, + "theoretical_loss": 3.4080198859593143, + "tokens_seen": 2164588544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035, + "loss": 2.8726, + "theoretical_loss": 3.4080114161468495, + "tokens_seen": 2164654080 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349979797979798, + "loss": 2.5693, + "theoretical_loss": 3.4080029466626067, + "tokens_seen": 2164719616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034995959595959596, + "loss": 2.4669, + "theoretical_loss": 3.4079944775065623, + "tokens_seen": 2164785152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499393939393939, + "loss": 2.7631, + "theoretical_loss": 3.407986008678694, + "tokens_seen": 2164850688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034991919191919193, + "loss": 2.7671, + "theoretical_loss": 3.4079775401789787, + "tokens_seen": 2164916224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498989898989899, + "loss": 2.6294, + "theoretical_loss": 3.4079690720073943, + "tokens_seen": 2164981760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498787878787879, + "loss": 2.4798, + "theoretical_loss": 3.407960604163918, + "tokens_seen": 2165047296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034985858585858586, + "loss": 2.5692, + "theoretical_loss": 3.4079521366485275, + "tokens_seen": 2165112832 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034983838383838387, + "loss": 2.6114, + "theoretical_loss": 3.4079436694611993, + "tokens_seen": 2165178368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498181818181818, + "loss": 2.7437, + "theoretical_loss": 3.4079352026019114, + "tokens_seen": 2165243904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497979797979798, + "loss": 2.7779, + "theoretical_loss": 3.4079267360706407, + "tokens_seen": 2165309440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497777777777778, + "loss": 2.4274, + "theoretical_loss": 3.4079182698673653, + "tokens_seen": 2165374976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034975757575757575, + "loss": 2.5162, + "theoretical_loss": 3.407909803992062, + "tokens_seen": 2165440512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034973737373737376, + "loss": 2.5543, + "theoretical_loss": 3.407901338444708, + "tokens_seen": 2165506048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497171717171717, + "loss": 2.5477, + "theoretical_loss": 3.4078928732252813, + "tokens_seen": 2165571584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496969696969697, + "loss": 2.5499, + "theoretical_loss": 3.407884408333759, + "tokens_seen": 2165637120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496767676767677, + "loss": 2.6026, + "theoretical_loss": 3.407875943770118, + "tokens_seen": 2165702656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496565656565657, + "loss": 2.6215, + "theoretical_loss": 3.407867479534336, + "tokens_seen": 2165768192 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034963636363636365, + "loss": 2.4358, + "theoretical_loss": 3.407859015626391, + "tokens_seen": 2165833728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496161616161616, + "loss": 2.4935, + "theoretical_loss": 3.40785055204626, + "tokens_seen": 2165899264 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034959595959595957, + "loss": 2.4719, + "theoretical_loss": 3.40784208879392, + "tokens_seen": 2165964800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495757575757576, + "loss": 2.5277, + "theoretical_loss": 3.4078336258693485, + "tokens_seen": 2166030336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495555555555556, + "loss": 2.585, + "theoretical_loss": 3.407825163272523, + "tokens_seen": 2166095872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034953535353535355, + "loss": 2.7183, + "theoretical_loss": 3.407816701003421, + "tokens_seen": 2166161408 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1193465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5298290252685547, + "objective/train/theoretical_loss": 3.4078082390620197, + "objective/train/tokens_used": 525086176, + "theoretical_loss": 3.4078082390620197, + "tokens_seen": 2166226944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495151515151515, + "loss": 2.5874, + "theoretical_loss": 3.4078082390620197, + "tokens_seen": 2166226944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034949494949494946, + "loss": 2.56, + "theoretical_loss": 3.4077997774482967, + "tokens_seen": 2166292480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494747474747475, + "loss": 2.5426, + "theoretical_loss": 3.4077913161622293, + "tokens_seen": 2166358016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494545454545455, + "loss": 2.7146, + "theoretical_loss": 3.407782855203795, + "tokens_seen": 2166423552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034943434343434344, + "loss": 2.5124, + "theoretical_loss": 3.4077743945729706, + "tokens_seen": 2166489088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494141414141414, + "loss": 2.6556, + "theoretical_loss": 3.4077659342697344, + "tokens_seen": 2166554624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034939393939393935, + "loss": 2.5807, + "theoretical_loss": 3.4077574742940637, + "tokens_seen": 2166620160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493737373737374, + "loss": 2.629, + "theoretical_loss": 3.407749014645935, + "tokens_seen": 2166685696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493535353535354, + "loss": 2.538, + "theoretical_loss": 3.4077405553253266, + "tokens_seen": 2166751232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034933333333333333, + "loss": 2.544, + "theoretical_loss": 3.407732096332216, + "tokens_seen": 2166816768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493131313131313, + "loss": 2.6393, + "theoretical_loss": 3.40772363766658, + "tokens_seen": 2166882304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492929292929293, + "loss": 2.5813, + "theoretical_loss": 3.407715179328396, + "tokens_seen": 2166947840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492727272727273, + "loss": 2.7715, + "theoretical_loss": 3.407706721317642, + "tokens_seen": 2167013376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034925252525252527, + "loss": 2.5057, + "theoretical_loss": 3.4076982636342956, + "tokens_seen": 2167078912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492323232323232, + "loss": 2.8016, + "theoretical_loss": 3.407689806278333, + "tokens_seen": 2167144448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492121212121212, + "loss": 2.8176, + "theoretical_loss": 3.407681349249733, + "tokens_seen": 2167209984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491919191919192, + "loss": 2.459, + "theoretical_loss": 3.407672892548472, + "tokens_seen": 2167275520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491717171717172, + "loss": 2.3919, + "theoretical_loss": 3.4076644361745276, + "tokens_seen": 2167341056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034915151515151516, + "loss": 2.7155, + "theoretical_loss": 3.4076559801278776, + "tokens_seen": 2167406592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491313131313131, + "loss": 2.6292, + "theoretical_loss": 3.4076475244084996, + "tokens_seen": 2167472128 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034911111111111113, + "loss": 2.7854, + "theoretical_loss": 3.407639069016371, + "tokens_seen": 2167537664 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034909090909090914, + "loss": 2.6411, + "theoretical_loss": 3.4076306139514685, + "tokens_seen": 2167603200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490707070707071, + "loss": 2.6537, + "theoretical_loss": 3.40762215921377, + "tokens_seen": 2167668736 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034905050505050505, + "loss": 2.8707, + "theoretical_loss": 3.4076137048032535, + "tokens_seen": 2167734272 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349030303030303, + "loss": 2.5394, + "theoretical_loss": 3.4076052507198957, + "tokens_seen": 2167799808 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1194124, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.193073272705078, + "objective/train/theoretical_loss": 3.4075967969636745, + "objective/train/tokens_used": 526724576, + "theoretical_loss": 3.4075967969636745, + "tokens_seen": 2167865344 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349010101010101, + "loss": 2.4182, + "theoretical_loss": 3.4075967969636745, + "tokens_seen": 2167865344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034898989898989903, + "loss": 2.7835, + "theoretical_loss": 3.4075883435345666, + "tokens_seen": 2167930880 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348969696969697, + "loss": 2.639, + "theoretical_loss": 3.40757989043255, + "tokens_seen": 2167996416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034894949494949495, + "loss": 2.6562, + "theoretical_loss": 3.4075714376576025, + "tokens_seen": 2168061952 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034892929292929296, + "loss": 2.6586, + "theoretical_loss": 3.4075629852097014, + "tokens_seen": 2168127488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489090909090909, + "loss": 2.4807, + "theoretical_loss": 3.4075545330888235, + "tokens_seen": 2168193024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488888888888889, + "loss": 2.6037, + "theoretical_loss": 3.4075460812949467, + "tokens_seen": 2168258560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488686868686869, + "loss": 2.633, + "theoretical_loss": 3.407537629828049, + "tokens_seen": 2168324096 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034884848484848484, + "loss": 2.5063, + "theoretical_loss": 3.4075291786881072, + "tokens_seen": 2168389632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034882828282828285, + "loss": 2.5902, + "theoretical_loss": 3.4075207278750987, + "tokens_seen": 2168455168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488080808080808, + "loss": 2.7776, + "theoretical_loss": 3.407512277389001, + "tokens_seen": 2168520704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487878787878788, + "loss": 2.4823, + "theoretical_loss": 3.4075038272297924, + "tokens_seen": 2168586240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487676767676768, + "loss": 2.5794, + "theoretical_loss": 3.4074953773974492, + "tokens_seen": 2168651776 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034874747474747473, + "loss": 2.59, + "theoretical_loss": 3.40748692789195, + "tokens_seen": 2168717312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034872727272727274, + "loss": 2.4795, + "theoretical_loss": 3.4074784787132715, + "tokens_seen": 2168782848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487070707070707, + "loss": 2.4988, + "theoretical_loss": 3.407470029861391, + "tokens_seen": 2168848384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486868686868687, + "loss": 2.6872, + "theoretical_loss": 3.407461581336287, + "tokens_seen": 2168913920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034866666666666667, + "loss": 2.5071, + "theoretical_loss": 3.407453133137936, + "tokens_seen": 2168979456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486464646464647, + "loss": 2.8315, + "theoretical_loss": 3.407444685266316, + "tokens_seen": 2169044992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034862626262626263, + "loss": 2.5917, + "theoretical_loss": 3.407436237721404, + "tokens_seen": 2169110528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486060606060606, + "loss": 2.7336, + "theoretical_loss": 3.4074277905031782, + "tokens_seen": 2169176064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485858585858586, + "loss": 2.463, + "theoretical_loss": 3.407419343611616, + "tokens_seen": 2169241600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034856565656565656, + "loss": 2.4789, + "theoretical_loss": 3.4074108970466943, + "tokens_seen": 2169307136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034854545454545457, + "loss": 2.6116, + "theoretical_loss": 3.407402450808391, + "tokens_seen": 2169372672 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034852525252525253, + "loss": 2.6967, + "theoretical_loss": 3.4073940048966835, + "tokens_seen": 2169438208 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1195527, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4617276191711426, + "objective/train/theoretical_loss": 3.4073855593115496, + "objective/train/tokens_used": 528362976, + "theoretical_loss": 3.4073855593115496, + "tokens_seen": 2169503744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485050505050505, + "loss": 2.602, + "theoretical_loss": 3.4073855593115496, + "tokens_seen": 2169503744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484848484848485, + "loss": 2.567, + "theoretical_loss": 3.4073771140529665, + "tokens_seen": 2169569280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484646464646465, + "loss": 2.5603, + "theoretical_loss": 3.4073686691209115, + "tokens_seen": 2169634816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034844444444444446, + "loss": 2.6815, + "theoretical_loss": 3.4073602245153625, + "tokens_seen": 2169700352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484242424242424, + "loss": 2.6966, + "theoretical_loss": 3.407351780236297, + "tokens_seen": 2169765888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484040404040404, + "loss": 2.9067, + "theoretical_loss": 3.4073433362836925, + "tokens_seen": 2169831424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483838383838384, + "loss": 2.5778, + "theoretical_loss": 3.4073348926575266, + "tokens_seen": 2169896960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483636363636364, + "loss": 2.5324, + "theoretical_loss": 3.407326449357776, + "tokens_seen": 2169962496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034834343434343436, + "loss": 2.7399, + "theoretical_loss": 3.4073180063844197, + "tokens_seen": 2170028032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483232323232323, + "loss": 2.5623, + "theoretical_loss": 3.4073095637374338, + "tokens_seen": 2170093568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034830303030303027, + "loss": 2.5096, + "theoretical_loss": 3.407301121416797, + "tokens_seen": 2170159104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034828282828282834, + "loss": 2.717, + "theoretical_loss": 3.407292679422486, + "tokens_seen": 2170224640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482626262626263, + "loss": 2.5496, + "theoretical_loss": 3.407284237754478, + "tokens_seen": 2170290176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034824242424242425, + "loss": 2.6856, + "theoretical_loss": 3.407275796412752, + "tokens_seen": 2170355712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482222222222222, + "loss": 2.6982, + "theoretical_loss": 3.407267355397284, + "tokens_seen": 2170421248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034820202020202016, + "loss": 2.6895, + "theoretical_loss": 3.407258914708053, + "tokens_seen": 2170486784 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034818181818181823, + "loss": 2.554, + "theoretical_loss": 3.407250474345035, + "tokens_seen": 2170552320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481616161616162, + "loss": 2.62, + "theoretical_loss": 3.407242034308209, + "tokens_seen": 2170617856 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034814141414141414, + "loss": 2.6023, + "theoretical_loss": 3.4072335945975514, + "tokens_seen": 2170683392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481212121212121, + "loss": 2.6248, + "theoretical_loss": 3.40722515521304, + "tokens_seen": 2170748928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481010101010101, + "loss": 2.6655, + "theoretical_loss": 3.407216716154653, + "tokens_seen": 2170814464 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480808080808081, + "loss": 2.6005, + "theoretical_loss": 3.4072082774223675, + "tokens_seen": 2170880000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480606060606061, + "loss": 2.4526, + "theoretical_loss": 3.407199839016161, + "tokens_seen": 2170945536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034804040404040403, + "loss": 2.634, + "theoretical_loss": 3.4071914009360107, + "tokens_seen": 2171011072 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348020202020202, + "loss": 2.5642, + "theoretical_loss": 3.407182963181895, + "tokens_seen": 2171076608 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1196718, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4015064239501953, + "objective/train/theoretical_loss": 3.4071745257537907, + "objective/train/tokens_used": 530001376, + "theoretical_loss": 3.4071745257537907, + "tokens_seen": 2171142144 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348, + "loss": 2.6376, + "theoretical_loss": 3.4071745257537907, + "tokens_seen": 2171142144 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347979797979798, + "loss": 2.5995, + "theoretical_loss": 3.407166088651676, + "tokens_seen": 2171207680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034795959595959597, + "loss": 2.6409, + "theoretical_loss": 3.407157651875528, + "tokens_seen": 2171273216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479393939393939, + "loss": 2.5, + "theoretical_loss": 3.4071492154253242, + "tokens_seen": 2171338752 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034791919191919194, + "loss": 2.6478, + "theoretical_loss": 3.407140779301043, + "tokens_seen": 2171404288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478989898989899, + "loss": 2.527, + "theoretical_loss": 3.407132343502661, + "tokens_seen": 2171469824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478787878787879, + "loss": 2.5339, + "theoretical_loss": 3.4071239080301563, + "tokens_seen": 2171535360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034785858585858586, + "loss": 2.4777, + "theoretical_loss": 3.4071154728835062, + "tokens_seen": 2171600896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478383838383838, + "loss": 2.3114, + "theoretical_loss": 3.407107038062688, + "tokens_seen": 2171666432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034781818181818183, + "loss": 2.6652, + "theoretical_loss": 3.4070986035676802, + "tokens_seen": 2171731968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477979797979798, + "loss": 2.5996, + "theoretical_loss": 3.40709016939846, + "tokens_seen": 2171797504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477777777777778, + "loss": 2.6978, + "theoretical_loss": 3.4070817355550047, + "tokens_seen": 2171863040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034775757575757576, + "loss": 2.4345, + "theoretical_loss": 3.4070733020372916, + "tokens_seen": 2171928576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034773737373737377, + "loss": 2.6146, + "theoretical_loss": 3.4070648688452994, + "tokens_seen": 2171994112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477171717171717, + "loss": 2.4398, + "theoretical_loss": 3.4070564359790048, + "tokens_seen": 2172059648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476969696969697, + "loss": 2.5982, + "theoretical_loss": 3.407048003438385, + "tokens_seen": 2172125184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476767676767677, + "loss": 2.7686, + "theoretical_loss": 3.407039571223419, + "tokens_seen": 2172190720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034765656565656565, + "loss": 2.8074, + "theoretical_loss": 3.4070311393340837, + "tokens_seen": 2172256256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034763636363636366, + "loss": 2.6708, + "theoretical_loss": 3.407022707770356, + "tokens_seen": 2172321792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476161616161616, + "loss": 2.6059, + "theoretical_loss": 3.4070142765322147, + "tokens_seen": 2172387328 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034759595959595957, + "loss": 2.4886, + "theoretical_loss": 3.4070058456196364, + "tokens_seen": 2172452864 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475757575757576, + "loss": 2.4637, + "theoretical_loss": 3.406997415032599, + "tokens_seen": 2172518400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475555555555556, + "loss": 2.6471, + "theoretical_loss": 3.406988984771081, + "tokens_seen": 2172583936 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034753535353535355, + "loss": 2.5464, + "theoretical_loss": 3.4069805548350587, + "tokens_seen": 2172649472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475151515151515, + "loss": 2.7893, + "theoretical_loss": 3.40697212522451, + "tokens_seen": 2172715008 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1197347, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8142454624176025, + "objective/train/theoretical_loss": 3.4069636959394134, + "objective/train/tokens_used": 531639776, + "theoretical_loss": 3.4069636959394134, + "tokens_seen": 2172780544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034749494949494947, + "loss": 2.5133, + "theoretical_loss": 3.4069636959394134, + "tokens_seen": 2172780544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474747474747475, + "loss": 2.6388, + "theoretical_loss": 3.4069552669797454, + "tokens_seen": 2172846080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474545454545455, + "loss": 2.3698, + "theoretical_loss": 3.4069468383454846, + "tokens_seen": 2172911616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034743434343434344, + "loss": 2.4434, + "theoretical_loss": 3.4069384100366076, + "tokens_seen": 2172977152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474141414141414, + "loss": 2.4434, + "theoretical_loss": 3.406929982053093, + "tokens_seen": 2173042688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473939393939394, + "loss": 2.5736, + "theoretical_loss": 3.406921554394918, + "tokens_seen": 2173108224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473737373737374, + "loss": 2.6301, + "theoretical_loss": 3.40691312706206, + "tokens_seen": 2173173760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473535353535354, + "loss": 2.6759, + "theoretical_loss": 3.406904700054497, + "tokens_seen": 2173239296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034733333333333334, + "loss": 2.6137, + "theoretical_loss": 3.406896273372207, + "tokens_seen": 2173304832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473131313131313, + "loss": 2.5298, + "theoretical_loss": 3.4068878470151662, + "tokens_seen": 2173370368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472929292929293, + "loss": 2.6246, + "theoretical_loss": 3.406879420983354, + "tokens_seen": 2173435904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472727272727273, + "loss": 2.5862, + "theoretical_loss": 3.406870995276747, + "tokens_seen": 2173501440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472525252525253, + "loss": 2.594, + "theoretical_loss": 3.4068625698953223, + "tokens_seen": 2173566976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034723232323232323, + "loss": 2.4562, + "theoretical_loss": 3.406854144839059, + "tokens_seen": 2173632512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472121212121212, + "loss": 2.5802, + "theoretical_loss": 3.4068457201079343, + "tokens_seen": 2173698048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034719191919191925, + "loss": 2.6286, + "theoretical_loss": 3.406837295701925, + "tokens_seen": 2173763584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471717171717172, + "loss": 2.6744, + "theoretical_loss": 3.40682887162101, + "tokens_seen": 2173829120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034715151515151517, + "loss": 2.4594, + "theoretical_loss": 3.406820447865166, + "tokens_seen": 2173894656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471313131313131, + "loss": 2.4142, + "theoretical_loss": 3.406812024434371, + "tokens_seen": 2173960192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471111111111111, + "loss": 2.5439, + "theoretical_loss": 3.4068036013286025, + "tokens_seen": 2174025728 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034709090909090914, + "loss": 2.4671, + "theoretical_loss": 3.406795178547838, + "tokens_seen": 2174091264 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470707070707071, + "loss": 2.5607, + "theoretical_loss": 3.406786756092056, + "tokens_seen": 2174156800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034705050505050506, + "loss": 2.5012, + "theoretical_loss": 3.4067783339612334, + "tokens_seen": 2174222336 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347030303030303, + "loss": 2.5176, + "theoretical_loss": 3.4067699121553483, + "tokens_seen": 2174287872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034701010101010097, + "loss": 2.5296, + "theoretical_loss": 3.406761490674378, + "tokens_seen": 2174353408 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1198617, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.766582489013672, + "objective/train/theoretical_loss": 3.4067530695183, + "objective/train/tokens_used": 533278176, + "theoretical_loss": 3.4067530695183, + "tokens_seen": 2174418944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034698989898989904, + "loss": 2.5671, + "theoretical_loss": 3.4067530695183, + "tokens_seen": 2174418944 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346969696969697, + "loss": 2.4026, + "theoretical_loss": 3.4067446486870927, + "tokens_seen": 2174484480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034694949494949495, + "loss": 2.7165, + "theoretical_loss": 3.4067362281807334, + "tokens_seen": 2174550016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003469292929292929, + "loss": 2.6798, + "theoretical_loss": 3.4067278079991996, + "tokens_seen": 2174615552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003469090909090909, + "loss": 2.6543, + "theoretical_loss": 3.4067193881424696, + "tokens_seen": 2174681088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034688888888888893, + "loss": 2.5014, + "theoretical_loss": 3.40671096861052, + "tokens_seen": 2174746624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468686868686869, + "loss": 2.502, + "theoretical_loss": 3.406702549403329, + "tokens_seen": 2174812160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034684848484848484, + "loss": 2.5578, + "theoretical_loss": 3.4066941305208753, + "tokens_seen": 2174877696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468282828282828, + "loss": 2.5906, + "theoretical_loss": 3.406685711963135, + "tokens_seen": 2174943232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468080808080808, + "loss": 2.6116, + "theoretical_loss": 3.4066772937300867, + "tokens_seen": 2175008768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467878787878788, + "loss": 2.3412, + "theoretical_loss": 3.406668875821708, + "tokens_seen": 2175074304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467676767676768, + "loss": 2.541, + "theoretical_loss": 3.406660458237976, + "tokens_seen": 2175139840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034674747474747474, + "loss": 2.5753, + "theoretical_loss": 3.406652040978869, + "tokens_seen": 2175205376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034672727272727275, + "loss": 2.4651, + "theoretical_loss": 3.406643624044365, + "tokens_seen": 2175270912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467070707070707, + "loss": 2.4329, + "theoretical_loss": 3.406635207434441, + "tokens_seen": 2175336448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466868686868687, + "loss": 2.6881, + "theoretical_loss": 3.406626791149075, + "tokens_seen": 2175401984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034666666666666667, + "loss": 2.5079, + "theoretical_loss": 3.4066183751882444, + "tokens_seen": 2175467520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034664646464646463, + "loss": 2.6497, + "theoretical_loss": 3.4066099595519272, + "tokens_seen": 2175533056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034662626262626264, + "loss": 2.6699, + "theoretical_loss": 3.4066015442401016, + "tokens_seen": 2175598592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466060606060606, + "loss": 2.6234, + "theoretical_loss": 3.4065931292527445, + "tokens_seen": 2175664128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465858585858586, + "loss": 2.453, + "theoretical_loss": 3.406584714589834, + "tokens_seen": 2175729664 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034656565656565657, + "loss": 2.5747, + "theoretical_loss": 3.406576300251348, + "tokens_seen": 2175795200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465454545454546, + "loss": 2.7449, + "theoretical_loss": 3.4065678862372635, + "tokens_seen": 2175860736 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034652525252525253, + "loss": 2.6427, + "theoretical_loss": 3.406559472547559, + "tokens_seen": 2175926272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465050505050505, + "loss": 2.509, + "theoretical_loss": 3.406551059182212, + "tokens_seen": 2175991808 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1199271, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.564194440841675, + "objective/train/theoretical_loss": 3.4065426461412, + "objective/train/tokens_used": 534916576, + "theoretical_loss": 3.4065426461412, + "tokens_seen": 2176057344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464848484848485, + "loss": 2.619, + "theoretical_loss": 3.4065426461412, + "tokens_seen": 2176057344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034646464646464646, + "loss": 2.5785, + "theoretical_loss": 3.406534233424501, + "tokens_seen": 2176122880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034644444444444447, + "loss": 2.5945, + "theoretical_loss": 3.4065258210320923, + "tokens_seen": 2176188416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464242424242424, + "loss": 2.3552, + "theoretical_loss": 3.4065174089639525, + "tokens_seen": 2176253952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464040404040404, + "loss": 2.5352, + "theoretical_loss": 3.406508997220058, + "tokens_seen": 2176319488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463838383838384, + "loss": 2.661, + "theoretical_loss": 3.406500585800388, + "tokens_seen": 2176385024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463636363636364, + "loss": 2.5717, + "theoretical_loss": 3.406492174704919, + "tokens_seen": 2176450560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034634343434343436, + "loss": 2.6531, + "theoretical_loss": 3.40648376393363, + "tokens_seen": 2176516096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463232323232323, + "loss": 2.5537, + "theoretical_loss": 3.4064753534864973, + "tokens_seen": 2176581632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463030303030303, + "loss": 2.6112, + "theoretical_loss": 3.4064669433635, + "tokens_seen": 2176647168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462828282828283, + "loss": 2.5072, + "theoretical_loss": 3.4064585335646154, + "tokens_seen": 2176712704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462626262626263, + "loss": 2.4548, + "theoretical_loss": 3.4064501240898206, + "tokens_seen": 2176778240 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034624242424242425, + "loss": 2.5108, + "theoretical_loss": 3.4064417149390938, + "tokens_seen": 2176843776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462222222222222, + "loss": 2.4037, + "theoretical_loss": 3.406433306112413, + "tokens_seen": 2176909312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034620202020202017, + "loss": 2.3914, + "theoretical_loss": 3.4064248976097558, + "tokens_seen": 2176974848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034618181818181823, + "loss": 2.4264, + "theoretical_loss": 3.4064164894310998, + "tokens_seen": 2177040384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461616161616162, + "loss": 2.6455, + "theoretical_loss": 3.406408081576423, + "tokens_seen": 2177105920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034614141414141415, + "loss": 2.4222, + "theoretical_loss": 3.406399674045703, + "tokens_seen": 2177171456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461212121212121, + "loss": 2.6485, + "theoretical_loss": 3.4063912668389174, + "tokens_seen": 2177236992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034610101010101006, + "loss": 2.7297, + "theoretical_loss": 3.406382859956045, + "tokens_seen": 2177302528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460808080808081, + "loss": 2.4382, + "theoretical_loss": 3.4063744533970617, + "tokens_seen": 2177368064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460606060606061, + "loss": 2.2843, + "theoretical_loss": 3.406366047161947, + "tokens_seen": 2177433600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034604040404040404, + "loss": 2.8041, + "theoretical_loss": 3.4063576412506777, + "tokens_seen": 2177499136 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346020202020202, + "loss": 2.6357, + "theoretical_loss": 3.406349235663232, + "tokens_seen": 2177564672 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346, + "loss": 2.5708, + "theoretical_loss": 3.4063408303995875, + "tokens_seen": 2177630208 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1200420, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5001168251037598, + "objective/train/theoretical_loss": 3.4063324254597225, + "objective/train/tokens_used": 536554976, + "theoretical_loss": 3.4063324254597225, + "tokens_seen": 2177695744 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345979797979798, + "loss": 2.5511, + "theoretical_loss": 3.4063324254597225, + "tokens_seen": 2177695744 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345959595959596, + "loss": 2.4979, + "theoretical_loss": 3.4063240208436136, + "tokens_seen": 2177761280 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034593939393939393, + "loss": 2.6736, + "theoretical_loss": 3.40631561655124, + "tokens_seen": 2177826816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003459191919191919, + "loss": 2.628, + "theoretical_loss": 3.406307212582578, + "tokens_seen": 2177892352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458989898989899, + "loss": 2.6927, + "theoretical_loss": 3.406298808937607, + "tokens_seen": 2177957888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458787878787879, + "loss": 2.505, + "theoretical_loss": 3.4062904056163035, + "tokens_seen": 2178023424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034585858585858587, + "loss": 2.5307, + "theoretical_loss": 3.406282002618646, + "tokens_seen": 2178088960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458383838383838, + "loss": 2.6472, + "theoretical_loss": 3.4062735999446123, + "tokens_seen": 2178154496 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034581818181818184, + "loss": 2.4959, + "theoretical_loss": 3.4062651975941796, + "tokens_seen": 2178220032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457979797979798, + "loss": 2.4972, + "theoretical_loss": 3.4062567955673266, + "tokens_seen": 2178285568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457777777777778, + "loss": 2.6403, + "theoretical_loss": 3.40624839386403, + "tokens_seen": 2178351104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034575757575757576, + "loss": 2.8023, + "theoretical_loss": 3.4062399924842683, + "tokens_seen": 2178416640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457373737373737, + "loss": 2.6405, + "theoretical_loss": 3.4062315914280195, + "tokens_seen": 2178482176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034571717171717173, + "loss": 2.4173, + "theoretical_loss": 3.406223190695261, + "tokens_seen": 2178547712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456969696969697, + "loss": 2.5385, + "theoretical_loss": 3.4062147902859707, + "tokens_seen": 2178613248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456767676767677, + "loss": 2.6363, + "theoretical_loss": 3.4062063902001265, + "tokens_seen": 2178678784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034565656565656565, + "loss": 2.6864, + "theoretical_loss": 3.4061979904377058, + "tokens_seen": 2178744320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034563636363636366, + "loss": 2.6469, + "theoretical_loss": 3.4061895909986872, + "tokens_seen": 2178809856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456161616161616, + "loss": 2.4382, + "theoretical_loss": 3.4061811918830482, + "tokens_seen": 2178875392 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034559595959595963, + "loss": 2.5073, + "theoretical_loss": 3.406172793090766, + "tokens_seen": 2178940928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455757575757576, + "loss": 2.5229, + "theoretical_loss": 3.406164394621819, + "tokens_seen": 2179006464 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034555555555555555, + "loss": 2.3853, + "theoretical_loss": 3.4061559964761856, + "tokens_seen": 2179072000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034553535353535356, + "loss": 2.7215, + "theoretical_loss": 3.4061475986538428, + "tokens_seen": 2179137536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455151515151515, + "loss": 2.4677, + "theoretical_loss": 3.4061392011547684, + "tokens_seen": 2179203072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454949494949495, + "loss": 2.7491, + "theoretical_loss": 3.4061308039789404, + "tokens_seen": 2179268608 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1201163, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7939035892486572, + "objective/train/theoretical_loss": 3.406122407126337, + "objective/train/tokens_used": 538193376, + "theoretical_loss": 3.406122407126337, + "tokens_seen": 2179334144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454747474747475, + "loss": 2.5683, + "theoretical_loss": 3.406122407126337, + "tokens_seen": 2179334144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034545454545454544, + "loss": 2.5964, + "theoretical_loss": 3.406114010596936, + "tokens_seen": 2179399680 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034543434343434345, + "loss": 2.4462, + "theoretical_loss": 3.4061056143907145, + "tokens_seen": 2179465216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454141414141414, + "loss": 2.6405, + "theoretical_loss": 3.406097218507651, + "tokens_seen": 2179530752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453939393939394, + "loss": 2.4423, + "theoretical_loss": 3.406088822947723, + "tokens_seen": 2179596288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453737373737374, + "loss": 2.4505, + "theoretical_loss": 3.4060804277109087, + "tokens_seen": 2179661824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453535353535354, + "loss": 2.5004, + "theoretical_loss": 3.406072032797186, + "tokens_seen": 2179727360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034533333333333334, + "loss": 2.4309, + "theoretical_loss": 3.406063638206532, + "tokens_seen": 2179792896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453131313131313, + "loss": 2.7099, + "theoretical_loss": 3.4060552439389253, + "tokens_seen": 2179858432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452929292929293, + "loss": 2.3582, + "theoretical_loss": 3.4060468499943437, + "tokens_seen": 2179923968 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034527272727272727, + "loss": 2.4526, + "theoretical_loss": 3.406038456372765, + "tokens_seen": 2179989504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452525252525253, + "loss": 2.6437, + "theoretical_loss": 3.4060300630741667, + "tokens_seen": 2180055040 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034523232323232324, + "loss": 2.6118, + "theoretical_loss": 3.4060216700985273, + "tokens_seen": 2180120576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452121212121212, + "loss": 2.5669, + "theoretical_loss": 3.4060132774458243, + "tokens_seen": 2180186112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451919191919192, + "loss": 2.625, + "theoretical_loss": 3.406004885116035, + "tokens_seen": 2180251648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451717171717172, + "loss": 2.4939, + "theoretical_loss": 3.4059964931091384, + "tokens_seen": 2180317184 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034515151515151517, + "loss": 2.7215, + "theoretical_loss": 3.405988101425111, + "tokens_seen": 2180382720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034513131313131313, + "loss": 2.525, + "theoretical_loss": 3.4059797100639324, + "tokens_seen": 2180448256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451111111111111, + "loss": 2.6231, + "theoretical_loss": 3.4059713190255794, + "tokens_seen": 2180513792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450909090909091, + "loss": 2.6351, + "theoretical_loss": 3.4059629283100294, + "tokens_seen": 2180579328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450707070707071, + "loss": 2.4628, + "theoretical_loss": 3.4059545379172618, + "tokens_seen": 2180644864 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034505050505050506, + "loss": 2.4169, + "theoretical_loss": 3.405946147847253, + "tokens_seen": 2180710400 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345030303030303, + "loss": 2.5559, + "theoretical_loss": 3.4059377580999817, + "tokens_seen": 2180775936 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345010101010101, + "loss": 2.289, + "theoretical_loss": 3.4059293686754253, + "tokens_seen": 2180841472 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034498989898989904, + "loss": 2.8415, + "theoretical_loss": 3.4059209795735623, + "tokens_seen": 2180907008 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1202317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.683361768722534, + "objective/train/theoretical_loss": 3.40591259079437, + "objective/train/tokens_used": 539831776, + "theoretical_loss": 3.40591259079437, + "tokens_seen": 2180972544 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344969696969697, + "loss": 2.5561, + "theoretical_loss": 3.40591259079437, + "tokens_seen": 2180972544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034494949494949496, + "loss": 2.8251, + "theoretical_loss": 3.405904202337827, + "tokens_seen": 2181038080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449292929292929, + "loss": 2.8263, + "theoretical_loss": 3.40589581420391, + "tokens_seen": 2181103616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034490909090909087, + "loss": 2.3851, + "theoretical_loss": 3.405887426392598, + "tokens_seen": 2181169152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034488888888888894, + "loss": 2.6011, + "theoretical_loss": 3.4058790389038687, + "tokens_seen": 2181234688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448686868686869, + "loss": 2.5665, + "theoretical_loss": 3.4058706517376995, + "tokens_seen": 2181300224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034484848484848485, + "loss": 2.6355, + "theoretical_loss": 3.4058622648940693, + "tokens_seen": 2181365760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448282828282828, + "loss": 2.6874, + "theoretical_loss": 3.4058538783729544, + "tokens_seen": 2181431296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448080808080808, + "loss": 2.2167, + "theoretical_loss": 3.4058454921743344, + "tokens_seen": 2181496832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034478787878787883, + "loss": 2.5234, + "theoretical_loss": 3.4058371062981863, + "tokens_seen": 2181562368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447676767676768, + "loss": 2.4368, + "theoretical_loss": 3.405828720744488, + "tokens_seen": 2181627904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034474747474747474, + "loss": 2.4132, + "theoretical_loss": 3.4058203355132175, + "tokens_seen": 2181693440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447272727272727, + "loss": 2.4337, + "theoretical_loss": 3.4058119506043534, + "tokens_seen": 2181758976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447070707070707, + "loss": 2.6031, + "theoretical_loss": 3.4058035660178727, + "tokens_seen": 2181824512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446868686868687, + "loss": 2.5448, + "theoretical_loss": 3.4057951817537537, + "tokens_seen": 2181890048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446666666666667, + "loss": 2.4287, + "theoretical_loss": 3.405786797811974, + "tokens_seen": 2181955584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034464646464646463, + "loss": 2.5903, + "theoretical_loss": 3.405778414192512, + "tokens_seen": 2182021120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034462626262626265, + "loss": 2.5824, + "theoretical_loss": 3.4057700308953454, + "tokens_seen": 2182086656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446060606060606, + "loss": 2.5581, + "theoretical_loss": 3.4057616479204524, + "tokens_seen": 2182152192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445858585858586, + "loss": 2.7394, + "theoretical_loss": 3.4057532652678106, + "tokens_seen": 2182217728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034456565656565657, + "loss": 2.4358, + "theoretical_loss": 3.405744882937398, + "tokens_seen": 2182283264 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034454545454545453, + "loss": 2.5903, + "theoretical_loss": 3.4057365009291924, + "tokens_seen": 2182348800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034452525252525254, + "loss": 2.4804, + "theoretical_loss": 3.405728119243172, + "tokens_seen": 2182414336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445050505050505, + "loss": 2.5562, + "theoretical_loss": 3.405719737879315, + "tokens_seen": 2182479872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444848484848485, + "loss": 2.4465, + "theoretical_loss": 3.4057113568375987, + "tokens_seen": 2182545408 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1203427, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4712274074554443, + "objective/train/theoretical_loss": 3.4057029761180013, + "objective/train/tokens_used": 541470176, + "theoretical_loss": 3.4057029761180013, + "tokens_seen": 2182610944 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034446464646464646, + "loss": 2.5553, + "theoretical_loss": 3.4057029761180013, + "tokens_seen": 2182610944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444444444444445, + "loss": 2.7063, + "theoretical_loss": 3.405694595720501, + "tokens_seen": 2182676480 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034442424242424243, + "loss": 2.6877, + "theoretical_loss": 3.4056862156450753, + "tokens_seen": 2182742016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444040404040404, + "loss": 2.5239, + "theoretical_loss": 3.4056778358917024, + "tokens_seen": 2182807552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443838383838384, + "loss": 2.564, + "theoretical_loss": 3.4056694564603607, + "tokens_seen": 2182873088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034436363636363636, + "loss": 2.3851, + "theoretical_loss": 3.405661077351027, + "tokens_seen": 2182938624 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034434343434343437, + "loss": 2.463, + "theoretical_loss": 3.4056526985636806, + "tokens_seen": 2183004160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443232323232323, + "loss": 2.5116, + "theoretical_loss": 3.405644320098298, + "tokens_seen": 2183069696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443030303030303, + "loss": 2.7442, + "theoretical_loss": 3.405635941954859, + "tokens_seen": 2183135232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442828282828283, + "loss": 2.5131, + "theoretical_loss": 3.40562756413334, + "tokens_seen": 2183200768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442626262626263, + "loss": 2.6491, + "theoretical_loss": 3.4056191866337198, + "tokens_seen": 2183266304 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034424242424242426, + "loss": 2.4617, + "theoretical_loss": 3.405610809455976, + "tokens_seen": 2183331840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442222222222222, + "loss": 2.5203, + "theoretical_loss": 3.4056024326000864, + "tokens_seen": 2183397376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442020202020202, + "loss": 2.5003, + "theoretical_loss": 3.4055940560660294, + "tokens_seen": 2183462912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441818181818182, + "loss": 2.528, + "theoretical_loss": 3.405585679853783, + "tokens_seen": 2183528448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441616161616162, + "loss": 2.6176, + "theoretical_loss": 3.405577303963325, + "tokens_seen": 2183593984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034414141414141415, + "loss": 2.6952, + "theoretical_loss": 3.405568928394633, + "tokens_seen": 2183659520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441212121212121, + "loss": 2.5119, + "theoretical_loss": 3.4055605531476854, + "tokens_seen": 2183725056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034410101010101007, + "loss": 2.5662, + "theoretical_loss": 3.4055521782224605, + "tokens_seen": 2183790592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034408080808080813, + "loss": 2.428, + "theoretical_loss": 3.4055438036189356, + "tokens_seen": 2183856128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440606060606061, + "loss": 2.2457, + "theoretical_loss": 3.405535429337089, + "tokens_seen": 2183921664 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034404040404040405, + "loss": 2.6719, + "theoretical_loss": 3.405527055376899, + "tokens_seen": 2183987200 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344020202020202, + "loss": 2.5255, + "theoretical_loss": 3.4055186817383434, + "tokens_seen": 2184052736 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034399999999999996, + "loss": 2.7181, + "theoretical_loss": 3.4055103084213996, + "tokens_seen": 2184118272 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343979797979798, + "loss": 2.6533, + "theoretical_loss": 3.4055019354260465, + "tokens_seen": 2184183808 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1204022, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1142489910125732, + "objective/train/theoretical_loss": 3.4054935627522616, + "objective/train/tokens_used": 543108576, + "theoretical_loss": 3.4054935627522616, + "tokens_seen": 2184249344 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343959595959596, + "loss": 2.4261, + "theoretical_loss": 3.4054935627522616, + "tokens_seen": 2184249344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034393939393939394, + "loss": 2.616, + "theoretical_loss": 3.405485190400023, + "tokens_seen": 2184314880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439191919191919, + "loss": 2.3905, + "theoretical_loss": 3.4054768183693085, + "tokens_seen": 2184380416 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034389898989898996, + "loss": 2.3627, + "theoretical_loss": 3.4054684466600964, + "tokens_seen": 2184445952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438787878787879, + "loss": 2.7071, + "theoretical_loss": 3.405460075272365, + "tokens_seen": 2184511488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438585858585859, + "loss": 2.6045, + "theoretical_loss": 3.4054517042060914, + "tokens_seen": 2184577024 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034383838383838383, + "loss": 2.707, + "theoretical_loss": 3.4054433334612546, + "tokens_seen": 2184642560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438181818181818, + "loss": 2.6189, + "theoretical_loss": 3.4054349630378318, + "tokens_seen": 2184708096 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034379797979797985, + "loss": 2.5901, + "theoretical_loss": 3.4054265929358016, + "tokens_seen": 2184773632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437777777777778, + "loss": 2.4351, + "theoretical_loss": 3.4054182231551415, + "tokens_seen": 2184839168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034375757575757577, + "loss": 2.5164, + "theoretical_loss": 3.40540985369583, + "tokens_seen": 2184904704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437373737373737, + "loss": 2.519, + "theoretical_loss": 3.4054014845578453, + "tokens_seen": 2184970240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437171717171717, + "loss": 2.6878, + "theoretical_loss": 3.4053931157411648, + "tokens_seen": 2185035776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034369696969696975, + "loss": 2.6094, + "theoretical_loss": 3.405384747245767, + "tokens_seen": 2185101312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436767676767677, + "loss": 2.4693, + "theoretical_loss": 3.405376379071629, + "tokens_seen": 2185166848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034365656565656566, + "loss": 2.4004, + "theoretical_loss": 3.40536801121873, + "tokens_seen": 2185232384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436363636363636, + "loss": 2.6808, + "theoretical_loss": 3.405359643687048, + "tokens_seen": 2185297920 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034361616161616163, + "loss": 2.4997, + "theoretical_loss": 3.4053512764765603, + "tokens_seen": 2185363456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034359595959595964, + "loss": 2.448, + "theoretical_loss": 3.405342909587245, + "tokens_seen": 2185428992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435757575757576, + "loss": 2.6548, + "theoretical_loss": 3.4053345430190807, + "tokens_seen": 2185494528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034355555555555555, + "loss": 2.3814, + "theoretical_loss": 3.405326176772045, + "tokens_seen": 2185560064 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435353535353535, + "loss": 2.5529, + "theoretical_loss": 3.405317810846116, + "tokens_seen": 2185625600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435151515151515, + "loss": 2.5558, + "theoretical_loss": 3.405309445241272, + "tokens_seen": 2185691136 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034349494949494953, + "loss": 2.4149, + "theoretical_loss": 3.4053010799574914, + "tokens_seen": 2185756672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434747474747475, + "loss": 2.6333, + "theoretical_loss": 3.405292714994751, + "tokens_seen": 2185822208 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1205294, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4200518131256104, + "objective/train/theoretical_loss": 3.40528435035303, + "objective/train/tokens_used": 544746976, + "theoretical_loss": 3.40528435035303, + "tokens_seen": 2185887744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034345454545454544, + "loss": 2.4917, + "theoretical_loss": 3.40528435035303, + "tokens_seen": 2185887744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034343434343434346, + "loss": 2.7631, + "theoretical_loss": 3.405275986032306, + "tokens_seen": 2185953280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434141414141414, + "loss": 2.5811, + "theoretical_loss": 3.4052676220325573, + "tokens_seen": 2186018816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433939393939394, + "loss": 2.6179, + "theoretical_loss": 3.4052592583537615, + "tokens_seen": 2186084352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433737373737374, + "loss": 2.6252, + "theoretical_loss": 3.405250894995897, + "tokens_seen": 2186149888 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034335353535353534, + "loss": 2.8562, + "theoretical_loss": 3.4052425319589417, + "tokens_seen": 2186215424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034333333333333335, + "loss": 2.4894, + "theoretical_loss": 3.405234169242874, + "tokens_seen": 2186280960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433131313131313, + "loss": 2.5904, + "theoretical_loss": 3.4052258068476715, + "tokens_seen": 2186346496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432929292929293, + "loss": 2.6932, + "theoretical_loss": 3.405217444773313, + "tokens_seen": 2186412032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432727272727273, + "loss": 2.3908, + "theoretical_loss": 3.4052090830197757, + "tokens_seen": 2186477568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432525252525253, + "loss": 2.6912, + "theoretical_loss": 3.405200721587038, + "tokens_seen": 2186543104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034323232323232324, + "loss": 2.5519, + "theoretical_loss": 3.405192360475078, + "tokens_seen": 2186608640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432121212121212, + "loss": 2.783, + "theoretical_loss": 3.405183999683874, + "tokens_seen": 2186674176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431919191919192, + "loss": 2.5782, + "theoretical_loss": 3.405175639213404, + "tokens_seen": 2186739712 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034317171717171717, + "loss": 2.3531, + "theoretical_loss": 3.4051672790636456, + "tokens_seen": 2186805248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431515151515152, + "loss": 2.6121, + "theoretical_loss": 3.4051589192345775, + "tokens_seen": 2186870784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034313131313131313, + "loss": 2.5944, + "theoretical_loss": 3.4051505597261778, + "tokens_seen": 2186936320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431111111111111, + "loss": 2.507, + "theoretical_loss": 3.405142200538424, + "tokens_seen": 2187001856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430909090909091, + "loss": 2.7761, + "theoretical_loss": 3.405133841671295, + "tokens_seen": 2187067392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430707070707071, + "loss": 2.6267, + "theoretical_loss": 3.405125483124768, + "tokens_seen": 2187132928 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034305050505050507, + "loss": 2.5586, + "theoretical_loss": 3.4051171248988217, + "tokens_seen": 2187198464 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343030303030303, + "loss": 2.4967, + "theoretical_loss": 3.4051087669934343, + "tokens_seen": 2187264000 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343010101010101, + "loss": 2.5115, + "theoretical_loss": 3.405100409408583, + "tokens_seen": 2187329536 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342989898989899, + "loss": 2.5764, + "theoretical_loss": 3.405092052144247, + "tokens_seen": 2187395072 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342969696969697, + "loss": 2.6482, + "theoretical_loss": 3.405083695200404, + "tokens_seen": 2187460608 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1205951, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2435500621795654, + "objective/train/theoretical_loss": 3.405075338577032, + "objective/train/tokens_used": 546385376, + "theoretical_loss": 3.405075338577032, + "tokens_seen": 2187526144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034294949494949496, + "loss": 2.0967, + "theoretical_loss": 3.405075338577032, + "tokens_seen": 2187526144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429292929292929, + "loss": 2.4133, + "theoretical_loss": 3.405066982274109, + "tokens_seen": 2187591680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429090909090909, + "loss": 2.478, + "theoretical_loss": 3.4050586262916136, + "tokens_seen": 2187657216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034288888888888894, + "loss": 2.5769, + "theoretical_loss": 3.405050270629523, + "tokens_seen": 2187722752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428686868686869, + "loss": 2.441, + "theoretical_loss": 3.4050419152878164, + "tokens_seen": 2187788288 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034284848484848486, + "loss": 2.5369, + "theoretical_loss": 3.4050335602664714, + "tokens_seen": 2187853824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428282828282828, + "loss": 2.6225, + "theoretical_loss": 3.405025205565466, + "tokens_seen": 2187919360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034280808080808077, + "loss": 2.4762, + "theoretical_loss": 3.4050168511847785, + "tokens_seen": 2187984896 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034278787878787883, + "loss": 2.482, + "theoretical_loss": 3.405008497124387, + "tokens_seen": 2188050432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427676767676768, + "loss": 2.6874, + "theoretical_loss": 3.4050001433842696, + "tokens_seen": 2188115968 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034274747474747475, + "loss": 2.7971, + "theoretical_loss": 3.404991789964405, + "tokens_seen": 2188181504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427272727272727, + "loss": 2.7184, + "theoretical_loss": 3.4049834368647702, + "tokens_seen": 2188247040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427070707070707, + "loss": 2.633, + "theoretical_loss": 3.404975084085344, + "tokens_seen": 2188312576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426868686868687, + "loss": 2.6838, + "theoretical_loss": 3.4049667316261045, + "tokens_seen": 2188378112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426666666666667, + "loss": 2.5053, + "theoretical_loss": 3.40495837948703, + "tokens_seen": 2188443648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034264646464646464, + "loss": 2.4909, + "theoretical_loss": 3.404950027668098, + "tokens_seen": 2188509184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426262626262626, + "loss": 2.6741, + "theoretical_loss": 3.4049416761692872, + "tokens_seen": 2188574720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426060606060606, + "loss": 2.6215, + "theoretical_loss": 3.404933324990576, + "tokens_seen": 2188640256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425858585858586, + "loss": 2.4817, + "theoretical_loss": 3.4049249741319416, + "tokens_seen": 2188705792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425656565656566, + "loss": 2.6325, + "theoretical_loss": 3.4049166235933632, + "tokens_seen": 2188771328 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034254545454545453, + "loss": 2.4636, + "theoretical_loss": 3.4049082733748186, + "tokens_seen": 2188836864 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034252525252525254, + "loss": 2.5871, + "theoretical_loss": 3.404899923476285, + "tokens_seen": 2188902400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425050505050505, + "loss": 2.6458, + "theoretical_loss": 3.4048915738977423, + "tokens_seen": 2188967936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424848484848485, + "loss": 2.6708, + "theoretical_loss": 3.4048832246391676, + "tokens_seen": 2189033472 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034246464646464647, + "loss": 2.3265, + "theoretical_loss": 3.4048748757005387, + "tokens_seen": 2189099008 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1207093, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.545804262161255, + "objective/train/theoretical_loss": 3.4048665270818343, + "objective/train/tokens_used": 548023776, + "theoretical_loss": 3.4048665270818343, + "tokens_seen": 2189164544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424444444444444, + "loss": 2.6491, + "theoretical_loss": 3.4048665270818343, + "tokens_seen": 2189164544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034242424242424244, + "loss": 2.376, + "theoretical_loss": 3.4048581787830328, + "tokens_seen": 2189230080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424040404040404, + "loss": 2.4601, + "theoretical_loss": 3.404849830804112, + "tokens_seen": 2189295616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423838383838384, + "loss": 2.5238, + "theoretical_loss": 3.4048414831450504, + "tokens_seen": 2189361152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034236363636363636, + "loss": 2.7257, + "theoretical_loss": 3.4048331358058257, + "tokens_seen": 2189426688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423434343434343, + "loss": 2.5514, + "theoretical_loss": 3.4048247887864163, + "tokens_seen": 2189492224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034232323232323233, + "loss": 2.5212, + "theoretical_loss": 3.4048164420868003, + "tokens_seen": 2189557760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423030303030303, + "loss": 2.7112, + "theoretical_loss": 3.404808095706956, + "tokens_seen": 2189623296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422828282828283, + "loss": 2.4652, + "theoretical_loss": 3.4047997496468616, + "tokens_seen": 2189688832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034226262626262625, + "loss": 2.3664, + "theoretical_loss": 3.4047914039064953, + "tokens_seen": 2189754368 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034224242424242427, + "loss": 2.5596, + "theoretical_loss": 3.404783058485835, + "tokens_seen": 2189819904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422222222222222, + "loss": 2.6098, + "theoretical_loss": 3.404774713384859, + "tokens_seen": 2189885440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034220202020202023, + "loss": 2.4988, + "theoretical_loss": 3.404766368603546, + "tokens_seen": 2189950976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421818181818182, + "loss": 2.5537, + "theoretical_loss": 3.404758024141873, + "tokens_seen": 2190016512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034216161616161615, + "loss": 2.584, + "theoretical_loss": 3.40474967999982, + "tokens_seen": 2190082048 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034214141414141416, + "loss": 2.5137, + "theoretical_loss": 3.4047413361773633, + "tokens_seen": 2190147584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421212121212121, + "loss": 2.7074, + "theoretical_loss": 3.4047329926744823, + "tokens_seen": 2190213120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421010101010101, + "loss": 2.6088, + "theoretical_loss": 3.4047246494911545, + "tokens_seen": 2190278656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420808080808081, + "loss": 2.4021, + "theoretical_loss": 3.4047163066273587, + "tokens_seen": 2190344192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420606060606061, + "loss": 2.5995, + "theoretical_loss": 3.404707964083073, + "tokens_seen": 2190409728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034204040404040405, + "loss": 2.5752, + "theoretical_loss": 3.404699621858275, + "tokens_seen": 2190475264 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342020202020202, + "loss": 2.8434, + "theoretical_loss": 3.4046912799529436, + "tokens_seen": 2190540800 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342, + "loss": 2.4992, + "theoretical_loss": 3.404682938367057, + "tokens_seen": 2190606336 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341979797979798, + "loss": 2.5464, + "theoretical_loss": 3.404674597100593, + "tokens_seen": 2190671872 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341959595959596, + "loss": 2.6982, + "theoretical_loss": 3.4046662561535297, + "tokens_seen": 2190737408 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1208190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.561566114425659, + "objective/train/theoretical_loss": 3.4046579155258456, + "objective/train/tokens_used": 549662176, + "theoretical_loss": 3.4046579155258456, + "tokens_seen": 2190802944 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034193939393939394, + "loss": 2.5418, + "theoretical_loss": 3.4046579155258456, + "tokens_seen": 2190802944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419191919191919, + "loss": 2.2125, + "theoretical_loss": 3.4046495752175194, + "tokens_seen": 2190868480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418989898989899, + "loss": 2.6678, + "theoretical_loss": 3.4046412352285285, + "tokens_seen": 2190934016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418787878787879, + "loss": 2.4224, + "theoretical_loss": 3.4046328955588514, + "tokens_seen": 2190999552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418585858585859, + "loss": 2.5616, + "theoretical_loss": 3.4046245562084665, + "tokens_seen": 2191065088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034183838383838384, + "loss": 2.5746, + "theoretical_loss": 3.404616217177352, + "tokens_seen": 2191130624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418181818181818, + "loss": 2.6096, + "theoretical_loss": 3.404607878465486, + "tokens_seen": 2191196160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417979797979798, + "loss": 2.3851, + "theoretical_loss": 3.4045995400728466, + "tokens_seen": 2191261696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417777777777778, + "loss": 2.5008, + "theoretical_loss": 3.4045912019994122, + "tokens_seen": 2191327232 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034175757575757577, + "loss": 2.4656, + "theoretical_loss": 3.404582864245161, + "tokens_seen": 2191392768 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034173737373737373, + "loss": 2.4672, + "theoretical_loss": 3.4045745268100718, + "tokens_seen": 2191458304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417171717171717, + "loss": 2.6212, + "theoretical_loss": 3.4045661896941217, + "tokens_seen": 2191523840 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034169696969696975, + "loss": 2.4015, + "theoretical_loss": 3.40455785289729, + "tokens_seen": 2191589376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416767676767677, + "loss": 2.591, + "theoretical_loss": 3.404549516419554, + "tokens_seen": 2191654912 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034165656565656566, + "loss": 2.7445, + "theoretical_loss": 3.404541180260893, + "tokens_seen": 2191720448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416363636363636, + "loss": 2.4693, + "theoretical_loss": 3.404532844421284, + "tokens_seen": 2191785984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416161616161616, + "loss": 2.2786, + "theoretical_loss": 3.404524508900707, + "tokens_seen": 2191851520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034159595959595964, + "loss": 2.6081, + "theoretical_loss": 3.404516173699138, + "tokens_seen": 2191917056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415757575757576, + "loss": 2.5328, + "theoretical_loss": 3.404507838816557, + "tokens_seen": 2191982592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034155555555555556, + "loss": 2.6958, + "theoretical_loss": 3.4044995042529416, + "tokens_seen": 2192048128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415353535353535, + "loss": 2.8967, + "theoretical_loss": 3.40449117000827, + "tokens_seen": 2192113664 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415151515151515, + "loss": 2.4535, + "theoretical_loss": 3.404482836082521, + "tokens_seen": 2192179200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034149494949494954, + "loss": 2.4954, + "theoretical_loss": 3.404474502475672, + "tokens_seen": 2192244736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414747474747475, + "loss": 2.5461, + "theoretical_loss": 3.4044661691877023, + "tokens_seen": 2192310272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034145454545454545, + "loss": 2.8149, + "theoretical_loss": 3.4044578362185893, + "tokens_seen": 2192375808 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1208841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7217459678649902, + "objective/train/theoretical_loss": 3.4044495035683116, + "objective/train/tokens_used": 551300576, + "theoretical_loss": 3.4044495035683116, + "tokens_seen": 2192441344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414343434343434, + "loss": 2.5229, + "theoretical_loss": 3.4044495035683116, + "tokens_seen": 2192441344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414141414141414, + "loss": 2.7207, + "theoretical_loss": 3.404441171236847, + "tokens_seen": 2192506880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034139393939393943, + "loss": 2.8168, + "theoretical_loss": 3.404432839224175, + "tokens_seen": 2192572416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413737373737374, + "loss": 2.67, + "theoretical_loss": 3.4044245075302726, + "tokens_seen": 2192637952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034135353535353534, + "loss": 2.4568, + "theoretical_loss": 3.404416176155119, + "tokens_seen": 2192703488 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034133333333333335, + "loss": 2.649, + "theoretical_loss": 3.4044078450986914, + "tokens_seen": 2192769024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413131313131313, + "loss": 2.4792, + "theoretical_loss": 3.404399514360969, + "tokens_seen": 2192834560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412929292929293, + "loss": 2.4585, + "theoretical_loss": 3.40439118394193, + "tokens_seen": 2192900096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412727272727273, + "loss": 2.6644, + "theoretical_loss": 3.4043828538415526, + "tokens_seen": 2192965632 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034125252525252524, + "loss": 2.7125, + "theoretical_loss": 3.4043745240598144, + "tokens_seen": 2193031168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034123232323232325, + "loss": 2.8195, + "theoretical_loss": 3.4043661945966948, + "tokens_seen": 2193096704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412121212121212, + "loss": 2.6109, + "theoretical_loss": 3.4043578654521713, + "tokens_seen": 2193162240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411919191919192, + "loss": 2.5872, + "theoretical_loss": 3.404349536626223, + "tokens_seen": 2193227776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034117171717171717, + "loss": 2.6027, + "theoretical_loss": 3.404341208118827, + "tokens_seen": 2193293312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411515151515152, + "loss": 2.457, + "theoretical_loss": 3.4043328799299624, + "tokens_seen": 2193358848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034113131313131314, + "loss": 2.6272, + "theoretical_loss": 3.4043245520596077, + "tokens_seen": 2193424384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411111111111111, + "loss": 2.5235, + "theoretical_loss": 3.4043162245077405, + "tokens_seen": 2193489920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410909090909091, + "loss": 2.6016, + "theoretical_loss": 3.4043078972743395, + "tokens_seen": 2193555456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034107070707070706, + "loss": 2.6477, + "theoretical_loss": 3.4042995703593832, + "tokens_seen": 2193620992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410505050505051, + "loss": 2.4979, + "theoretical_loss": 3.4042912437628496, + "tokens_seen": 2193686528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034103030303030303, + "loss": 2.5399, + "theoretical_loss": 3.404282917484717, + "tokens_seen": 2193752064 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341010101010101, + "loss": 2.6218, + "theoretical_loss": 3.4042745915249637, + "tokens_seen": 2193817600 + }, + { + "epoch": 0.32, + "learning_rate": 0.000340989898989899, + "loss": 2.5061, + "theoretical_loss": 3.4042662658835683, + "tokens_seen": 2193883136 + }, + { + "epoch": 0.32, + "learning_rate": 0.000340969696969697, + "loss": 2.6345, + "theoretical_loss": 3.404257940560509, + "tokens_seen": 2193948672 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034094949494949497, + "loss": 2.7469, + "theoretical_loss": 3.4042496155557638, + "tokens_seen": 2194014208 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1210098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.358370542526245, + "objective/train/theoretical_loss": 3.4042412908693116, + "objective/train/tokens_used": 552938976, + "theoretical_loss": 3.4042412908693116, + "tokens_seen": 2194079744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003409292929292929, + "loss": 2.4407, + "theoretical_loss": 3.4042412908693116, + "tokens_seen": 2194079744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003409090909090909, + "loss": 2.5013, + "theoretical_loss": 3.40423296650113, + "tokens_seen": 2194145280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408888888888889, + "loss": 2.3507, + "theoretical_loss": 3.404224642451198, + "tokens_seen": 2194210816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408686868686869, + "loss": 2.6394, + "theoretical_loss": 3.404216318719494, + "tokens_seen": 2194276352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034084848484848486, + "loss": 2.6258, + "theoretical_loss": 3.4042079953059954, + "tokens_seen": 2194341888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408282828282828, + "loss": 2.5948, + "theoretical_loss": 3.4041996722106815, + "tokens_seen": 2194407424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408080808080808, + "loss": 2.5158, + "theoretical_loss": 3.40419134943353, + "tokens_seen": 2194472960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034078787878787884, + "loss": 2.4552, + "theoretical_loss": 3.4041830269745197, + "tokens_seen": 2194538496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407676767676768, + "loss": 2.6235, + "theoretical_loss": 3.4041747048336286, + "tokens_seen": 2194604032 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034074747474747475, + "loss": 2.5909, + "theoretical_loss": 3.404166383010835, + "tokens_seen": 2194669568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407272727272727, + "loss": 2.7844, + "theoretical_loss": 3.4041580615061178, + "tokens_seen": 2194735104 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034070707070707067, + "loss": 2.4736, + "theoretical_loss": 3.404149740319455, + "tokens_seen": 2194800640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034068686868686873, + "loss": 2.7426, + "theoretical_loss": 3.4041414194508244, + "tokens_seen": 2194866176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406666666666667, + "loss": 2.5016, + "theoretical_loss": 3.404133098900205, + "tokens_seen": 2194931712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034064646464646465, + "loss": 2.4223, + "theoretical_loss": 3.4041247786675752, + "tokens_seen": 2194997248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406262626262626, + "loss": 2.6775, + "theoretical_loss": 3.404116458752913, + "tokens_seen": 2195062784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034060606060606056, + "loss": 2.7619, + "theoretical_loss": 3.404108139156197, + "tokens_seen": 2195128320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405858585858586, + "loss": 2.5595, + "theoretical_loss": 3.4040998198774055, + "tokens_seen": 2195193856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405656565656566, + "loss": 2.4847, + "theoretical_loss": 3.4040915009165174, + "tokens_seen": 2195259392 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034054545454545454, + "loss": 2.5576, + "theoretical_loss": 3.4040831822735096, + "tokens_seen": 2195324928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405252525252525, + "loss": 2.6036, + "theoretical_loss": 3.404074863948362, + "tokens_seen": 2195390464 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034050505050505056, + "loss": 2.7201, + "theoretical_loss": 3.4040665459410517, + "tokens_seen": 2195456000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404848484848485, + "loss": 2.6916, + "theoretical_loss": 3.404058228251558, + "tokens_seen": 2195521536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404646464646465, + "loss": 2.5531, + "theoretical_loss": 3.404049910879859, + "tokens_seen": 2195587072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034044444444444443, + "loss": 2.4956, + "theoretical_loss": 3.404041593825933, + "tokens_seen": 2195652608 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1211496, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.74421763420105, + "objective/train/theoretical_loss": 3.404033277089759, + "objective/train/tokens_used": 554577376, + "theoretical_loss": 3.404033277089759, + "tokens_seen": 2195718144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404242424242424, + "loss": 2.6424, + "theoretical_loss": 3.404033277089759, + "tokens_seen": 2195718144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034040404040404045, + "loss": 2.6356, + "theoretical_loss": 3.4040249606713138, + "tokens_seen": 2195783680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403838383838384, + "loss": 2.5341, + "theoretical_loss": 3.4040166445705773, + "tokens_seen": 2195849216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034036363636363637, + "loss": 2.7132, + "theoretical_loss": 3.404008328787527, + "tokens_seen": 2195914752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403434343434343, + "loss": 2.6677, + "theoretical_loss": 3.4040000133221424, + "tokens_seen": 2195980288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034032323232323234, + "loss": 2.4513, + "theoretical_loss": 3.4039916981744005, + "tokens_seen": 2196045824 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034030303030303035, + "loss": 2.4427, + "theoretical_loss": 3.4039833833442805, + "tokens_seen": 2196111360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402828282828283, + "loss": 2.5379, + "theoretical_loss": 3.4039750688317603, + "tokens_seen": 2196176896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034026262626262626, + "loss": 2.4633, + "theoretical_loss": 3.403966754636819, + "tokens_seen": 2196242432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402424242424242, + "loss": 2.3333, + "theoretical_loss": 3.4039584407594345, + "tokens_seen": 2196307968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034022222222222223, + "loss": 2.4106, + "theoretical_loss": 3.403950127199585, + "tokens_seen": 2196373504 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034020202020202024, + "loss": 2.76, + "theoretical_loss": 3.4039418139572493, + "tokens_seen": 2196439040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401818181818182, + "loss": 2.6414, + "theoretical_loss": 3.403933501032406, + "tokens_seen": 2196504576 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034016161616161615, + "loss": 2.4559, + "theoretical_loss": 3.403925188425033, + "tokens_seen": 2196570112 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034014141414141416, + "loss": 2.3674, + "theoretical_loss": 3.4039168761351086, + "tokens_seen": 2196635648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401212121212121, + "loss": 2.4677, + "theoretical_loss": 3.4039085641626117, + "tokens_seen": 2196701184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034010101010101013, + "loss": 2.3859, + "theoretical_loss": 3.4039002525075204, + "tokens_seen": 2196766720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400808080808081, + "loss": 2.5589, + "theoretical_loss": 3.4038919411698134, + "tokens_seen": 2196832256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034006060606060605, + "loss": 2.6093, + "theoretical_loss": 3.403883630149469, + "tokens_seen": 2196897792 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034004040404040406, + "loss": 2.5632, + "theoretical_loss": 3.403875319446465, + "tokens_seen": 2196963328 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340020202020202, + "loss": 2.4322, + "theoretical_loss": 3.403867009060781, + "tokens_seen": 2197028864 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034, + "loss": 2.8422, + "theoretical_loss": 3.4038586989923942, + "tokens_seen": 2197094400 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339979797979798, + "loss": 2.6952, + "theoretical_loss": 3.4038503892412844, + "tokens_seen": 2197159936 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339959595959596, + "loss": 2.4847, + "theoretical_loss": 3.4038420798074283, + "tokens_seen": 2197225472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033993939393939395, + "loss": 2.551, + "theoretical_loss": 3.403833770690806, + "tokens_seen": 2197291008 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1212273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0346760749816895, + "objective/train/theoretical_loss": 3.403825461891395, + "objective/train/tokens_used": 556215776, + "theoretical_loss": 3.403825461891395, + "tokens_seen": 2197356544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399191919191919, + "loss": 2.7611, + "theoretical_loss": 3.403825461891395, + "tokens_seen": 2197356544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398989898989899, + "loss": 2.4756, + "theoretical_loss": 3.4038171534091735, + "tokens_seen": 2197422080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398787878787879, + "loss": 2.6328, + "theoretical_loss": 3.4038088452441206, + "tokens_seen": 2197487616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398585858585859, + "loss": 2.7409, + "theoretical_loss": 3.403800537396214, + "tokens_seen": 2197553152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033983838383838384, + "loss": 2.5518, + "theoretical_loss": 3.4037922298654335, + "tokens_seen": 2197618688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398181818181818, + "loss": 2.6347, + "theoretical_loss": 3.403783922651756, + "tokens_seen": 2197684224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397979797979798, + "loss": 2.4669, + "theoretical_loss": 3.403775615755161, + "tokens_seen": 2197749760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397777777777778, + "loss": 2.5773, + "theoretical_loss": 3.403767309175626, + "tokens_seen": 2197815296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397575757575758, + "loss": 2.6887, + "theoretical_loss": 3.4037590029131306, + "tokens_seen": 2197880832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033973737373737373, + "loss": 2.4716, + "theoretical_loss": 3.403750696967652, + "tokens_seen": 2197946368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397171717171717, + "loss": 2.5483, + "theoretical_loss": 3.40374239133917, + "tokens_seen": 2198011904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396969696969697, + "loss": 2.4754, + "theoretical_loss": 3.4037340860276615, + "tokens_seen": 2198077440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396767676767677, + "loss": 2.6292, + "theoretical_loss": 3.4037257810331063, + "tokens_seen": 2198142976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033965656565656567, + "loss": 2.6845, + "theoretical_loss": 3.403717476355482, + "tokens_seen": 2198208512 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033963636363636363, + "loss": 2.6357, + "theoretical_loss": 3.403709171994768, + "tokens_seen": 2198274048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396161616161616, + "loss": 2.3748, + "theoretical_loss": 3.4037008679509415, + "tokens_seen": 2198339584 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033959595959595965, + "loss": 2.4924, + "theoretical_loss": 3.4036925642239817, + "tokens_seen": 2198405120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395757575757576, + "loss": 2.7771, + "theoretical_loss": 3.4036842608138667, + "tokens_seen": 2198470656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033955555555555556, + "loss": 2.506, + "theoretical_loss": 3.403675957720576, + "tokens_seen": 2198536192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395353535353535, + "loss": 2.4778, + "theoretical_loss": 3.4036676549440865, + "tokens_seen": 2198601728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395151515151515, + "loss": 2.5482, + "theoretical_loss": 3.403659352484378, + "tokens_seen": 2198667264 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033949494949494954, + "loss": 2.5968, + "theoretical_loss": 3.4036510503414283, + "tokens_seen": 2198732800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394747474747475, + "loss": 2.6466, + "theoretical_loss": 3.403642748515216, + "tokens_seen": 2198798336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033945454545454546, + "loss": 2.4165, + "theoretical_loss": 3.40363444700572, + "tokens_seen": 2198863872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394343434343434, + "loss": 2.4413, + "theoretical_loss": 3.403626145812918, + "tokens_seen": 2198929408 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1213496, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7383503913879395, + "objective/train/theoretical_loss": 3.4036178449367887, + "objective/train/tokens_used": 557854176, + "theoretical_loss": 3.4036178449367887, + "tokens_seen": 2198994944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394141414141414, + "loss": 2.5066, + "theoretical_loss": 3.4036178449367887, + "tokens_seen": 2198994944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033939393939393943, + "loss": 2.4764, + "theoretical_loss": 3.4036095443773107, + "tokens_seen": 2199060480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393737373737374, + "loss": 2.5683, + "theoretical_loss": 3.4036012441344625, + "tokens_seen": 2199126016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033935353535353535, + "loss": 2.591, + "theoretical_loss": 3.403592944208223, + "tokens_seen": 2199191552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393333333333333, + "loss": 2.5321, + "theoretical_loss": 3.40358464459857, + "tokens_seen": 2199257088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393131313131313, + "loss": 2.2204, + "theoretical_loss": 3.403576345305482, + "tokens_seen": 2199322624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033929292929292933, + "loss": 2.6112, + "theoretical_loss": 3.4035680463289384, + "tokens_seen": 2199388160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392727272727273, + "loss": 2.5665, + "theoretical_loss": 3.4035597476689166, + "tokens_seen": 2199453696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033925252525252524, + "loss": 2.5351, + "theoretical_loss": 3.4035514493253958, + "tokens_seen": 2199519232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033923232323232325, + "loss": 2.7012, + "theoretical_loss": 3.4035431512983543, + "tokens_seen": 2199584768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392121212121212, + "loss": 2.5129, + "theoretical_loss": 3.4035348535877707, + "tokens_seen": 2199650304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391919191919192, + "loss": 2.7113, + "theoretical_loss": 3.403526556193623, + "tokens_seen": 2199715840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391717171717172, + "loss": 2.4473, + "theoretical_loss": 3.40351825911589, + "tokens_seen": 2199781376 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033915151515151513, + "loss": 2.5923, + "theoretical_loss": 3.403509962354551, + "tokens_seen": 2199846912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033913131313131315, + "loss": 2.6168, + "theoretical_loss": 3.403501665909583, + "tokens_seen": 2199912448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391111111111111, + "loss": 2.5153, + "theoretical_loss": 3.403493369780966, + "tokens_seen": 2199977984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390909090909091, + "loss": 2.5009, + "theoretical_loss": 3.4034850739686773, + "tokens_seen": 2200043520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033907070707070707, + "loss": 2.6043, + "theoretical_loss": 3.403476778472696, + "tokens_seen": 2200109056 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339050505050505, + "loss": 2.2239, + "theoretical_loss": 3.4034684832930004, + "tokens_seen": 2200174592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033903030303030304, + "loss": 2.4125, + "theoretical_loss": 3.4034601884295697, + "tokens_seen": 2200240128 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339010101010101, + "loss": 2.661, + "theoretical_loss": 3.4034518938823815, + "tokens_seen": 2200305664 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338989898989899, + "loss": 2.6988, + "theoretical_loss": 3.403443599651415, + "tokens_seen": 2200371200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033896969696969696, + "loss": 2.4141, + "theoretical_loss": 3.4034353057366484, + "tokens_seen": 2200436736 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338949494949495, + "loss": 2.3495, + "theoretical_loss": 3.40342701213806, + "tokens_seen": 2200502272 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033892929292929293, + "loss": 2.6577, + "theoretical_loss": 3.4034187188556286, + "tokens_seen": 2200567808 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1214048, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5324034690856934, + "objective/train/theoretical_loss": 3.403410425889333, + "objective/train/tokens_used": 559492576, + "theoretical_loss": 3.403410425889333, + "tokens_seen": 2200633344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389090909090909, + "loss": 2.6052, + "theoretical_loss": 3.403410425889333, + "tokens_seen": 2200633344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388888888888889, + "loss": 2.8732, + "theoretical_loss": 3.4034021332391515, + "tokens_seen": 2200698880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033886868686868686, + "loss": 2.3591, + "theoretical_loss": 3.4033938409050624, + "tokens_seen": 2200764416 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033884848484848487, + "loss": 2.3755, + "theoretical_loss": 3.4033855488870444, + "tokens_seen": 2200829952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388282828282828, + "loss": 2.5591, + "theoretical_loss": 3.403377257185076, + "tokens_seen": 2200895488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388080808080808, + "loss": 2.4537, + "theoretical_loss": 3.403368965799136, + "tokens_seen": 2200961024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387878787878788, + "loss": 2.6963, + "theoretical_loss": 3.4033606747292033, + "tokens_seen": 2201026560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387676767676768, + "loss": 2.5424, + "theoretical_loss": 3.4033523839752555, + "tokens_seen": 2201092096 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033874747474747476, + "loss": 2.6247, + "theoretical_loss": 3.403344093537272, + "tokens_seen": 2201157632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387272727272727, + "loss": 2.7775, + "theoretical_loss": 3.40333580341523, + "tokens_seen": 2201223168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387070707070707, + "loss": 2.404, + "theoretical_loss": 3.4033275136091095, + "tokens_seen": 2201288704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386868686868687, + "loss": 2.528, + "theoretical_loss": 3.4033192241188885, + "tokens_seen": 2201354240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386666666666667, + "loss": 2.6364, + "theoretical_loss": 3.4033109349445456, + "tokens_seen": 2201419776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033864646464646465, + "loss": 2.7123, + "theoretical_loss": 3.4033026460860594, + "tokens_seen": 2201485312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386262626262626, + "loss": 2.6844, + "theoretical_loss": 3.4032943575434085, + "tokens_seen": 2201550848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386060606060606, + "loss": 2.433, + "theoretical_loss": 3.403286069316571, + "tokens_seen": 2201616384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033858585858585863, + "loss": 2.8952, + "theoretical_loss": 3.4032777814055266, + "tokens_seen": 2201681920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385656565656566, + "loss": 2.6094, + "theoretical_loss": 3.4032694938102526, + "tokens_seen": 2201747456 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033854545454545454, + "loss": 2.6436, + "theoretical_loss": 3.403261206530728, + "tokens_seen": 2201812992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385252525252525, + "loss": 2.4752, + "theoretical_loss": 3.4032529195669317, + "tokens_seen": 2201878528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385050505050505, + "loss": 2.4291, + "theoretical_loss": 3.4032446329188417, + "tokens_seen": 2201944064 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384848484848485, + "loss": 2.9939, + "theoretical_loss": 3.4032363465864375, + "tokens_seen": 2202009600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384646464646465, + "loss": 2.618, + "theoretical_loss": 3.4032280605696967, + "tokens_seen": 2202075136 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033844444444444444, + "loss": 2.5891, + "theoretical_loss": 3.403219774868598, + "tokens_seen": 2202140672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384242424242424, + "loss": 2.612, + "theoretical_loss": 3.403211489483121, + "tokens_seen": 2202206208 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1214686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2311205863952637, + "objective/train/theoretical_loss": 3.403203204413243, + "objective/train/tokens_used": 561130976, + "theoretical_loss": 3.403203204413243, + "tokens_seen": 2202271744 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033840404040404046, + "loss": 2.316, + "theoretical_loss": 3.403203204413243, + "tokens_seen": 2202271744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383838383838384, + "loss": 2.7264, + "theoretical_loss": 3.4031949196589433, + "tokens_seen": 2202337280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383636363636364, + "loss": 2.5682, + "theoretical_loss": 3.4031866352202003, + "tokens_seen": 2202402816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033834343434343433, + "loss": 2.6374, + "theoretical_loss": 3.4031783510969924, + "tokens_seen": 2202468352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383232323232323, + "loss": 2.6837, + "theoretical_loss": 3.403170067289299, + "tokens_seen": 2202533888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033830303030303035, + "loss": 2.4823, + "theoretical_loss": 3.4031617837970973, + "tokens_seen": 2202599424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382828282828283, + "loss": 2.4551, + "theoretical_loss": 3.403153500620367, + "tokens_seen": 2202664960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033826262626262627, + "loss": 2.7068, + "theoretical_loss": 3.4031452177590866, + "tokens_seen": 2202730496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382424242424242, + "loss": 2.3841, + "theoretical_loss": 3.4031369352132343, + "tokens_seen": 2202796032 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033822222222222223, + "loss": 2.6533, + "theoretical_loss": 3.4031286529827893, + "tokens_seen": 2202861568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033820202020202024, + "loss": 2.5688, + "theoretical_loss": 3.4031203710677294, + "tokens_seen": 2202927104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381818181818182, + "loss": 2.5709, + "theoretical_loss": 3.4031120894680336, + "tokens_seen": 2202992640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033816161616161616, + "loss": 2.6249, + "theoretical_loss": 3.4031038081836806, + "tokens_seen": 2203058176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381414141414141, + "loss": 2.5922, + "theoretical_loss": 3.4030955272146493, + "tokens_seen": 2203123712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381212121212121, + "loss": 2.4562, + "theoretical_loss": 3.403087246560917, + "tokens_seen": 2203189248 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033810101010101014, + "loss": 2.4766, + "theoretical_loss": 3.4030789662224645, + "tokens_seen": 2203254784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380808080808081, + "loss": 2.4682, + "theoretical_loss": 3.4030706861992686, + "tokens_seen": 2203320320 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033806060606060605, + "loss": 2.7268, + "theoretical_loss": 3.4030624064913084, + "tokens_seen": 2203385856 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033804040404040406, + "loss": 2.3349, + "theoretical_loss": 3.403054127098563, + "tokens_seen": 2203451392 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338020202020202, + "loss": 2.449, + "theoretical_loss": 3.40304584802101, + "tokens_seen": 2203516928 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033800000000000003, + "loss": 2.6137, + "theoretical_loss": 3.403037569258629, + "tokens_seen": 2203582464 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337979797979798, + "loss": 2.3822, + "theoretical_loss": 3.4030292908113986, + "tokens_seen": 2203648000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033795959595959594, + "loss": 2.521, + "theoretical_loss": 3.403021012679297, + "tokens_seen": 2203713536 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033793939393939395, + "loss": 2.4525, + "theoretical_loss": 3.403012734862303, + "tokens_seen": 2203779072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379191919191919, + "loss": 2.4292, + "theoretical_loss": 3.403004457360395, + "tokens_seen": 2203844608 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1215685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3675143718719482, + "objective/train/theoretical_loss": 3.402996180173552, + "objective/train/tokens_used": 562769376, + "theoretical_loss": 3.402996180173552, + "tokens_seen": 2203910144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378989898989899, + "loss": 2.612, + "theoretical_loss": 3.402996180173552, + "tokens_seen": 2203910144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378787878787879, + "loss": 2.6328, + "theoretical_loss": 3.4029879033017525, + "tokens_seen": 2203975680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378585858585859, + "loss": 2.5141, + "theoretical_loss": 3.4029796267449752, + "tokens_seen": 2204041216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033783838383838385, + "loss": 2.5341, + "theoretical_loss": 3.4029713505031984, + "tokens_seen": 2204106752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378181818181818, + "loss": 2.698, + "theoretical_loss": 3.4029630745764012, + "tokens_seen": 2204172288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377979797979798, + "loss": 2.671, + "theoretical_loss": 3.4029547989645623, + "tokens_seen": 2204237824 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033777777777777777, + "loss": 2.6546, + "theoretical_loss": 3.4029465236676595, + "tokens_seen": 2204303360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377575757575758, + "loss": 2.5107, + "theoretical_loss": 3.4029382486856727, + "tokens_seen": 2204368896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033773737373737374, + "loss": 2.5914, + "theoretical_loss": 3.40292997401858, + "tokens_seen": 2204434432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377171717171717, + "loss": 2.4237, + "theoretical_loss": 3.4029216996663596, + "tokens_seen": 2204499968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376969696969697, + "loss": 2.5588, + "theoretical_loss": 3.4029134256289906, + "tokens_seen": 2204565504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376767676767677, + "loss": 2.5284, + "theoretical_loss": 3.4029051519064515, + "tokens_seen": 2204631040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376565656565657, + "loss": 2.3153, + "theoretical_loss": 3.402896878498721, + "tokens_seen": 2204696576 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033763636363636363, + "loss": 2.8618, + "theoretical_loss": 3.402888605405778, + "tokens_seen": 2204762112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376161616161616, + "loss": 2.607, + "theoretical_loss": 3.402880332627601, + "tokens_seen": 2204827648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375959595959596, + "loss": 2.4707, + "theoretical_loss": 3.4028720601641687, + "tokens_seen": 2204893184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375757575757576, + "loss": 2.5004, + "theoretical_loss": 3.4028637880154595, + "tokens_seen": 2204958720 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033755555555555557, + "loss": 2.6331, + "theoretical_loss": 3.4028555161814524, + "tokens_seen": 2205024256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375353535353535, + "loss": 2.4841, + "theoretical_loss": 3.4028472446621265, + "tokens_seen": 2205089792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375151515151515, + "loss": 2.4584, + "theoretical_loss": 3.4028389734574596, + "tokens_seen": 2205155328 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033749494949494955, + "loss": 2.7482, + "theoretical_loss": 3.4028307025674303, + "tokens_seen": 2205220864 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374747474747475, + "loss": 2.3796, + "theoretical_loss": 3.402822431992018, + "tokens_seen": 2205286400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033745454545454546, + "loss": 2.6053, + "theoretical_loss": 3.402814161731201, + "tokens_seen": 2205351936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374343434343434, + "loss": 2.5552, + "theoretical_loss": 3.4028058917849586, + "tokens_seen": 2205417472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374141414141414, + "loss": 2.416, + "theoretical_loss": 3.4027976221532685, + "tokens_seen": 2205483008 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1216619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1585769653320312, + "objective/train/theoretical_loss": 3.40278935283611, + "objective/train/tokens_used": 564407776, + "theoretical_loss": 3.40278935283611, + "tokens_seen": 2205548544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033739393939393944, + "loss": 2.3125, + "theoretical_loss": 3.40278935283611, + "tokens_seen": 2205548544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373737373737374, + "loss": 2.8216, + "theoretical_loss": 3.402781083833462, + "tokens_seen": 2205614080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033735353535353535, + "loss": 2.5437, + "theoretical_loss": 3.402772815145302, + "tokens_seen": 2205679616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373333333333333, + "loss": 2.3755, + "theoretical_loss": 3.4027645467716106, + "tokens_seen": 2205745152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033731313131313127, + "loss": 2.5188, + "theoretical_loss": 3.4027562787123644, + "tokens_seen": 2205810688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033729292929292933, + "loss": 2.4873, + "theoretical_loss": 3.4027480109675436, + "tokens_seen": 2205876224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372727272727273, + "loss": 2.5146, + "theoretical_loss": 3.4027397435371265, + "tokens_seen": 2205941760 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033725252525252525, + "loss": 2.6188, + "theoretical_loss": 3.402731476421092, + "tokens_seen": 2206007296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372323232323232, + "loss": 2.4189, + "theoretical_loss": 3.4027232096194178, + "tokens_seen": 2206072832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372121212121212, + "loss": 2.684, + "theoretical_loss": 3.402714943132084, + "tokens_seen": 2206138368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371919191919192, + "loss": 2.6022, + "theoretical_loss": 3.4027066769590686, + "tokens_seen": 2206203904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371717171717172, + "loss": 2.6802, + "theoretical_loss": 3.40269841110035, + "tokens_seen": 2206269440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033715151515151514, + "loss": 2.6851, + "theoretical_loss": 3.4026901455559075, + "tokens_seen": 2206334976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371313131313131, + "loss": 2.46, + "theoretical_loss": 3.40268188032572, + "tokens_seen": 2206400512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371111111111111, + "loss": 2.5575, + "theoretical_loss": 3.4026736154097654, + "tokens_seen": 2206466048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370909090909091, + "loss": 2.4875, + "theoretical_loss": 3.402665350808023, + "tokens_seen": 2206531584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370707070707071, + "loss": 2.2709, + "theoretical_loss": 3.4026570865204713, + "tokens_seen": 2206597120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033705050505050503, + "loss": 2.6232, + "theoretical_loss": 3.4026488225470892, + "tokens_seen": 2206662656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033703030303030304, + "loss": 2.7487, + "theoretical_loss": 3.402640558887855, + "tokens_seen": 2206728192 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033701010101010105, + "loss": 2.3714, + "theoretical_loss": 3.402632295542748, + "tokens_seen": 2206793728 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336989898989899, + "loss": 2.7876, + "theoretical_loss": 3.402624032511747, + "tokens_seen": 2206859264 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033696969696969697, + "loss": 2.3925, + "theoretical_loss": 3.40261576979483, + "tokens_seen": 2206924800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369494949494949, + "loss": 2.511, + "theoretical_loss": 3.402607507391976, + "tokens_seen": 2206990336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033692929292929294, + "loss": 2.6294, + "theoretical_loss": 3.4025992453031644, + "tokens_seen": 2207055872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033690909090909095, + "loss": 2.6245, + "theoretical_loss": 3.402590983528373, + "tokens_seen": 2207121408 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1217122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3241126537323, + "objective/train/theoretical_loss": 3.402582722067581, + "objective/train/tokens_used": 566046176, + "theoretical_loss": 3.402582722067581, + "tokens_seen": 2207186944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368888888888889, + "loss": 2.5174, + "theoretical_loss": 3.402582722067581, + "tokens_seen": 2207186944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033686868686868686, + "loss": 2.6031, + "theoretical_loss": 3.4025744609207673, + "tokens_seen": 2207252480 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033684848484848487, + "loss": 2.6602, + "theoretical_loss": 3.4025662000879104, + "tokens_seen": 2207318016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033682828282828283, + "loss": 2.4869, + "theoretical_loss": 3.402557939568989, + "tokens_seen": 2207383552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033680808080808084, + "loss": 2.4876, + "theoretical_loss": 3.4025496793639824, + "tokens_seen": 2207449088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367878787878788, + "loss": 2.7501, + "theoretical_loss": 3.4025414194728683, + "tokens_seen": 2207514624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033676767676767675, + "loss": 2.3752, + "theoretical_loss": 3.4025331598956265, + "tokens_seen": 2207580160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033674747474747476, + "loss": 2.7045, + "theoretical_loss": 3.402524900632235, + "tokens_seen": 2207645696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367272727272727, + "loss": 2.6499, + "theoretical_loss": 3.402516641682673, + "tokens_seen": 2207711232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033670707070707073, + "loss": 2.6278, + "theoretical_loss": 3.4025083830469196, + "tokens_seen": 2207776768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366868686868687, + "loss": 2.5024, + "theoretical_loss": 3.4025001247249524, + "tokens_seen": 2207842304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366666666666667, + "loss": 2.5961, + "theoretical_loss": 3.402491866716751, + "tokens_seen": 2207907840 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033664646464646466, + "loss": 2.3533, + "theoretical_loss": 3.402483609022294, + "tokens_seen": 2207973376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366262626262626, + "loss": 2.5906, + "theoretical_loss": 3.4024753516415607, + "tokens_seen": 2208038912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366060606060606, + "loss": 2.6582, + "theoretical_loss": 3.402467094574529, + "tokens_seen": 2208104448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365858585858586, + "loss": 2.6773, + "theoretical_loss": 3.402458837821178, + "tokens_seen": 2208169984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365656565656566, + "loss": 2.4831, + "theoretical_loss": 3.4024505813814865, + "tokens_seen": 2208235520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033654545454545455, + "loss": 2.2427, + "theoretical_loss": 3.402442325255433, + "tokens_seen": 2208301056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365252525252525, + "loss": 2.6192, + "theoretical_loss": 3.402434069442997, + "tokens_seen": 2208366592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365050505050505, + "loss": 2.6418, + "theoretical_loss": 3.402425813944157, + "tokens_seen": 2208432128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033648484848484853, + "loss": 2.5576, + "theoretical_loss": 3.4024175587588914, + "tokens_seen": 2208497664 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364646464646465, + "loss": 2.6503, + "theoretical_loss": 3.402409303887179, + "tokens_seen": 2208563200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033644444444444444, + "loss": 2.4943, + "theoretical_loss": 3.402401049328999, + "tokens_seen": 2208628736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364242424242424, + "loss": 2.4334, + "theoretical_loss": 3.40239279508433, + "tokens_seen": 2208694272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364040404040404, + "loss": 2.694, + "theoretical_loss": 3.402384541153151, + "tokens_seen": 2208759808 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1218187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.472285509109497, + "objective/train/theoretical_loss": 3.40237628753544, + "objective/train/tokens_used": 567684576, + "theoretical_loss": 3.40237628753544, + "tokens_seen": 2208825344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363838383838384, + "loss": 2.5674, + "theoretical_loss": 3.40237628753544, + "tokens_seen": 2208825344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363636363636364, + "loss": 2.4357, + "theoretical_loss": 3.402368034231177, + "tokens_seen": 2208890880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033634343434343434, + "loss": 2.5027, + "theoretical_loss": 3.4023597812403397, + "tokens_seen": 2208956416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363232323232323, + "loss": 2.57, + "theoretical_loss": 3.4023515285629076, + "tokens_seen": 2209021952 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033630303030303036, + "loss": 2.6849, + "theoretical_loss": 3.402343276198859, + "tokens_seen": 2209087488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362828282828283, + "loss": 2.6881, + "theoretical_loss": 3.402335024148173, + "tokens_seen": 2209153024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033626262626262627, + "loss": 2.4768, + "theoretical_loss": 3.4023267724108286, + "tokens_seen": 2209218560 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033624242424242423, + "loss": 2.4945, + "theoretical_loss": 3.4023185209868045, + "tokens_seen": 2209284096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362222222222222, + "loss": 2.6198, + "theoretical_loss": 3.402310269876079, + "tokens_seen": 2209349632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033620202020202025, + "loss": 2.716, + "theoretical_loss": 3.4023020190786317, + "tokens_seen": 2209415168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361818181818182, + "loss": 2.6395, + "theoretical_loss": 3.4022937685944408, + "tokens_seen": 2209480704 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033616161616161616, + "loss": 2.6683, + "theoretical_loss": 3.4022855184234855, + "tokens_seen": 2209546240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361414141414141, + "loss": 2.4532, + "theoretical_loss": 3.402277268565744, + "tokens_seen": 2209611776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033612121212121213, + "loss": 2.7676, + "theoretical_loss": 3.4022690190211957, + "tokens_seen": 2209677312 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033610101010101014, + "loss": 2.6974, + "theoretical_loss": 3.4022607697898195, + "tokens_seen": 2209742848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360808080808081, + "loss": 2.4207, + "theoretical_loss": 3.402252520871594, + "tokens_seen": 2209808384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033606060606060606, + "loss": 2.4476, + "theoretical_loss": 3.4022442722664983, + "tokens_seen": 2209873920 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336040404040404, + "loss": 2.5548, + "theoretical_loss": 3.4022360239745106, + "tokens_seen": 2209939456 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336020202020202, + "loss": 2.4813, + "theoretical_loss": 3.40222777599561, + "tokens_seen": 2210004992 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033600000000000004, + "loss": 2.5984, + "theoretical_loss": 3.4022195283297756, + "tokens_seen": 2210070528 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335979797979798, + "loss": 2.7201, + "theoretical_loss": 3.402211280976986, + "tokens_seen": 2210136064 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033595959595959595, + "loss": 2.4198, + "theoretical_loss": 3.4022030339372202, + "tokens_seen": 2210201600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033593939393939396, + "loss": 2.6951, + "theoretical_loss": 3.402194787210457, + "tokens_seen": 2210267136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359191919191919, + "loss": 2.6134, + "theoretical_loss": 3.402186540796675, + "tokens_seen": 2210332672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033589898989898993, + "loss": 2.4243, + "theoretical_loss": 3.402178294695853, + "tokens_seen": 2210398208 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1219601, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.494659900665283, + "objective/train/theoretical_loss": 3.4021700489079705, + "objective/train/tokens_used": 569322976, + "theoretical_loss": 3.4021700489079705, + "tokens_seen": 2210463744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358787878787879, + "loss": 2.4444, + "theoretical_loss": 3.4021700489079705, + "tokens_seen": 2210463744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033585858585858584, + "loss": 2.5409, + "theoretical_loss": 3.402161803433006, + "tokens_seen": 2210529280 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033583838383838385, + "loss": 2.5478, + "theoretical_loss": 3.402153558270938, + "tokens_seen": 2210594816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358181818181818, + "loss": 2.6713, + "theoretical_loss": 3.4021453134217454, + "tokens_seen": 2210660352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357979797979798, + "loss": 2.5541, + "theoretical_loss": 3.4021370688854073, + "tokens_seen": 2210725888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357777777777778, + "loss": 2.7063, + "theoretical_loss": 3.402128824661903, + "tokens_seen": 2210791424 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033575757575757573, + "loss": 2.4704, + "theoretical_loss": 3.4021205807512107, + "tokens_seen": 2210856960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033573737373737375, + "loss": 2.583, + "theoretical_loss": 3.402112337153309, + "tokens_seen": 2210922496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357171717171717, + "loss": 2.8334, + "theoretical_loss": 3.4021040938681777, + "tokens_seen": 2210988032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356969696969697, + "loss": 2.5484, + "theoretical_loss": 3.402095850895795, + "tokens_seen": 2211053568 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033567676767676767, + "loss": 2.6949, + "theoretical_loss": 3.40208760823614, + "tokens_seen": 2211119104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356565656565657, + "loss": 2.58, + "theoretical_loss": 3.4020793658891915, + "tokens_seen": 2211184640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033563636363636364, + "loss": 2.5433, + "theoretical_loss": 3.402071123854928, + "tokens_seen": 2211250176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356161616161616, + "loss": 2.7101, + "theoretical_loss": 3.402062882133329, + "tokens_seen": 2211315712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355959595959596, + "loss": 2.5147, + "theoretical_loss": 3.4020546407243732, + "tokens_seen": 2211381248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033557575757575756, + "loss": 2.4719, + "theoretical_loss": 3.402046399628039, + "tokens_seen": 2211446784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355555555555556, + "loss": 2.6339, + "theoretical_loss": 3.4020381588443063, + "tokens_seen": 2211512320 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033553535353535353, + "loss": 2.4807, + "theoretical_loss": 3.402029918373153, + "tokens_seen": 2211577856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355151515151515, + "loss": 2.4844, + "theoretical_loss": 3.402021678214558, + "tokens_seen": 2211643392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354949494949495, + "loss": 2.6246, + "theoretical_loss": 3.402013438368501, + "tokens_seen": 2211708928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354747474747475, + "loss": 2.3825, + "theoretical_loss": 3.40200519883496, + "tokens_seen": 2211774464 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033545454545454547, + "loss": 2.3494, + "theoretical_loss": 3.401996959613914, + "tokens_seen": 2211840000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354343434343434, + "loss": 2.5031, + "theoretical_loss": 3.4019887207053428, + "tokens_seen": 2211905536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354141414141414, + "loss": 2.484, + "theoretical_loss": 3.4019804821092245, + "tokens_seen": 2211971072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353939393939394, + "loss": 2.884, + "theoretical_loss": 3.401972243825538, + "tokens_seen": 2212036608 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1220207, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6348462104797363, + "objective/train/theoretical_loss": 3.4019640058542624, + "objective/train/tokens_used": 570961376, + "theoretical_loss": 3.4019640058542624, + "tokens_seen": 2212102144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353737373737374, + "loss": 2.3554, + "theoretical_loss": 3.4019640058542624, + "tokens_seen": 2212102144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033535353535353536, + "loss": 2.5459, + "theoretical_loss": 3.4019557681953767, + "tokens_seen": 2212167680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353333333333333, + "loss": 2.4922, + "theoretical_loss": 3.4019475308488594, + "tokens_seen": 2212233216 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033531313131313133, + "loss": 2.7239, + "theoretical_loss": 3.4019392938146895, + "tokens_seen": 2212298752 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033529292929292934, + "loss": 2.414, + "theoretical_loss": 3.4019310570928463, + "tokens_seen": 2212364288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352727272727273, + "loss": 2.8008, + "theoretical_loss": 3.4019228206833088, + "tokens_seen": 2212429824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033525252525252525, + "loss": 2.6254, + "theoretical_loss": 3.4019145845860552, + "tokens_seen": 2212495360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352323232323232, + "loss": 2.5691, + "theoretical_loss": 3.401906348801065, + "tokens_seen": 2212560896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352121212121212, + "loss": 2.8517, + "theoretical_loss": 3.401898113328316, + "tokens_seen": 2212626432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033519191919191923, + "loss": 2.6577, + "theoretical_loss": 3.401889878167789, + "tokens_seen": 2212691968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351717171717172, + "loss": 2.5093, + "theoretical_loss": 3.4018816433194616, + "tokens_seen": 2212757504 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033515151515151515, + "loss": 2.5333, + "theoretical_loss": 3.401873408783313, + "tokens_seen": 2212823040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351313131313131, + "loss": 2.7045, + "theoretical_loss": 3.401865174559322, + "tokens_seen": 2212888576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033511111111111117, + "loss": 2.5791, + "theoretical_loss": 3.401856940647468, + "tokens_seen": 2212954112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350909090909091, + "loss": 2.4296, + "theoretical_loss": 3.401848707047729, + "tokens_seen": 2213019648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350707070707071, + "loss": 2.6218, + "theoretical_loss": 3.4018404737600854, + "tokens_seen": 2213085184 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033505050505050504, + "loss": 2.6225, + "theoretical_loss": 3.4018322407845147, + "tokens_seen": 2213150720 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335030303030303, + "loss": 2.6526, + "theoretical_loss": 3.4018240081209963, + "tokens_seen": 2213216256 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033501010101010106, + "loss": 2.6658, + "theoretical_loss": 3.4018157757695096, + "tokens_seen": 2213281792 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334989898989899, + "loss": 2.3678, + "theoretical_loss": 3.401807543730033, + "tokens_seen": 2213347328 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334969696969697, + "loss": 2.5685, + "theoretical_loss": 3.401799312002545, + "tokens_seen": 2213412864 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033494949494949493, + "loss": 2.6017, + "theoretical_loss": 3.401791080587026, + "tokens_seen": 2213478400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033492929292929294, + "loss": 2.6115, + "theoretical_loss": 3.4017828494834537, + "tokens_seen": 2213543936 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033490909090909095, + "loss": 2.5889, + "theoretical_loss": 3.4017746186918076, + "tokens_seen": 2213609472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348888888888889, + "loss": 2.6011, + "theoretical_loss": 3.4017663882120663, + "tokens_seen": 2213675008 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1221489, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4770944118499756, + "objective/train/theoretical_loss": 3.401758158044209, + "objective/train/tokens_used": 572599776, + "theoretical_loss": 3.401758158044209, + "tokens_seen": 2213740544 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033486868686868687, + "loss": 2.4942, + "theoretical_loss": 3.401758158044209, + "tokens_seen": 2213740544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348484848484848, + "loss": 2.5457, + "theoretical_loss": 3.4017499281882144, + "tokens_seen": 2213806080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033482828282828283, + "loss": 2.5708, + "theoretical_loss": 3.4017416986440616, + "tokens_seen": 2213871616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033480808080808085, + "loss": 2.5185, + "theoretical_loss": 3.4017334694117296, + "tokens_seen": 2213937152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347878787878788, + "loss": 2.5026, + "theoretical_loss": 3.4017252404911975, + "tokens_seen": 2214002688 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033476767676767676, + "loss": 2.5462, + "theoretical_loss": 3.4017170118824436, + "tokens_seen": 2214068224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033474747474747477, + "loss": 2.5386, + "theoretical_loss": 3.4017087835854474, + "tokens_seen": 2214133760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347272727272727, + "loss": 2.5107, + "theoretical_loss": 3.401700555600188, + "tokens_seen": 2214199296 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033470707070707074, + "loss": 2.3553, + "theoretical_loss": 3.401692327926644, + "tokens_seen": 2214264832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346868686868687, + "loss": 2.5025, + "theoretical_loss": 3.401684100564795, + "tokens_seen": 2214330368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033466666666666665, + "loss": 2.4847, + "theoretical_loss": 3.4016758735146193, + "tokens_seen": 2214395904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033464646464646466, + "loss": 2.5301, + "theoretical_loss": 3.4016676467760956, + "tokens_seen": 2214461440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346262626262626, + "loss": 2.5679, + "theoretical_loss": 3.401659420349204, + "tokens_seen": 2214526976 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033460606060606063, + "loss": 2.5658, + "theoretical_loss": 3.4016511942339225, + "tokens_seen": 2214592512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345858585858586, + "loss": 2.453, + "theoretical_loss": 3.40164296843023, + "tokens_seen": 2214658048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345656565656566, + "loss": 2.5215, + "theoretical_loss": 3.401634742938106, + "tokens_seen": 2214723584 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033454545454545456, + "loss": 2.5998, + "theoretical_loss": 3.40162651775753, + "tokens_seen": 2214789120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345252525252525, + "loss": 2.7373, + "theoretical_loss": 3.4016182928884797, + "tokens_seen": 2214854656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345050505050505, + "loss": 2.2561, + "theoretical_loss": 3.4016100683309345, + "tokens_seen": 2214920192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344848484848485, + "loss": 2.4015, + "theoretical_loss": 3.4016018440848743, + "tokens_seen": 2214985728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344646464646465, + "loss": 2.746, + "theoretical_loss": 3.401593620150277, + "tokens_seen": 2215051264 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033444444444444445, + "loss": 2.3898, + "theoretical_loss": 3.4015853965271217, + "tokens_seen": 2215116800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344242424242424, + "loss": 2.5943, + "theoretical_loss": 3.401577173215388, + "tokens_seen": 2215182336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344040404040404, + "loss": 2.4772, + "theoretical_loss": 3.4015689502150543, + "tokens_seen": 2215247872 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033438383838383843, + "loss": 2.5286, + "theoretical_loss": 3.4015607275260997, + "tokens_seen": 2215313408 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1222176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.791827440261841, + "objective/train/theoretical_loss": 3.401552505148504, + "objective/train/tokens_used": 574238176, + "theoretical_loss": 3.401552505148504, + "tokens_seen": 2215378944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343636363636364, + "loss": 2.6742, + "theoretical_loss": 3.401552505148504, + "tokens_seen": 2215378944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033434343434343434, + "loss": 2.4913, + "theoretical_loss": 3.401544283082245, + "tokens_seen": 2215444480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343232323232323, + "loss": 2.6004, + "theoretical_loss": 3.4015360613273025, + "tokens_seen": 2215510016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343030303030303, + "loss": 2.3775, + "theoretical_loss": 3.401527839883655, + "tokens_seen": 2215575552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342828282828283, + "loss": 2.4678, + "theoretical_loss": 3.4015196187512817, + "tokens_seen": 2215641088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342626262626263, + "loss": 2.4456, + "theoretical_loss": 3.401511397930162, + "tokens_seen": 2215706624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033424242424242423, + "loss": 2.5144, + "theoretical_loss": 3.4015031774202744, + "tokens_seen": 2215772160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342222222222222, + "loss": 2.2735, + "theoretical_loss": 3.401494957221598, + "tokens_seen": 2215837696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033420202020202026, + "loss": 2.4292, + "theoretical_loss": 3.401486737334112, + "tokens_seen": 2215903232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341818181818182, + "loss": 2.4665, + "theoretical_loss": 3.4014785177577953, + "tokens_seen": 2215968768 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033416161616161617, + "loss": 2.4636, + "theoretical_loss": 3.401470298492627, + "tokens_seen": 2216034304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341414141414141, + "loss": 2.5201, + "theoretical_loss": 3.401462079538586, + "tokens_seen": 2216099840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341212121212121, + "loss": 2.6508, + "theoretical_loss": 3.4014538608956513, + "tokens_seen": 2216165376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033410101010101015, + "loss": 2.7864, + "theoretical_loss": 3.401445642563802, + "tokens_seen": 2216230912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340808080808081, + "loss": 2.6858, + "theoretical_loss": 3.4014374245430172, + "tokens_seen": 2216296448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033406060606060606, + "loss": 2.4874, + "theoretical_loss": 3.401429206833276, + "tokens_seen": 2216361984 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334040404040404, + "loss": 2.3966, + "theoretical_loss": 3.4014209894345573, + "tokens_seen": 2216427520 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334020202020202, + "loss": 2.562, + "theoretical_loss": 3.40141277234684, + "tokens_seen": 2216493056 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033400000000000004, + "loss": 2.7305, + "theoretical_loss": 3.401404555570103, + "tokens_seen": 2216558592 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333979797979798, + "loss": 2.4205, + "theoretical_loss": 3.401396339104326, + "tokens_seen": 2216624128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033395959595959595, + "loss": 2.6097, + "theoretical_loss": 3.4013881229494873, + "tokens_seen": 2216689664 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339393939393939, + "loss": 2.5154, + "theoretical_loss": 3.401379907105566, + "tokens_seen": 2216755200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339191919191919, + "loss": 2.604, + "theoretical_loss": 3.401371691572542, + "tokens_seen": 2216820736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033389898989898993, + "loss": 2.5253, + "theoretical_loss": 3.4013634763503937, + "tokens_seen": 2216886272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338787878787879, + "loss": 2.4062, + "theoretical_loss": 3.4013552614391003, + "tokens_seen": 2216951808 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1223215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9110116958618164, + "objective/train/theoretical_loss": 3.40134704683864, + "objective/train/tokens_used": 575876576, + "theoretical_loss": 3.40134704683864, + "tokens_seen": 2217017344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033385858585858585, + "loss": 2.5671, + "theoretical_loss": 3.40134704683864, + "tokens_seen": 2217017344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338383838383838, + "loss": 2.771, + "theoretical_loss": 3.4013388325489933, + "tokens_seen": 2217082880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338181818181818, + "loss": 2.5389, + "theoretical_loss": 3.4013306185701384, + "tokens_seen": 2217148416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337979797979798, + "loss": 2.7053, + "theoretical_loss": 3.4013224049020545, + "tokens_seen": 2217213952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337777777777778, + "loss": 2.5321, + "theoretical_loss": 3.4013141915447207, + "tokens_seen": 2217279488 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033375757575757574, + "loss": 2.5079, + "theoretical_loss": 3.401305978498116, + "tokens_seen": 2217345024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033373737373737375, + "loss": 2.5116, + "theoretical_loss": 3.4012977657622194, + "tokens_seen": 2217410560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337171717171717, + "loss": 2.6053, + "theoretical_loss": 3.40128955333701, + "tokens_seen": 2217476096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336969696969697, + "loss": 2.498, + "theoretical_loss": 3.401281341222467, + "tokens_seen": 2217541632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336767676767677, + "loss": 2.6724, + "theoretical_loss": 3.4012731294185694, + "tokens_seen": 2217607168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033365656565656563, + "loss": 2.5151, + "theoretical_loss": 3.4012649179252965, + "tokens_seen": 2217672704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033363636363636364, + "loss": 2.6096, + "theoretical_loss": 3.4012567067426267, + "tokens_seen": 2217738240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033361616161616166, + "loss": 2.7375, + "theoretical_loss": 3.4012484958705396, + "tokens_seen": 2217803776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335959595959596, + "loss": 2.3421, + "theoretical_loss": 3.4012402853090142, + "tokens_seen": 2217869312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033357575757575757, + "loss": 2.5651, + "theoretical_loss": 3.40123207505803, + "tokens_seen": 2217934848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335555555555556, + "loss": 2.5675, + "theoretical_loss": 3.401223865117565, + "tokens_seen": 2218000384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033353535353535354, + "loss": 2.5096, + "theoretical_loss": 3.401215655487599, + "tokens_seen": 2218065920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033351515151515155, + "loss": 2.7468, + "theoretical_loss": 3.401207446168111, + "tokens_seen": 2218131456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334949494949495, + "loss": 2.5889, + "theoretical_loss": 3.4011992371590805, + "tokens_seen": 2218196992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033347474747474746, + "loss": 2.639, + "theoretical_loss": 3.401191028460486, + "tokens_seen": 2218262528 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033345454545454547, + "loss": 2.392, + "theoretical_loss": 3.4011828200723064, + "tokens_seen": 2218328064 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033343434343434343, + "loss": 2.6493, + "theoretical_loss": 3.4011746119945214, + "tokens_seen": 2218393600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033341414141414144, + "loss": 2.5369, + "theoretical_loss": 3.40116640422711, + "tokens_seen": 2218459136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333939393939394, + "loss": 2.5511, + "theoretical_loss": 3.4011581967700506, + "tokens_seen": 2218524672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333737373737374, + "loss": 2.7442, + "theoretical_loss": 3.401149989623323, + "tokens_seen": 2218590208 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1224236, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.486609935760498, + "objective/train/theoretical_loss": 3.4011417827869064, + "objective/train/tokens_used": 577514976, + "theoretical_loss": 3.4011417827869064, + "tokens_seen": 2218655744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033335353535353537, + "loss": 2.6855, + "theoretical_loss": 3.4011417827869064, + "tokens_seen": 2218655744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333333333333333, + "loss": 2.6871, + "theoretical_loss": 3.4011335762607797, + "tokens_seen": 2218721280 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033331313131313133, + "loss": 2.5971, + "theoretical_loss": 3.4011253700449218, + "tokens_seen": 2218786816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332929292929293, + "loss": 2.4757, + "theoretical_loss": 3.401117164139312, + "tokens_seen": 2218852352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332727272727273, + "loss": 2.6521, + "theoretical_loss": 3.4011089585439294, + "tokens_seen": 2218917888 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033325252525252526, + "loss": 2.6123, + "theoretical_loss": 3.4011007532587527, + "tokens_seen": 2218983424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332323232323232, + "loss": 2.6107, + "theoretical_loss": 3.4010925482837617, + "tokens_seen": 2219048960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332121212121212, + "loss": 2.6139, + "theoretical_loss": 3.401084343618935, + "tokens_seen": 2219114496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033319191919191924, + "loss": 2.6066, + "theoretical_loss": 3.401076139264252, + "tokens_seen": 2219180032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331717171717172, + "loss": 2.475, + "theoretical_loss": 3.4010679352196918, + "tokens_seen": 2219245568 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033315151515151515, + "loss": 2.5528, + "theoretical_loss": 3.4010597314852333, + "tokens_seen": 2219311104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331313131313131, + "loss": 2.704, + "theoretical_loss": 3.4010515280608558, + "tokens_seen": 2219376640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331111111111111, + "loss": 2.4247, + "theoretical_loss": 3.401043324946538, + "tokens_seen": 2219442176 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033309090909090913, + "loss": 2.7846, + "theoretical_loss": 3.40103512214226, + "tokens_seen": 2219507712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330707070707071, + "loss": 2.826, + "theoretical_loss": 3.4010269196480003, + "tokens_seen": 2219573248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033305050505050504, + "loss": 2.6933, + "theoretical_loss": 3.401018717463738, + "tokens_seen": 2219638784 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333030303030303, + "loss": 2.6265, + "theoretical_loss": 3.4010105155894523, + "tokens_seen": 2219704320 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033301010101010107, + "loss": 2.4605, + "theoretical_loss": 3.401002314025122, + "tokens_seen": 2219769856 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332989898989899, + "loss": 2.7084, + "theoretical_loss": 3.400994112770727, + "tokens_seen": 2219835392 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332969696969697, + "loss": 2.6849, + "theoretical_loss": 3.4009859118262455, + "tokens_seen": 2219900928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033294949494949494, + "loss": 2.57, + "theoretical_loss": 3.4009777111916577, + "tokens_seen": 2219966464 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329292929292929, + "loss": 2.3423, + "theoretical_loss": 3.400969510866942, + "tokens_seen": 2220032000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033290909090909096, + "loss": 2.6705, + "theoretical_loss": 3.4009613108520775, + "tokens_seen": 2220097536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328888888888889, + "loss": 2.5765, + "theoretical_loss": 3.400953111147044, + "tokens_seen": 2220163072 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033286868686868687, + "loss": 2.7474, + "theoretical_loss": 3.40094491175182, + "tokens_seen": 2220228608 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1224894, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.333254337310791, + "objective/train/theoretical_loss": 3.4009367126663848, + "objective/train/tokens_used": 579153376, + "theoretical_loss": 3.4009367126663848, + "tokens_seen": 2220294144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033284848484848483, + "loss": 2.6027, + "theoretical_loss": 3.4009367126663848, + "tokens_seen": 2220294144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033282828282828284, + "loss": 2.6271, + "theoretical_loss": 3.400928513890718, + "tokens_seen": 2220359680 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033280808080808085, + "loss": 2.6168, + "theoretical_loss": 3.4009203154247976, + "tokens_seen": 2220425216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327878787878788, + "loss": 2.7739, + "theoretical_loss": 3.400912117268604, + "tokens_seen": 2220490752 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033276767676767676, + "loss": 2.774, + "theoretical_loss": 3.4009039194221162, + "tokens_seen": 2220556288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327474747474747, + "loss": 2.5009, + "theoretical_loss": 3.4008957218853126, + "tokens_seen": 2220621824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033272727272727273, + "loss": 2.8622, + "theoretical_loss": 3.4008875246581733, + "tokens_seen": 2220687360 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033270707070707074, + "loss": 2.7762, + "theoretical_loss": 3.4008793277406766, + "tokens_seen": 2220752896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326868686868687, + "loss": 2.5999, + "theoretical_loss": 3.400871131132802, + "tokens_seen": 2220818432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033266666666666666, + "loss": 2.4279, + "theoretical_loss": 3.400862934834529, + "tokens_seen": 2220883968 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033264646464646467, + "loss": 2.5061, + "theoretical_loss": 3.4008547388458363, + "tokens_seen": 2220949504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326262626262626, + "loss": 2.5058, + "theoretical_loss": 3.400846543166703, + "tokens_seen": 2221015040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033260606060606064, + "loss": 2.3584, + "theoretical_loss": 3.400838347797109, + "tokens_seen": 2221080576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325858585858586, + "loss": 2.3845, + "theoretical_loss": 3.4008301527370333, + "tokens_seen": 2221146112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033256565656565655, + "loss": 2.5764, + "theoretical_loss": 3.4008219579864543, + "tokens_seen": 2221211648 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033254545454545456, + "loss": 2.646, + "theoretical_loss": 3.4008137635453517, + "tokens_seen": 2221277184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325252525252525, + "loss": 2.5971, + "theoretical_loss": 3.4008055694137047, + "tokens_seen": 2221342720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033250505050505053, + "loss": 2.6967, + "theoretical_loss": 3.4007973755914924, + "tokens_seen": 2221408256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324848484848485, + "loss": 2.727, + "theoretical_loss": 3.400789182078694, + "tokens_seen": 2221473792 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033246464646464644, + "loss": 2.7374, + "theoretical_loss": 3.400780988875289, + "tokens_seen": 2221539328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033244444444444445, + "loss": 2.5549, + "theoretical_loss": 3.400772795981256, + "tokens_seen": 2221604864 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324242424242424, + "loss": 2.7307, + "theoretical_loss": 3.4007646033965746, + "tokens_seen": 2221670400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324040404040404, + "loss": 2.5225, + "theoretical_loss": 3.4007564111212236, + "tokens_seen": 2221735936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323838383838384, + "loss": 2.3545, + "theoretical_loss": 3.400748219155183, + "tokens_seen": 2221801472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323636363636364, + "loss": 2.5976, + "theoretical_loss": 3.400740027498431, + "tokens_seen": 2221867008 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1225752, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6411614418029785, + "objective/train/theoretical_loss": 3.4007318361509475, + "objective/train/tokens_used": 580791776, + "theoretical_loss": 3.4007318361509475, + "tokens_seen": 2221932544 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033234343434343435, + "loss": 2.7676, + "theoretical_loss": 3.4007318361509475, + "tokens_seen": 2221932544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323232323232323, + "loss": 2.787, + "theoretical_loss": 3.400723645112712, + "tokens_seen": 2221998080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323030303030303, + "loss": 2.5546, + "theoretical_loss": 3.400715454383702, + "tokens_seen": 2222063616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033228282828282827, + "loss": 2.5891, + "theoretical_loss": 3.400707263963899, + "tokens_seen": 2222129152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322626262626263, + "loss": 2.5609, + "theoretical_loss": 3.4006990738532807, + "tokens_seen": 2222194688 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033224242424242424, + "loss": 2.4844, + "theoretical_loss": 3.400690884051827, + "tokens_seen": 2222260224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322222222222222, + "loss": 2.8366, + "theoretical_loss": 3.4006826945595163, + "tokens_seen": 2222325760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322020202020202, + "loss": 2.6884, + "theoretical_loss": 3.4006745053763288, + "tokens_seen": 2222391296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321818181818182, + "loss": 2.5898, + "theoretical_loss": 3.400666316502243, + "tokens_seen": 2222456832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321616161616162, + "loss": 2.6397, + "theoretical_loss": 3.4006581279372385, + "tokens_seen": 2222522368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033214141414141413, + "loss": 2.7176, + "theoretical_loss": 3.400649939681294, + "tokens_seen": 2222587904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321212121212121, + "loss": 2.7142, + "theoretical_loss": 3.4006417517343897, + "tokens_seen": 2222653440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321010101010101, + "loss": 2.6039, + "theoretical_loss": 3.4006335640965037, + "tokens_seen": 2222718976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320808080808081, + "loss": 2.6894, + "theoretical_loss": 3.4006253767676164, + "tokens_seen": 2222784512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033206060606060607, + "loss": 2.5729, + "theoretical_loss": 3.400617189747706, + "tokens_seen": 2222850048 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332040404040404, + "loss": 2.486, + "theoretical_loss": 3.4006090030367524, + "tokens_seen": 2222915584 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332020202020202, + "loss": 2.3843, + "theoretical_loss": 3.4006008166347343, + "tokens_seen": 2222981120 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033200000000000005, + "loss": 2.7624, + "theoretical_loss": 3.4005926305416314, + "tokens_seen": 2223046656 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331979797979798, + "loss": 2.5662, + "theoretical_loss": 3.4005844447574227, + "tokens_seen": 2223112192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033195959595959596, + "loss": 2.5462, + "theoretical_loss": 3.4005762592820874, + "tokens_seen": 2223177728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319393939393939, + "loss": 2.6776, + "theoretical_loss": 3.400568074115605, + "tokens_seen": 2223243264 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319191919191919, + "loss": 2.4568, + "theoretical_loss": 3.4005598892579543, + "tokens_seen": 2223308800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033189898989898994, + "loss": 2.4441, + "theoretical_loss": 3.4005517047091147, + "tokens_seen": 2223374336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318787878787879, + "loss": 2.8038, + "theoretical_loss": 3.400543520469066, + "tokens_seen": 2223439872 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033185858585858585, + "loss": 2.3434, + "theoretical_loss": 3.4005353365377866, + "tokens_seen": 2223505408 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1226492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.975679636001587, + "objective/train/theoretical_loss": 3.4005271529152563, + "objective/train/tokens_used": 582430176, + "theoretical_loss": 3.4005271529152563, + "tokens_seen": 2223570944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318383838383838, + "loss": 2.5701, + "theoretical_loss": 3.4005271529152563, + "tokens_seen": 2223570944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318181818181819, + "loss": 2.6182, + "theoretical_loss": 3.400518969601454, + "tokens_seen": 2223636480 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033179797979797983, + "loss": 2.5841, + "theoretical_loss": 3.4005107865963597, + "tokens_seen": 2223702016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317777777777778, + "loss": 2.6592, + "theoretical_loss": 3.4005026038999517, + "tokens_seen": 2223767552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033175757575757575, + "loss": 2.7015, + "theoretical_loss": 3.4004944215122097, + "tokens_seen": 2223833088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317373737373737, + "loss": 2.647, + "theoretical_loss": 3.4004862394331132, + "tokens_seen": 2223898624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033171717171717177, + "loss": 2.629, + "theoretical_loss": 3.400478057662641, + "tokens_seen": 2223964160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316969696969697, + "loss": 2.6428, + "theoretical_loss": 3.4004698762007726, + "tokens_seen": 2224029696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316767676767677, + "loss": 2.5553, + "theoretical_loss": 3.400461695047487, + "tokens_seen": 2224095232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033165656565656564, + "loss": 2.7775, + "theoretical_loss": 3.400453514202764, + "tokens_seen": 2224160768 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033163636363636365, + "loss": 2.5992, + "theoretical_loss": 3.4004453336665827, + "tokens_seen": 2224226304 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033161616161616166, + "loss": 2.5181, + "theoretical_loss": 3.4004371534389217, + "tokens_seen": 2224291840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315959595959596, + "loss": 2.8832, + "theoretical_loss": 3.4004289735197615, + "tokens_seen": 2224357376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315757575757576, + "loss": 2.6969, + "theoretical_loss": 3.4004207939090803, + "tokens_seen": 2224422912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033155555555555553, + "loss": 2.5016, + "theoretical_loss": 3.4004126146068576, + "tokens_seen": 2224488448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033153535353535354, + "loss": 2.6914, + "theoretical_loss": 3.400404435613073, + "tokens_seen": 2224553984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033151515151515155, + "loss": 2.6616, + "theoretical_loss": 3.4003962569277055, + "tokens_seen": 2224619520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314949494949495, + "loss": 2.5709, + "theoretical_loss": 3.400388078550735, + "tokens_seen": 2224685056 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033147474747474747, + "loss": 2.5039, + "theoretical_loss": 3.40037990048214, + "tokens_seen": 2224750592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314545454545455, + "loss": 2.6247, + "theoretical_loss": 3.4003717227219, + "tokens_seen": 2224816128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033143434343434344, + "loss": 2.8543, + "theoretical_loss": 3.4003635452699945, + "tokens_seen": 2224881664 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033141414141414145, + "loss": 2.8326, + "theoretical_loss": 3.4003553681264025, + "tokens_seen": 2224947200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313939393939394, + "loss": 2.6964, + "theoretical_loss": 3.400347191291104, + "tokens_seen": 2225012736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033137373737373736, + "loss": 2.7465, + "theoretical_loss": 3.4003390147640773, + "tokens_seen": 2225078272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033135353535353537, + "loss": 2.8577, + "theoretical_loss": 3.4003308385453024, + "tokens_seen": 2225143808 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1227694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9130122661590576, + "objective/train/theoretical_loss": 3.4003226626347582, + "objective/train/tokens_used": 584068576, + "theoretical_loss": 3.4003226626347582, + "tokens_seen": 2225209344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033133333333333333, + "loss": 2.6959, + "theoretical_loss": 3.4003226626347582, + "tokens_seen": 2225209344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033131313131313134, + "loss": 2.7366, + "theoretical_loss": 3.400314487032424, + "tokens_seen": 2225274880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312929292929293, + "loss": 2.5441, + "theoretical_loss": 3.4003063117382797, + "tokens_seen": 2225340416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312727272727273, + "loss": 2.48, + "theoretical_loss": 3.400298136752304, + "tokens_seen": 2225405952 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033125252525252526, + "loss": 2.715, + "theoretical_loss": 3.4002899620744764, + "tokens_seen": 2225471488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312323232323232, + "loss": 2.5601, + "theoretical_loss": 3.400281787704776, + "tokens_seen": 2225537024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033121212121212123, + "loss": 2.5835, + "theoretical_loss": 3.400273613643183, + "tokens_seen": 2225602560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311919191919192, + "loss": 2.7125, + "theoretical_loss": 3.4002654398896754, + "tokens_seen": 2225668096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311717171717172, + "loss": 2.5304, + "theoretical_loss": 3.400257266444233, + "tokens_seen": 2225733632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033115151515151516, + "loss": 2.4401, + "theoretical_loss": 3.4002490933068357, + "tokens_seen": 2225799168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311313131313131, + "loss": 2.6642, + "theoretical_loss": 3.4002409204774624, + "tokens_seen": 2225864704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311111111111111, + "loss": 2.7592, + "theoretical_loss": 3.4002327479560925, + "tokens_seen": 2225930240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033109090909090914, + "loss": 2.5815, + "theoretical_loss": 3.4002245757427048, + "tokens_seen": 2225995776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310707070707071, + "loss": 2.6235, + "theoretical_loss": 3.4002164038372795, + "tokens_seen": 2226061312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033105050505050505, + "loss": 2.478, + "theoretical_loss": 3.400208232239795, + "tokens_seen": 2226126848 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331030303030303, + "loss": 2.8519, + "theoretical_loss": 3.4002000609502314, + "tokens_seen": 2226192384 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331010101010101, + "loss": 2.5961, + "theoretical_loss": 3.4001918899685677, + "tokens_seen": 2226257920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033098989898989903, + "loss": 2.6191, + "theoretical_loss": 3.4001837192947835, + "tokens_seen": 2226323456 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330969696969697, + "loss": 2.6254, + "theoretical_loss": 3.4001755489288574, + "tokens_seen": 2226388992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033094949494949494, + "loss": 2.5801, + "theoretical_loss": 3.40016737887077, + "tokens_seen": 2226454528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309292929292929, + "loss": 2.4333, + "theoretical_loss": 3.400159209120499, + "tokens_seen": 2226520064 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033090909090909096, + "loss": 2.7306, + "theoretical_loss": 3.4001510396780255, + "tokens_seen": 2226585600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308888888888889, + "loss": 2.7405, + "theoretical_loss": 3.400142870543328, + "tokens_seen": 2226651136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308686868686869, + "loss": 2.6858, + "theoretical_loss": 3.400134701716385, + "tokens_seen": 2226716672 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033084848484848483, + "loss": 2.6114, + "theoretical_loss": 3.4001265331971773, + "tokens_seen": 2226782208 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1229090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5205607414245605, + "objective/train/theoretical_loss": 3.400118364985684, + "objective/train/tokens_used": 585706976, + "theoretical_loss": 3.400118364985684, + "tokens_seen": 2226847744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308282828282828, + "loss": 2.6387, + "theoretical_loss": 3.400118364985684, + "tokens_seen": 2226847744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033080808080808086, + "loss": 2.3516, + "theoretical_loss": 3.4001101970818834, + "tokens_seen": 2226913280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307878787878788, + "loss": 2.6906, + "theoretical_loss": 3.4001020294857556, + "tokens_seen": 2226978816 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033076767676767677, + "loss": 2.5544, + "theoretical_loss": 3.4000938621972803, + "tokens_seen": 2227044352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033074747474747473, + "loss": 2.7082, + "theoretical_loss": 3.400085695216436, + "tokens_seen": 2227109888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307272727272727, + "loss": 2.7331, + "theoretical_loss": 3.400077528543203, + "tokens_seen": 2227175424 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033070707070707075, + "loss": 2.6219, + "theoretical_loss": 3.40006936217756, + "tokens_seen": 2227240960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306868686868687, + "loss": 2.7171, + "theoretical_loss": 3.4000611961194864, + "tokens_seen": 2227306496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033066666666666666, + "loss": 2.5398, + "theoretical_loss": 3.400053030368962, + "tokens_seen": 2227372032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306464646464646, + "loss": 2.5068, + "theoretical_loss": 3.400044864925966, + "tokens_seen": 2227437568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033062626262626263, + "loss": 2.6589, + "theoretical_loss": 3.400036699790477, + "tokens_seen": 2227503104 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033060606060606064, + "loss": 2.5127, + "theoretical_loss": 3.4000285349624755, + "tokens_seen": 2227568640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305858585858586, + "loss": 2.5166, + "theoretical_loss": 3.4000203704419403, + "tokens_seen": 2227634176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033056565656565656, + "loss": 2.5547, + "theoretical_loss": 3.4000122062288507, + "tokens_seen": 2227699712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305454545454545, + "loss": 2.6215, + "theoretical_loss": 3.4000040423231863, + "tokens_seen": 2227765248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305252525252525, + "loss": 2.7859, + "theoretical_loss": 3.3999958787249267, + "tokens_seen": 2227830784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033050505050505053, + "loss": 2.5376, + "theoretical_loss": 3.399987715434051, + "tokens_seen": 2227896320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304848484848485, + "loss": 2.513, + "theoretical_loss": 3.3999795524505387, + "tokens_seen": 2227961856 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033046464646464645, + "loss": 2.4192, + "theoretical_loss": 3.3999713897743686, + "tokens_seen": 2228027392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033044444444444446, + "loss": 2.7682, + "theoretical_loss": 3.3999632274055207, + "tokens_seen": 2228092928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304242424242424, + "loss": 2.7186, + "theoretical_loss": 3.399955065343975, + "tokens_seen": 2228158464 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033040404040404043, + "loss": 2.6879, + "theoretical_loss": 3.399946903589709, + "tokens_seen": 2228224000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303838383838384, + "loss": 2.5556, + "theoretical_loss": 3.3999387421427043, + "tokens_seen": 2228289536 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033036363636363634, + "loss": 2.6052, + "theoretical_loss": 3.3999305810029385, + "tokens_seen": 2228355072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033034343434343435, + "loss": 2.5945, + "theoretical_loss": 3.3999224201703924, + "tokens_seen": 2228420608 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1229755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9436287879943848, + "objective/train/theoretical_loss": 3.3999142596450445, + "objective/train/tokens_used": 587345376, + "theoretical_loss": 3.3999142596450445, + "tokens_seen": 2228486144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303232323232323, + "loss": 2.7154, + "theoretical_loss": 3.3999142596450445, + "tokens_seen": 2228486144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303030303030303, + "loss": 2.5713, + "theoretical_loss": 3.3999060994268744, + "tokens_seen": 2228551680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302828282828283, + "loss": 2.6092, + "theoretical_loss": 3.3998979395158613, + "tokens_seen": 2228617216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302626262626263, + "loss": 2.5645, + "theoretical_loss": 3.399889779911985, + "tokens_seen": 2228682752 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033024242424242424, + "loss": 2.5129, + "theoretical_loss": 3.399881620615225, + "tokens_seen": 2228748288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302222222222222, + "loss": 2.5417, + "theoretical_loss": 3.3998734616255604, + "tokens_seen": 2228813824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302020202020202, + "loss": 2.5971, + "theoretical_loss": 3.399865302942971, + "tokens_seen": 2228879360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033018181818181817, + "loss": 2.5154, + "theoretical_loss": 3.3998571445674353, + "tokens_seen": 2228944896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301616161616162, + "loss": 2.5734, + "theoretical_loss": 3.3998489864989336, + "tokens_seen": 2229010432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033014141414141414, + "loss": 2.5061, + "theoretical_loss": 3.399840828737445, + "tokens_seen": 2229075968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033012121212121215, + "loss": 2.5298, + "theoretical_loss": 3.399832671282949, + "tokens_seen": 2229141504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301010101010101, + "loss": 2.5969, + "theoretical_loss": 3.3998245141354255, + "tokens_seen": 2229207040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300808080808081, + "loss": 2.5479, + "theoretical_loss": 3.3998163572948528, + "tokens_seen": 2229272576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300606060606061, + "loss": 2.7211, + "theoretical_loss": 3.3998082007612114, + "tokens_seen": 2229338112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033004040404040403, + "loss": 2.4358, + "theoretical_loss": 3.3998000445344796, + "tokens_seen": 2229403648 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033002020202020204, + "loss": 2.7302, + "theoretical_loss": 3.3997918886146383, + "tokens_seen": 2229469184 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033, + "loss": 2.5349, + "theoretical_loss": 3.3997837330016654, + "tokens_seen": 2229534720 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329979797979798, + "loss": 2.5502, + "theoretical_loss": 3.399775577695541, + "tokens_seen": 2229600256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032995959595959597, + "loss": 2.807, + "theoretical_loss": 3.3997674226962453, + "tokens_seen": 2229665792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299393939393939, + "loss": 2.5437, + "theoretical_loss": 3.3997592680037565, + "tokens_seen": 2229731328 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032991919191919193, + "loss": 2.6995, + "theoretical_loss": 3.399751113618055, + "tokens_seen": 2229796864 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032989898989898995, + "loss": 2.5486, + "theoretical_loss": 3.3997429595391195, + "tokens_seen": 2229862400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298787878787879, + "loss": 2.5481, + "theoretical_loss": 3.39973480576693, + "tokens_seen": 2229927936 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032985858585858586, + "loss": 2.4959, + "theoretical_loss": 3.3997266523014655, + "tokens_seen": 2229993472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298383838383838, + "loss": 2.6664, + "theoretical_loss": 3.399718499142706, + "tokens_seen": 2230059008 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1230979, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6503684520721436, + "objective/train/theoretical_loss": 3.39971034629063, + "objective/train/tokens_used": 588983776, + "theoretical_loss": 3.39971034629063, + "tokens_seen": 2230124544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298181818181818, + "loss": 2.78, + "theoretical_loss": 3.39971034629063, + "tokens_seen": 2230124544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032979797979797984, + "loss": 2.6312, + "theoretical_loss": 3.399702193745218, + "tokens_seen": 2230190080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297777777777778, + "loss": 2.632, + "theoretical_loss": 3.399694041506449, + "tokens_seen": 2230255616 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032975757575757575, + "loss": 2.5803, + "theoretical_loss": 3.3996858895743025, + "tokens_seen": 2230321152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297373737373737, + "loss": 2.44, + "theoretical_loss": 3.3996777379487577, + "tokens_seen": 2230386688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297171717171718, + "loss": 2.4862, + "theoretical_loss": 3.399669586629795, + "tokens_seen": 2230452224 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032969696969696973, + "loss": 2.5718, + "theoretical_loss": 3.3996614356173924, + "tokens_seen": 2230517760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296767676767677, + "loss": 2.5235, + "theoretical_loss": 3.39965328491153, + "tokens_seen": 2230583296 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032965656565656564, + "loss": 2.5377, + "theoretical_loss": 3.3996451345121876, + "tokens_seen": 2230648832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296363636363636, + "loss": 2.6386, + "theoretical_loss": 3.399636984419345, + "tokens_seen": 2230714368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032961616161616167, + "loss": 2.6945, + "theoretical_loss": 3.3996288346329804, + "tokens_seen": 2230779904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295959595959596, + "loss": 2.7157, + "theoretical_loss": 3.3996206851530744, + "tokens_seen": 2230845440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295757575757576, + "loss": 2.434, + "theoretical_loss": 3.399612535979606, + "tokens_seen": 2230910976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032955555555555554, + "loss": 2.6753, + "theoretical_loss": 3.3996043871125545, + "tokens_seen": 2230976512 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032953535353535355, + "loss": 2.6915, + "theoretical_loss": 3.3995962385519, + "tokens_seen": 2231042048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032951515151515156, + "loss": 2.7246, + "theoretical_loss": 3.3995880902976214, + "tokens_seen": 2231107584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294949494949495, + "loss": 2.5912, + "theoretical_loss": 3.399579942349698, + "tokens_seen": 2231173120 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032947474747474747, + "loss": 2.8135, + "theoretical_loss": 3.3995717947081103, + "tokens_seen": 2231238656 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032945454545454543, + "loss": 2.8094, + "theoretical_loss": 3.399563647372837, + "tokens_seen": 2231304192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032943434343434344, + "loss": 2.6365, + "theoretical_loss": 3.3995555003438573, + "tokens_seen": 2231369728 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032941414141414145, + "loss": 2.6156, + "theoretical_loss": 3.3995473536211516, + "tokens_seen": 2231435264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293939393939394, + "loss": 2.5703, + "theoretical_loss": 3.3995392072046986, + "tokens_seen": 2231500800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032937373737373737, + "loss": 2.7854, + "theoretical_loss": 3.3995310610944784, + "tokens_seen": 2231566336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293535353535354, + "loss": 2.7232, + "theoretical_loss": 3.3995229152904702, + "tokens_seen": 2231631872 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032933333333333333, + "loss": 2.4552, + "theoretical_loss": 3.3995147697926535, + "tokens_seen": 2231697408 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1231622, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8242321014404297, + "objective/train/theoretical_loss": 3.3995066246010075, + "objective/train/tokens_used": 590622176, + "theoretical_loss": 3.3995066246010075, + "tokens_seen": 2231762944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032931313131313134, + "loss": 2.7745, + "theoretical_loss": 3.3995066246010075, + "tokens_seen": 2231762944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292929292929293, + "loss": 2.5516, + "theoretical_loss": 3.399498479715512, + "tokens_seen": 2231828480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032927272727272726, + "loss": 2.6586, + "theoretical_loss": 3.399490335136147, + "tokens_seen": 2231894016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032925252525252527, + "loss": 2.5252, + "theoretical_loss": 3.399482190862891, + "tokens_seen": 2231959552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292323232323232, + "loss": 2.4889, + "theoretical_loss": 3.3994740468957243, + "tokens_seen": 2232025088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032921212121212124, + "loss": 2.6499, + "theoretical_loss": 3.3994659032346264, + "tokens_seen": 2232090624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291919191919192, + "loss": 2.6323, + "theoretical_loss": 3.399457759879576, + "tokens_seen": 2232156160 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032917171717171715, + "loss": 2.5841, + "theoretical_loss": 3.3994496168305535, + "tokens_seen": 2232221696 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032915151515151516, + "loss": 2.6963, + "theoretical_loss": 3.399441474087538, + "tokens_seen": 2232287232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291313131313131, + "loss": 2.5118, + "theoretical_loss": 3.399433331650509, + "tokens_seen": 2232352768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032911111111111113, + "loss": 2.4879, + "theoretical_loss": 3.3994251895194463, + "tokens_seen": 2232418304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290909090909091, + "loss": 2.5526, + "theoretical_loss": 3.399417047694329, + "tokens_seen": 2232483840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290707070707071, + "loss": 2.5397, + "theoretical_loss": 3.399408906175137, + "tokens_seen": 2232549376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032905050505050505, + "loss": 2.6151, + "theoretical_loss": 3.3994007649618494, + "tokens_seen": 2232614912 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329030303030303, + "loss": 2.6126, + "theoretical_loss": 3.3993926240544465, + "tokens_seen": 2232680448 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329010101010101, + "loss": 2.6836, + "theoretical_loss": 3.399384483452907, + "tokens_seen": 2232745984 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328989898989899, + "loss": 2.5229, + "theoretical_loss": 3.399376343157211, + "tokens_seen": 2232811520 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328969696969697, + "loss": 2.3784, + "theoretical_loss": 3.3993682031673376, + "tokens_seen": 2232877056 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032894949494949495, + "loss": 2.7474, + "theoretical_loss": 3.3993600634832664, + "tokens_seen": 2232942592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289292929292929, + "loss": 2.6362, + "theoretical_loss": 3.399351924104977, + "tokens_seen": 2233008128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289090909090909, + "loss": 2.5158, + "theoretical_loss": 3.3993437850324497, + "tokens_seen": 2233073664 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288888888888889, + "loss": 2.3868, + "theoretical_loss": 3.399335646265663, + "tokens_seen": 2233139200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288686868686869, + "loss": 2.5855, + "theoretical_loss": 3.3993275078045966, + "tokens_seen": 2233204736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032884848484848484, + "loss": 2.5106, + "theoretical_loss": 3.3993193696492305, + "tokens_seen": 2233270272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288282828282828, + "loss": 2.6512, + "theoretical_loss": 3.3993112317995435, + "tokens_seen": 2233335808 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1232627, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4419264793395996, + "objective/train/theoretical_loss": 3.399303094255516, + "objective/train/tokens_used": 592260576, + "theoretical_loss": 3.399303094255516, + "tokens_seen": 2233401344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288080808080808, + "loss": 2.5994, + "theoretical_loss": 3.399303094255516, + "tokens_seen": 2233401344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287878787878788, + "loss": 2.585, + "theoretical_loss": 3.399294957017127, + "tokens_seen": 2233466880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287676767676768, + "loss": 2.6031, + "theoretical_loss": 3.3992868200843565, + "tokens_seen": 2233532416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032874747474747473, + "loss": 2.4659, + "theoretical_loss": 3.3992786834571835, + "tokens_seen": 2233597952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287272727272727, + "loss": 2.4812, + "theoretical_loss": 3.399270547135588, + "tokens_seen": 2233663488 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032870707070707075, + "loss": 2.5293, + "theoretical_loss": 3.3992624111195497, + "tokens_seen": 2233729024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286868686868687, + "loss": 2.5891, + "theoretical_loss": 3.3992542754090476, + "tokens_seen": 2233794560 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032866666666666667, + "loss": 2.5712, + "theoretical_loss": 3.3992461400040614, + "tokens_seen": 2233860096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286464646464646, + "loss": 2.5188, + "theoretical_loss": 3.399238004904571, + "tokens_seen": 2233925632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286262626262626, + "loss": 2.4667, + "theoretical_loss": 3.3992298701105557, + "tokens_seen": 2233991168 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032860606060606065, + "loss": 2.5163, + "theoretical_loss": 3.399221735621995, + "tokens_seen": 2234056704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285858585858586, + "loss": 2.867, + "theoretical_loss": 3.399213601438869, + "tokens_seen": 2234122240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032856565656565656, + "loss": 2.527, + "theoretical_loss": 3.3992054675611563, + "tokens_seen": 2234187776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285454545454545, + "loss": 2.6442, + "theoretical_loss": 3.3991973339888375, + "tokens_seen": 2234253312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032852525252525253, + "loss": 2.5602, + "theoretical_loss": 3.3991892007218913, + "tokens_seen": 2234318848 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032850505050505054, + "loss": 2.6464, + "theoretical_loss": 3.3991810677602983, + "tokens_seen": 2234384384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284848484848485, + "loss": 2.549, + "theoretical_loss": 3.399172935104037, + "tokens_seen": 2234449920 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032846464646464645, + "loss": 2.7451, + "theoretical_loss": 3.399164802753088, + "tokens_seen": 2234515456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284444444444444, + "loss": 2.5182, + "theoretical_loss": 3.3991566707074297, + "tokens_seen": 2234580992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284242424242425, + "loss": 2.8511, + "theoretical_loss": 3.3991485389670424, + "tokens_seen": 2234646528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032840404040404043, + "loss": 2.4048, + "theoretical_loss": 3.399140407531906, + "tokens_seen": 2234712064 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283838383838384, + "loss": 2.6627, + "theoretical_loss": 3.3991322764019993, + "tokens_seen": 2234777600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032836363636363635, + "loss": 2.6656, + "theoretical_loss": 3.3991241455773027, + "tokens_seen": 2234843136 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032834343434343436, + "loss": 2.5002, + "theoretical_loss": 3.3991160150577953, + "tokens_seen": 2234908672 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032832323232323237, + "loss": 2.6034, + "theoretical_loss": 3.3991078848434566, + "tokens_seen": 2234974208 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1233498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7271065711975098, + "objective/train/theoretical_loss": 3.3990997549342667, + "objective/train/tokens_used": 593898976, + "theoretical_loss": 3.3990997549342667, + "tokens_seen": 2235039744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283030303030303, + "loss": 2.5348, + "theoretical_loss": 3.3990997549342667, + "tokens_seen": 2235039744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282828282828283, + "loss": 2.5156, + "theoretical_loss": 3.399091625330205, + "tokens_seen": 2235105280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032826262626262624, + "loss": 2.818, + "theoretical_loss": 3.3990834960312504, + "tokens_seen": 2235170816 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032824242424242425, + "loss": 2.6451, + "theoretical_loss": 3.3990753670373834, + "tokens_seen": 2235236352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032822222222222226, + "loss": 2.2549, + "theoretical_loss": 3.399067238348583, + "tokens_seen": 2235301888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282020202020202, + "loss": 2.4261, + "theoretical_loss": 3.3990591099648295, + "tokens_seen": 2235367424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281818181818182, + "loss": 2.6187, + "theoretical_loss": 3.399050981886102, + "tokens_seen": 2235432960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281616161616162, + "loss": 2.442, + "theoretical_loss": 3.3990428541123805, + "tokens_seen": 2235498496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032814141414141414, + "loss": 2.6671, + "theoretical_loss": 3.3990347266436443, + "tokens_seen": 2235564032 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032812121212121215, + "loss": 2.6224, + "theoretical_loss": 3.399026599479873, + "tokens_seen": 2235629568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281010101010101, + "loss": 2.8171, + "theoretical_loss": 3.399018472621046, + "tokens_seen": 2235695104 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032808080808080807, + "loss": 2.487, + "theoretical_loss": 3.3990103460671435, + "tokens_seen": 2235760640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280606060606061, + "loss": 2.6067, + "theoretical_loss": 3.3990022198181444, + "tokens_seen": 2235826176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032804040404040404, + "loss": 2.5759, + "theoretical_loss": 3.398994093874029, + "tokens_seen": 2235891712 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032802020202020205, + "loss": 2.4193, + "theoretical_loss": 3.3989859682347765, + "tokens_seen": 2235957248 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328, + "loss": 2.6055, + "theoretical_loss": 3.398977842900367, + "tokens_seen": 2236022784 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327979797979798, + "loss": 2.8154, + "theoretical_loss": 3.39896971787078, + "tokens_seen": 2236088320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032795959595959597, + "loss": 2.6897, + "theoretical_loss": 3.3989615931459944, + "tokens_seen": 2236153856 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032793939393939393, + "loss": 2.665, + "theoretical_loss": 3.398953468725991, + "tokens_seen": 2236219392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032791919191919194, + "loss": 2.5713, + "theoretical_loss": 3.3989453446107483, + "tokens_seen": 2236284928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278989898989899, + "loss": 2.6761, + "theoretical_loss": 3.3989372208002466, + "tokens_seen": 2236350464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278787878787879, + "loss": 2.6578, + "theoretical_loss": 3.3989290972944657, + "tokens_seen": 2236416000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032785858585858586, + "loss": 2.6511, + "theoretical_loss": 3.398920974093385, + "tokens_seen": 2236481536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278383838383838, + "loss": 2.6278, + "theoretical_loss": 3.3989128511969833, + "tokens_seen": 2236547072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032781818181818183, + "loss": 2.5598, + "theoretical_loss": 3.398904728605242, + "tokens_seen": 2236612608 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1234140, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.896017551422119, + "objective/train/theoretical_loss": 3.398896606318139, + "objective/train/tokens_used": 595537376, + "theoretical_loss": 3.398896606318139, + "tokens_seen": 2236678144 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032779797979797984, + "loss": 2.4891, + "theoretical_loss": 3.398896606318139, + "tokens_seen": 2236678144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277777777777778, + "loss": 2.7951, + "theoretical_loss": 3.398888484335655, + "tokens_seen": 2236743680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032775757575757576, + "loss": 2.6724, + "theoretical_loss": 3.3988803626577697, + "tokens_seen": 2236809216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277373737373737, + "loss": 2.452, + "theoretical_loss": 3.398872241284462, + "tokens_seen": 2236874752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277171717171717, + "loss": 2.7162, + "theoretical_loss": 3.398864120215712, + "tokens_seen": 2236940288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032769696969696974, + "loss": 2.6553, + "theoretical_loss": 3.3988559994514995, + "tokens_seen": 2237005824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276767676767677, + "loss": 2.6449, + "theoretical_loss": 3.398847878991804, + "tokens_seen": 2237071360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032765656565656565, + "loss": 2.5481, + "theoretical_loss": 3.3988397588366053, + "tokens_seen": 2237136896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276363636363636, + "loss": 2.4543, + "theoretical_loss": 3.398831638985883, + "tokens_seen": 2237202432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032761616161616167, + "loss": 2.4903, + "theoretical_loss": 3.398823519439616, + "tokens_seen": 2237267968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032759595959595963, + "loss": 2.4446, + "theoretical_loss": 3.398815400197785, + "tokens_seen": 2237333504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275757575757576, + "loss": 2.487, + "theoretical_loss": 3.3988072812603694, + "tokens_seen": 2237399040 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032755555555555554, + "loss": 2.4346, + "theoretical_loss": 3.3987991626273493, + "tokens_seen": 2237464576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275353535353535, + "loss": 2.681, + "theoretical_loss": 3.398791044298703, + "tokens_seen": 2237530112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032751515151515156, + "loss": 2.6709, + "theoretical_loss": 3.3987829262744116, + "tokens_seen": 2237595648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274949494949495, + "loss": 2.8194, + "theoretical_loss": 3.398774808554454, + "tokens_seen": 2237661184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274747474747475, + "loss": 2.5114, + "theoretical_loss": 3.3987666911388104, + "tokens_seen": 2237726720 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032745454545454544, + "loss": 2.6755, + "theoretical_loss": 3.3987585740274593, + "tokens_seen": 2237792256 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274343434343434, + "loss": 2.6678, + "theoretical_loss": 3.398750457220382, + "tokens_seen": 2237857792 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032741414141414146, + "loss": 2.4602, + "theoretical_loss": 3.3987423407175577, + "tokens_seen": 2237923328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273939393939394, + "loss": 2.3153, + "theoretical_loss": 3.3987342245189653, + "tokens_seen": 2237988864 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032737373737373737, + "loss": 2.6252, + "theoretical_loss": 3.398726108624585, + "tokens_seen": 2238054400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032735353535353533, + "loss": 2.5537, + "theoretical_loss": 3.398717993034397, + "tokens_seen": 2238119936 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032733333333333334, + "loss": 2.5703, + "theoretical_loss": 3.39870987774838, + "tokens_seen": 2238185472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032731313131313135, + "loss": 2.8515, + "theoretical_loss": 3.3987017627665144, + "tokens_seen": 2238251008 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1235278, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7147154808044434, + "objective/train/theoretical_loss": 3.3986936480887793, + "objective/train/tokens_used": 597175776, + "theoretical_loss": 3.3986936480887793, + "tokens_seen": 2238316544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272929292929293, + "loss": 2.6231, + "theoretical_loss": 3.3986936480887793, + "tokens_seen": 2238316544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032727272727272726, + "loss": 2.7508, + "theoretical_loss": 3.398685533715155, + "tokens_seen": 2238382080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272525252525252, + "loss": 2.6738, + "theoretical_loss": 3.3986774196456215, + "tokens_seen": 2238447616 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032723232323232323, + "loss": 2.7251, + "theoretical_loss": 3.3986693058801576, + "tokens_seen": 2238513152 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032721212121212124, + "loss": 2.6471, + "theoretical_loss": 3.3986611924187433, + "tokens_seen": 2238578688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271919191919192, + "loss": 2.704, + "theoretical_loss": 3.3986530792613583, + "tokens_seen": 2238644224 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032717171717171716, + "loss": 2.7981, + "theoretical_loss": 3.3986449664079825, + "tokens_seen": 2238709760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032715151515151517, + "loss": 2.683, + "theoretical_loss": 3.3986368538585956, + "tokens_seen": 2238775296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271313131313131, + "loss": 2.5068, + "theoretical_loss": 3.398628741613177, + "tokens_seen": 2238840832 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032711111111111114, + "loss": 2.5823, + "theoretical_loss": 3.398620629671707, + "tokens_seen": 2238906368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270909090909091, + "loss": 2.5502, + "theoretical_loss": 3.398612518034165, + "tokens_seen": 2238971904 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032707070707070705, + "loss": 2.5768, + "theoretical_loss": 3.3986044067005303, + "tokens_seen": 2239037440 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032705050505050506, + "loss": 2.4648, + "theoretical_loss": 3.3985962956707834, + "tokens_seen": 2239102976 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327030303030303, + "loss": 2.5492, + "theoretical_loss": 3.398588184944903, + "tokens_seen": 2239168512 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032701010101010103, + "loss": 2.6662, + "theoretical_loss": 3.39858007452287, + "tokens_seen": 2239234048 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326989898989899, + "loss": 2.637, + "theoretical_loss": 3.3985719644046632, + "tokens_seen": 2239299584 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326969696969697, + "loss": 2.5445, + "theoretical_loss": 3.398563854590263, + "tokens_seen": 2239365120 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032694949494949495, + "loss": 2.5525, + "theoretical_loss": 3.3985557450796486, + "tokens_seen": 2239430656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269292929292929, + "loss": 2.8025, + "theoretical_loss": 3.3985476358728, + "tokens_seen": 2239496192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269090909090909, + "loss": 2.6419, + "theoretical_loss": 3.3985395269696967, + "tokens_seen": 2239561728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268888888888889, + "loss": 2.4434, + "theoretical_loss": 3.398531418370319, + "tokens_seen": 2239627264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268686868686869, + "loss": 2.4144, + "theoretical_loss": 3.398523310074646, + "tokens_seen": 2239692800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032684848484848485, + "loss": 2.6922, + "theoretical_loss": 3.3985152020826574, + "tokens_seen": 2239758336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268282828282828, + "loss": 2.5596, + "theoretical_loss": 3.3985070943943336, + "tokens_seen": 2239823872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268080808080808, + "loss": 2.5989, + "theoretical_loss": 3.3984989870096536, + "tokens_seen": 2239889408 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1235978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.565056800842285, + "objective/train/theoretical_loss": 3.398490879928598, + "objective/train/tokens_used": 598814176, + "theoretical_loss": 3.398490879928598, + "tokens_seen": 2239954944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267878787878788, + "loss": 2.6903, + "theoretical_loss": 3.398490879928598, + "tokens_seen": 2239954944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267676767676768, + "loss": 2.5615, + "theoretical_loss": 3.3984827731511458, + "tokens_seen": 2240020480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032674747474747474, + "loss": 2.5883, + "theoretical_loss": 3.398474666677277, + "tokens_seen": 2240086016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267272727272727, + "loss": 2.466, + "theoretical_loss": 3.3984665605069715, + "tokens_seen": 2240151552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267070707070707, + "loss": 2.5902, + "theoretical_loss": 3.398458454640209, + "tokens_seen": 2240217088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266868686868687, + "loss": 2.3745, + "theoretical_loss": 3.3984503490769686, + "tokens_seen": 2240282624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266666666666667, + "loss": 2.5181, + "theoretical_loss": 3.3984422438172315, + "tokens_seen": 2240348160 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032664646464646463, + "loss": 2.486, + "theoretical_loss": 3.3984341388609756, + "tokens_seen": 2240413696 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032662626262626264, + "loss": 2.594, + "theoretical_loss": 3.3984260342081822, + "tokens_seen": 2240479232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032660606060606065, + "loss": 2.8373, + "theoretical_loss": 3.3984179298588306, + "tokens_seen": 2240544768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265858585858586, + "loss": 2.4156, + "theoretical_loss": 3.3984098258129003, + "tokens_seen": 2240610304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032656565656565657, + "loss": 2.7603, + "theoretical_loss": 3.3984017220703713, + "tokens_seen": 2240675840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265454545454545, + "loss": 2.7061, + "theoretical_loss": 3.398393618631223, + "tokens_seen": 2240741376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032652525252525253, + "loss": 2.7418, + "theoretical_loss": 3.398385515495436, + "tokens_seen": 2240806912 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032650505050505055, + "loss": 2.6515, + "theoretical_loss": 3.398377412662989, + "tokens_seen": 2240872448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264848484848485, + "loss": 2.6523, + "theoretical_loss": 3.3983693101338623, + "tokens_seen": 2240937984 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032646464646464646, + "loss": 2.4976, + "theoretical_loss": 3.398361207908036, + "tokens_seen": 2241003520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264444444444444, + "loss": 2.7018, + "theoretical_loss": 3.3983531059854895, + "tokens_seen": 2241069056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264242424242425, + "loss": 2.5705, + "theoretical_loss": 3.3983450043662025, + "tokens_seen": 2241134592 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032640404040404044, + "loss": 2.7231, + "theoretical_loss": 3.398336903050155, + "tokens_seen": 2241200128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263838383838384, + "loss": 2.5503, + "theoretical_loss": 3.398328802037327, + "tokens_seen": 2241265664 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032636363636363635, + "loss": 2.4337, + "theoretical_loss": 3.3983207013276977, + "tokens_seen": 2241331200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263434343434343, + "loss": 2.8032, + "theoretical_loss": 3.398312600921247, + "tokens_seen": 2241396736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263232323232324, + "loss": 2.5385, + "theoretical_loss": 3.3983045008179555, + "tokens_seen": 2241462272 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032630303030303033, + "loss": 2.2273, + "theoretical_loss": 3.3982964010178014, + "tokens_seen": 2241527808 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1237176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4149110317230225, + "objective/train/theoretical_loss": 3.3982883015207666, + "objective/train/tokens_used": 600452576, + "theoretical_loss": 3.3982883015207666, + "tokens_seen": 2241593344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262828282828283, + "loss": 2.4795, + "theoretical_loss": 3.3982883015207666, + "tokens_seen": 2241593344 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032626262626262624, + "loss": 2.603, + "theoretical_loss": 3.398280202326829, + "tokens_seen": 2241658880 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032624242424242426, + "loss": 2.5269, + "theoretical_loss": 3.398272103435969, + "tokens_seen": 2241724416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032622222222222227, + "loss": 2.5095, + "theoretical_loss": 3.398264004848167, + "tokens_seen": 2241789952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262020202020202, + "loss": 2.4808, + "theoretical_loss": 3.398255906563402, + "tokens_seen": 2241855488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261818181818182, + "loss": 2.65, + "theoretical_loss": 3.3982478085816545, + "tokens_seen": 2241921024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032616161616161614, + "loss": 2.4468, + "theoretical_loss": 3.398239710902904, + "tokens_seen": 2241986560 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032614141414141415, + "loss": 2.4904, + "theoretical_loss": 3.3982316135271295, + "tokens_seen": 2242052096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032612121212121216, + "loss": 2.5256, + "theoretical_loss": 3.3982235164543124, + "tokens_seen": 2242117632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261010101010101, + "loss": 2.7606, + "theoretical_loss": 3.3982154196844316, + "tokens_seen": 2242183168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260808080808081, + "loss": 2.6065, + "theoretical_loss": 3.3982073232174663, + "tokens_seen": 2242248704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260606060606061, + "loss": 2.6405, + "theoretical_loss": 3.398199227053398, + "tokens_seen": 2242314240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032604040404040404, + "loss": 2.554, + "theoretical_loss": 3.398191131192205, + "tokens_seen": 2242379776 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032602020202020205, + "loss": 2.6045, + "theoretical_loss": 3.3981830356338674, + "tokens_seen": 2242445312 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326, + "loss": 2.5911, + "theoretical_loss": 3.3981749403783654, + "tokens_seen": 2242510848 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032597979797979797, + "loss": 2.4874, + "theoretical_loss": 3.3981668454256786, + "tokens_seen": 2242576384 + }, + { + "epoch": 0.35, + "learning_rate": 0.000325959595959596, + "loss": 2.5332, + "theoretical_loss": 3.3981587507757873, + "tokens_seen": 2242641920 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032593939393939393, + "loss": 2.6727, + "theoretical_loss": 3.3981506564286708, + "tokens_seen": 2242707456 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032591919191919195, + "loss": 2.688, + "theoretical_loss": 3.398142562384309, + "tokens_seen": 2242772992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258989898989899, + "loss": 2.6281, + "theoretical_loss": 3.3981344686426818, + "tokens_seen": 2242838528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032587878787878786, + "loss": 2.3954, + "theoretical_loss": 3.398126375203769, + "tokens_seen": 2242904064 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032585858585858587, + "loss": 2.5233, + "theoretical_loss": 3.3981182820675504, + "tokens_seen": 2242969600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258383838383838, + "loss": 2.6406, + "theoretical_loss": 3.3981101892340058, + "tokens_seen": 2243035136 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032581818181818184, + "loss": 2.7894, + "theoretical_loss": 3.3981020967031155, + "tokens_seen": 2243100672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003257979797979798, + "loss": 2.6066, + "theoretical_loss": 3.398094004474858, + "tokens_seen": 2243166208 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1238466, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8873229026794434, + "objective/train/theoretical_loss": 3.3980859125492153, + "objective/train/tokens_used": 602090976, + "theoretical_loss": 3.3980859125492153, + "tokens_seen": 2243231744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003257777777777778, + "loss": 2.6947, + "theoretical_loss": 3.3980859125492153, + "tokens_seen": 2243231744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032575757575757576, + "loss": 2.8324, + "theoretical_loss": 3.3980778209261655, + "tokens_seen": 2243297280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257373737373737, + "loss": 2.6064, + "theoretical_loss": 3.398069729605689, + "tokens_seen": 2243362816 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032571717171717173, + "loss": 2.7454, + "theoretical_loss": 3.3980616385877656, + "tokens_seen": 2243428352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256969696969697, + "loss": 2.7327, + "theoretical_loss": 3.3980535478723755, + "tokens_seen": 2243493888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256767676767677, + "loss": 2.7072, + "theoretical_loss": 3.398045457459498, + "tokens_seen": 2243559424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032565656565656566, + "loss": 2.7652, + "theoretical_loss": 3.3980373673491133, + "tokens_seen": 2243624960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256363636363636, + "loss": 2.4725, + "theoretical_loss": 3.398029277541201, + "tokens_seen": 2243690496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256161616161616, + "loss": 2.3954, + "theoretical_loss": 3.398021188035741, + "tokens_seen": 2243756032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032559595959595963, + "loss": 2.2931, + "theoretical_loss": 3.3980130988327133, + "tokens_seen": 2243821568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255757575757576, + "loss": 2.7455, + "theoretical_loss": 3.398005009932098, + "tokens_seen": 2243887104 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032555555555555555, + "loss": 2.5907, + "theoretical_loss": 3.3979969213338745, + "tokens_seen": 2243952640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255353535353535, + "loss": 2.7126, + "theoretical_loss": 3.3979888330380232, + "tokens_seen": 2244018176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255151515151515, + "loss": 2.6299, + "theoretical_loss": 3.397980745044523, + "tokens_seen": 2244083712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032549494949494953, + "loss": 2.564, + "theoretical_loss": 3.3979726573533546, + "tokens_seen": 2244149248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254747474747475, + "loss": 2.5305, + "theoretical_loss": 3.397964569964498, + "tokens_seen": 2244214784 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032545454545454544, + "loss": 2.6263, + "theoretical_loss": 3.3979564828779325, + "tokens_seen": 2244280320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254343434343434, + "loss": 2.7636, + "theoretical_loss": 3.3979483960936383, + "tokens_seen": 2244345856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032541414141414146, + "loss": 2.5791, + "theoretical_loss": 3.3979403096115948, + "tokens_seen": 2244411392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253939393939394, + "loss": 2.6575, + "theoretical_loss": 3.397932223431783, + "tokens_seen": 2244476928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253737373737374, + "loss": 2.4282, + "theoretical_loss": 3.397924137554181, + "tokens_seen": 2244542464 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032535353535353533, + "loss": 2.4786, + "theoretical_loss": 3.3979160519787706, + "tokens_seen": 2244608000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253333333333333, + "loss": 2.6864, + "theoretical_loss": 3.3979079667055303, + "tokens_seen": 2244673536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032531313131313136, + "loss": 2.6525, + "theoretical_loss": 3.397899881734441, + "tokens_seen": 2244739072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252929292929293, + "loss": 2.7195, + "theoretical_loss": 3.3978917970654816, + "tokens_seen": 2244804608 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1238885, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1809070110321045, + "objective/train/theoretical_loss": 3.3978837126986328, + "objective/train/tokens_used": 603729376, + "theoretical_loss": 3.3978837126986328, + "tokens_seen": 2244870144 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032527272727272727, + "loss": 2.5608, + "theoretical_loss": 3.3978837126986328, + "tokens_seen": 2244870144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252525252525252, + "loss": 2.7261, + "theoretical_loss": 3.397875628633874, + "tokens_seen": 2244935680 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032523232323232324, + "loss": 2.6386, + "theoretical_loss": 3.397867544871185, + "tokens_seen": 2245001216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032521212121212125, + "loss": 2.4322, + "theoretical_loss": 3.397859461410546, + "tokens_seen": 2245066752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251919191919192, + "loss": 2.5448, + "theoretical_loss": 3.397851378251937, + "tokens_seen": 2245132288 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032517171717171716, + "loss": 2.7242, + "theoretical_loss": 3.397843295395338, + "tokens_seen": 2245197824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251515151515151, + "loss": 2.48, + "theoretical_loss": 3.3978352128407283, + "tokens_seen": 2245263360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032513131313131313, + "loss": 2.5817, + "theoretical_loss": 3.3978271305880883, + "tokens_seen": 2245328896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032511111111111114, + "loss": 2.7349, + "theoretical_loss": 3.3978190486373974, + "tokens_seen": 2245394432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250909090909091, + "loss": 2.56, + "theoretical_loss": 3.3978109669886365, + "tokens_seen": 2245459968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032507070707070705, + "loss": 2.7129, + "theoretical_loss": 3.3978028856417843, + "tokens_seen": 2245525504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032505050505050507, + "loss": 2.4765, + "theoretical_loss": 3.3977948045968214, + "tokens_seen": 2245591040 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325030303030303, + "loss": 2.5874, + "theoretical_loss": 3.397786723853728, + "tokens_seen": 2245656576 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032501010101010103, + "loss": 2.4721, + "theoretical_loss": 3.397778643412483, + "tokens_seen": 2245722112 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324989898989899, + "loss": 2.6481, + "theoretical_loss": 3.3977705632730673, + "tokens_seen": 2245787648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032496969696969695, + "loss": 2.6252, + "theoretical_loss": 3.3977624834354603, + "tokens_seen": 2245853184 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032494949494949496, + "loss": 2.3284, + "theoretical_loss": 3.397754403899642, + "tokens_seen": 2245918720 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032492929292929297, + "loss": 2.6229, + "theoretical_loss": 3.397746324665593, + "tokens_seen": 2245984256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249090909090909, + "loss": 2.6977, + "theoretical_loss": 3.397738245733292, + "tokens_seen": 2246049792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248888888888889, + "loss": 2.5982, + "theoretical_loss": 3.3977301671027194, + "tokens_seen": 2246115328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248686868686869, + "loss": 2.6435, + "theoretical_loss": 3.3977220887738557, + "tokens_seen": 2246180864 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032484848484848485, + "loss": 2.5501, + "theoretical_loss": 3.39771401074668, + "tokens_seen": 2246246400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032482828282828286, + "loss": 2.6576, + "theoretical_loss": 3.3977059330211725, + "tokens_seen": 2246311936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248080808080808, + "loss": 2.6838, + "theoretical_loss": 3.397697855597314, + "tokens_seen": 2246377472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247878787878788, + "loss": 2.4642, + "theoretical_loss": 3.3976897784750832, + "tokens_seen": 2246443008 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1239988, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6558778285980225, + "objective/train/theoretical_loss": 3.3976817016544603, + "objective/train/tokens_used": 605367776, + "theoretical_loss": 3.3976817016544603, + "tokens_seen": 2246508544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247676767676768, + "loss": 2.5854, + "theoretical_loss": 3.3976817016544603, + "tokens_seen": 2246508544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032474747474747474, + "loss": 2.5479, + "theoretical_loss": 3.397673625135426, + "tokens_seen": 2246574080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032472727272727276, + "loss": 2.4586, + "theoretical_loss": 3.3976655489179595, + "tokens_seen": 2246639616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247070707070707, + "loss": 2.7558, + "theoretical_loss": 3.397657473002041, + "tokens_seen": 2246705152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246868686868687, + "loss": 2.6734, + "theoretical_loss": 3.3976493973876503, + "tokens_seen": 2246770688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246666666666667, + "loss": 2.5989, + "theoretical_loss": 3.397641322074767, + "tokens_seen": 2246836224 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032464646464646464, + "loss": 2.4503, + "theoretical_loss": 3.397633247063372, + "tokens_seen": 2246901760 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032462626262626265, + "loss": 2.4268, + "theoretical_loss": 3.397625172353445, + "tokens_seen": 2246967296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246060606060606, + "loss": 2.7411, + "theoretical_loss": 3.3976170979449654, + "tokens_seen": 2247032832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245858585858586, + "loss": 2.5106, + "theoretical_loss": 3.397609023837913, + "tokens_seen": 2247098368 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032456565656565657, + "loss": 2.5413, + "theoretical_loss": 3.397600950032269, + "tokens_seen": 2247163904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032454545454545453, + "loss": 2.5425, + "theoretical_loss": 3.397592876528012, + "tokens_seen": 2247229440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032452525252525254, + "loss": 2.6852, + "theoretical_loss": 3.3975848033251226, + "tokens_seen": 2247294976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032450505050505055, + "loss": 2.5082, + "theoretical_loss": 3.397576730423581, + "tokens_seen": 2247360512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244848484848485, + "loss": 2.6075, + "theoretical_loss": 3.3975686578233666, + "tokens_seen": 2247426048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032446464646464647, + "loss": 2.5743, + "theoretical_loss": 3.39756058552446, + "tokens_seen": 2247491584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244444444444444, + "loss": 2.566, + "theoretical_loss": 3.3975525135268403, + "tokens_seen": 2247557120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032442424242424243, + "loss": 2.6145, + "theoretical_loss": 3.397544441830488, + "tokens_seen": 2247622656 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032440404040404044, + "loss": 2.585, + "theoretical_loss": 3.3975363704353834, + "tokens_seen": 2247688192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243838383838384, + "loss": 2.4886, + "theoretical_loss": 3.3975282993415057, + "tokens_seen": 2247753728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032436363636363636, + "loss": 2.6176, + "theoretical_loss": 3.397520228548836, + "tokens_seen": 2247819264 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243434343434343, + "loss": 2.6563, + "theoretical_loss": 3.397512158057353, + "tokens_seen": 2247884800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243232323232324, + "loss": 2.6648, + "theoretical_loss": 3.397504087867037, + "tokens_seen": 2247950336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032430303030303034, + "loss": 2.6209, + "theoretical_loss": 3.3974960179778684, + "tokens_seen": 2248015872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242828282828283, + "loss": 2.6033, + "theoretical_loss": 3.397487948389827, + "tokens_seen": 2248081408 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1241252, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.037294864654541, + "objective/train/theoretical_loss": 3.3974798791028933, + "objective/train/tokens_used": 607006176, + "theoretical_loss": 3.3974798791028933, + "tokens_seen": 2248146944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032426262626262625, + "loss": 2.7454, + "theoretical_loss": 3.3974798791028933, + "tokens_seen": 2248146944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242424242424242, + "loss": 2.5246, + "theoretical_loss": 3.3974718101170462, + "tokens_seen": 2248212480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032422222222222227, + "loss": 2.6371, + "theoretical_loss": 3.3974637414322664, + "tokens_seen": 2248278016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032420202020202023, + "loss": 2.8248, + "theoretical_loss": 3.397455673048534, + "tokens_seen": 2248343552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241818181818182, + "loss": 2.7675, + "theoretical_loss": 3.397447604965828, + "tokens_seen": 2248409088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032416161616161614, + "loss": 2.5905, + "theoretical_loss": 3.3974395371841304, + "tokens_seen": 2248474624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241414141414141, + "loss": 2.6519, + "theoretical_loss": 3.397431469703419, + "tokens_seen": 2248540160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032412121212121217, + "loss": 2.4771, + "theoretical_loss": 3.397423402523675, + "tokens_seen": 2248605696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241010101010101, + "loss": 2.5812, + "theoretical_loss": 3.397415335644878, + "tokens_seen": 2248671232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240808080808081, + "loss": 2.7362, + "theoretical_loss": 3.3974072690670085, + "tokens_seen": 2248736768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032406060606060604, + "loss": 2.7021, + "theoretical_loss": 3.397399202790046, + "tokens_seen": 2248802304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032404040404040405, + "loss": 2.5885, + "theoretical_loss": 3.3973911368139706, + "tokens_seen": 2248867840 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032402020202020206, + "loss": 2.7223, + "theoretical_loss": 3.3973830711387625, + "tokens_seen": 2248933376 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324, + "loss": 2.7474, + "theoretical_loss": 3.397375005764401, + "tokens_seen": 2248998912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032397979797979797, + "loss": 2.5893, + "theoretical_loss": 3.3973669406908673, + "tokens_seen": 2249064448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032395959595959593, + "loss": 2.6812, + "theoretical_loss": 3.397358875918141, + "tokens_seen": 2249129984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032393939393939394, + "loss": 2.6245, + "theoretical_loss": 3.397350811446201, + "tokens_seen": 2249195520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032391919191919195, + "loss": 2.246, + "theoretical_loss": 3.397342747275029, + "tokens_seen": 2249261056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238989898989899, + "loss": 2.5908, + "theoretical_loss": 3.397334683404604, + "tokens_seen": 2249326592 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032387878787878786, + "loss": 2.6178, + "theoretical_loss": 3.3973266198349066, + "tokens_seen": 2249392128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238585858585859, + "loss": 2.6177, + "theoretical_loss": 3.3973185565659163, + "tokens_seen": 2249457664 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032383838383838383, + "loss": 2.5661, + "theoretical_loss": 3.397310493597613, + "tokens_seen": 2249523200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032381818181818184, + "loss": 2.6387, + "theoretical_loss": 3.3973024309299773, + "tokens_seen": 2249588736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237979797979798, + "loss": 2.6505, + "theoretical_loss": 3.397294368562989, + "tokens_seen": 2249654272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032377777777777776, + "loss": 2.6009, + "theoretical_loss": 3.3972863064966283, + "tokens_seen": 2249719808 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1241776, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0098400115966797, + "objective/train/theoretical_loss": 3.3972782447308747, + "objective/train/tokens_used": 608644576, + "theoretical_loss": 3.3972782447308747, + "tokens_seen": 2249785344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032375757575757577, + "loss": 2.7253, + "theoretical_loss": 3.3972782447308747, + "tokens_seen": 2249785344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237373737373737, + "loss": 2.6584, + "theoretical_loss": 3.397270183265709, + "tokens_seen": 2249850880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032371717171717174, + "loss": 2.6867, + "theoretical_loss": 3.3972621221011106, + "tokens_seen": 2249916416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236969696969697, + "loss": 2.6644, + "theoretical_loss": 3.3972540612370596, + "tokens_seen": 2249981952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236767676767677, + "loss": 2.7775, + "theoretical_loss": 3.3972460006735363, + "tokens_seen": 2250047488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032365656565656566, + "loss": 2.5241, + "theoretical_loss": 3.3972379404105206, + "tokens_seen": 2250113024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236363636363636, + "loss": 2.5691, + "theoretical_loss": 3.397229880447993, + "tokens_seen": 2250178560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032361616161616163, + "loss": 2.5051, + "theoretical_loss": 3.3972218207859326, + "tokens_seen": 2250244096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235959595959596, + "loss": 2.7053, + "theoretical_loss": 3.3972137614243203, + "tokens_seen": 2250309632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235757575757576, + "loss": 2.6714, + "theoretical_loss": 3.3972057023631357, + "tokens_seen": 2250375168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032355555555555555, + "loss": 2.5531, + "theoretical_loss": 3.397197643602359, + "tokens_seen": 2250440704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235353535353535, + "loss": 2.1475, + "theoretical_loss": 3.3971895851419704, + "tokens_seen": 2250506240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235151515151515, + "loss": 2.6421, + "theoretical_loss": 3.3971815269819494, + "tokens_seen": 2250571776 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032349494949494953, + "loss": 2.7133, + "theoretical_loss": 3.397173469122277, + "tokens_seen": 2250637312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234747474747475, + "loss": 2.6396, + "theoretical_loss": 3.3971654115629324, + "tokens_seen": 2250702848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032345454545454545, + "loss": 2.7516, + "theoretical_loss": 3.397157354303896, + "tokens_seen": 2250768384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234343434343434, + "loss": 2.5693, + "theoretical_loss": 3.397149297345148, + "tokens_seen": 2250833920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234141414141414, + "loss": 2.5616, + "theoretical_loss": 3.397141240686668, + "tokens_seen": 2250899456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233939393939394, + "loss": 2.5604, + "theoretical_loss": 3.3971331843284367, + "tokens_seen": 2250964992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233737373737374, + "loss": 2.647, + "theoretical_loss": 3.3971251282704333, + "tokens_seen": 2251030528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032335353535353534, + "loss": 2.6121, + "theoretical_loss": 3.397117072512639, + "tokens_seen": 2251096064 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233333333333333, + "loss": 2.7934, + "theoretical_loss": 3.3971090170550333, + "tokens_seen": 2251161600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032331313131313136, + "loss": 2.7487, + "theoretical_loss": 3.397100961897596, + "tokens_seen": 2251227136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232929292929293, + "loss": 2.6383, + "theoretical_loss": 3.3970929070403075, + "tokens_seen": 2251292672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232727272727273, + "loss": 2.6685, + "theoretical_loss": 3.3970848524831476, + "tokens_seen": 2251358208 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1242897, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5388667583465576, + "objective/train/theoretical_loss": 3.397076798226097, + "objective/train/tokens_used": 610282976, + "theoretical_loss": 3.397076798226097, + "tokens_seen": 2251423744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032325252525252523, + "loss": 2.8134, + "theoretical_loss": 3.397076798226097, + "tokens_seen": 2251423744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032323232323232324, + "loss": 2.8143, + "theoretical_loss": 3.397068744269135, + "tokens_seen": 2251489280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032321212121212125, + "loss": 2.8504, + "theoretical_loss": 3.3970606906122423, + "tokens_seen": 2251554816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231919191919192, + "loss": 2.746, + "theoretical_loss": 3.397052637255399, + "tokens_seen": 2251620352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032317171717171717, + "loss": 2.5253, + "theoretical_loss": 3.3970445841985843, + "tokens_seen": 2251685888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231515151515151, + "loss": 2.6397, + "theoretical_loss": 3.3970365314417794, + "tokens_seen": 2251751424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231313131313132, + "loss": 2.4531, + "theoretical_loss": 3.397028478984964, + "tokens_seen": 2251816960 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032311111111111115, + "loss": 2.5787, + "theoretical_loss": 3.397020426828118, + "tokens_seen": 2251882496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230909090909091, + "loss": 2.3132, + "theoretical_loss": 3.3970123749712213, + "tokens_seen": 2251948032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032307070707070706, + "loss": 2.6369, + "theoretical_loss": 3.397004323414255, + "tokens_seen": 2252013568 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323050505050505, + "loss": 2.7356, + "theoretical_loss": 3.396996272157198, + "tokens_seen": 2252079104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230303030303031, + "loss": 2.5593, + "theoretical_loss": 3.396988221200031, + "tokens_seen": 2252144640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032301010101010104, + "loss": 2.5869, + "theoretical_loss": 3.396980170542734, + "tokens_seen": 2252210176 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322989898989899, + "loss": 2.5806, + "theoretical_loss": 3.3969721201852874, + "tokens_seen": 2252275712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032296969696969695, + "loss": 2.6232, + "theoretical_loss": 3.3969640701276704, + "tokens_seen": 2252341248 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032294949494949496, + "loss": 2.517, + "theoretical_loss": 3.3969560203698643, + "tokens_seen": 2252406784 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322929292929293, + "loss": 2.5175, + "theoretical_loss": 3.3969479709118486, + "tokens_seen": 2252472320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032290909090909093, + "loss": 2.609, + "theoretical_loss": 3.3969399217536034, + "tokens_seen": 2252537856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228888888888889, + "loss": 2.6138, + "theoretical_loss": 3.3969318728951086, + "tokens_seen": 2252603392 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032286868686868685, + "loss": 2.8452, + "theoretical_loss": 3.396923824336345, + "tokens_seen": 2252668928 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032284848484848486, + "loss": 2.9954, + "theoretical_loss": 3.396915776077292, + "tokens_seen": 2252734464 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032282828282828287, + "loss": 2.5626, + "theoretical_loss": 3.39690772811793, + "tokens_seen": 2252800000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228080808080808, + "loss": 2.6104, + "theoretical_loss": 3.39689968045824, + "tokens_seen": 2252865536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227878787878788, + "loss": 2.5124, + "theoretical_loss": 3.3968916330982006, + "tokens_seen": 2252931072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032276767676767674, + "loss": 2.7513, + "theoretical_loss": 3.3968835860377924, + "tokens_seen": 2252996608 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1243463, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8337297439575195, + "objective/train/theoretical_loss": 3.396875539276996, + "objective/train/tokens_used": 611921376, + "theoretical_loss": 3.396875539276996, + "tokens_seen": 2253062144 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032274747474747475, + "loss": 2.6388, + "theoretical_loss": 3.396875539276996, + "tokens_seen": 2253062144 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032272727272727276, + "loss": 2.8779, + "theoretical_loss": 3.396867492815791, + "tokens_seen": 2253127680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227070707070707, + "loss": 2.5202, + "theoretical_loss": 3.3968594466541586, + "tokens_seen": 2253193216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226868686868687, + "loss": 2.6369, + "theoretical_loss": 3.3968514007920776, + "tokens_seen": 2253258752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226666666666667, + "loss": 2.5562, + "theoretical_loss": 3.3968433552295285, + "tokens_seen": 2253324288 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032264646464646464, + "loss": 2.785, + "theoretical_loss": 3.396835309966492, + "tokens_seen": 2253389824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032262626262626265, + "loss": 2.5689, + "theoretical_loss": 3.3968272650029476, + "tokens_seen": 2253455360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226060606060606, + "loss": 2.7323, + "theoretical_loss": 3.396819220338876, + "tokens_seen": 2253520896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032258585858585857, + "loss": 2.6385, + "theoretical_loss": 3.3968111759742565, + "tokens_seen": 2253586432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225656565656566, + "loss": 2.5644, + "theoretical_loss": 3.39680313190907, + "tokens_seen": 2253651968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032254545454545453, + "loss": 2.5264, + "theoretical_loss": 3.3967950881432967, + "tokens_seen": 2253717504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032252525252525255, + "loss": 2.5731, + "theoretical_loss": 3.396787044676916, + "tokens_seen": 2253783040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225050505050505, + "loss": 2.7812, + "theoretical_loss": 3.396779001509909, + "tokens_seen": 2253848576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224848484848485, + "loss": 2.7024, + "theoretical_loss": 3.3967709586422554, + "tokens_seen": 2253914112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032246464646464647, + "loss": 2.8075, + "theoretical_loss": 3.3967629160739348, + "tokens_seen": 2253979648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032244444444444443, + "loss": 2.526, + "theoretical_loss": 3.3967548738049285, + "tokens_seen": 2254045184 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032242424242424244, + "loss": 2.5916, + "theoretical_loss": 3.3967468318352156, + "tokens_seen": 2254110720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224040404040404, + "loss": 2.9027, + "theoretical_loss": 3.3967387901647768, + "tokens_seen": 2254176256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223838383838384, + "loss": 2.3612, + "theoretical_loss": 3.3967307487935923, + "tokens_seen": 2254241792 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032236363636363636, + "loss": 2.4982, + "theoretical_loss": 3.396722707721642, + "tokens_seen": 2254307328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223434343434343, + "loss": 2.6329, + "theoretical_loss": 3.3967146669489066, + "tokens_seen": 2254372864 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032232323232323233, + "loss": 2.5713, + "theoretical_loss": 3.396706626475366, + "tokens_seen": 2254438400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032230303030303034, + "loss": 2.6205, + "theoretical_loss": 3.3966985863009995, + "tokens_seen": 2254503936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222828282828283, + "loss": 2.6128, + "theoretical_loss": 3.396690546425788, + "tokens_seen": 2254569472 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032226262626262626, + "loss": 2.625, + "theoretical_loss": 3.396682506849712, + "tokens_seen": 2254635008 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1244562, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.452125072479248, + "objective/train/theoretical_loss": 3.3966744675727516, + "objective/train/tokens_used": 613559776, + "theoretical_loss": 3.3966744675727516, + "tokens_seen": 2254700544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222424242424242, + "loss": 2.7192, + "theoretical_loss": 3.3966744675727516, + "tokens_seen": 2254700544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222222222222222, + "loss": 2.8083, + "theoretical_loss": 3.3966664285948864, + "tokens_seen": 2254766080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032220202020202024, + "loss": 2.3679, + "theoretical_loss": 3.396658389916097, + "tokens_seen": 2254831616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221818181818182, + "loss": 2.6082, + "theoretical_loss": 3.3966503515363637, + "tokens_seen": 2254897152 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032216161616161615, + "loss": 2.6304, + "theoretical_loss": 3.3966423134556667, + "tokens_seen": 2254962688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221414141414141, + "loss": 2.5595, + "theoretical_loss": 3.3966342756739856, + "tokens_seen": 2255028224 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032212121212121217, + "loss": 2.6471, + "theoretical_loss": 3.396626238191301, + "tokens_seen": 2255093760 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032210101010101013, + "loss": 2.7279, + "theoretical_loss": 3.396618201007593, + "tokens_seen": 2255159296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220808080808081, + "loss": 2.5665, + "theoretical_loss": 3.396610164122842, + "tokens_seen": 2255224832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032206060606060604, + "loss": 2.573, + "theoretical_loss": 3.3966021275370277, + "tokens_seen": 2255290368 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322040404040404, + "loss": 2.3949, + "theoretical_loss": 3.396594091250131, + "tokens_seen": 2255355904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032202020202020206, + "loss": 2.5372, + "theoretical_loss": 3.3965860552621314, + "tokens_seen": 2255421440 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322, + "loss": 2.5403, + "theoretical_loss": 3.3965780195730098, + "tokens_seen": 2255486976 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321979797979798, + "loss": 2.629, + "theoretical_loss": 3.3965699841827455, + "tokens_seen": 2255552512 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032195959595959593, + "loss": 2.5865, + "theoretical_loss": 3.39656194909132, + "tokens_seen": 2255618048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032193939393939395, + "loss": 2.8571, + "theoretical_loss": 3.3965539142987122, + "tokens_seen": 2255683584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032191919191919196, + "loss": 2.6604, + "theoretical_loss": 3.396545879804903, + "tokens_seen": 2255749120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218989898989899, + "loss": 2.7366, + "theoretical_loss": 3.3965378456098723, + "tokens_seen": 2255814656 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032187878787878787, + "loss": 2.4626, + "theoretical_loss": 3.3965298117136005, + "tokens_seen": 2255880192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218585858585858, + "loss": 2.5269, + "theoretical_loss": 3.3965217781160675, + "tokens_seen": 2255945728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032183838383838384, + "loss": 2.773, + "theoretical_loss": 3.3965137448172538, + "tokens_seen": 2256011264 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032181818181818185, + "loss": 2.769, + "theoretical_loss": 3.39650571181714, + "tokens_seen": 2256076800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217979797979798, + "loss": 2.6846, + "theoretical_loss": 3.396497679115706, + "tokens_seen": 2256142336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032177777777777776, + "loss": 2.6453, + "theoretical_loss": 3.3964896467129315, + "tokens_seen": 2256207872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217575757575758, + "loss": 2.6132, + "theoretical_loss": 3.3964816146087973, + "tokens_seen": 2256273408 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1245051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4939658641815186, + "objective/train/theoretical_loss": 3.3964735828032833, + "objective/train/tokens_used": 615198176, + "theoretical_loss": 3.3964735828032833, + "tokens_seen": 2256338944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032173737373737373, + "loss": 2.5558, + "theoretical_loss": 3.3964735828032833, + "tokens_seen": 2256338944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032171717171717174, + "loss": 2.6278, + "theoretical_loss": 3.3964655512963704, + "tokens_seen": 2256404480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216969696969697, + "loss": 2.7621, + "theoretical_loss": 3.396457520088038, + "tokens_seen": 2256470016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032167676767676766, + "loss": 2.4765, + "theoretical_loss": 3.3964494891782664, + "tokens_seen": 2256535552 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032165656565656567, + "loss": 2.5545, + "theoretical_loss": 3.3964414585670366, + "tokens_seen": 2256601088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216363636363636, + "loss": 2.6117, + "theoretical_loss": 3.396433428254328, + "tokens_seen": 2256666624 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032161616161616163, + "loss": 2.4907, + "theoretical_loss": 3.396425398240121, + "tokens_seen": 2256732160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215959595959596, + "loss": 2.5676, + "theoretical_loss": 3.3964173685243964, + "tokens_seen": 2256797696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215757575757576, + "loss": 2.568, + "theoretical_loss": 3.3964093391071337, + "tokens_seen": 2256863232 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032155555555555556, + "loss": 2.6658, + "theoretical_loss": 3.3964013099883137, + "tokens_seen": 2256928768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032153535353535357, + "loss": 2.5634, + "theoretical_loss": 3.3963932811679163, + "tokens_seen": 2256994304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032151515151515153, + "loss": 2.587, + "theoretical_loss": 3.3963852526459215, + "tokens_seen": 2257059840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214949494949495, + "loss": 2.6791, + "theoretical_loss": 3.3963772244223107, + "tokens_seen": 2257125376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214747474747475, + "loss": 2.5604, + "theoretical_loss": 3.3963691964970626, + "tokens_seen": 2257190912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032145454545454545, + "loss": 2.5811, + "theoretical_loss": 3.3963611688701585, + "tokens_seen": 2257256448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032143434343434346, + "loss": 2.4722, + "theoretical_loss": 3.3963531415415784, + "tokens_seen": 2257321984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214141414141414, + "loss": 2.7084, + "theoretical_loss": 3.3963451145113024, + "tokens_seen": 2257387520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032139393939393943, + "loss": 2.6442, + "theoretical_loss": 3.396337087779311, + "tokens_seen": 2257453056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213737373737374, + "loss": 2.6079, + "theoretical_loss": 3.396329061345584, + "tokens_seen": 2257518592 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032135353535353534, + "loss": 2.5841, + "theoretical_loss": 3.396321035210102, + "tokens_seen": 2257584128 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032133333333333336, + "loss": 2.6716, + "theoretical_loss": 3.3963130093728453, + "tokens_seen": 2257649664 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213131313131313, + "loss": 2.5498, + "theoretical_loss": 3.396304983833794, + "tokens_seen": 2257715200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212929292929293, + "loss": 2.5937, + "theoretical_loss": 3.396296958592929, + "tokens_seen": 2257780736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212727272727273, + "loss": 2.5268, + "theoretical_loss": 3.3962889336502293, + "tokens_seen": 2257846272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032125252525252524, + "loss": 2.4315, + "theoretical_loss": 3.396280909005676, + "tokens_seen": 2257911808 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1246329, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1417624950408936, + "objective/train/theoretical_loss": 3.3962728846592496, + "objective/train/tokens_used": 616836576, + "theoretical_loss": 3.3962728846592496, + "tokens_seen": 2257977344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032123232323232325, + "loss": 2.5338, + "theoretical_loss": 3.3962728846592496, + "tokens_seen": 2257977344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032121212121212126, + "loss": 2.4146, + "theoretical_loss": 3.39626486061093, + "tokens_seen": 2258042880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211919191919192, + "loss": 2.5625, + "theoretical_loss": 3.3962568368606973, + "tokens_seen": 2258108416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211717171717172, + "loss": 2.498, + "theoretical_loss": 3.396248813408532, + "tokens_seen": 2258173952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032115151515151513, + "loss": 2.4951, + "theoretical_loss": 3.3962407902544145, + "tokens_seen": 2258239488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032113131313131314, + "loss": 2.3995, + "theoretical_loss": 3.396232767398325, + "tokens_seen": 2258305024 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032111111111111115, + "loss": 2.4617, + "theoretical_loss": 3.3962247448402434, + "tokens_seen": 2258370560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210909090909091, + "loss": 2.5313, + "theoretical_loss": 3.3962167225801503, + "tokens_seen": 2258436096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032107070707070707, + "loss": 2.3825, + "theoretical_loss": 3.3962087006180264, + "tokens_seen": 2258501632 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321050505050505, + "loss": 2.6772, + "theoretical_loss": 3.3962006789538512, + "tokens_seen": 2258567168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210303030303031, + "loss": 2.4764, + "theoretical_loss": 3.3961926575876054, + "tokens_seen": 2258632704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032101010101010104, + "loss": 2.6091, + "theoretical_loss": 3.396184636519269, + "tokens_seen": 2258698240 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320989898989899, + "loss": 2.4795, + "theoretical_loss": 3.396176615748823, + "tokens_seen": 2258763776 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032096969696969696, + "loss": 2.6561, + "theoretical_loss": 3.3961685952762473, + "tokens_seen": 2258829312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209494949494949, + "loss": 2.4325, + "theoretical_loss": 3.396160575101522, + "tokens_seen": 2258894848 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320929292929293, + "loss": 2.713, + "theoretical_loss": 3.3961525552246274, + "tokens_seen": 2258960384 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032090909090909094, + "loss": 2.5842, + "theoretical_loss": 3.396144535645544, + "tokens_seen": 2259025920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208888888888889, + "loss": 2.8008, + "theoretical_loss": 3.396136516364252, + "tokens_seen": 2259091456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032086868686868685, + "loss": 2.6915, + "theoretical_loss": 3.3961284973807313, + "tokens_seen": 2259156992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208484848484848, + "loss": 2.4567, + "theoretical_loss": 3.3961204786949635, + "tokens_seen": 2259222528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208282828282829, + "loss": 2.6053, + "theoretical_loss": 3.3961124603069277, + "tokens_seen": 2259288064 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032080808080808083, + "loss": 2.7385, + "theoretical_loss": 3.3961044422166045, + "tokens_seen": 2259353600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207878787878788, + "loss": 2.425, + "theoretical_loss": 3.396096424423974, + "tokens_seen": 2259419136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032076767676767674, + "loss": 2.6334, + "theoretical_loss": 3.3960884069290174, + "tokens_seen": 2259484672 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032074747474747476, + "loss": 2.5307, + "theoretical_loss": 3.3960803897317136, + "tokens_seen": 2259550208 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1246940, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8410935401916504, + "objective/train/theoretical_loss": 3.396072372832044, + "objective/train/tokens_used": 618474976, + "theoretical_loss": 3.396072372832044, + "tokens_seen": 2259615744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032072727272727277, + "loss": 2.6552, + "theoretical_loss": 3.396072372832044, + "tokens_seen": 2259615744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207070707070707, + "loss": 2.4731, + "theoretical_loss": 3.396064356229989, + "tokens_seen": 2259681280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206868686868687, + "loss": 2.6754, + "theoretical_loss": 3.3960563399255284, + "tokens_seen": 2259746816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032066666666666664, + "loss": 2.6882, + "theoretical_loss": 3.3960483239186425, + "tokens_seen": 2259812352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032064646464646465, + "loss": 2.5887, + "theoretical_loss": 3.396040308209312, + "tokens_seen": 2259877888 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032062626262626266, + "loss": 2.593, + "theoretical_loss": 3.396032292797517, + "tokens_seen": 2259943424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206060606060606, + "loss": 2.6753, + "theoretical_loss": 3.396024277683238, + "tokens_seen": 2260008960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032058585858585857, + "loss": 2.5601, + "theoretical_loss": 3.3960162628664547, + "tokens_seen": 2260074496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205656565656566, + "loss": 2.3922, + "theoretical_loss": 3.3960082483471483, + "tokens_seen": 2260140032 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032054545454545454, + "loss": 2.7442, + "theoretical_loss": 3.3960002341252986, + "tokens_seen": 2260205568 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032052525252525255, + "loss": 2.6287, + "theoretical_loss": 3.395992220200886, + "tokens_seen": 2260271104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205050505050505, + "loss": 2.7109, + "theoretical_loss": 3.3959842065738908, + "tokens_seen": 2260336640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032048484848484847, + "loss": 2.6713, + "theoretical_loss": 3.395976193244294, + "tokens_seen": 2260402176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204646464646465, + "loss": 2.7537, + "theoretical_loss": 3.395968180212075, + "tokens_seen": 2260467712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032044444444444443, + "loss": 2.5858, + "theoretical_loss": 3.395960167477215, + "tokens_seen": 2260533248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032042424242424244, + "loss": 2.5759, + "theoretical_loss": 3.3959521550396934, + "tokens_seen": 2260598784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204040404040404, + "loss": 2.4519, + "theoretical_loss": 3.395944142899491, + "tokens_seen": 2260664320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203838383838384, + "loss": 2.3593, + "theoretical_loss": 3.395936131056588, + "tokens_seen": 2260729856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032036363636363637, + "loss": 2.6424, + "theoretical_loss": 3.3959281195109656, + "tokens_seen": 2260795392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203434343434343, + "loss": 2.7046, + "theoretical_loss": 3.395920108262603, + "tokens_seen": 2260860928 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032032323232323234, + "loss": 2.6292, + "theoretical_loss": 3.3959120973114816, + "tokens_seen": 2260926464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203030303030303, + "loss": 2.3901, + "theoretical_loss": 3.3959040866575805, + "tokens_seen": 2260992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202828282828283, + "loss": 2.607, + "theoretical_loss": 3.395896076300881, + "tokens_seen": 2261057536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032026262626262626, + "loss": 2.6355, + "theoretical_loss": 3.3958880662413633, + "tokens_seen": 2261123072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202424242424242, + "loss": 2.6611, + "theoretical_loss": 3.3958800564790077, + "tokens_seen": 2261188608 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1248049, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4141077995300293, + "objective/train/theoretical_loss": 3.3958720470137944, + "objective/train/tokens_used": 620113376, + "theoretical_loss": 3.3958720470137944, + "tokens_seen": 2261254144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032022222222222223, + "loss": 2.3786, + "theoretical_loss": 3.3958720470137944, + "tokens_seen": 2261254144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032020202020202024, + "loss": 2.4124, + "theoretical_loss": 3.3958640378457043, + "tokens_seen": 2261319680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201818181818182, + "loss": 2.6535, + "theoretical_loss": 3.395856028974717, + "tokens_seen": 2261385216 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032016161616161615, + "loss": 2.5763, + "theoretical_loss": 3.3958480204008135, + "tokens_seen": 2261450752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201414141414141, + "loss": 2.6341, + "theoretical_loss": 3.3958400121239736, + "tokens_seen": 2261516288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201212121212121, + "loss": 2.666, + "theoretical_loss": 3.395832004144178, + "tokens_seen": 2261581824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032010101010101013, + "loss": 2.802, + "theoretical_loss": 3.395823996461407, + "tokens_seen": 2261647360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200808080808081, + "loss": 2.5193, + "theoretical_loss": 3.3958159890756416, + "tokens_seen": 2261712896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032006060606060605, + "loss": 2.5524, + "theoretical_loss": 3.395807981986861, + "tokens_seen": 2261778432 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320040404040404, + "loss": 2.6795, + "theoretical_loss": 3.3957999751950467, + "tokens_seen": 2261843968 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032002020202020207, + "loss": 2.3938, + "theoretical_loss": 3.3957919687001783, + "tokens_seen": 2261909504 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032, + "loss": 2.7354, + "theoretical_loss": 3.3957839625022364, + "tokens_seen": 2261975040 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319979797979798, + "loss": 2.5789, + "theoretical_loss": 3.3957759566012014, + "tokens_seen": 2262040576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031995959595959594, + "loss": 2.618, + "theoretical_loss": 3.395767950997054, + "tokens_seen": 2262106112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199393939393939, + "loss": 2.5708, + "theoretical_loss": 3.3957599456897745, + "tokens_seen": 2262171648 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031991919191919196, + "loss": 2.6591, + "theoretical_loss": 3.3957519406793426, + "tokens_seen": 2262237184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198989898989899, + "loss": 2.6823, + "theoretical_loss": 3.3957439359657395, + "tokens_seen": 2262302720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198787878787879, + "loss": 2.8268, + "theoretical_loss": 3.3957359315489453, + "tokens_seen": 2262368256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031985858585858583, + "loss": 2.7604, + "theoretical_loss": 3.3957279274289403, + "tokens_seen": 2262433792 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031983838383838384, + "loss": 2.7604, + "theoretical_loss": 3.395719923605705, + "tokens_seen": 2262499328 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031981818181818185, + "loss": 2.6101, + "theoretical_loss": 3.39571192007922, + "tokens_seen": 2262564864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197979797979798, + "loss": 2.6851, + "theoretical_loss": 3.395703916849465, + "tokens_seen": 2262630400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031977777777777777, + "loss": 2.6499, + "theoretical_loss": 3.3956959139164216, + "tokens_seen": 2262695936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197575757575757, + "loss": 2.3554, + "theoretical_loss": 3.395687911280069, + "tokens_seen": 2262761472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197373737373738, + "loss": 2.7722, + "theoretical_loss": 3.395679908940388, + "tokens_seen": 2262827008 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1248635, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3614001274108887, + "objective/train/theoretical_loss": 3.39567190689736, + "objective/train/tokens_used": 621751776, + "theoretical_loss": 3.39567190689736, + "tokens_seen": 2262892544 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031971717171717175, + "loss": 2.4185, + "theoretical_loss": 3.39567190689736, + "tokens_seen": 2262892544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196969696969697, + "loss": 2.6732, + "theoretical_loss": 3.395663905150964, + "tokens_seen": 2262958080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031967676767676766, + "loss": 2.6997, + "theoretical_loss": 3.3956559037011806, + "tokens_seen": 2263023616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031965656565656567, + "loss": 2.3822, + "theoretical_loss": 3.395647902547991, + "tokens_seen": 2263089152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196363636363637, + "loss": 2.6004, + "theoretical_loss": 3.395639901691375, + "tokens_seen": 2263154688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031961616161616164, + "loss": 2.627, + "theoretical_loss": 3.3956319011313134, + "tokens_seen": 2263220224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195959595959596, + "loss": 2.4729, + "theoretical_loss": 3.395623900867786, + "tokens_seen": 2263285760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031957575757575755, + "loss": 2.5198, + "theoretical_loss": 3.3956159009007743, + "tokens_seen": 2263351296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031955555555555556, + "loss": 2.666, + "theoretical_loss": 3.395607901230258, + "tokens_seen": 2263416832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195353535353536, + "loss": 2.3702, + "theoretical_loss": 3.3955999018562175, + "tokens_seen": 2263482368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031951515151515153, + "loss": 2.5827, + "theoretical_loss": 3.395591902778633, + "tokens_seen": 2263547904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194949494949495, + "loss": 2.5407, + "theoretical_loss": 3.3955839039974856, + "tokens_seen": 2263613440 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031947474747474745, + "loss": 2.4561, + "theoretical_loss": 3.3955759055127555, + "tokens_seen": 2263678976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031945454545454546, + "loss": 2.701, + "theoretical_loss": 3.3955679073244225, + "tokens_seen": 2263744512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031943434343434347, + "loss": 2.6367, + "theoretical_loss": 3.3955599094324684, + "tokens_seen": 2263810048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194141414141414, + "loss": 2.7365, + "theoretical_loss": 3.395551911836872, + "tokens_seen": 2263875584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193939393939394, + "loss": 2.5264, + "theoretical_loss": 3.395543914537615, + "tokens_seen": 2263941120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193737373737374, + "loss": 2.603, + "theoretical_loss": 3.3955359175346773, + "tokens_seen": 2264006656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031935353535353535, + "loss": 2.5396, + "theoretical_loss": 3.3955279208280396, + "tokens_seen": 2264072192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031933333333333336, + "loss": 2.604, + "theoretical_loss": 3.395519924417682, + "tokens_seen": 2264137728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193131313131313, + "loss": 2.8452, + "theoretical_loss": 3.395511928303585, + "tokens_seen": 2264203264 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192929292929293, + "loss": 2.6547, + "theoretical_loss": 3.3955039324857292, + "tokens_seen": 2264268800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192727272727273, + "loss": 2.3664, + "theoretical_loss": 3.395495936964095, + "tokens_seen": 2264334336 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031925252525252524, + "loss": 2.5833, + "theoretical_loss": 3.395487941738663, + "tokens_seen": 2264399872 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031923232323232325, + "loss": 2.688, + "theoretical_loss": 3.395479946809414, + "tokens_seen": 2264465408 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1249740, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7208354473114014, + "objective/train/theoretical_loss": 3.3954719521763277, + "objective/train/tokens_used": 623390176, + "theoretical_loss": 3.3954719521763277, + "tokens_seen": 2264530944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192121212121212, + "loss": 2.5151, + "theoretical_loss": 3.3954719521763277, + "tokens_seen": 2264530944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191919191919192, + "loss": 2.8347, + "theoretical_loss": 3.3954639578393846, + "tokens_seen": 2264596480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191717171717172, + "loss": 2.8171, + "theoretical_loss": 3.3954559637985655, + "tokens_seen": 2264662016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031915151515151514, + "loss": 2.4953, + "theoretical_loss": 3.3954479700538505, + "tokens_seen": 2264727552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031913131313131315, + "loss": 2.7396, + "theoretical_loss": 3.395439976605221, + "tokens_seen": 2264793088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191111111111111, + "loss": 2.5031, + "theoretical_loss": 3.3954319834526565, + "tokens_seen": 2264858624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190909090909091, + "loss": 2.3527, + "theoretical_loss": 3.3954239905961376, + "tokens_seen": 2264924160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031907070707070707, + "loss": 2.5388, + "theoretical_loss": 3.395415998035645, + "tokens_seen": 2264989696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031905050505050503, + "loss": 2.7247, + "theoretical_loss": 3.3954080057711593, + "tokens_seen": 2265055232 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031903030303030304, + "loss": 2.5036, + "theoretical_loss": 3.3954000138026603, + "tokens_seen": 2265120768 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031901010101010105, + "loss": 2.5102, + "theoretical_loss": 3.3953920221301295, + "tokens_seen": 2265186304 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318989898989899, + "loss": 2.6913, + "theoretical_loss": 3.395384030753547, + "tokens_seen": 2265251840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031896969696969696, + "loss": 2.5066, + "theoretical_loss": 3.3953760396728923, + "tokens_seen": 2265317376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189494949494949, + "loss": 2.7384, + "theoretical_loss": 3.3953680488881472, + "tokens_seen": 2265382912 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031892929292929293, + "loss": 2.5828, + "theoretical_loss": 3.395360058399292, + "tokens_seen": 2265448448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031890909090909094, + "loss": 2.5857, + "theoretical_loss": 3.3953520682063063, + "tokens_seen": 2265513984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188888888888889, + "loss": 2.5849, + "theoretical_loss": 3.3953440783091713, + "tokens_seen": 2265579520 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031886868686868686, + "loss": 2.5293, + "theoretical_loss": 3.395336088707867, + "tokens_seen": 2265645056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188484848484848, + "loss": 2.6149, + "theoretical_loss": 3.395328099402375, + "tokens_seen": 2265710592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188282828282829, + "loss": 2.7865, + "theoretical_loss": 3.3953201103926745, + "tokens_seen": 2265776128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031880808080808084, + "loss": 2.5799, + "theoretical_loss": 3.3953121216787467, + "tokens_seen": 2265841664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187878787878788, + "loss": 2.425, + "theoretical_loss": 3.395304133260572, + "tokens_seen": 2265907200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031876767676767675, + "loss": 2.6586, + "theoretical_loss": 3.3952961451381305, + "tokens_seen": 2265972736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187474747474747, + "loss": 2.5189, + "theoretical_loss": 3.3952881573114033, + "tokens_seen": 2266038272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031872727272727277, + "loss": 2.569, + "theoretical_loss": 3.3952801697803707, + "tokens_seen": 2266103808 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1250565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.438335418701172, + "objective/train/theoretical_loss": 3.3952721825450127, + "objective/train/tokens_used": 625028576, + "theoretical_loss": 3.3952721825450127, + "tokens_seen": 2266169344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031870707070707073, + "loss": 2.6011, + "theoretical_loss": 3.3952721825450127, + "tokens_seen": 2266169344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186868686868687, + "loss": 2.8084, + "theoretical_loss": 3.3952641956053102, + "tokens_seen": 2266234880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031866666666666664, + "loss": 2.483, + "theoretical_loss": 3.395256208961244, + "tokens_seen": 2266300416 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031864646464646465, + "loss": 2.6591, + "theoretical_loss": 3.3952482226127945, + "tokens_seen": 2266365952 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031862626262626266, + "loss": 2.486, + "theoretical_loss": 3.3952402365599417, + "tokens_seen": 2266431488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186060606060606, + "loss": 2.6736, + "theoretical_loss": 3.3952322508026667, + "tokens_seen": 2266497024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185858585858586, + "loss": 2.5937, + "theoretical_loss": 3.3952242653409495, + "tokens_seen": 2266562560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031856565656565654, + "loss": 2.5626, + "theoretical_loss": 3.395216280174771, + "tokens_seen": 2266628096 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031854545454545455, + "loss": 2.7643, + "theoretical_loss": 3.3952082953041116, + "tokens_seen": 2266693632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031852525252525256, + "loss": 2.5137, + "theoretical_loss": 3.395200310728952, + "tokens_seen": 2266759168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185050505050505, + "loss": 2.5405, + "theoretical_loss": 3.395192326449272, + "tokens_seen": 2266824704 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031848484848484847, + "loss": 2.6636, + "theoretical_loss": 3.395184342465053, + "tokens_seen": 2266890240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184646464646465, + "loss": 2.8203, + "theoretical_loss": 3.395176358776275, + "tokens_seen": 2266955776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031844444444444444, + "loss": 2.8766, + "theoretical_loss": 3.3951683753829194, + "tokens_seen": 2267021312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031842424242424245, + "loss": 2.7388, + "theoretical_loss": 3.3951603922849656, + "tokens_seen": 2267086848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184040404040404, + "loss": 2.7884, + "theoretical_loss": 3.395152409482394, + "tokens_seen": 2267152384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031838383838383836, + "loss": 2.4545, + "theoretical_loss": 3.3951444269751865, + "tokens_seen": 2267217920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183636363636364, + "loss": 2.6157, + "theoretical_loss": 3.395136444763323, + "tokens_seen": 2267283456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031834343434343433, + "loss": 2.478, + "theoretical_loss": 3.3951284628467833, + "tokens_seen": 2267348992 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031832323232323234, + "loss": 2.6954, + "theoretical_loss": 3.3951204812255487, + "tokens_seen": 2267414528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183030303030303, + "loss": 2.6418, + "theoretical_loss": 3.395112499899599, + "tokens_seen": 2267480064 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182828282828283, + "loss": 2.7822, + "theoretical_loss": 3.395104518868916, + "tokens_seen": 2267545600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031826262626262627, + "loss": 2.7872, + "theoretical_loss": 3.3950965381334797, + "tokens_seen": 2267611136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182424242424242, + "loss": 2.7075, + "theoretical_loss": 3.39508855769327, + "tokens_seen": 2267676672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031822222222222224, + "loss": 2.2708, + "theoretical_loss": 3.395080577548268, + "tokens_seen": 2267742208 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1251506, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.883519411087036, + "objective/train/theoretical_loss": 3.3950725976984546, + "objective/train/tokens_used": 626666976, + "theoretical_loss": 3.3950725976984546, + "tokens_seen": 2267807744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182020202020202, + "loss": 2.7139, + "theoretical_loss": 3.3950725976984546, + "tokens_seen": 2267807744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181818181818182, + "loss": 2.4992, + "theoretical_loss": 3.3950646181438096, + "tokens_seen": 2267873280 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031816161616161616, + "loss": 2.5939, + "theoretical_loss": 3.3950566388843137, + "tokens_seen": 2267938816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181414141414141, + "loss": 2.6849, + "theoretical_loss": 3.395048659919948, + "tokens_seen": 2268004352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031812121212121213, + "loss": 2.5904, + "theoretical_loss": 3.3950406812506926, + "tokens_seen": 2268069888 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031810101010101014, + "loss": 2.5358, + "theoretical_loss": 3.395032702876528, + "tokens_seen": 2268135424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180808080808081, + "loss": 2.502, + "theoretical_loss": 3.3950247247974352, + "tokens_seen": 2268200960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031806060606060605, + "loss": 2.4941, + "theoretical_loss": 3.3950167470133943, + "tokens_seen": 2268266496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031804040404040406, + "loss": 2.6482, + "theoretical_loss": 3.395008769524386, + "tokens_seen": 2268332032 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318020202020202, + "loss": 2.6035, + "theoretical_loss": 3.3950007923303906, + "tokens_seen": 2268397568 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031800000000000003, + "loss": 2.4701, + "theoretical_loss": 3.3949928154313893, + "tokens_seen": 2268463104 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317979797979798, + "loss": 2.5139, + "theoretical_loss": 3.3949848388273627, + "tokens_seen": 2268528640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031795959595959595, + "loss": 2.6684, + "theoretical_loss": 3.3949768625182903, + "tokens_seen": 2268594176 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031793939393939396, + "loss": 2.6147, + "theoretical_loss": 3.394968886504154, + "tokens_seen": 2268659712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031791919191919197, + "loss": 2.5804, + "theoretical_loss": 3.3949609107849334, + "tokens_seen": 2268725248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178989898989899, + "loss": 2.4854, + "theoretical_loss": 3.39495293536061, + "tokens_seen": 2268790784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178787878787879, + "loss": 2.5769, + "theoretical_loss": 3.394944960231163, + "tokens_seen": 2268856320 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031785858585858584, + "loss": 2.6394, + "theoretical_loss": 3.3949369853965745, + "tokens_seen": 2268921856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031783838383838385, + "loss": 2.5502, + "theoretical_loss": 3.3949290108568237, + "tokens_seen": 2268987392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031781818181818186, + "loss": 2.5496, + "theoretical_loss": 3.3949210366118923, + "tokens_seen": 2269052928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177979797979798, + "loss": 2.518, + "theoretical_loss": 3.3949130626617605, + "tokens_seen": 2269118464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177777777777778, + "loss": 2.6216, + "theoretical_loss": 3.3949050890064085, + "tokens_seen": 2269184000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031775757575757573, + "loss": 2.3819, + "theoretical_loss": 3.3948971156458176, + "tokens_seen": 2269249536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177373737373738, + "loss": 2.8586, + "theoretical_loss": 3.394889142579968, + "tokens_seen": 2269315072 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031771717171717175, + "loss": 2.6064, + "theoretical_loss": 3.39488116980884, + "tokens_seen": 2269380608 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1252104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.231213092803955, + "objective/train/theoretical_loss": 3.394873197332415, + "objective/train/tokens_used": 628305376, + "theoretical_loss": 3.394873197332415, + "tokens_seen": 2269446144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176969696969697, + "loss": 2.4436, + "theoretical_loss": 3.394873197332415, + "tokens_seen": 2269446144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031767676767676767, + "loss": 2.4369, + "theoretical_loss": 3.3948652251506726, + "tokens_seen": 2269511680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176565656565656, + "loss": 2.7869, + "theoretical_loss": 3.394857253263594, + "tokens_seen": 2269577216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176363636363637, + "loss": 2.379, + "theoretical_loss": 3.39484928167116, + "tokens_seen": 2269642752 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031761616161616165, + "loss": 2.6299, + "theoretical_loss": 3.3948413103733506, + "tokens_seen": 2269708288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175959595959596, + "loss": 2.6942, + "theoretical_loss": 3.394833339370147, + "tokens_seen": 2269773824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031757575757575756, + "loss": 2.6772, + "theoretical_loss": 3.394825368661529, + "tokens_seen": 2269839360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175555555555555, + "loss": 2.5731, + "theoretical_loss": 3.3948173982474783, + "tokens_seen": 2269904896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175353535353536, + "loss": 2.7969, + "theoretical_loss": 3.3948094281279744, + "tokens_seen": 2269970432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031751515151515154, + "loss": 2.4478, + "theoretical_loss": 3.394801458302999, + "tokens_seen": 2270035968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174949494949495, + "loss": 2.6576, + "theoretical_loss": 3.3947934887725317, + "tokens_seen": 2270101504 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031747474747474745, + "loss": 2.5791, + "theoretical_loss": 3.394785519536554, + "tokens_seen": 2270167040 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031745454545454546, + "loss": 2.6718, + "theoretical_loss": 3.3947775505950455, + "tokens_seen": 2270232576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174343434343435, + "loss": 2.4232, + "theoretical_loss": 3.3947695819479877, + "tokens_seen": 2270298112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031741414141414143, + "loss": 2.575, + "theoretical_loss": 3.394761613595361, + "tokens_seen": 2270363648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173939393939394, + "loss": 2.5744, + "theoretical_loss": 3.394753645537146, + "tokens_seen": 2270429184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031737373737373734, + "loss": 2.535, + "theoretical_loss": 3.394745677773323, + "tokens_seen": 2270494720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031735353535353536, + "loss": 2.4876, + "theoretical_loss": 3.3947377103038736, + "tokens_seen": 2270560256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031733333333333337, + "loss": 2.6554, + "theoretical_loss": 3.394729743128777, + "tokens_seen": 2270625792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173131313131313, + "loss": 2.7185, + "theoretical_loss": 3.394721776248015, + "tokens_seen": 2270691328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172929292929293, + "loss": 2.4903, + "theoretical_loss": 3.394713809661568, + "tokens_seen": 2270756864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172727272727273, + "loss": 2.5748, + "theoretical_loss": 3.3947058433694157, + "tokens_seen": 2270822400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031725252525252525, + "loss": 2.3493, + "theoretical_loss": 3.3946978773715397, + "tokens_seen": 2270887936 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031723232323232326, + "loss": 2.4245, + "theoretical_loss": 3.394689911667921, + "tokens_seen": 2270953472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172121212121212, + "loss": 2.4038, + "theoretical_loss": 3.394681946258539, + "tokens_seen": 2271019008 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1253238, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6408684253692627, + "objective/train/theoretical_loss": 3.394673981143375, + "objective/train/tokens_used": 629943776, + "theoretical_loss": 3.394673981143375, + "tokens_seen": 2271084544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171919191919192, + "loss": 2.5847, + "theoretical_loss": 3.394673981143375, + "tokens_seen": 2271084544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171717171717172, + "loss": 2.5202, + "theoretical_loss": 3.39466601632241, + "tokens_seen": 2271150080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031715151515151514, + "loss": 2.4437, + "theoretical_loss": 3.394658051795624, + "tokens_seen": 2271215616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031713131313131315, + "loss": 2.4978, + "theoretical_loss": 3.3946500875629977, + "tokens_seen": 2271281152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171111111111111, + "loss": 2.6541, + "theoretical_loss": 3.3946421236245126, + "tokens_seen": 2271346688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170909090909091, + "loss": 2.6193, + "theoretical_loss": 3.394634159980148, + "tokens_seen": 2271412224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170707070707071, + "loss": 2.3838, + "theoretical_loss": 3.394626196629886, + "tokens_seen": 2271477760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031705050505050503, + "loss": 2.4387, + "theoretical_loss": 3.3946182335737056, + "tokens_seen": 2271543296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031703030303030305, + "loss": 2.5306, + "theoretical_loss": 3.3946102708115893, + "tokens_seen": 2271608832 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317010101010101, + "loss": 2.5514, + "theoretical_loss": 3.3946023083435164, + "tokens_seen": 2271674368 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316989898989899, + "loss": 2.4871, + "theoretical_loss": 3.394594346169468, + "tokens_seen": 2271739904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031696969696969697, + "loss": 2.6184, + "theoretical_loss": 3.3945863842894246, + "tokens_seen": 2271805440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169494949494949, + "loss": 2.6487, + "theoretical_loss": 3.3945784227033675, + "tokens_seen": 2271870976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031692929292929294, + "loss": 2.6708, + "theoretical_loss": 3.394570461411276, + "tokens_seen": 2271936512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031690909090909095, + "loss": 2.6389, + "theoretical_loss": 3.394562500413133, + "tokens_seen": 2272002048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168888888888889, + "loss": 2.5883, + "theoretical_loss": 3.3945545397089165, + "tokens_seen": 2272067584 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031686868686868686, + "loss": 2.6557, + "theoretical_loss": 3.394546579298609, + "tokens_seen": 2272133120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168484848484848, + "loss": 2.6129, + "theoretical_loss": 3.3945386191821907, + "tokens_seen": 2272198656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031682828282828283, + "loss": 2.5841, + "theoretical_loss": 3.394530659359642, + "tokens_seen": 2272264192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031680808080808084, + "loss": 2.4224, + "theoretical_loss": 3.3945226998309446, + "tokens_seen": 2272329728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167878787878788, + "loss": 2.5636, + "theoretical_loss": 3.3945147405960774, + "tokens_seen": 2272395264 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031676767676767676, + "loss": 2.3931, + "theoretical_loss": 3.3945067816550223, + "tokens_seen": 2272460800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167474747474747, + "loss": 2.6219, + "theoretical_loss": 3.39449882300776, + "tokens_seen": 2272526336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167272727272728, + "loss": 2.2824, + "theoretical_loss": 3.394490864654271, + "tokens_seen": 2272591872 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031670707070707073, + "loss": 2.4839, + "theoretical_loss": 3.394482906594536, + "tokens_seen": 2272657408 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1253856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.699746608734131, + "objective/train/theoretical_loss": 3.394474948828535, + "objective/train/tokens_used": 631582176, + "theoretical_loss": 3.394474948828535, + "tokens_seen": 2272722944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166868686868687, + "loss": 2.582, + "theoretical_loss": 3.394474948828535, + "tokens_seen": 2272722944 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031666666666666665, + "loss": 2.5631, + "theoretical_loss": 3.39446699135625, + "tokens_seen": 2272788480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166464646464646, + "loss": 2.6504, + "theoretical_loss": 3.39445903417766, + "tokens_seen": 2272854016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031662626262626267, + "loss": 2.6006, + "theoretical_loss": 3.394451077292748, + "tokens_seen": 2272919552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166060606060606, + "loss": 2.567, + "theoretical_loss": 3.3944431207014922, + "tokens_seen": 2272985088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165858585858586, + "loss": 2.744, + "theoretical_loss": 3.394435164403875, + "tokens_seen": 2273050624 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031656565656565654, + "loss": 2.679, + "theoretical_loss": 3.3944272083998763, + "tokens_seen": 2273116160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031654545454545455, + "loss": 2.3544, + "theoretical_loss": 3.3944192526894774, + "tokens_seen": 2273181696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031652525252525256, + "loss": 2.791, + "theoretical_loss": 3.394411297272659, + "tokens_seen": 2273247232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165050505050505, + "loss": 2.684, + "theoretical_loss": 3.3944033421494004, + "tokens_seen": 2273312768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164848484848485, + "loss": 2.5749, + "theoretical_loss": 3.394395387319684, + "tokens_seen": 2273378304 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031646464646464643, + "loss": 2.6576, + "theoretical_loss": 3.39438743278349, + "tokens_seen": 2273443840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031644444444444444, + "loss": 2.4809, + "theoretical_loss": 3.394379478540799, + "tokens_seen": 2273509376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031642424242424246, + "loss": 2.5898, + "theoretical_loss": 3.394371524591591, + "tokens_seen": 2273574912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164040404040404, + "loss": 2.5077, + "theoretical_loss": 3.394363570935848, + "tokens_seen": 2273640448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031638383838383837, + "loss": 2.4831, + "theoretical_loss": 3.39435561757355, + "tokens_seen": 2273705984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163636363636364, + "loss": 2.581, + "theoretical_loss": 3.3943476645046777, + "tokens_seen": 2273771520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163434343434344, + "loss": 2.5112, + "theoretical_loss": 3.394339711729212, + "tokens_seen": 2273837056 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031632323232323235, + "loss": 2.6041, + "theoretical_loss": 3.394331759247134, + "tokens_seen": 2273902592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163030303030303, + "loss": 2.5754, + "theoretical_loss": 3.3943238070584236, + "tokens_seen": 2273968128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031628282828282826, + "loss": 2.701, + "theoretical_loss": 3.394315855163062, + "tokens_seen": 2274033664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162626262626263, + "loss": 2.4644, + "theoretical_loss": 3.39430790356103, + "tokens_seen": 2274099200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162424242424243, + "loss": 2.5318, + "theoretical_loss": 3.394299952252308, + "tokens_seen": 2274164736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031622222222222224, + "loss": 2.7892, + "theoretical_loss": 3.3942920012368765, + "tokens_seen": 2274230272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162020202020202, + "loss": 2.5052, + "theoretical_loss": 3.394284050514717, + "tokens_seen": 2274295808 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1254938, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9091999530792236, + "objective/train/theoretical_loss": 3.39427610008581, + "objective/train/tokens_used": 633220576, + "theoretical_loss": 3.39427610008581, + "tokens_seen": 2274361344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031618181818181815, + "loss": 2.8026, + "theoretical_loss": 3.39427610008581, + "tokens_seen": 2274361344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031616161616161617, + "loss": 2.6426, + "theoretical_loss": 3.394268149950136, + "tokens_seen": 2274426880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161414141414142, + "loss": 2.3218, + "theoretical_loss": 3.394260200107676, + "tokens_seen": 2274492416 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031612121212121213, + "loss": 2.4472, + "theoretical_loss": 3.3942522505584103, + "tokens_seen": 2274557952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161010101010101, + "loss": 2.801, + "theoretical_loss": 3.39424430130232, + "tokens_seen": 2274623488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160808080808081, + "loss": 2.6436, + "theoretical_loss": 3.3942363523393855, + "tokens_seen": 2274689024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031606060606060606, + "loss": 2.6681, + "theoretical_loss": 3.3942284036695884, + "tokens_seen": 2274754560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031604040404040407, + "loss": 2.4765, + "theoretical_loss": 3.394220455292908, + "tokens_seen": 2274820096 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316020202020202, + "loss": 2.4176, + "theoretical_loss": 3.3942125072093265, + "tokens_seen": 2274885632 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316, + "loss": 2.6185, + "theoretical_loss": 3.394204559418824, + "tokens_seen": 2274951168 + }, + { + "epoch": 0.37, + "learning_rate": 0.000315979797979798, + "loss": 2.5395, + "theoretical_loss": 3.394196611921381, + "tokens_seen": 2275016704 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031595959595959595, + "loss": 2.8416, + "theoretical_loss": 3.394188664716979, + "tokens_seen": 2275082240 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031593939393939396, + "loss": 2.58, + "theoretical_loss": 3.3941807178055976, + "tokens_seen": 2275147776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159191919191919, + "loss": 2.4582, + "theoretical_loss": 3.3941727711872187, + "tokens_seen": 2275213312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031589898989898993, + "loss": 2.5861, + "theoretical_loss": 3.394164824861823, + "tokens_seen": 2275278848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158787878787879, + "loss": 2.6339, + "theoretical_loss": 3.3941568788293903, + "tokens_seen": 2275344384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031585858585858584, + "loss": 2.588, + "theoretical_loss": 3.3941489330899017, + "tokens_seen": 2275409920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031583838383838385, + "loss": 2.5906, + "theoretical_loss": 3.394140987643339, + "tokens_seen": 2275475456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158181818181818, + "loss": 2.6659, + "theoretical_loss": 3.394133042489681, + "tokens_seen": 2275540992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157979797979798, + "loss": 2.6415, + "theoretical_loss": 3.3941250976289106, + "tokens_seen": 2275606528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157777777777778, + "loss": 2.4657, + "theoretical_loss": 3.3941171530610075, + "tokens_seen": 2275672064 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031575757575757574, + "loss": 2.2953, + "theoretical_loss": 3.3941092087859523, + "tokens_seen": 2275737600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031573737373737375, + "loss": 2.6089, + "theoretical_loss": 3.394101264803726, + "tokens_seen": 2275803136 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031571717171717176, + "loss": 2.6701, + "theoretical_loss": 3.3940933211143096, + "tokens_seen": 2275868672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003156969696969697, + "loss": 2.6224, + "theoretical_loss": 3.3940853777176834, + "tokens_seen": 2275934208 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1256312, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5594100952148438, + "objective/train/theoretical_loss": 3.3940774346138287, + "objective/train/tokens_used": 634858976, + "theoretical_loss": 3.3940774346138287, + "tokens_seen": 2275999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031567676767676767, + "loss": 2.5569, + "theoretical_loss": 3.3940774346138287, + "tokens_seen": 2275999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031565656565656563, + "loss": 2.5351, + "theoretical_loss": 3.394069491802726, + "tokens_seen": 2276065280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031563636363636364, + "loss": 2.6004, + "theoretical_loss": 3.394061549284356, + "tokens_seen": 2276130816 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031561616161616165, + "loss": 2.5797, + "theoretical_loss": 3.3940536070586997, + "tokens_seen": 2276196352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155959595959596, + "loss": 2.5316, + "theoretical_loss": 3.394045665125738, + "tokens_seen": 2276261888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031557575757575757, + "loss": 2.2698, + "theoretical_loss": 3.394037723485451, + "tokens_seen": 2276327424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155555555555555, + "loss": 2.6141, + "theoretical_loss": 3.39402978213782, + "tokens_seen": 2276392960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155353535353536, + "loss": 2.5721, + "theoretical_loss": 3.394021841082826, + "tokens_seen": 2276458496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031551515151515154, + "loss": 2.562, + "theoretical_loss": 3.3940139003204495, + "tokens_seen": 2276524032 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154949494949495, + "loss": 2.5509, + "theoretical_loss": 3.3940059598506718, + "tokens_seen": 2276589568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031547474747474746, + "loss": 2.5295, + "theoretical_loss": 3.3939980196734725, + "tokens_seen": 2276655104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154545454545454, + "loss": 2.5415, + "theoretical_loss": 3.3939900797888334, + "tokens_seen": 2276720640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154343434343435, + "loss": 2.6936, + "theoretical_loss": 3.3939821401967354, + "tokens_seen": 2276786176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031541414141414144, + "loss": 2.5969, + "theoretical_loss": 3.3939742008971585, + "tokens_seen": 2276851712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153939393939394, + "loss": 2.6559, + "theoretical_loss": 3.393966261890084, + "tokens_seen": 2276917248 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031537373737373735, + "loss": 2.5614, + "theoretical_loss": 3.3939583231754926, + "tokens_seen": 2276982784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031535353535353536, + "loss": 2.5683, + "theoretical_loss": 3.3939503847533654, + "tokens_seen": 2277048320 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031533333333333337, + "loss": 2.746, + "theoretical_loss": 3.393942446623683, + "tokens_seen": 2277113856 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031531313131313133, + "loss": 2.7708, + "theoretical_loss": 3.3939345087864257, + "tokens_seen": 2277179392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152929292929293, + "loss": 2.586, + "theoretical_loss": 3.393926571241575, + "tokens_seen": 2277244928 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031527272727272724, + "loss": 2.5205, + "theoretical_loss": 3.3939186339891116, + "tokens_seen": 2277310464 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031525252525252525, + "loss": 2.6002, + "theoretical_loss": 3.3939106970290163, + "tokens_seen": 2277376000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031523232323232327, + "loss": 2.538, + "theoretical_loss": 3.3939027603612697, + "tokens_seen": 2277441536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152121212121212, + "loss": 2.5522, + "theoretical_loss": 3.3938948239858524, + "tokens_seen": 2277507072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151919191919192, + "loss": 2.4125, + "theoretical_loss": 3.393886887902746, + "tokens_seen": 2277572608 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1256724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7904176712036133, + "objective/train/theoretical_loss": 3.3938789521119306, + "objective/train/tokens_used": 636497376, + "theoretical_loss": 3.3938789521119306, + "tokens_seen": 2277638144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151717171717172, + "loss": 2.5879, + "theoretical_loss": 3.3938789521119306, + "tokens_seen": 2277638144 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031515151515151515, + "loss": 2.6244, + "theoretical_loss": 3.393871016613388, + "tokens_seen": 2277703680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031513131313131316, + "loss": 2.824, + "theoretical_loss": 3.3938630814070976, + "tokens_seen": 2277769216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151111111111111, + "loss": 2.6226, + "theoretical_loss": 3.393855146493041, + "tokens_seen": 2277834752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031509090909090907, + "loss": 2.5232, + "theoretical_loss": 3.3938472118711993, + "tokens_seen": 2277900288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150707070707071, + "loss": 2.5386, + "theoretical_loss": 3.393839277541553, + "tokens_seen": 2277965824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031505050505050504, + "loss": 2.4923, + "theoretical_loss": 3.393831343504083, + "tokens_seen": 2278031360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031503030303030305, + "loss": 2.3651, + "theoretical_loss": 3.3938234097587703, + "tokens_seen": 2278096896 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315010101010101, + "loss": 2.5789, + "theoretical_loss": 3.393815476305595, + "tokens_seen": 2278162432 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314989898989899, + "loss": 2.4717, + "theoretical_loss": 3.393807543144539, + "tokens_seen": 2278227968 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314969696969697, + "loss": 2.3875, + "theoretical_loss": 3.393799610275582, + "tokens_seen": 2278293504 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031494949494949493, + "loss": 2.6223, + "theoretical_loss": 3.3937916776987063, + "tokens_seen": 2278359040 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031492929292929294, + "loss": 2.7733, + "theoretical_loss": 3.3937837454138915, + "tokens_seen": 2278424576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149090909090909, + "loss": 2.5053, + "theoretical_loss": 3.3937758134211187, + "tokens_seen": 2278490112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148888888888889, + "loss": 2.2793, + "theoretical_loss": 3.393767881720369, + "tokens_seen": 2278555648 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031486868686868687, + "loss": 2.5787, + "theoretical_loss": 3.3937599503116234, + "tokens_seen": 2278621184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148484848484848, + "loss": 2.6155, + "theoretical_loss": 3.3937520191948622, + "tokens_seen": 2278686720 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031482828282828284, + "loss": 2.5502, + "theoretical_loss": 3.3937440883700667, + "tokens_seen": 2278752256 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031480808080808085, + "loss": 2.6447, + "theoretical_loss": 3.3937361578372176, + "tokens_seen": 2278817792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147878787878788, + "loss": 2.3951, + "theoretical_loss": 3.393728227596296, + "tokens_seen": 2278883328 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031476767676767676, + "loss": 2.685, + "theoretical_loss": 3.393720297647282, + "tokens_seen": 2278948864 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147474747474747, + "loss": 2.6433, + "theoretical_loss": 3.3937123679901573, + "tokens_seen": 2279014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031472727272727273, + "loss": 2.5549, + "theoretical_loss": 3.3937044386249022, + "tokens_seen": 2279079936 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031470707070707074, + "loss": 2.6744, + "theoretical_loss": 3.393696509551498, + "tokens_seen": 2279145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146868686868687, + "loss": 2.6514, + "theoretical_loss": 3.3936885807699255, + "tokens_seen": 2279211008 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1257757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0417563915252686, + "objective/train/theoretical_loss": 3.3936806522801657, + "objective/train/tokens_used": 638135776, + "theoretical_loss": 3.3936806522801657, + "tokens_seen": 2279276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031466666666666665, + "loss": 2.3856, + "theoretical_loss": 3.3936806522801657, + "tokens_seen": 2279276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031464646464646466, + "loss": 2.771, + "theoretical_loss": 3.3936727240821987, + "tokens_seen": 2279342080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146262626262627, + "loss": 2.5876, + "theoretical_loss": 3.3936647961760062, + "tokens_seen": 2279407616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031460606060606063, + "loss": 2.7307, + "theoretical_loss": 3.393656868561569, + "tokens_seen": 2279473152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145858585858586, + "loss": 2.5587, + "theoretical_loss": 3.393648941238867, + "tokens_seen": 2279538688 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031456565656565655, + "loss": 2.5594, + "theoretical_loss": 3.3936410142078826, + "tokens_seen": 2279604224 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031454545454545456, + "loss": 2.6003, + "theoretical_loss": 3.3936330874685954, + "tokens_seen": 2279669760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031452525252525257, + "loss": 2.6426, + "theoretical_loss": 3.393625161020987, + "tokens_seen": 2279735296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145050505050505, + "loss": 2.613, + "theoretical_loss": 3.393617234865038, + "tokens_seen": 2279800832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144848484848485, + "loss": 2.761, + "theoretical_loss": 3.3936093090007295, + "tokens_seen": 2279866368 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031446464646464644, + "loss": 2.478, + "theoretical_loss": 3.393601383428042, + "tokens_seen": 2279931904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144444444444445, + "loss": 2.5253, + "theoretical_loss": 3.393593458146957, + "tokens_seen": 2279997440 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031442424242424246, + "loss": 2.5476, + "theoretical_loss": 3.393585533157455, + "tokens_seen": 2280062976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144040404040404, + "loss": 2.3795, + "theoretical_loss": 3.393577608459516, + "tokens_seen": 2280128512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143838383838384, + "loss": 2.6271, + "theoretical_loss": 3.393569684053123, + "tokens_seen": 2280194048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031436363636363633, + "loss": 2.5094, + "theoretical_loss": 3.393561759938255, + "tokens_seen": 2280259584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143434343434344, + "loss": 2.4484, + "theoretical_loss": 3.393553836114894, + "tokens_seen": 2280325120 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031432323232323235, + "loss": 2.5701, + "theoretical_loss": 3.3935459125830203, + "tokens_seen": 2280390656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143030303030303, + "loss": 2.6625, + "theoretical_loss": 3.393537989342615, + "tokens_seen": 2280456192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031428282828282827, + "loss": 2.6173, + "theoretical_loss": 3.3935300663936587, + "tokens_seen": 2280521728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142626262626262, + "loss": 2.6235, + "theoretical_loss": 3.393522143736133, + "tokens_seen": 2280587264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142424242424243, + "loss": 2.599, + "theoretical_loss": 3.3935142213700185, + "tokens_seen": 2280652800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031422222222222225, + "loss": 2.448, + "theoretical_loss": 3.3935062992952956, + "tokens_seen": 2280718336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142020202020202, + "loss": 2.5578, + "theoretical_loss": 3.393498377511946, + "tokens_seen": 2280783872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031418181818181816, + "loss": 2.6501, + "theoretical_loss": 3.39349045601995, + "tokens_seen": 2280849408 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1258472, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2951927185058594, + "objective/train/theoretical_loss": 3.393482534819289, + "objective/train/tokens_used": 639774176, + "theoretical_loss": 3.393482534819289, + "tokens_seen": 2280914944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031416161616161617, + "loss": 2.5746, + "theoretical_loss": 3.393482534819289, + "tokens_seen": 2280914944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141414141414142, + "loss": 2.5196, + "theoretical_loss": 3.393474613909943, + "tokens_seen": 2280980480 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031412121212121214, + "loss": 2.5398, + "theoretical_loss": 3.393466693291894, + "tokens_seen": 2281046016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141010101010101, + "loss": 2.618, + "theoretical_loss": 3.393458772965123, + "tokens_seen": 2281111552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031408080808080805, + "loss": 2.6383, + "theoretical_loss": 3.3934508529296097, + "tokens_seen": 2281177088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031406060606060606, + "loss": 2.6989, + "theoretical_loss": 3.3934429331853364, + "tokens_seen": 2281242624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140404040404041, + "loss": 2.5702, + "theoretical_loss": 3.393435013732283, + "tokens_seen": 2281308160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031402020202020203, + "loss": 2.9019, + "theoretical_loss": 3.3934270945704306, + "tokens_seen": 2281373696 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314, + "loss": 2.5032, + "theoretical_loss": 3.3934191756997603, + "tokens_seen": 2281439232 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313979797979798, + "loss": 2.6049, + "theoretical_loss": 3.3934112571202535, + "tokens_seen": 2281504768 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031395959595959596, + "loss": 2.5911, + "theoretical_loss": 3.3934033388318903, + "tokens_seen": 2281570304 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031393939393939397, + "loss": 2.4743, + "theoretical_loss": 3.3933954208346524, + "tokens_seen": 2281635840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139191919191919, + "loss": 2.7516, + "theoretical_loss": 3.39338750312852, + "tokens_seen": 2281701376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138989898989899, + "loss": 2.5268, + "theoretical_loss": 3.3933795857134745, + "tokens_seen": 2281766912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138787878787879, + "loss": 2.753, + "theoretical_loss": 3.393371668589497, + "tokens_seen": 2281832448 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031385858585858585, + "loss": 2.6866, + "theoretical_loss": 3.3933637517565676, + "tokens_seen": 2281897984 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031383838383838386, + "loss": 2.6907, + "theoretical_loss": 3.393355835214668, + "tokens_seen": 2281963520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138181818181818, + "loss": 2.49, + "theoretical_loss": 3.393347918963779, + "tokens_seen": 2282029056 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031379797979797983, + "loss": 2.5764, + "theoretical_loss": 3.393340003003882, + "tokens_seen": 2282094592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137777777777778, + "loss": 2.6265, + "theoretical_loss": 3.3933320873349566, + "tokens_seen": 2282160128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031375757575757574, + "loss": 2.485, + "theoretical_loss": 3.393324171956985, + "tokens_seen": 2282225664 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031373737373737375, + "loss": 2.4023, + "theoretical_loss": 3.3933162568699475, + "tokens_seen": 2282291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137171717171717, + "loss": 2.559, + "theoretical_loss": 3.3933083420738255, + "tokens_seen": 2282356736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136969696969697, + "loss": 2.5539, + "theoretical_loss": 3.3933004275686, + "tokens_seen": 2282422272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136767676767677, + "loss": 2.6842, + "theoretical_loss": 3.3932925133542513, + "tokens_seen": 2282487808 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1259673, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.045727491378784, + "objective/train/theoretical_loss": 3.393284599430761, + "objective/train/tokens_used": 641412576, + "theoretical_loss": 3.393284599430761, + "tokens_seen": 2282553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031365656565656563, + "loss": 2.9195, + "theoretical_loss": 3.393284599430761, + "tokens_seen": 2282553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031363636363636365, + "loss": 2.5363, + "theoretical_loss": 3.39327668579811, + "tokens_seen": 2282618880 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031361616161616166, + "loss": 2.575, + "theoretical_loss": 3.3932687724562784, + "tokens_seen": 2282684416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135959595959596, + "loss": 2.6229, + "theoretical_loss": 3.393260859405249, + "tokens_seen": 2282749952 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031357575757575757, + "loss": 2.5216, + "theoretical_loss": 3.3932529466450005, + "tokens_seen": 2282815488 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031355555555555553, + "loss": 2.5041, + "theoretical_loss": 3.3932450341755156, + "tokens_seen": 2282881024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031353535353535354, + "loss": 2.7757, + "theoretical_loss": 3.3932371219967745, + "tokens_seen": 2282946560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031351515151515155, + "loss": 2.5232, + "theoretical_loss": 3.393229210108758, + "tokens_seen": 2283012096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134949494949495, + "loss": 2.4608, + "theoretical_loss": 3.393221298511448, + "tokens_seen": 2283077632 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031347474747474746, + "loss": 2.6039, + "theoretical_loss": 3.3932133872048245, + "tokens_seen": 2283143168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134545454545454, + "loss": 2.5171, + "theoretical_loss": 3.393205476188869, + "tokens_seen": 2283208704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134343434343435, + "loss": 2.5416, + "theoretical_loss": 3.393197565463562, + "tokens_seen": 2283274240 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031341414141414144, + "loss": 2.6953, + "theoretical_loss": 3.393189655028885, + "tokens_seen": 2283339776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133939393939394, + "loss": 2.5893, + "theoretical_loss": 3.393181744884819, + "tokens_seen": 2283405312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031337373737373736, + "loss": 2.7462, + "theoretical_loss": 3.3931738350313445, + "tokens_seen": 2283470848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133535353535353, + "loss": 2.6833, + "theoretical_loss": 3.393165925468443, + "tokens_seen": 2283536384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133333333333334, + "loss": 2.5097, + "theoretical_loss": 3.393158016196095, + "tokens_seen": 2283601920 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031331313131313134, + "loss": 2.5382, + "theoretical_loss": 3.393150107214282, + "tokens_seen": 2283667456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132929292929293, + "loss": 2.5648, + "theoretical_loss": 3.393142198522984, + "tokens_seen": 2283732992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031327272727272725, + "loss": 2.6896, + "theoretical_loss": 3.3931342901221835, + "tokens_seen": 2283798528 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031325252525252526, + "loss": 2.566, + "theoretical_loss": 3.3931263820118605, + "tokens_seen": 2283864064 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031323232323232327, + "loss": 2.5805, + "theoretical_loss": 3.393118474191996, + "tokens_seen": 2283929600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031321212121212123, + "loss": 2.615, + "theoretical_loss": 3.393110566662571, + "tokens_seen": 2283995136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131919191919192, + "loss": 2.6735, + "theoretical_loss": 3.393102659423567, + "tokens_seen": 2284060672 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031317171717171714, + "loss": 2.6565, + "theoretical_loss": 3.393094752474965, + "tokens_seen": 2284126208 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1260242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6039111614227295, + "objective/train/theoretical_loss": 3.393086845816746, + "objective/train/tokens_used": 643050976, + "theoretical_loss": 3.393086845816746, + "tokens_seen": 2284191744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031315151515151515, + "loss": 2.5128, + "theoretical_loss": 3.393086845816746, + "tokens_seen": 2284191744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031313131313131316, + "loss": 2.6107, + "theoretical_loss": 3.39307893944889, + "tokens_seen": 2284257280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131111111111111, + "loss": 2.7187, + "theoretical_loss": 3.3930710333713785, + "tokens_seen": 2284322816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130909090909091, + "loss": 2.5281, + "theoretical_loss": 3.393063127584193, + "tokens_seen": 2284388352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130707070707071, + "loss": 2.5965, + "theoretical_loss": 3.3930552220873147, + "tokens_seen": 2284453888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031305050505050505, + "loss": 2.4025, + "theoretical_loss": 3.3930473168807236, + "tokens_seen": 2284519424 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031303030303030306, + "loss": 2.7882, + "theoretical_loss": 3.3930394119644016, + "tokens_seen": 2284584960 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313010101010101, + "loss": 2.7073, + "theoretical_loss": 3.3930315073383293, + "tokens_seen": 2284650496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031298989898989897, + "loss": 2.5197, + "theoretical_loss": 3.3930236030024874, + "tokens_seen": 2284716032 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312969696969697, + "loss": 2.449, + "theoretical_loss": 3.393015698956858, + "tokens_seen": 2284781568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031294949494949494, + "loss": 2.4184, + "theoretical_loss": 3.393007795201421, + "tokens_seen": 2284847104 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031292929292929295, + "loss": 2.5209, + "theoretical_loss": 3.3929998917361575, + "tokens_seen": 2284912640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129090909090909, + "loss": 2.4928, + "theoretical_loss": 3.3929919885610493, + "tokens_seen": 2284978176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031288888888888886, + "loss": 2.4869, + "theoretical_loss": 3.392984085676077, + "tokens_seen": 2285043712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128686868686869, + "loss": 2.4231, + "theoretical_loss": 3.392976183081222, + "tokens_seen": 2285109248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128484848484849, + "loss": 2.5543, + "theoretical_loss": 3.3929682807764645, + "tokens_seen": 2285174784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031282828282828284, + "loss": 2.6339, + "theoretical_loss": 3.392960378761786, + "tokens_seen": 2285240320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128080808080808, + "loss": 2.6631, + "theoretical_loss": 3.3929524770371677, + "tokens_seen": 2285305856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127878787878788, + "loss": 2.5875, + "theoretical_loss": 3.3929445756025904, + "tokens_seen": 2285371392 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031276767676767677, + "loss": 2.4776, + "theoretical_loss": 3.392936674458035, + "tokens_seen": 2285436928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127474747474748, + "loss": 2.7435, + "theoretical_loss": 3.392928773603483, + "tokens_seen": 2285502464 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031272727272727273, + "loss": 2.5056, + "theoretical_loss": 3.392920873038915, + "tokens_seen": 2285568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127070707070707, + "loss": 2.5386, + "theoretical_loss": 3.3929129727643126, + "tokens_seen": 2285633536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126868686868687, + "loss": 2.7133, + "theoretical_loss": 3.392905072779656, + "tokens_seen": 2285699072 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031266666666666666, + "loss": 2.7472, + "theoretical_loss": 3.392897173084927, + "tokens_seen": 2285764608 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1261616, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.160874605178833, + "objective/train/theoretical_loss": 3.392889273680106, + "objective/train/tokens_used": 644689376, + "theoretical_loss": 3.392889273680106, + "tokens_seen": 2285830144 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031264646464646467, + "loss": 2.7724, + "theoretical_loss": 3.392889273680106, + "tokens_seen": 2285830144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126262626262626, + "loss": 2.6675, + "theoretical_loss": 3.3928813745651745, + "tokens_seen": 2285895680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031260606060606064, + "loss": 2.5848, + "theoretical_loss": 3.3928734757401138, + "tokens_seen": 2285961216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125858585858586, + "loss": 2.4852, + "theoretical_loss": 3.3928655772049043, + "tokens_seen": 2286026752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031256565656565655, + "loss": 2.459, + "theoretical_loss": 3.3928576789595275, + "tokens_seen": 2286092288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031254545454545456, + "loss": 2.3874, + "theoretical_loss": 3.3928497810039646, + "tokens_seen": 2286157824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125252525252525, + "loss": 2.742, + "theoretical_loss": 3.3928418833381957, + "tokens_seen": 2286223360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031250505050505053, + "loss": 2.5145, + "theoretical_loss": 3.392833985962203, + "tokens_seen": 2286288896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124848484848485, + "loss": 2.6008, + "theoretical_loss": 3.392826088875967, + "tokens_seen": 2286354432 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031246464646464644, + "loss": 2.258, + "theoretical_loss": 3.392818192079469, + "tokens_seen": 2286419968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031244444444444446, + "loss": 2.5079, + "theoretical_loss": 3.3928102955726898, + "tokens_seen": 2286485504 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031242424242424247, + "loss": 2.5061, + "theoretical_loss": 3.3928023993556105, + "tokens_seen": 2286551040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124040404040404, + "loss": 2.8721, + "theoretical_loss": 3.3927945034282123, + "tokens_seen": 2286616576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123838383838384, + "loss": 2.6242, + "theoretical_loss": 3.3927866077904767, + "tokens_seen": 2286682112 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031236363636363634, + "loss": 2.5288, + "theoretical_loss": 3.3927787124423836, + "tokens_seen": 2286747648 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031234343434343435, + "loss": 2.5572, + "theoretical_loss": 3.3927708173839153, + "tokens_seen": 2286813184 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031232323232323236, + "loss": 2.6219, + "theoretical_loss": 3.392762922615052, + "tokens_seen": 2286878720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123030303030303, + "loss": 2.6096, + "theoretical_loss": 3.3927550281357757, + "tokens_seen": 2286944256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122828282828283, + "loss": 2.4923, + "theoretical_loss": 3.3927471339460666, + "tokens_seen": 2287009792 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031226262626262623, + "loss": 2.6047, + "theoretical_loss": 3.3927392400459055, + "tokens_seen": 2287075328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122424242424243, + "loss": 2.5471, + "theoretical_loss": 3.392731346435275, + "tokens_seen": 2287140864 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031222222222222225, + "loss": 2.7516, + "theoretical_loss": 3.392723453114155, + "tokens_seen": 2287206400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122020202020202, + "loss": 2.5674, + "theoretical_loss": 3.3927155600825265, + "tokens_seen": 2287271936 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031218181818181817, + "loss": 2.5033, + "theoretical_loss": 3.3927076673403715, + "tokens_seen": 2287337472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121616161616161, + "loss": 2.6554, + "theoretical_loss": 3.39269977488767, + "tokens_seen": 2287403008 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1262743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.618569850921631, + "objective/train/theoretical_loss": 3.392691882724404, + "objective/train/tokens_used": 646327776, + "theoretical_loss": 3.392691882724404, + "tokens_seen": 2287468544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121414141414142, + "loss": 2.7417, + "theoretical_loss": 3.392691882724404, + "tokens_seen": 2287468544 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031212121212121214, + "loss": 2.7722, + "theoretical_loss": 3.392683990850554, + "tokens_seen": 2287534080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121010101010101, + "loss": 2.5364, + "theoretical_loss": 3.3926760992661014, + "tokens_seen": 2287599616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031208080808080806, + "loss": 2.5532, + "theoretical_loss": 3.3926682079710275, + "tokens_seen": 2287665152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031206060606060607, + "loss": 2.5556, + "theoretical_loss": 3.3926603169653125, + "tokens_seen": 2287730688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120404040404041, + "loss": 2.549, + "theoretical_loss": 3.3926524262489384, + "tokens_seen": 2287796224 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031202020202020204, + "loss": 2.4129, + "theoretical_loss": 3.392644535821886, + "tokens_seen": 2287861760 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312, + "loss": 2.4981, + "theoretical_loss": 3.392636645684137, + "tokens_seen": 2287927296 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031197979797979795, + "loss": 2.5447, + "theoretical_loss": 3.392628755835671, + "tokens_seen": 2287992832 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031195959595959596, + "loss": 2.584, + "theoretical_loss": 3.3926208662764705, + "tokens_seen": 2288058368 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311939393939394, + "loss": 2.6751, + "theoretical_loss": 3.392612977006516, + "tokens_seen": 2288123904 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031191919191919193, + "loss": 2.6755, + "theoretical_loss": 3.392605088025789, + "tokens_seen": 2288189440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118989898989899, + "loss": 2.5299, + "theoretical_loss": 3.3925971993342703, + "tokens_seen": 2288254976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118787878787879, + "loss": 2.415, + "theoretical_loss": 3.3925893109319407, + "tokens_seen": 2288320512 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031185858585858585, + "loss": 2.5081, + "theoretical_loss": 3.392581422818782, + "tokens_seen": 2288386048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031183838383838387, + "loss": 2.6131, + "theoretical_loss": 3.392573534994775, + "tokens_seen": 2288451584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118181818181818, + "loss": 2.6848, + "theoretical_loss": 3.392565647459901, + "tokens_seen": 2288517120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117979797979798, + "loss": 2.5136, + "theoretical_loss": 3.3925577602141406, + "tokens_seen": 2288582656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117777777777778, + "loss": 2.4394, + "theoretical_loss": 3.392549873257476, + "tokens_seen": 2288648192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031175757575757575, + "loss": 2.5428, + "theoretical_loss": 3.3925419865898867, + "tokens_seen": 2288713728 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031173737373737376, + "loss": 2.6776, + "theoretical_loss": 3.392534100211355, + "tokens_seen": 2288779264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117171717171717, + "loss": 2.4304, + "theoretical_loss": 3.392526214121862, + "tokens_seen": 2288844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116969696969697, + "loss": 2.4941, + "theoretical_loss": 3.3925183283213887, + "tokens_seen": 2288910336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116767676767677, + "loss": 2.6966, + "theoretical_loss": 3.392510442809916, + "tokens_seen": 2288975872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031165656565656564, + "loss": 2.648, + "theoretical_loss": 3.3925025575874255, + "tokens_seen": 2289041408 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1263359, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.576524019241333, + "objective/train/theoretical_loss": 3.392494672653897, + "objective/train/tokens_used": 647966176, + "theoretical_loss": 3.392494672653897, + "tokens_seen": 2289106944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031163636363636365, + "loss": 2.7716, + "theoretical_loss": 3.392494672653897, + "tokens_seen": 2289106944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116161616161616, + "loss": 2.6873, + "theoretical_loss": 3.3924867880093137, + "tokens_seen": 2289172480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115959595959596, + "loss": 2.5042, + "theoretical_loss": 3.392478903653655, + "tokens_seen": 2289238016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115757575757576, + "loss": 2.4627, + "theoretical_loss": 3.3924710195869032, + "tokens_seen": 2289303552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031155555555555553, + "loss": 2.3332, + "theoretical_loss": 3.392463135809039, + "tokens_seen": 2289369088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031153535353535354, + "loss": 2.6457, + "theoretical_loss": 3.392455252320043, + "tokens_seen": 2289434624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031151515151515156, + "loss": 2.3721, + "theoretical_loss": 3.392447369119897, + "tokens_seen": 2289500160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114949494949495, + "loss": 2.8084, + "theoretical_loss": 3.3924394862085823, + "tokens_seen": 2289565696 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031147474747474747, + "loss": 2.4445, + "theoretical_loss": 3.3924316035860795, + "tokens_seen": 2289631232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114545454545454, + "loss": 2.4715, + "theoretical_loss": 3.3924237212523702, + "tokens_seen": 2289696768 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031143434343434344, + "loss": 2.6434, + "theoretical_loss": 3.3924158392074353, + "tokens_seen": 2289762304 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031141414141414145, + "loss": 2.5525, + "theoretical_loss": 3.392407957451256, + "tokens_seen": 2289827840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113939393939394, + "loss": 2.5955, + "theoretical_loss": 3.3924000759838133, + "tokens_seen": 2289893376 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031137373737373736, + "loss": 2.6712, + "theoretical_loss": 3.392392194805089, + "tokens_seen": 2289958912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113535353535353, + "loss": 2.5935, + "theoretical_loss": 3.3923843139150636, + "tokens_seen": 2290024448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113333333333334, + "loss": 2.5464, + "theoretical_loss": 3.3923764333137183, + "tokens_seen": 2290089984 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031131313131313134, + "loss": 2.4847, + "theoretical_loss": 3.3923685530010346, + "tokens_seen": 2290155520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112929292929293, + "loss": 2.7156, + "theoretical_loss": 3.3923606729769933, + "tokens_seen": 2290221056 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031127272727272725, + "loss": 2.509, + "theoretical_loss": 3.3923527932415762, + "tokens_seen": 2290286592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112525252525252, + "loss": 2.9099, + "theoretical_loss": 3.3923449137947634, + "tokens_seen": 2290352128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112323232323233, + "loss": 2.4327, + "theoretical_loss": 3.392337034636537, + "tokens_seen": 2290417664 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031121212121212123, + "loss": 2.863, + "theoretical_loss": 3.3923291557668778, + "tokens_seen": 2290483200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111919191919192, + "loss": 2.5169, + "theoretical_loss": 3.3923212771857676, + "tokens_seen": 2290548736 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031117171717171715, + "loss": 2.6663, + "theoretical_loss": 3.3923133988931866, + "tokens_seen": 2290614272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111515151515152, + "loss": 2.4132, + "theoretical_loss": 3.392305520889116, + "tokens_seen": 2290679808 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1264250, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7503297328948975, + "objective/train/theoretical_loss": 3.392297643173538, + "objective/train/tokens_used": 649604576, + "theoretical_loss": 3.392297643173538, + "tokens_seen": 2290745344 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031113131313131317, + "loss": 2.5558, + "theoretical_loss": 3.392297643173538, + "tokens_seen": 2290745344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111111111111111, + "loss": 2.8482, + "theoretical_loss": 3.3922897657464333, + "tokens_seen": 2290810880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110909090909091, + "loss": 2.6156, + "theoretical_loss": 3.3922818886077826, + "tokens_seen": 2290876416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031107070707070704, + "loss": 2.6552, + "theoretical_loss": 3.392274011757567, + "tokens_seen": 2290941952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110505050505051, + "loss": 2.5031, + "theoretical_loss": 3.3922661351957686, + "tokens_seen": 2291007488 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031103030303030306, + "loss": 2.6322, + "theoretical_loss": 3.3922582589223684, + "tokens_seen": 2291073024 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311010101010101, + "loss": 2.5946, + "theoretical_loss": 3.392250382937347, + "tokens_seen": 2291138560 + }, + { + "epoch": 0.38, + "learning_rate": 0.000310989898989899, + "loss": 2.6454, + "theoretical_loss": 3.392242507240686, + "tokens_seen": 2291204096 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031096969696969693, + "loss": 2.4293, + "theoretical_loss": 3.392234631832366, + "tokens_seen": 2291269632 + }, + { + "epoch": 0.38, + "learning_rate": 0.000310949494949495, + "loss": 2.4459, + "theoretical_loss": 3.3922267567123696, + "tokens_seen": 2291335168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031092929292929295, + "loss": 2.6167, + "theoretical_loss": 3.3922188818806767, + "tokens_seen": 2291400704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109090909090909, + "loss": 2.5918, + "theoretical_loss": 3.3922110073372687, + "tokens_seen": 2291466240 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031088888888888887, + "loss": 2.6297, + "theoretical_loss": 3.392203133082127, + "tokens_seen": 2291531776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108686868686869, + "loss": 2.7358, + "theoretical_loss": 3.392195259115233, + "tokens_seen": 2291597312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108484848484849, + "loss": 2.5908, + "theoretical_loss": 3.392187385436568, + "tokens_seen": 2291662848 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031082828282828285, + "loss": 2.8274, + "theoretical_loss": 3.3921795120461122, + "tokens_seen": 2291728384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108080808080808, + "loss": 2.6506, + "theoretical_loss": 3.392171638943848, + "tokens_seen": 2291793920 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031078787878787876, + "loss": 2.5598, + "theoretical_loss": 3.392163766129756, + "tokens_seen": 2291859456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031076767676767677, + "loss": 2.8199, + "theoretical_loss": 3.3921558936038174, + "tokens_seen": 2291924992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107474747474748, + "loss": 2.602, + "theoretical_loss": 3.3921480213660136, + "tokens_seen": 2291990528 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031072727272727274, + "loss": 2.6791, + "theoretical_loss": 3.392140149416326, + "tokens_seen": 2292056064 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107070707070707, + "loss": 2.6833, + "theoretical_loss": 3.3921322777547354, + "tokens_seen": 2292121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003106868686868687, + "loss": 2.4246, + "theoretical_loss": 3.3921244063812233, + "tokens_seen": 2292187136 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031066666666666666, + "loss": 2.6528, + "theoretical_loss": 3.392116535295771, + "tokens_seen": 2292252672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003106464646464647, + "loss": 2.6907, + "theoretical_loss": 3.3921086644983593, + "tokens_seen": 2292318208 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1264872, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.41400146484375, + "objective/train/theoretical_loss": 3.39210079398897, + "objective/train/tokens_used": 651242976, + "theoretical_loss": 3.39210079398897, + "tokens_seen": 2292383744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031062626262626263, + "loss": 2.4975, + "theoretical_loss": 3.39210079398897, + "tokens_seen": 2292383744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003106060606060606, + "loss": 2.6498, + "theoretical_loss": 3.3920929237675836, + "tokens_seen": 2292449280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105858585858586, + "loss": 2.5796, + "theoretical_loss": 3.392085053834182, + "tokens_seen": 2292514816 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031056565656565656, + "loss": 2.8169, + "theoretical_loss": 3.392077184188746, + "tokens_seen": 2292580352 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031054545454545457, + "loss": 2.7175, + "theoretical_loss": 3.3920693148312573, + "tokens_seen": 2292645888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105252525252525, + "loss": 2.669, + "theoretical_loss": 3.392061445761696, + "tokens_seen": 2292711424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031050505050505054, + "loss": 2.5452, + "theoretical_loss": 3.392053576980045, + "tokens_seen": 2292776960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104848484848485, + "loss": 2.669, + "theoretical_loss": 3.3920457084862843, + "tokens_seen": 2292842496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031046464646464645, + "loss": 2.4938, + "theoretical_loss": 3.3920378402803957, + "tokens_seen": 2292908032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031044444444444446, + "loss": 2.647, + "theoretical_loss": 3.3920299723623604, + "tokens_seen": 2292973568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104242424242424, + "loss": 2.6427, + "theoretical_loss": 3.3920221047321597, + "tokens_seen": 2293039104 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031040404040404043, + "loss": 2.5542, + "theoretical_loss": 3.392014237389774, + "tokens_seen": 2293104640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103838383838384, + "loss": 2.6219, + "theoretical_loss": 3.392006370335186, + "tokens_seen": 2293170176 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031036363636363634, + "loss": 2.8148, + "theoretical_loss": 3.3919985035683755, + "tokens_seen": 2293235712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031034343434343435, + "loss": 2.8275, + "theoretical_loss": 3.3919906370893247, + "tokens_seen": 2293301248 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031032323232323236, + "loss": 2.5837, + "theoretical_loss": 3.391982770898015, + "tokens_seen": 2293366784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103030303030303, + "loss": 2.6612, + "theoretical_loss": 3.3919749049944263, + "tokens_seen": 2293432320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102828282828283, + "loss": 2.5767, + "theoretical_loss": 3.3919670393785415, + "tokens_seen": 2293497856 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031026262626262624, + "loss": 2.5719, + "theoretical_loss": 3.3919591740503408, + "tokens_seen": 2293563392 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031024242424242425, + "loss": 2.388, + "theoretical_loss": 3.391951309009806, + "tokens_seen": 2293628928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031022222222222226, + "loss": 2.6829, + "theoretical_loss": 3.391943444256918, + "tokens_seen": 2293694464 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102020202020202, + "loss": 2.6251, + "theoretical_loss": 3.391935579791658, + "tokens_seen": 2293760000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031018181818181817, + "loss": 2.5934, + "theoretical_loss": 3.3919277156140075, + "tokens_seen": 2293825536 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031016161616161613, + "loss": 2.4721, + "theoretical_loss": 3.391919851723948, + "tokens_seen": 2293891072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101414141414142, + "loss": 2.6051, + "theoretical_loss": 3.3919119881214606, + "tokens_seen": 2293956608 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1266089, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6598479747772217, + "objective/train/theoretical_loss": 3.3919041248065263, + "objective/train/tokens_used": 652881376, + "theoretical_loss": 3.3919041248065263, + "tokens_seen": 2294022144 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031012121212121215, + "loss": 2.5609, + "theoretical_loss": 3.3919041248065263, + "tokens_seen": 2294022144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101010101010101, + "loss": 2.6973, + "theoretical_loss": 3.3918962617791264, + "tokens_seen": 2294087680 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031008080808080806, + "loss": 2.5322, + "theoretical_loss": 3.3918883990392423, + "tokens_seen": 2294153216 + }, + { + "epoch": 0.39, + "learning_rate": 0.000310060606060606, + "loss": 2.6408, + "theoretical_loss": 3.3918805365868554, + "tokens_seen": 2294218752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100404040404041, + "loss": 2.5532, + "theoretical_loss": 3.391872674421947, + "tokens_seen": 2294284288 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031002020202020204, + "loss": 2.6585, + "theoretical_loss": 3.391864812544498, + "tokens_seen": 2294349824 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031, + "loss": 2.4281, + "theoretical_loss": 3.39185695095449, + "tokens_seen": 2294415360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030997979797979796, + "loss": 2.7691, + "theoretical_loss": 3.3918490896519042, + "tokens_seen": 2294480896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030995959595959597, + "loss": 2.6763, + "theoretical_loss": 3.391841228636722, + "tokens_seen": 2294546432 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309939393939394, + "loss": 2.5084, + "theoretical_loss": 3.3918333679089243, + "tokens_seen": 2294611968 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030991919191919194, + "loss": 2.6088, + "theoretical_loss": 3.391825507468493, + "tokens_seen": 2294677504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098989898989899, + "loss": 2.5089, + "theoretical_loss": 3.391817647315409, + "tokens_seen": 2294743040 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030987878787878785, + "loss": 2.5429, + "theoretical_loss": 3.3918097874496533, + "tokens_seen": 2294808576 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030985858585858586, + "loss": 2.5231, + "theoretical_loss": 3.391801927871208, + "tokens_seen": 2294874112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030983838383838387, + "loss": 2.6438, + "theoretical_loss": 3.3917940685800536, + "tokens_seen": 2294939648 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030981818181818183, + "loss": 2.575, + "theoretical_loss": 3.3917862095761717, + "tokens_seen": 2295005184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097979797979798, + "loss": 2.5795, + "theoretical_loss": 3.3917783508595436, + "tokens_seen": 2295070720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097777777777778, + "loss": 2.6748, + "theoretical_loss": 3.3917704924301506, + "tokens_seen": 2295136256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030975757575757575, + "loss": 2.5086, + "theoretical_loss": 3.391762634287974, + "tokens_seen": 2295201792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030973737373737376, + "loss": 2.7201, + "theoretical_loss": 3.3917547764329954, + "tokens_seen": 2295267328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097171717171717, + "loss": 2.7846, + "theoretical_loss": 3.391746918865196, + "tokens_seen": 2295332864 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096969696969697, + "loss": 2.6534, + "theoretical_loss": 3.3917390615845564, + "tokens_seen": 2295398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096767676767677, + "loss": 2.6957, + "theoretical_loss": 3.3917312045910584, + "tokens_seen": 2295463936 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030965656565656565, + "loss": 2.3621, + "theoretical_loss": 3.3917233478846835, + "tokens_seen": 2295529472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030963636363636366, + "loss": 2.497, + "theoretical_loss": 3.391715491465413, + "tokens_seen": 2295595008 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1266610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5779285430908203, + "objective/train/theoretical_loss": 3.3917076353332276, + "objective/train/tokens_used": 654519776, + "theoretical_loss": 3.3917076353332276, + "tokens_seen": 2295660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096161616161616, + "loss": 2.6769, + "theoretical_loss": 3.3917076353332276, + "tokens_seen": 2295660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030959595959595957, + "loss": 2.5646, + "theoretical_loss": 3.3916997794881096, + "tokens_seen": 2295726080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095757575757576, + "loss": 2.5626, + "theoretical_loss": 3.39169192393004, + "tokens_seen": 2295791616 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030955555555555554, + "loss": 2.5432, + "theoretical_loss": 3.391684068658999, + "tokens_seen": 2295857152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030953535353535355, + "loss": 2.6588, + "theoretical_loss": 3.391676213674969, + "tokens_seen": 2295922688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095151515151515, + "loss": 2.6954, + "theoretical_loss": 3.391668358977932, + "tokens_seen": 2295988224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094949494949495, + "loss": 2.705, + "theoretical_loss": 3.391660504567868, + "tokens_seen": 2296053760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094747474747475, + "loss": 2.5259, + "theoretical_loss": 3.391652650444758, + "tokens_seen": 2296119296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094545454545455, + "loss": 2.713, + "theoretical_loss": 3.391644796608585, + "tokens_seen": 2296184832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030943434343434344, + "loss": 2.759, + "theoretical_loss": 3.3916369430593294, + "tokens_seen": 2296250368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094141414141414, + "loss": 2.4451, + "theoretical_loss": 3.3916290897969725, + "tokens_seen": 2296315904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093939393939394, + "loss": 2.7109, + "theoretical_loss": 3.3916212368214955, + "tokens_seen": 2296381440 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030937373737373737, + "loss": 2.7819, + "theoretical_loss": 3.39161338413288, + "tokens_seen": 2296446976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093535353535354, + "loss": 2.7268, + "theoretical_loss": 3.391605531731108, + "tokens_seen": 2296512512 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030933333333333334, + "loss": 2.6137, + "theoretical_loss": 3.3915976796161593, + "tokens_seen": 2296578048 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030931313131313135, + "loss": 2.8267, + "theoretical_loss": 3.391589827788016, + "tokens_seen": 2296643584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092929292929293, + "loss": 2.8173, + "theoretical_loss": 3.39158197624666, + "tokens_seen": 2296709120 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030927272727272726, + "loss": 2.6758, + "theoretical_loss": 3.3915741249920717, + "tokens_seen": 2296774656 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030925252525252527, + "loss": 2.4154, + "theoretical_loss": 3.391566274024233, + "tokens_seen": 2296840192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030923232323232323, + "loss": 2.6211, + "theoretical_loss": 3.3915584233431253, + "tokens_seen": 2296905728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030921212121212124, + "loss": 2.5942, + "theoretical_loss": 3.39155057294873, + "tokens_seen": 2296971264 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091919191919192, + "loss": 2.6141, + "theoretical_loss": 3.3915427228410278, + "tokens_seen": 2297036800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030917171717171715, + "loss": 2.5705, + "theoretical_loss": 3.3915348730200003, + "tokens_seen": 2297102336 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030915151515151516, + "loss": 2.8109, + "theoretical_loss": 3.3915270234856294, + "tokens_seen": 2297167872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091313131313132, + "loss": 2.5373, + "theoretical_loss": 3.3915191742378963, + "tokens_seen": 2297233408 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1267818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8280587196350098, + "objective/train/theoretical_loss": 3.391511325276782, + "objective/train/tokens_used": 656158176, + "theoretical_loss": 3.391511325276782, + "tokens_seen": 2297298944 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030911111111111113, + "loss": 2.7048, + "theoretical_loss": 3.391511325276782, + "tokens_seen": 2297298944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090909090909091, + "loss": 2.4976, + "theoretical_loss": 3.3915034766022676, + "tokens_seen": 2297364480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030907070707070705, + "loss": 2.6139, + "theoretical_loss": 3.3914956282143347, + "tokens_seen": 2297430016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030905050505050506, + "loss": 2.4821, + "theoretical_loss": 3.3914877801129655, + "tokens_seen": 2297495552 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030903030303030307, + "loss": 2.5747, + "theoretical_loss": 3.3914799322981404, + "tokens_seen": 2297561088 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309010101010101, + "loss": 2.6617, + "theoretical_loss": 3.391472084769841, + "tokens_seen": 2297626624 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308989898989899, + "loss": 2.589, + "theoretical_loss": 3.391464237528049, + "tokens_seen": 2297692160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030896969696969694, + "loss": 2.6321, + "theoretical_loss": 3.391456390572745, + "tokens_seen": 2297757696 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308949494949495, + "loss": 2.6723, + "theoretical_loss": 3.391448543903911, + "tokens_seen": 2297823232 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030892929292929296, + "loss": 2.6143, + "theoretical_loss": 3.3914406975215283, + "tokens_seen": 2297888768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089090909090909, + "loss": 2.6186, + "theoretical_loss": 3.391432851425578, + "tokens_seen": 2297954304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088888888888889, + "loss": 2.6953, + "theoretical_loss": 3.391425005616042, + "tokens_seen": 2298019840 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030886868686868683, + "loss": 2.4945, + "theoretical_loss": 3.3914171600929013, + "tokens_seen": 2298085376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088484848484849, + "loss": 2.6718, + "theoretical_loss": 3.3914093148561375, + "tokens_seen": 2298150912 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030882828282828285, + "loss": 2.7374, + "theoretical_loss": 3.3914014699057313, + "tokens_seen": 2298216448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088080808080808, + "loss": 2.5461, + "theoretical_loss": 3.391393625241665, + "tokens_seen": 2298281984 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030878787878787877, + "loss": 2.5156, + "theoretical_loss": 3.3913857808639194, + "tokens_seen": 2298347520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087676767676768, + "loss": 2.8457, + "theoretical_loss": 3.391377936772476, + "tokens_seen": 2298413056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087474747474748, + "loss": 2.4838, + "theoretical_loss": 3.3913700929673163, + "tokens_seen": 2298478592 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030872727272727275, + "loss": 2.7192, + "theoretical_loss": 3.391362249448422, + "tokens_seen": 2298544128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087070707070707, + "loss": 2.7096, + "theoretical_loss": 3.3913544062157737, + "tokens_seen": 2298609664 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030868686868686866, + "loss": 2.6366, + "theoretical_loss": 3.3913465632693534, + "tokens_seen": 2298675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030866666666666667, + "loss": 2.4499, + "theoretical_loss": 3.391338720609142, + "tokens_seen": 2298740736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086464646464647, + "loss": 2.5605, + "theoretical_loss": 3.3913308782351215, + "tokens_seen": 2298806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030862626262626264, + "loss": 2.8173, + "theoretical_loss": 3.391323036147273, + "tokens_seen": 2298871808 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1268401, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1293718814849854, + "objective/train/theoretical_loss": 3.391315194345578, + "objective/train/tokens_used": 657796576, + "theoretical_loss": 3.391315194345578, + "tokens_seen": 2298937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086060606060606, + "loss": 2.3771, + "theoretical_loss": 3.391315194345578, + "tokens_seen": 2298937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085858585858586, + "loss": 2.8841, + "theoretical_loss": 3.3913073528300175, + "tokens_seen": 2299002880 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030856565656565656, + "loss": 2.6842, + "theoretical_loss": 3.3912995116005735, + "tokens_seen": 2299068416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085454545454546, + "loss": 2.5984, + "theoretical_loss": 3.391291670657227, + "tokens_seen": 2299133952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030852525252525253, + "loss": 2.5788, + "theoretical_loss": 3.3912838299999595, + "tokens_seen": 2299199488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085050505050505, + "loss": 2.4784, + "theoretical_loss": 3.3912759896287525, + "tokens_seen": 2299265024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084848484848485, + "loss": 2.7181, + "theoretical_loss": 3.391268149543587, + "tokens_seen": 2299330560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030846464646464646, + "loss": 2.5747, + "theoretical_loss": 3.391260309744445, + "tokens_seen": 2299396096 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030844444444444447, + "loss": 2.4784, + "theoretical_loss": 3.391252470231308, + "tokens_seen": 2299461632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084242424242424, + "loss": 2.4493, + "theoretical_loss": 3.391244631004157, + "tokens_seen": 2299527168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030840404040404043, + "loss": 2.5682, + "theoretical_loss": 3.391236792062973, + "tokens_seen": 2299592704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083838383838384, + "loss": 2.3225, + "theoretical_loss": 3.3912289534077384, + "tokens_seen": 2299658240 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030836363636363635, + "loss": 2.6492, + "theoretical_loss": 3.391221115038434, + "tokens_seen": 2299723776 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030834343434343436, + "loss": 2.4695, + "theoretical_loss": 3.3912132769550416, + "tokens_seen": 2299789312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083232323232323, + "loss": 2.2739, + "theoretical_loss": 3.3912054391575417, + "tokens_seen": 2299854848 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030830303030303033, + "loss": 2.5069, + "theoretical_loss": 3.391197601645917, + "tokens_seen": 2299920384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082828282828283, + "loss": 2.435, + "theoretical_loss": 3.3911897644201483, + "tokens_seen": 2299985920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030826262626262624, + "loss": 2.6603, + "theoretical_loss": 3.391181927480217, + "tokens_seen": 2300051456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030824242424242425, + "loss": 2.6379, + "theoretical_loss": 3.3911740908261043, + "tokens_seen": 2300116992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030822222222222226, + "loss": 2.4672, + "theoretical_loss": 3.3911662544577927, + "tokens_seen": 2300182528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082020202020202, + "loss": 2.5501, + "theoretical_loss": 3.3911584183752623, + "tokens_seen": 2300248064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081818181818182, + "loss": 2.4988, + "theoretical_loss": 3.391150582578495, + "tokens_seen": 2300313600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030816161616161613, + "loss": 2.7939, + "theoretical_loss": 3.3911427470674727, + "tokens_seen": 2300379136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030814141414141414, + "loss": 2.6005, + "theoretical_loss": 3.391134911842176, + "tokens_seen": 2300444672 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030812121212121216, + "loss": 2.5733, + "theoretical_loss": 3.3911270769025874, + "tokens_seen": 2300510208 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1269713, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.40155291557312, + "objective/train/theoretical_loss": 3.3911192422486875, + "objective/train/tokens_used": 659434976, + "theoretical_loss": 3.3911192422486875, + "tokens_seen": 2300575744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081010101010101, + "loss": 2.5398, + "theoretical_loss": 3.3911192422486875, + "tokens_seen": 2300575744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030808080808080807, + "loss": 2.5321, + "theoretical_loss": 3.391111407880458, + "tokens_seen": 2300641280 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308060606060606, + "loss": 2.77, + "theoretical_loss": 3.3911035737978805, + "tokens_seen": 2300706816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080404040404041, + "loss": 2.622, + "theoretical_loss": 3.391095740000936, + "tokens_seen": 2300772352 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030802020202020205, + "loss": 2.5246, + "theoretical_loss": 3.3910879064896067, + "tokens_seen": 2300837888 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308, + "loss": 2.6132, + "theoretical_loss": 3.3910800732638733, + "tokens_seen": 2300903424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030797979797979796, + "loss": 2.5629, + "theoretical_loss": 3.3910722403237177, + "tokens_seen": 2300968960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079595959595959, + "loss": 2.7191, + "theoretical_loss": 3.3910644076691208, + "tokens_seen": 2301034496 + }, + { + "epoch": 0.39, + "learning_rate": 0.000307939393939394, + "loss": 2.535, + "theoretical_loss": 3.391056575300065, + "tokens_seen": 2301100032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030791919191919194, + "loss": 2.4897, + "theoretical_loss": 3.391048743216531, + "tokens_seen": 2301165568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078989898989899, + "loss": 2.72, + "theoretical_loss": 3.3910409114185005, + "tokens_seen": 2301231104 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030787878787878786, + "loss": 2.4124, + "theoretical_loss": 3.391033079905955, + "tokens_seen": 2301296640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078585858585858, + "loss": 2.4873, + "theoretical_loss": 3.3910252486788757, + "tokens_seen": 2301362176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078383838383839, + "loss": 2.3633, + "theoretical_loss": 3.3910174177372445, + "tokens_seen": 2301427712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030781818181818183, + "loss": 2.597, + "theoretical_loss": 3.391009587081043, + "tokens_seen": 2301493248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077979797979798, + "loss": 2.5238, + "theoretical_loss": 3.391001756710252, + "tokens_seen": 2301558784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030777777777777775, + "loss": 2.4102, + "theoretical_loss": 3.390993926624853, + "tokens_seen": 2301624320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077575757575758, + "loss": 2.6404, + "theoretical_loss": 3.390986096824828, + "tokens_seen": 2301689856 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030773737373737377, + "loss": 2.5578, + "theoretical_loss": 3.3909782673101585, + "tokens_seen": 2301755392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077171717171717, + "loss": 2.6326, + "theoretical_loss": 3.390970438080825, + "tokens_seen": 2301820928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076969696969697, + "loss": 2.451, + "theoretical_loss": 3.390962609136811, + "tokens_seen": 2301886464 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030767676767676764, + "loss": 2.4678, + "theoretical_loss": 3.3909547804780953, + "tokens_seen": 2301952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076565656565657, + "loss": 2.2928, + "theoretical_loss": 3.3909469521046613, + "tokens_seen": 2302017536 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030763636363636366, + "loss": 2.4691, + "theoretical_loss": 3.39093912401649, + "tokens_seen": 2302083072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076161616161616, + "loss": 2.3614, + "theoretical_loss": 3.390931296213563, + "tokens_seen": 2302148608 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1270398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.385732412338257, + "objective/train/theoretical_loss": 3.3909234686958616, + "objective/train/tokens_used": 661073376, + "theoretical_loss": 3.3909234686958616, + "tokens_seen": 2302214144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075959595959596, + "loss": 2.6673, + "theoretical_loss": 3.3909234686958616, + "tokens_seen": 2302214144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075757575757576, + "loss": 2.7333, + "theoretical_loss": 3.3909156414633665, + "tokens_seen": 2302279680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075555555555556, + "loss": 2.4734, + "theoretical_loss": 3.390907814516061, + "tokens_seen": 2302345216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030753535353535356, + "loss": 2.6199, + "theoretical_loss": 3.390899987853925, + "tokens_seen": 2302410752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075151515151515, + "loss": 2.6377, + "theoretical_loss": 3.3908921614769407, + "tokens_seen": 2302476288 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030749494949494947, + "loss": 2.499, + "theoretical_loss": 3.3908843353850893, + "tokens_seen": 2302541824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074747474747475, + "loss": 2.448, + "theoretical_loss": 3.390876509578353, + "tokens_seen": 2302607360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074545454545455, + "loss": 2.4048, + "theoretical_loss": 3.3908686840567124, + "tokens_seen": 2302672896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030743434343434345, + "loss": 2.537, + "theoretical_loss": 3.390860858820149, + "tokens_seen": 2302738432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074141414141414, + "loss": 2.5484, + "theoretical_loss": 3.3908530338686456, + "tokens_seen": 2302803968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073939393939394, + "loss": 2.4612, + "theoretical_loss": 3.390845209202182, + "tokens_seen": 2302869504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030737373737373737, + "loss": 2.5923, + "theoretical_loss": 3.390837384820741, + "tokens_seen": 2302935040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073535353535354, + "loss": 2.436, + "theoretical_loss": 3.390829560724303, + "tokens_seen": 2303000576 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030733333333333334, + "loss": 2.648, + "theoretical_loss": 3.3908217369128506, + "tokens_seen": 2303066112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073131313131313, + "loss": 2.5518, + "theoretical_loss": 3.3908139133863644, + "tokens_seen": 2303131648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072929292929293, + "loss": 2.6575, + "theoretical_loss": 3.3908060901448263, + "tokens_seen": 2303197184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030727272727272727, + "loss": 2.4643, + "theoretical_loss": 3.390798267188218, + "tokens_seen": 2303262720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072525252525253, + "loss": 2.5023, + "theoretical_loss": 3.390790444516521, + "tokens_seen": 2303328256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030723232323232323, + "loss": 2.5001, + "theoretical_loss": 3.3907826221297164, + "tokens_seen": 2303393792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030721212121212124, + "loss": 2.4754, + "theoretical_loss": 3.390774800027786, + "tokens_seen": 2303459328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071919191919192, + "loss": 2.616, + "theoretical_loss": 3.3907669782107113, + "tokens_seen": 2303524864 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030717171717171716, + "loss": 2.377, + "theoretical_loss": 3.390759156678474, + "tokens_seen": 2303590400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030715151515151517, + "loss": 2.2582, + "theoretical_loss": 3.3907513354310557, + "tokens_seen": 2303655936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071313131313131, + "loss": 2.6651, + "theoretical_loss": 3.3907435144684372, + "tokens_seen": 2303721472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030711111111111114, + "loss": 2.512, + "theoretical_loss": 3.3907356937906004, + "tokens_seen": 2303787008 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1271635, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6408653259277344, + "objective/train/theoretical_loss": 3.3907278733975272, + "objective/train/tokens_used": 662711776, + "theoretical_loss": 3.3907278733975272, + "tokens_seen": 2303852544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070909090909091, + "loss": 2.3809, + "theoretical_loss": 3.3907278733975272, + "tokens_seen": 2303852544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030707070707070705, + "loss": 2.4918, + "theoretical_loss": 3.390720053289199, + "tokens_seen": 2303918080 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030705050505050506, + "loss": 2.3329, + "theoretical_loss": 3.390712233465597, + "tokens_seen": 2303983616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070303030303031, + "loss": 2.6718, + "theoretical_loss": 3.390704413926703, + "tokens_seen": 2304049152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030701010101010103, + "loss": 2.5126, + "theoretical_loss": 3.390696594672498, + "tokens_seen": 2304114688 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306989898989899, + "loss": 2.2646, + "theoretical_loss": 3.390688775702965, + "tokens_seen": 2304180224 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030696969696969694, + "loss": 2.5245, + "theoretical_loss": 3.3906809570180836, + "tokens_seen": 2304245760 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030694949494949495, + "loss": 2.85, + "theoretical_loss": 3.390673138617837, + "tokens_seen": 2304311296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030692929292929297, + "loss": 2.5639, + "theoretical_loss": 3.3906653205022055, + "tokens_seen": 2304376832 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069090909090909, + "loss": 2.4869, + "theoretical_loss": 3.3906575026711714, + "tokens_seen": 2304442368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068888888888889, + "loss": 2.6592, + "theoretical_loss": 3.390649685124716, + "tokens_seen": 2304507904 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030686868686868684, + "loss": 2.5078, + "theoretical_loss": 3.3906418678628207, + "tokens_seen": 2304573440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068484848484849, + "loss": 2.5299, + "theoretical_loss": 3.3906340508854673, + "tokens_seen": 2304638976 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030682828282828286, + "loss": 2.3896, + "theoretical_loss": 3.3906262341926374, + "tokens_seen": 2304704512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068080808080808, + "loss": 2.4701, + "theoretical_loss": 3.3906184177843124, + "tokens_seen": 2304770048 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030678787878787877, + "loss": 2.6287, + "theoretical_loss": 3.390610601660474, + "tokens_seen": 2304835584 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030676767676767673, + "loss": 2.583, + "theoretical_loss": 3.3906027858211036, + "tokens_seen": 2304901120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067474747474748, + "loss": 2.8207, + "theoretical_loss": 3.3905949702661826, + "tokens_seen": 2304966656 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030672727272727275, + "loss": 2.2541, + "theoretical_loss": 3.3905871549956927, + "tokens_seen": 2305032192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067070707070707, + "loss": 2.465, + "theoretical_loss": 3.390579340009616, + "tokens_seen": 2305097728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030668686868686866, + "loss": 2.2704, + "theoretical_loss": 3.390571525307933, + "tokens_seen": 2305163264 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066666666666667, + "loss": 2.5124, + "theoretical_loss": 3.390563710890626, + "tokens_seen": 2305228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066464646464647, + "loss": 2.8017, + "theoretical_loss": 3.3905558967576765, + "tokens_seen": 2305294336 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030662626262626264, + "loss": 2.7332, + "theoretical_loss": 3.390548082909066, + "tokens_seen": 2305359872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066060606060606, + "loss": 2.5018, + "theoretical_loss": 3.3905402693447764, + "tokens_seen": 2305425408 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1272482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6973299980163574, + "objective/train/theoretical_loss": 3.3905324560647885, + "objective/train/tokens_used": 664350176, + "theoretical_loss": 3.3905324560647885, + "tokens_seen": 2305490944 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030658585858585856, + "loss": 2.5932, + "theoretical_loss": 3.3905324560647885, + "tokens_seen": 2305490944 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030656565656565657, + "loss": 2.5374, + "theoretical_loss": 3.3905246430690843, + "tokens_seen": 2305556480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065454545454546, + "loss": 2.5087, + "theoretical_loss": 3.3905168303576456, + "tokens_seen": 2305622016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030652525252525254, + "loss": 2.5229, + "theoretical_loss": 3.3905090179304533, + "tokens_seen": 2305687552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065050505050505, + "loss": 2.6958, + "theoretical_loss": 3.39050120578749, + "tokens_seen": 2305753088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064848484848485, + "loss": 2.3973, + "theoretical_loss": 3.3904933939287365, + "tokens_seen": 2305818624 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030646464646464646, + "loss": 2.3898, + "theoretical_loss": 3.3904855823541746, + "tokens_seen": 2305884160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030644444444444447, + "loss": 2.6596, + "theoretical_loss": 3.390477771063786, + "tokens_seen": 2305949696 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030642424242424243, + "loss": 2.5784, + "theoretical_loss": 3.390469960057552, + "tokens_seen": 2306015232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064040404040404, + "loss": 2.5346, + "theoretical_loss": 3.3904621493354545, + "tokens_seen": 2306080768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063838383838384, + "loss": 2.5161, + "theoretical_loss": 3.3904543388974746, + "tokens_seen": 2306146304 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030636363636363635, + "loss": 2.4475, + "theoretical_loss": 3.3904465287435945, + "tokens_seen": 2306211840 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030634343434343437, + "loss": 2.7658, + "theoretical_loss": 3.3904387188737957, + "tokens_seen": 2306277376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063232323232323, + "loss": 2.4109, + "theoretical_loss": 3.3904309092880593, + "tokens_seen": 2306342912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063030303030303, + "loss": 2.4875, + "theoretical_loss": 3.390423099986368, + "tokens_seen": 2306408448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062828282828283, + "loss": 2.6148, + "theoretical_loss": 3.3904152909687015, + "tokens_seen": 2306473984 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030626262626262625, + "loss": 2.8049, + "theoretical_loss": 3.390407482235043, + "tokens_seen": 2306539520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030624242424242426, + "loss": 2.6674, + "theoretical_loss": 3.3903996737853737, + "tokens_seen": 2306605056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062222222222222, + "loss": 2.5598, + "theoretical_loss": 3.390391865619675, + "tokens_seen": 2306670592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062020202020202, + "loss": 2.4708, + "theoretical_loss": 3.3903840577379287, + "tokens_seen": 2306736128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061818181818182, + "loss": 2.6744, + "theoretical_loss": 3.3903762501401165, + "tokens_seen": 2306801664 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030616161616161614, + "loss": 2.5458, + "theoretical_loss": 3.3903684428262197, + "tokens_seen": 2306867200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030614141414141415, + "loss": 2.5156, + "theoretical_loss": 3.3903606357962204, + "tokens_seen": 2306932736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061212121212121, + "loss": 2.6062, + "theoretical_loss": 3.3903528290500993, + "tokens_seen": 2306998272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061010101010101, + "loss": 2.6043, + "theoretical_loss": 3.390345022587839, + "tokens_seen": 2307063808 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1273129, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3374531269073486, + "objective/train/theoretical_loss": 3.3903372164094208, + "objective/train/tokens_used": 665988576, + "theoretical_loss": 3.3903372164094208, + "tokens_seen": 2307129344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060808080808081, + "loss": 2.4758, + "theoretical_loss": 3.3903372164094208, + "tokens_seen": 2307129344 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030606060606060603, + "loss": 2.5465, + "theoretical_loss": 3.390329410514826, + "tokens_seen": 2307194880 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030604040404040404, + "loss": 2.3914, + "theoretical_loss": 3.3903216049040363, + "tokens_seen": 2307260416 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030602020202020205, + "loss": 2.5634, + "theoretical_loss": 3.3903137995770334, + "tokens_seen": 2307325952 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306, + "loss": 2.2768, + "theoretical_loss": 3.3903059945337994, + "tokens_seen": 2307391488 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030597979797979797, + "loss": 2.7051, + "theoretical_loss": 3.390298189774315, + "tokens_seen": 2307457024 + }, + { + "epoch": 0.39, + "learning_rate": 0.000305959595959596, + "loss": 2.4825, + "theoretical_loss": 3.3902903852985626, + "tokens_seen": 2307522560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030593939393939394, + "loss": 2.4341, + "theoretical_loss": 3.390282581106524, + "tokens_seen": 2307588096 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030591919191919195, + "loss": 2.6403, + "theoretical_loss": 3.39027477719818, + "tokens_seen": 2307653632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058989898989899, + "loss": 2.7605, + "theoretical_loss": 3.3902669735735125, + "tokens_seen": 2307719168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030587878787878786, + "loss": 2.4623, + "theoretical_loss": 3.390259170232503, + "tokens_seen": 2307784704 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030585858585858587, + "loss": 2.5519, + "theoretical_loss": 3.390251367175134, + "tokens_seen": 2307850240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058383838383839, + "loss": 2.4641, + "theoretical_loss": 3.390243564401386, + "tokens_seen": 2307915776 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030581818181818184, + "loss": 2.6751, + "theoretical_loss": 3.3902357619112413, + "tokens_seen": 2307981312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057979797979798, + "loss": 2.5488, + "theoretical_loss": 3.390227959704682, + "tokens_seen": 2308046848 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030577777777777775, + "loss": 2.723, + "theoretical_loss": 3.3902201577816884, + "tokens_seen": 2308112384 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030575757575757576, + "loss": 2.5285, + "theoretical_loss": 3.390212356142243, + "tokens_seen": 2308177920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057373737373738, + "loss": 2.645, + "theoretical_loss": 3.3902045547863273, + "tokens_seen": 2308243456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030571717171717173, + "loss": 2.6811, + "theoretical_loss": 3.3901967537139233, + "tokens_seen": 2308308992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056969696969697, + "loss": 2.5863, + "theoretical_loss": 3.390188952925012, + "tokens_seen": 2308374528 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030567676767676765, + "loss": 2.5006, + "theoretical_loss": 3.3901811524195757, + "tokens_seen": 2308440064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056565656565657, + "loss": 2.4715, + "theoretical_loss": 3.3901733521975954, + "tokens_seen": 2308505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030563636363636367, + "loss": 2.5382, + "theoretical_loss": 3.3901655522590532, + "tokens_seen": 2308571136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056161616161616, + "loss": 2.6117, + "theoretical_loss": 3.3901577526039306, + "tokens_seen": 2308636672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003055959595959596, + "loss": 2.6662, + "theoretical_loss": 3.3901499532322097, + "tokens_seen": 2308702208 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1274350, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.294728994369507, + "objective/train/theoretical_loss": 3.390142154143871, + "objective/train/tokens_used": 667626976, + "theoretical_loss": 3.390142154143871, + "tokens_seen": 2308767744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030557575757575754, + "loss": 2.5048, + "theoretical_loss": 3.390142154143871, + "tokens_seen": 2308767744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055555555555556, + "loss": 2.6972, + "theoretical_loss": 3.3901343553388976, + "tokens_seen": 2308833280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030553535353535356, + "loss": 2.5423, + "theoretical_loss": 3.39012655681727, + "tokens_seen": 2308898816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055151515151515, + "loss": 2.54, + "theoretical_loss": 3.3901187585789705, + "tokens_seen": 2308964352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054949494949495, + "loss": 2.5325, + "theoretical_loss": 3.3901109606239808, + "tokens_seen": 2309029888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054747474747475, + "loss": 2.5189, + "theoretical_loss": 3.3901031629522818, + "tokens_seen": 2309095424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054545454545455, + "loss": 2.7139, + "theoretical_loss": 3.390095365563856, + "tokens_seen": 2309160960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030543434343434345, + "loss": 2.5577, + "theoretical_loss": 3.3900875684586853, + "tokens_seen": 2309226496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054141414141414, + "loss": 2.5071, + "theoretical_loss": 3.3900797716367506, + "tokens_seen": 2309292032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030539393939393937, + "loss": 2.7558, + "theoretical_loss": 3.3900719750980333, + "tokens_seen": 2309357568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053737373737374, + "loss": 2.3048, + "theoretical_loss": 3.390064178842516, + "tokens_seen": 2309423104 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053535353535354, + "loss": 2.6012, + "theoretical_loss": 3.3900563828701804, + "tokens_seen": 2309488640 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030533333333333335, + "loss": 2.7033, + "theoretical_loss": 3.390048587181007, + "tokens_seen": 2309554176 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053131313131313, + "loss": 2.9447, + "theoretical_loss": 3.390040791774979, + "tokens_seen": 2309619712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052929292929293, + "loss": 2.3706, + "theoretical_loss": 3.390032996652077, + "tokens_seen": 2309685248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030527272727272727, + "loss": 2.5489, + "theoretical_loss": 3.390025201812283, + "tokens_seen": 2309750784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052525252525253, + "loss": 2.6724, + "theoretical_loss": 3.390017407255579, + "tokens_seen": 2309816320 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030523232323232324, + "loss": 2.6711, + "theoretical_loss": 3.3900096129819457, + "tokens_seen": 2309881856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052121212121212, + "loss": 2.8219, + "theoretical_loss": 3.390001818991366, + "tokens_seen": 2309947392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051919191919192, + "loss": 2.6027, + "theoretical_loss": 3.389994025283821, + "tokens_seen": 2310012928 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030517171717171716, + "loss": 2.5895, + "theoretical_loss": 3.3899862318592926, + "tokens_seen": 2310078464 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051515151515152, + "loss": 2.5967, + "theoretical_loss": 3.3899784387177623, + "tokens_seen": 2310144000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030513131313131313, + "loss": 2.5548, + "theoretical_loss": 3.389970645859212, + "tokens_seen": 2310209536 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030511111111111114, + "loss": 2.6856, + "theoretical_loss": 3.389962853283623, + "tokens_seen": 2310275072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050909090909091, + "loss": 2.6099, + "theoretical_loss": 3.389955060990977, + "tokens_seen": 2310340608 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1274732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.461940050125122, + "objective/train/theoretical_loss": 3.389947268981256, + "objective/train/tokens_used": 669265376, + "theoretical_loss": 3.389947268981256, + "tokens_seen": 2310406144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030507070707070706, + "loss": 2.354, + "theoretical_loss": 3.389947268981256, + "tokens_seen": 2310406144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030505050505050507, + "loss": 2.5882, + "theoretical_loss": 3.3899394772544422, + "tokens_seen": 2310471680 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305030303030303, + "loss": 2.517, + "theoretical_loss": 3.3899316858105166, + "tokens_seen": 2310537216 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030501010101010104, + "loss": 2.6366, + "theoretical_loss": 3.3899238946494608, + "tokens_seen": 2310602752 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304989898989899, + "loss": 2.6087, + "theoretical_loss": 3.3899161037712573, + "tokens_seen": 2310668288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030496969696969695, + "loss": 2.5083, + "theoretical_loss": 3.3899083131758867, + "tokens_seen": 2310733824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030494949494949496, + "loss": 2.4066, + "theoretical_loss": 3.3899005228633317, + "tokens_seen": 2310799360 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030492929292929297, + "loss": 2.3075, + "theoretical_loss": 3.389892732833573, + "tokens_seen": 2310864896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030490909090909093, + "loss": 2.4479, + "theoretical_loss": 3.3898849430865936, + "tokens_seen": 2310930432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048888888888889, + "loss": 2.635, + "theoretical_loss": 3.389877153622374, + "tokens_seen": 2310995968 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030486868686868684, + "loss": 2.6388, + "theoretical_loss": 3.389869364440897, + "tokens_seen": 2311061504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030484848484848485, + "loss": 2.4726, + "theoretical_loss": 3.3898615755421435, + "tokens_seen": 2311127040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030482828282828286, + "loss": 2.4258, + "theoretical_loss": 3.389853786926096, + "tokens_seen": 2311192576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048080808080808, + "loss": 2.554, + "theoretical_loss": 3.389845998592735, + "tokens_seen": 2311258112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047878787878788, + "loss": 2.6835, + "theoretical_loss": 3.3898382105420435, + "tokens_seen": 2311323648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030476767676767673, + "loss": 2.5725, + "theoretical_loss": 3.3898304227740024, + "tokens_seen": 2311389184 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047474747474748, + "loss": 2.4118, + "theoretical_loss": 3.3898226352885934, + "tokens_seen": 2311454720 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030472727272727276, + "loss": 2.6774, + "theoretical_loss": 3.389814848085799, + "tokens_seen": 2311520256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047070707070707, + "loss": 2.548, + "theoretical_loss": 3.3898070611656, + "tokens_seen": 2311585792 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030468686868686867, + "loss": 2.6404, + "theoretical_loss": 3.389799274527979, + "tokens_seen": 2311651328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030466666666666663, + "loss": 2.3368, + "theoretical_loss": 3.389791488172917, + "tokens_seen": 2311716864 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046464646464647, + "loss": 2.7248, + "theoretical_loss": 3.3897837021003965, + "tokens_seen": 2311782400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030462626262626265, + "loss": 2.5494, + "theoretical_loss": 3.3897759163103984, + "tokens_seen": 2311847936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046060606060606, + "loss": 2.7867, + "theoretical_loss": 3.3897681308029055, + "tokens_seen": 2311913472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030458585858585856, + "loss": 2.5334, + "theoretical_loss": 3.389760345577898, + "tokens_seen": 2311979008 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1275645, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.752490520477295, + "objective/train/theoretical_loss": 3.389752560635359, + "objective/train/tokens_used": 670903776, + "theoretical_loss": 3.389752560635359, + "tokens_seen": 2312044544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045656565656565, + "loss": 2.6714, + "theoretical_loss": 3.389752560635359, + "tokens_seen": 2312044544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045454545454546, + "loss": 2.6804, + "theoretical_loss": 3.38974477597527, + "tokens_seen": 2312110080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030452525252525254, + "loss": 2.7004, + "theoretical_loss": 3.389736991597612, + "tokens_seen": 2312175616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045050505050505, + "loss": 2.4943, + "theoretical_loss": 3.3897292075023677, + "tokens_seen": 2312241152 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030448484848484846, + "loss": 2.5386, + "theoretical_loss": 3.3897214236895183, + "tokens_seen": 2312306688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030446464646464647, + "loss": 2.6384, + "theoretical_loss": 3.3897136401590457, + "tokens_seen": 2312372224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044444444444445, + "loss": 2.5939, + "theoretical_loss": 3.3897058569109317, + "tokens_seen": 2312437760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030442424242424243, + "loss": 2.6308, + "theoretical_loss": 3.3896980739451577, + "tokens_seen": 2312503296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044040404040404, + "loss": 2.5855, + "theoretical_loss": 3.3896902912617057, + "tokens_seen": 2312568832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030438383838383835, + "loss": 2.6419, + "theoretical_loss": 3.3896825088605578, + "tokens_seen": 2312634368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030436363636363636, + "loss": 2.4153, + "theoretical_loss": 3.3896747267416956, + "tokens_seen": 2312699904 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030434343434343437, + "loss": 2.6084, + "theoretical_loss": 3.3896669449051, + "tokens_seen": 2312765440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030432323232323233, + "loss": 2.4422, + "theoretical_loss": 3.3896591633507542, + "tokens_seen": 2312830976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043030303030303, + "loss": 2.7781, + "theoretical_loss": 3.3896513820786387, + "tokens_seen": 2312896512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042828282828283, + "loss": 2.6358, + "theoretical_loss": 3.3896436010887365, + "tokens_seen": 2312962048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042626262626263, + "loss": 2.5163, + "theoretical_loss": 3.389635820381028, + "tokens_seen": 2313027584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030424242424242426, + "loss": 2.5995, + "theoretical_loss": 3.3896280399554963, + "tokens_seen": 2313093120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042222222222222, + "loss": 2.9271, + "theoretical_loss": 3.389620259812122, + "tokens_seen": 2313158656 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042020202020202, + "loss": 2.6007, + "theoretical_loss": 3.3896124799508875, + "tokens_seen": 2313224192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041818181818182, + "loss": 2.8641, + "theoretical_loss": 3.3896047003717746, + "tokens_seen": 2313289728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041616161616162, + "loss": 2.6169, + "theoretical_loss": 3.389596921074765, + "tokens_seen": 2313355264 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030414141414141416, + "loss": 2.7031, + "theoretical_loss": 3.3895891420598403, + "tokens_seen": 2313420800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041212121212121, + "loss": 2.7717, + "theoretical_loss": 3.3895813633269825, + "tokens_seen": 2313486336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041010101010101, + "loss": 2.7382, + "theoretical_loss": 3.389573584876173, + "tokens_seen": 2313551872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040808080808081, + "loss": 2.6476, + "theoretical_loss": 3.3895658067073944, + "tokens_seen": 2313617408 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1276618, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.560056209564209, + "objective/train/theoretical_loss": 3.3895580288206277, + "objective/train/tokens_used": 672542176, + "theoretical_loss": 3.3895580288206277, + "tokens_seen": 2313682944 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040606060606061, + "loss": 2.5657, + "theoretical_loss": 3.3895580288206277, + "tokens_seen": 2313682944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030404040404040405, + "loss": 2.5529, + "theoretical_loss": 3.389550251215855, + "tokens_seen": 2313748480 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304020202020202, + "loss": 2.6101, + "theoretical_loss": 3.389542473893058, + "tokens_seen": 2313814016 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304, + "loss": 2.7534, + "theoretical_loss": 3.389534696852219, + "tokens_seen": 2313879552 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303979797979798, + "loss": 2.5307, + "theoretical_loss": 3.3895269200933185, + "tokens_seen": 2313945088 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303959595959596, + "loss": 2.6574, + "theoretical_loss": 3.3895191436163397, + "tokens_seen": 2314010624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030393939393939394, + "loss": 2.5521, + "theoretical_loss": 3.3895113674212634, + "tokens_seen": 2314076160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030391919191919195, + "loss": 2.3814, + "theoretical_loss": 3.3895035915080722, + "tokens_seen": 2314141696 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038989898989899, + "loss": 2.594, + "theoretical_loss": 3.389495815876747, + "tokens_seen": 2314207232 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030387878787878787, + "loss": 2.6072, + "theoretical_loss": 3.389488040527271, + "tokens_seen": 2314272768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038585858585859, + "loss": 2.7825, + "theoretical_loss": 3.3894802654596243, + "tokens_seen": 2314338304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030383838383838383, + "loss": 2.6401, + "theoretical_loss": 3.38947249067379, + "tokens_seen": 2314403840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030381818181818185, + "loss": 2.6194, + "theoretical_loss": 3.389464716169749, + "tokens_seen": 2314469376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037979797979798, + "loss": 2.6753, + "theoretical_loss": 3.3894569419474836, + "tokens_seen": 2314534912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030377777777777776, + "loss": 2.8071, + "theoretical_loss": 3.389449168006976, + "tokens_seen": 2314600448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030375757575757577, + "loss": 2.5532, + "theoretical_loss": 3.389441394348207, + "tokens_seen": 2314665984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037373737373738, + "loss": 2.7082, + "theoretical_loss": 3.3894336209711593, + "tokens_seen": 2314731520 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030371717171717174, + "loss": 2.7112, + "theoretical_loss": 3.3894258478758146, + "tokens_seen": 2314797056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036969696969697, + "loss": 2.6809, + "theoretical_loss": 3.3894180750621543, + "tokens_seen": 2314862592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030367676767676765, + "loss": 2.6255, + "theoretical_loss": 3.3894103025301603, + "tokens_seen": 2314928128 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030365656565656566, + "loss": 2.6545, + "theoretical_loss": 3.3894025302798143, + "tokens_seen": 2314993664 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036363636363637, + "loss": 2.8286, + "theoretical_loss": 3.3893947583110986, + "tokens_seen": 2315059200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030361616161616163, + "loss": 2.5691, + "theoretical_loss": 3.389386986623995, + "tokens_seen": 2315124736 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035959595959596, + "loss": 2.7007, + "theoretical_loss": 3.3893792152184847, + "tokens_seen": 2315190272 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030357575757575754, + "loss": 2.5359, + "theoretical_loss": 3.3893714440945506, + "tokens_seen": 2315255808 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1277260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.181581735610962, + "objective/train/theoretical_loss": 3.3893636732521735, + "objective/train/tokens_used": 674180576, + "theoretical_loss": 3.3893636732521735, + "tokens_seen": 2315321344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035555555555556, + "loss": 2.7444, + "theoretical_loss": 3.3893636732521735, + "tokens_seen": 2315321344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030353535353535357, + "loss": 2.5262, + "theoretical_loss": 3.389355902691335, + "tokens_seen": 2315386880 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035151515151515, + "loss": 2.6335, + "theoretical_loss": 3.3893481324120183, + "tokens_seen": 2315452416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034949494949495, + "loss": 2.769, + "theoretical_loss": 3.389340362414204, + "tokens_seen": 2315517952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030347474747474744, + "loss": 2.7289, + "theoretical_loss": 3.389332592697875, + "tokens_seen": 2315583488 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034545454545455, + "loss": 2.6278, + "theoretical_loss": 3.3893248232630118, + "tokens_seen": 2315649024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030343434343434346, + "loss": 2.651, + "theoretical_loss": 3.3893170541095974, + "tokens_seen": 2315714560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034141414141414, + "loss": 2.7817, + "theoretical_loss": 3.389309285237613, + "tokens_seen": 2315780096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033939393939394, + "loss": 2.4756, + "theoretical_loss": 3.389301516647041, + "tokens_seen": 2315845632 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033737373737374, + "loss": 2.7765, + "theoretical_loss": 3.3892937483378622, + "tokens_seen": 2315911168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033535353535354, + "loss": 2.4798, + "theoretical_loss": 3.38928598031006, + "tokens_seen": 2315976704 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030333333333333335, + "loss": 2.6061, + "theoretical_loss": 3.3892782125636147, + "tokens_seen": 2316042240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033131313131313, + "loss": 2.6794, + "theoretical_loss": 3.389270445098509, + "tokens_seen": 2316107776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030329292929292927, + "loss": 2.5613, + "theoretical_loss": 3.389262677914725, + "tokens_seen": 2316173312 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032727272727273, + "loss": 2.6267, + "theoretical_loss": 3.3892549110122436, + "tokens_seen": 2316238848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032525252525253, + "loss": 2.7271, + "theoretical_loss": 3.3892471443910472, + "tokens_seen": 2316304384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030323232323232324, + "loss": 2.7138, + "theoretical_loss": 3.3892393780511174, + "tokens_seen": 2316369920 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032121212121212, + "loss": 2.7729, + "theoretical_loss": 3.389231611992437, + "tokens_seen": 2316435456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031919191919192, + "loss": 2.5525, + "theoretical_loss": 3.389223846214987, + "tokens_seen": 2316500992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030317171717171717, + "loss": 2.6584, + "theoretical_loss": 3.3892160807187492, + "tokens_seen": 2316566528 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031515151515152, + "loss": 2.6226, + "theoretical_loss": 3.389208315503706, + "tokens_seen": 2316632064 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030313131313131314, + "loss": 2.6754, + "theoretical_loss": 3.3892005505698384, + "tokens_seen": 2316697600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031111111111111, + "loss": 2.5844, + "theoretical_loss": 3.3891927859171287, + "tokens_seen": 2316763136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030909090909091, + "loss": 2.6093, + "theoretical_loss": 3.3891850215455595, + "tokens_seen": 2316828672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030307070707070706, + "loss": 2.695, + "theoretical_loss": 3.3891772574551116, + "tokens_seen": 2316894208 + }, + { + "debugging/Self-BLEU-5": 0.20619410364685406, + "debugging/distinct-1-grams": 0.7574324808367362, + "debugging/distinct-2-grams": 0.9451559705015926, + "debugging/entropy-1-grams": 4.6576422156928405, + "debugging/entropy-2-grams": 5.24391176970891, + "debugging/length": 578.6666666666666, + "debugging/num_segments": 3, + "epoch": 0.4, + "objective/train/docs_used": 1278459, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2756106853485107, + "objective/train/theoretical_loss": 3.3891694936457677, + "objective/train/tokens_used": 675818976, + "theoretical_loss": 3.3891694936457677, + "tokens_seen": 2316959744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030505050505051, + "loss": 2.4249, + "theoretical_loss": 3.3891694936457677, + "tokens_seen": 2316959744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030303030303030303, + "loss": 2.5714, + "theoretical_loss": 3.389161730117509, + "tokens_seen": 2317025280 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303010101010101, + "loss": 2.6624, + "theoretical_loss": 3.3891539668703174, + "tokens_seen": 2317090816 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302989898989899, + "loss": 2.5118, + "theoretical_loss": 3.3891462039041755, + "tokens_seen": 2317156352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030296969696969695, + "loss": 2.5119, + "theoretical_loss": 3.389138441219065, + "tokens_seen": 2317221888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030294949494949497, + "loss": 2.5936, + "theoretical_loss": 3.3891306788149667, + "tokens_seen": 2317287424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029292929292929, + "loss": 2.506, + "theoretical_loss": 3.3891229166918637, + "tokens_seen": 2317352960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030290909090909093, + "loss": 2.7018, + "theoretical_loss": 3.389115154849737, + "tokens_seen": 2317418496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028888888888889, + "loss": 2.5806, + "theoretical_loss": 3.3891073932885694, + "tokens_seen": 2317484032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030286868686868685, + "loss": 2.7083, + "theoretical_loss": 3.3890996320083424, + "tokens_seen": 2317549568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030284848484848486, + "loss": 2.5808, + "theoretical_loss": 3.3890918710090374, + "tokens_seen": 2317615104 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028282828282828, + "loss": 2.6755, + "theoretical_loss": 3.3890841102906366, + "tokens_seen": 2317680640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028080808080808, + "loss": 2.6778, + "theoretical_loss": 3.3890763498531227, + "tokens_seen": 2317746176 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027878787878788, + "loss": 2.6103, + "theoretical_loss": 3.3890685896964765, + "tokens_seen": 2317811712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030276767676767674, + "loss": 2.4966, + "theoretical_loss": 3.38906082982068, + "tokens_seen": 2317877248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030274747474747475, + "loss": 2.5976, + "theoretical_loss": 3.3890530702257156, + "tokens_seen": 2317942784 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030272727272727276, + "loss": 2.5301, + "theoretical_loss": 3.389045310911565, + "tokens_seen": 2318008320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027070707070707, + "loss": 2.6838, + "theoretical_loss": 3.38903755187821, + "tokens_seen": 2318073856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026868686868687, + "loss": 2.5929, + "theoretical_loss": 3.3890297931256326, + "tokens_seen": 2318139392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030266666666666663, + "loss": 2.2318, + "theoretical_loss": 3.3890220346538142, + "tokens_seen": 2318204928 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030264646464646464, + "loss": 2.4557, + "theoretical_loss": 3.3890142764627376, + "tokens_seen": 2318270464 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030262626262626266, + "loss": 2.6753, + "theoretical_loss": 3.3890065185523843, + "tokens_seen": 2318336000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026060606060606, + "loss": 2.7227, + "theoretical_loss": 3.3889987609227363, + "tokens_seen": 2318401536 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030258585858585857, + "loss": 2.6271, + "theoretical_loss": 3.388991003573775, + "tokens_seen": 2318467072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025656565656566, + "loss": 2.6011, + "theoretical_loss": 3.3889832465054828, + "tokens_seen": 2318532608 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1279139, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5781471729278564, + "objective/train/theoretical_loss": 3.3889754897178417, + "objective/train/tokens_used": 677457376, + "theoretical_loss": 3.3889754897178417, + "tokens_seen": 2318598144 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025454545454546, + "loss": 2.513, + "theoretical_loss": 3.3889754897178417, + "tokens_seen": 2318598144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030252525252525255, + "loss": 2.6527, + "theoretical_loss": 3.3889677332108334, + "tokens_seen": 2318663680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025050505050505, + "loss": 2.5918, + "theoretical_loss": 3.38895997698444, + "tokens_seen": 2318729216 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030248484848484846, + "loss": 2.5136, + "theoretical_loss": 3.3889522210386427, + "tokens_seen": 2318794752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030246464646464647, + "loss": 2.7214, + "theoretical_loss": 3.388944465373424, + "tokens_seen": 2318860288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024444444444445, + "loss": 2.2814, + "theoretical_loss": 3.388936709988766, + "tokens_seen": 2318925824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030242424242424244, + "loss": 2.3996, + "theoretical_loss": 3.388928954884651, + "tokens_seen": 2318991360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024040404040404, + "loss": 2.6588, + "theoretical_loss": 3.3889212000610596, + "tokens_seen": 2319056896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030238383838383835, + "loss": 2.7012, + "theoretical_loss": 3.3889134455179746, + "tokens_seen": 2319122432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023636363636364, + "loss": 2.488, + "theoretical_loss": 3.3889056912553777, + "tokens_seen": 2319187968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023434343434344, + "loss": 2.5442, + "theoretical_loss": 3.388897937273251, + "tokens_seen": 2319253504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030232323232323233, + "loss": 2.7, + "theoretical_loss": 3.3888901835715766, + "tokens_seen": 2319319040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023030303030303, + "loss": 2.6603, + "theoretical_loss": 3.388882430150336, + "tokens_seen": 2319384576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030228282828282825, + "loss": 2.3551, + "theoretical_loss": 3.3888746770095115, + "tokens_seen": 2319450112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022626262626263, + "loss": 2.6881, + "theoretical_loss": 3.3888669241490845, + "tokens_seen": 2319515648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030224242424242427, + "loss": 2.7914, + "theoretical_loss": 3.3888591715690377, + "tokens_seen": 2319581184 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022222222222222, + "loss": 2.6074, + "theoretical_loss": 3.3888514192693524, + "tokens_seen": 2319646720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022020202020202, + "loss": 2.7748, + "theoretical_loss": 3.388843667250011, + "tokens_seen": 2319712256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021818181818182, + "loss": 2.7027, + "theoretical_loss": 3.388835915510995, + "tokens_seen": 2319777792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021616161616162, + "loss": 2.5539, + "theoretical_loss": 3.3888281640522866, + "tokens_seen": 2319843328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030214141414141416, + "loss": 2.4172, + "theoretical_loss": 3.3888204128738675, + "tokens_seen": 2319908864 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021212121212121, + "loss": 2.5263, + "theoretical_loss": 3.3888126619757197, + "tokens_seen": 2319974400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021010101010101, + "loss": 2.5707, + "theoretical_loss": 3.388804911357826, + "tokens_seen": 2320039936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020808080808081, + "loss": 2.6024, + "theoretical_loss": 3.3887971610201673, + "tokens_seen": 2320105472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020606060606061, + "loss": 2.5436, + "theoretical_loss": 3.3887894109627257, + "tokens_seen": 2320171008 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1280192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5417916774749756, + "objective/train/theoretical_loss": 3.3887816611854835, + "objective/train/tokens_used": 679095776, + "theoretical_loss": 3.3887816611854835, + "tokens_seen": 2320236544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030204040404040405, + "loss": 2.9523, + "theoretical_loss": 3.3887816611854835, + "tokens_seen": 2320236544 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302020202020202, + "loss": 2.6333, + "theoretical_loss": 3.388773911688422, + "tokens_seen": 2320302080 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302, + "loss": 2.6155, + "theoretical_loss": 3.3887661624715246, + "tokens_seen": 2320367616 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301979797979798, + "loss": 2.6584, + "theoretical_loss": 3.388758413534772, + "tokens_seen": 2320433152 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301959595959596, + "loss": 2.4983, + "theoretical_loss": 3.3887506648781462, + "tokens_seen": 2320498688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030193939393939395, + "loss": 2.494, + "theoretical_loss": 3.38874291650163, + "tokens_seen": 2320564224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003019191919191919, + "loss": 2.516, + "theoretical_loss": 3.388735168405204, + "tokens_seen": 2320629760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018989898989899, + "loss": 2.2484, + "theoretical_loss": 3.3887274205888516, + "tokens_seen": 2320695296 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030187878787878787, + "loss": 2.6002, + "theoretical_loss": 3.388719673052554, + "tokens_seen": 2320760832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018585858585859, + "loss": 2.6474, + "theoretical_loss": 3.388711925796293, + "tokens_seen": 2320826368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030183838383838384, + "loss": 2.4902, + "theoretical_loss": 3.3887041788200514, + "tokens_seen": 2320891904 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030181818181818185, + "loss": 2.5106, + "theoretical_loss": 3.3886964321238104, + "tokens_seen": 2320957440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017979797979798, + "loss": 2.64, + "theoretical_loss": 3.388688685707552, + "tokens_seen": 2321022976 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030177777777777776, + "loss": 2.699, + "theoretical_loss": 3.388680939571259, + "tokens_seen": 2321088512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017575757575758, + "loss": 2.7201, + "theoretical_loss": 3.3886731937149124, + "tokens_seen": 2321154048 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030173737373737373, + "loss": 2.4053, + "theoretical_loss": 3.388665448138495, + "tokens_seen": 2321219584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030171717171717174, + "loss": 2.5413, + "theoretical_loss": 3.3886577028419875, + "tokens_seen": 2321285120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016969696969697, + "loss": 2.3769, + "theoretical_loss": 3.3886499578253737, + "tokens_seen": 2321350656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030167676767676766, + "loss": 2.6089, + "theoretical_loss": 3.388642213088634, + "tokens_seen": 2321416192 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030165656565656567, + "loss": 2.4305, + "theoretical_loss": 3.388634468631751, + "tokens_seen": 2321481728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016363636363637, + "loss": 2.586, + "theoretical_loss": 3.388626724454707, + "tokens_seen": 2321547264 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030161616161616164, + "loss": 2.6987, + "theoretical_loss": 3.3886189805574833, + "tokens_seen": 2321612800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015959595959596, + "loss": 2.6412, + "theoretical_loss": 3.3886112369400623, + "tokens_seen": 2321678336 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030157575757575755, + "loss": 2.5613, + "theoretical_loss": 3.3886034936024263, + "tokens_seen": 2321743872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030155555555555556, + "loss": 2.5951, + "theoretical_loss": 3.3885957505445568, + "tokens_seen": 2321809408 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1280928, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7444186210632324, + "objective/train/theoretical_loss": 3.388588007766436, + "objective/train/tokens_used": 680734176, + "theoretical_loss": 3.388588007766436, + "tokens_seen": 2321874944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030153535353535357, + "loss": 2.5264, + "theoretical_loss": 3.388588007766436, + "tokens_seen": 2321874944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030151515151515153, + "loss": 2.6344, + "theoretical_loss": 3.3885802652680455, + "tokens_seen": 2321940480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014949494949495, + "loss": 2.6376, + "theoretical_loss": 3.3885725230493677, + "tokens_seen": 2322006016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030147474747474744, + "loss": 2.5501, + "theoretical_loss": 3.388564781110385, + "tokens_seen": 2322071552 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014545454545455, + "loss": 2.6154, + "theoretical_loss": 3.3885570394510784, + "tokens_seen": 2322137088 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030143434343434346, + "loss": 2.7452, + "theoretical_loss": 3.388549298071431, + "tokens_seen": 2322202624 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014141414141414, + "loss": 2.6932, + "theoretical_loss": 3.3885415569714237, + "tokens_seen": 2322268160 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013939393939394, + "loss": 2.6463, + "theoretical_loss": 3.3885338161510394, + "tokens_seen": 2322333696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030137373737373734, + "loss": 2.7288, + "theoretical_loss": 3.3885260756102595, + "tokens_seen": 2322399232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013535353535354, + "loss": 2.5697, + "theoretical_loss": 3.3885183353490667, + "tokens_seen": 2322464768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030133333333333336, + "loss": 2.5758, + "theoretical_loss": 3.388510595367442, + "tokens_seen": 2322530304 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013131313131313, + "loss": 2.4679, + "theoretical_loss": 3.3885028556653687, + "tokens_seen": 2322595840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030129292929292927, + "loss": 2.5451, + "theoretical_loss": 3.3884951162428276, + "tokens_seen": 2322661376 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030127272727272723, + "loss": 2.5937, + "theoretical_loss": 3.3884873770998016, + "tokens_seen": 2322726912 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012525252525253, + "loss": 2.549, + "theoretical_loss": 3.388479638236272, + "tokens_seen": 2322792448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030123232323232325, + "loss": 2.672, + "theoretical_loss": 3.3884718996522216, + "tokens_seen": 2322857984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012121212121212, + "loss": 2.483, + "theoretical_loss": 3.388464161347632, + "tokens_seen": 2322923520 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030119191919191916, + "loss": 2.3783, + "theoretical_loss": 3.3884564233224848, + "tokens_seen": 2322989056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011717171717172, + "loss": 2.6028, + "theoretical_loss": 3.388448685576763, + "tokens_seen": 2323054592 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011515151515152, + "loss": 2.5197, + "theoretical_loss": 3.3884409481104476, + "tokens_seen": 2323120128 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030113131313131314, + "loss": 2.7962, + "theoretical_loss": 3.3884332109235213, + "tokens_seen": 2323185664 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011111111111111, + "loss": 2.619, + "theoretical_loss": 3.388425474015966, + "tokens_seen": 2323251200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030109090909090906, + "loss": 2.5069, + "theoretical_loss": 3.3884177373877633, + "tokens_seen": 2323316736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030107070707070707, + "loss": 2.7151, + "theoretical_loss": 3.388410001038896, + "tokens_seen": 2323382272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010505050505051, + "loss": 2.7642, + "theoretical_loss": 3.3884022649693453, + "tokens_seen": 2323447808 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1282083, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3946757316589355, + "objective/train/theoretical_loss": 3.3883945291790942, + "objective/train/tokens_used": 682372576, + "theoretical_loss": 3.3883945291790942, + "tokens_seen": 2323513344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030103030303030304, + "loss": 2.5281, + "theoretical_loss": 3.3883945291790942, + "tokens_seen": 2323513344 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301010101010101, + "loss": 2.4667, + "theoretical_loss": 3.3883867936681242, + "tokens_seen": 2323578880 + }, + { + "epoch": 0.4, + "learning_rate": 0.000300989898989899, + "loss": 2.551, + "theoretical_loss": 3.388379058436417, + "tokens_seen": 2323644416 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030096969696969696, + "loss": 2.7176, + "theoretical_loss": 3.3883713234839554, + "tokens_seen": 2323709952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030094949494949497, + "loss": 2.2652, + "theoretical_loss": 3.3883635888107206, + "tokens_seen": 2323775488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030092929292929293, + "loss": 2.5805, + "theoretical_loss": 3.3883558544166954, + "tokens_seen": 2323841024 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003009090909090909, + "loss": 2.433, + "theoretical_loss": 3.3883481203018615, + "tokens_seen": 2323906560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008888888888889, + "loss": 2.4721, + "theoretical_loss": 3.388340386466201, + "tokens_seen": 2323972096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030086868686868685, + "loss": 2.4395, + "theoretical_loss": 3.3883326529096958, + "tokens_seen": 2324037632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030084848484848486, + "loss": 2.4886, + "theoretical_loss": 3.3883249196323284, + "tokens_seen": 2324103168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008282828282828, + "loss": 2.6109, + "theoretical_loss": 3.3883171866340804, + "tokens_seen": 2324168704 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030080808080808083, + "loss": 2.8457, + "theoretical_loss": 3.3883094539149337, + "tokens_seen": 2324234240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007878787878788, + "loss": 2.6223, + "theoretical_loss": 3.3883017214748707, + "tokens_seen": 2324299776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007676767676768, + "loss": 2.4087, + "theoretical_loss": 3.388293989313874, + "tokens_seen": 2324365312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030074747474747476, + "loss": 2.6035, + "theoretical_loss": 3.3882862574319246, + "tokens_seen": 2324430848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007272727272727, + "loss": 2.507, + "theoretical_loss": 3.388278525829005, + "tokens_seen": 2324496384 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007070707070707, + "loss": 2.5625, + "theoretical_loss": 3.3882707945050976, + "tokens_seen": 2324561920 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006868686868687, + "loss": 2.3879, + "theoretical_loss": 3.388263063460184, + "tokens_seen": 2324627456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006666666666667, + "loss": 2.4293, + "theoretical_loss": 3.3882553326942464, + "tokens_seen": 2324692992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030064646464646465, + "loss": 2.5838, + "theoretical_loss": 3.388247602207267, + "tokens_seen": 2324758528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030062626262626266, + "loss": 2.5775, + "theoretical_loss": 3.3882398719992275, + "tokens_seen": 2324824064 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006060606060606, + "loss": 2.73, + "theoretical_loss": 3.3882321420701103, + "tokens_seen": 2324889600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003005858585858586, + "loss": 2.6222, + "theoretical_loss": 3.3882244124198975, + "tokens_seen": 2324955136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003005656565656566, + "loss": 2.586, + "theoretical_loss": 3.3882166830485714, + "tokens_seen": 2325020672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030054545454545454, + "loss": 2.7063, + "theoretical_loss": 3.3882089539561133, + "tokens_seen": 2325086208 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1282722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1986050605773926, + "objective/train/theoretical_loss": 3.388201225142506, + "objective/train/tokens_used": 684010976, + "theoretical_loss": 3.388201225142506, + "tokens_seen": 2325151744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030052525252525255, + "loss": 2.4966, + "theoretical_loss": 3.388201225142506, + "tokens_seen": 2325151744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003005050505050505, + "loss": 2.4373, + "theoretical_loss": 3.388193496607731, + "tokens_seen": 2325217280 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030048484848484847, + "loss": 2.512, + "theoretical_loss": 3.3881857683517715, + "tokens_seen": 2325282816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004646464646465, + "loss": 2.6173, + "theoretical_loss": 3.3881780403746085, + "tokens_seen": 2325348352 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004444444444445, + "loss": 2.4411, + "theoretical_loss": 3.388170312676224, + "tokens_seen": 2325413888 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030042424242424245, + "loss": 2.6006, + "theoretical_loss": 3.388162585256601, + "tokens_seen": 2325479424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004040404040404, + "loss": 2.5983, + "theoretical_loss": 3.3881548581157204, + "tokens_seen": 2325544960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030038383838383836, + "loss": 2.447, + "theoretical_loss": 3.3881471312535654, + "tokens_seen": 2325610496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030036363636363637, + "loss": 2.7848, + "theoretical_loss": 3.3881394046701176, + "tokens_seen": 2325676032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003434343434344, + "loss": 2.5488, + "theoretical_loss": 3.388131678365359, + "tokens_seen": 2325741568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030032323232323234, + "loss": 2.7041, + "theoretical_loss": 3.3881239523392717, + "tokens_seen": 2325807104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003030303030303, + "loss": 2.3782, + "theoretical_loss": 3.3881162265918383, + "tokens_seen": 2325872640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030028282828282825, + "loss": 2.5706, + "theoretical_loss": 3.3881085011230403, + "tokens_seen": 2325938176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002626262626263, + "loss": 2.6527, + "theoretical_loss": 3.38810077593286, + "tokens_seen": 2326003712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002424242424243, + "loss": 2.3732, + "theoretical_loss": 3.3880930510212797, + "tokens_seen": 2326069248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030022222222222223, + "loss": 2.4778, + "theoretical_loss": 3.388085326388281, + "tokens_seen": 2326134784 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002020202020202, + "loss": 2.3248, + "theoretical_loss": 3.388077602033847, + "tokens_seen": 2326200320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030018181818181815, + "loss": 2.7419, + "theoretical_loss": 3.3880698779579586, + "tokens_seen": 2326265856 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001616161616162, + "loss": 2.725, + "theoretical_loss": 3.3880621541605986, + "tokens_seen": 2326331392 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030014141414141417, + "loss": 2.6013, + "theoretical_loss": 3.388054430641749, + "tokens_seen": 2326396928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001212121212121, + "loss": 2.4874, + "theoretical_loss": 3.388046707401392, + "tokens_seen": 2326462464 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001010101010101, + "loss": 2.3981, + "theoretical_loss": 3.3880389844395093, + "tokens_seen": 2326528000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000808080808081, + "loss": 2.637, + "theoretical_loss": 3.3880312617560833, + "tokens_seen": 2326593536 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000606060606061, + "loss": 2.4284, + "theoretical_loss": 3.3880235393510962, + "tokens_seen": 2326659072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030004040404040406, + "loss": 2.5925, + "theoretical_loss": 3.38801581722453, + "tokens_seen": 2326724608 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1283581, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.384247064590454, + "objective/train/theoretical_loss": 3.388008095376367, + "objective/train/tokens_used": 685649376, + "theoretical_loss": 3.388008095376367, + "tokens_seen": 2326790144 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300020202020202, + "loss": 2.4081, + "theoretical_loss": 3.388008095376367, + "tokens_seen": 2326790144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003, + "loss": 2.6352, + "theoretical_loss": 3.388000373806589, + "tokens_seen": 2326855680 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299979797979798, + "loss": 2.6886, + "theoretical_loss": 3.3879926525151784, + "tokens_seen": 2326921216 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299959595959596, + "loss": 2.717, + "theoretical_loss": 3.3879849315021175, + "tokens_seen": 2326986752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029993939393939395, + "loss": 2.768, + "theoretical_loss": 3.387977210767388, + "tokens_seen": 2327052288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999191919191919, + "loss": 2.6944, + "theoretical_loss": 3.387969490310972, + "tokens_seen": 2327117824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029989898989898987, + "loss": 2.6222, + "theoretical_loss": 3.3879617701328515, + "tokens_seen": 2327183360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998787878787879, + "loss": 2.5626, + "theoretical_loss": 3.3879540502330094, + "tokens_seen": 2327248896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998585858585859, + "loss": 2.5228, + "theoretical_loss": 3.3879463306114275, + "tokens_seen": 2327314432 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029983838383838385, + "loss": 2.6241, + "theoretical_loss": 3.387938611268088, + "tokens_seen": 2327379968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998181818181818, + "loss": 2.4605, + "theoretical_loss": 3.3879308922029723, + "tokens_seen": 2327445504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997979797979798, + "loss": 2.5005, + "theoretical_loss": 3.387923173416063, + "tokens_seen": 2327511040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029977777777777777, + "loss": 2.7255, + "theoretical_loss": 3.387915454907343, + "tokens_seen": 2327576576 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997575757575758, + "loss": 2.5728, + "theoretical_loss": 3.387907736676793, + "tokens_seen": 2327642112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029973737373737374, + "loss": 2.5169, + "theoretical_loss": 3.3879000187243964, + "tokens_seen": 2327707648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997171717171717, + "loss": 2.6249, + "theoretical_loss": 3.387892301050135, + "tokens_seen": 2327773184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996969696969697, + "loss": 2.5634, + "theoretical_loss": 3.3878845836539906, + "tokens_seen": 2327838720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029967676767676766, + "loss": 2.5164, + "theoretical_loss": 3.3878768665359456, + "tokens_seen": 2327904256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996565656565657, + "loss": 2.5191, + "theoretical_loss": 3.387869149695982, + "tokens_seen": 2327969792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029963636363636363, + "loss": 2.3382, + "theoretical_loss": 3.387861433134082, + "tokens_seen": 2328035328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029961616161616164, + "loss": 2.497, + "theoretical_loss": 3.387853716850228, + "tokens_seen": 2328100864 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995959595959596, + "loss": 2.7257, + "theoretical_loss": 3.3878460008444016, + "tokens_seen": 2328166400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029957575757575756, + "loss": 2.579, + "theoretical_loss": 3.387838285116586, + "tokens_seen": 2328231936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029955555555555557, + "loss": 2.7085, + "theoretical_loss": 3.387830569666762, + "tokens_seen": 2328297472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995353535353535, + "loss": 2.5562, + "theoretical_loss": 3.3878228544949125, + "tokens_seen": 2328363008 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1284315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.855656147003174, + "objective/train/theoretical_loss": 3.38781513960102, + "objective/train/tokens_used": 687287776, + "theoretical_loss": 3.38781513960102, + "tokens_seen": 2328428544 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029951515151515153, + "loss": 2.5938, + "theoretical_loss": 3.38781513960102, + "tokens_seen": 2328428544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994949494949495, + "loss": 2.4312, + "theoretical_loss": 3.3878074249850663, + "tokens_seen": 2328494080 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029947474747474745, + "loss": 2.6684, + "theoretical_loss": 3.387799710647033, + "tokens_seen": 2328559616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029945454545454546, + "loss": 2.6161, + "theoretical_loss": 3.387791996586903, + "tokens_seen": 2328625152 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029943434343434347, + "loss": 2.8061, + "theoretical_loss": 3.3877842828046587, + "tokens_seen": 2328690688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029941414141414143, + "loss": 2.3686, + "theoretical_loss": 3.3877765693002813, + "tokens_seen": 2328756224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993939393939394, + "loss": 2.5241, + "theoretical_loss": 3.3877688560737536, + "tokens_seen": 2328821760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029937373737373734, + "loss": 2.5534, + "theoretical_loss": 3.387761143125058, + "tokens_seen": 2328887296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029935353535353535, + "loss": 2.4191, + "theoretical_loss": 3.387753430454176, + "tokens_seen": 2328952832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029933333333333336, + "loss": 2.4657, + "theoretical_loss": 3.3877457180610904, + "tokens_seen": 2329018368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993131313131313, + "loss": 2.4365, + "theoretical_loss": 3.3877380059457827, + "tokens_seen": 2329083904 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992929292929293, + "loss": 2.5397, + "theoretical_loss": 3.387730294108236, + "tokens_seen": 2329149440 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029927272727272723, + "loss": 2.5297, + "theoretical_loss": 3.3877225825484314, + "tokens_seen": 2329214976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992525252525253, + "loss": 2.6503, + "theoretical_loss": 3.387714871266352, + "tokens_seen": 2329280512 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029923232323232326, + "loss": 2.5606, + "theoretical_loss": 3.3877071602619795, + "tokens_seen": 2329346048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992121212121212, + "loss": 2.6814, + "theoretical_loss": 3.3876994495352966, + "tokens_seen": 2329411584 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029919191919191917, + "loss": 2.6162, + "theoretical_loss": 3.3876917390862844, + "tokens_seen": 2329477120 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991717171717171, + "loss": 2.5459, + "theoretical_loss": 3.387684028914926, + "tokens_seen": 2329542656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991515151515152, + "loss": 2.5083, + "theoretical_loss": 3.387676319021204, + "tokens_seen": 2329608192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029913131313131315, + "loss": 2.4128, + "theoretical_loss": 3.3876686094050994, + "tokens_seen": 2329673728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991111111111111, + "loss": 2.6386, + "theoretical_loss": 3.387660900066595, + "tokens_seen": 2329739264 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029909090909090906, + "loss": 2.641, + "theoretical_loss": 3.3876531910056733, + "tokens_seen": 2329804800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029907070707070713, + "loss": 2.2965, + "theoretical_loss": 3.387645482222316, + "tokens_seen": 2329870336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990505050505051, + "loss": 2.2496, + "theoretical_loss": 3.3876377737165053, + "tokens_seen": 2329935872 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029903030303030304, + "loss": 2.446, + "theoretical_loss": 3.387630065488224, + "tokens_seen": 2330001408 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1285456, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5356266498565674, + "objective/train/theoretical_loss": 3.3876223575374533, + "objective/train/tokens_used": 688926176, + "theoretical_loss": 3.3876223575374533, + "tokens_seen": 2330066944 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299010101010101, + "loss": 2.743, + "theoretical_loss": 3.3876223575374533, + "tokens_seen": 2330066944 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029898989898989895, + "loss": 2.5193, + "theoretical_loss": 3.387614649864176, + "tokens_seen": 2330132480 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298969696969697, + "loss": 2.4647, + "theoretical_loss": 3.387606942468375, + "tokens_seen": 2330198016 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298949494949495, + "loss": 2.553, + "theoretical_loss": 3.387599235350031, + "tokens_seen": 2330263552 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029892929292929293, + "loss": 2.4722, + "theoretical_loss": 3.387591528509127, + "tokens_seen": 2330329088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989090909090909, + "loss": 2.4273, + "theoretical_loss": 3.387583821945646, + "tokens_seen": 2330394624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988888888888889, + "loss": 2.6327, + "theoretical_loss": 3.3875761156595687, + "tokens_seen": 2330460160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988686868686869, + "loss": 2.7975, + "theoretical_loss": 3.3875684096508785, + "tokens_seen": 2330525696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029884848484848487, + "loss": 2.4514, + "theoretical_loss": 3.3875607039195565, + "tokens_seen": 2330591232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988282828282828, + "loss": 2.6605, + "theoretical_loss": 3.387552998465586, + "tokens_seen": 2330656768 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988080808080808, + "loss": 2.5183, + "theoretical_loss": 3.387545293288949, + "tokens_seen": 2330722304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987878787878788, + "loss": 2.3649, + "theoretical_loss": 3.3875375883896273, + "tokens_seen": 2330787840 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987676767676768, + "loss": 2.402, + "theoretical_loss": 3.387529883767603, + "tokens_seen": 2330853376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029874747474747476, + "loss": 2.8542, + "theoretical_loss": 3.3875221794228594, + "tokens_seen": 2330918912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987272727272727, + "loss": 2.5957, + "theoretical_loss": 3.387514475355377, + "tokens_seen": 2330984448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029870707070707073, + "loss": 2.7468, + "theoretical_loss": 3.38750677156514, + "tokens_seen": 2331049984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986868686868687, + "loss": 2.5161, + "theoretical_loss": 3.387499068052129, + "tokens_seen": 2331115520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986666666666667, + "loss": 2.5359, + "theoretical_loss": 3.3874913648163267, + "tokens_seen": 2331181056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029864646464646466, + "loss": 2.6, + "theoretical_loss": 3.3874836618577158, + "tokens_seen": 2331246592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986262626262626, + "loss": 2.761, + "theoretical_loss": 3.387475959176278, + "tokens_seen": 2331312128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986060606060606, + "loss": 2.4458, + "theoretical_loss": 3.3874682567719963, + "tokens_seen": 2331377664 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985858585858586, + "loss": 2.5724, + "theoretical_loss": 3.387460554644852, + "tokens_seen": 2331443200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985656565656566, + "loss": 2.5471, + "theoretical_loss": 3.387452852794828, + "tokens_seen": 2331508736 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029854545454545455, + "loss": 2.3454, + "theoretical_loss": 3.387445151221906, + "tokens_seen": 2331574272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029852525252525256, + "loss": 2.6932, + "theoretical_loss": 3.3874374499260687, + "tokens_seen": 2331639808 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1286600, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6554930210113525, + "objective/train/theoretical_loss": 3.387429748907298, + "objective/train/tokens_used": 690564576, + "theoretical_loss": 3.387429748907298, + "tokens_seen": 2331705344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985050505050505, + "loss": 2.5126, + "theoretical_loss": 3.387429748907298, + "tokens_seen": 2331705344 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029848484848484847, + "loss": 2.7253, + "theoretical_loss": 3.3874220481655763, + "tokens_seen": 2331770880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984646464646465, + "loss": 2.4796, + "theoretical_loss": 3.3874143477008865, + "tokens_seen": 2331836416 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029844444444444444, + "loss": 2.6368, + "theoretical_loss": 3.3874066475132096, + "tokens_seen": 2331901952 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029842424242424245, + "loss": 2.6352, + "theoretical_loss": 3.387398947602528, + "tokens_seen": 2331967488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984040404040404, + "loss": 2.6589, + "theoretical_loss": 3.387391247968825, + "tokens_seen": 2332033024 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029838383838383837, + "loss": 2.4235, + "theoretical_loss": 3.3873835486120822, + "tokens_seen": 2332098560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983636363636364, + "loss": 2.5454, + "theoretical_loss": 3.3873758495322823, + "tokens_seen": 2332164096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983434343434344, + "loss": 2.7885, + "theoretical_loss": 3.3873681507294067, + "tokens_seen": 2332229632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029832323232323234, + "loss": 2.6011, + "theoretical_loss": 3.387360452203438, + "tokens_seen": 2332295168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983030303030303, + "loss": 2.6292, + "theoretical_loss": 3.387352753954359, + "tokens_seen": 2332360704 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029828282828282826, + "loss": 2.7666, + "theoretical_loss": 3.387345055982151, + "tokens_seen": 2332426240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029826262626262627, + "loss": 2.6206, + "theoretical_loss": 3.387337358286797, + "tokens_seen": 2332491776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982424242424243, + "loss": 2.5381, + "theoretical_loss": 3.3873296608682795, + "tokens_seen": 2332557312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029822222222222224, + "loss": 2.6186, + "theoretical_loss": 3.3873219637265803, + "tokens_seen": 2332622848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982020202020202, + "loss": 2.4217, + "theoretical_loss": 3.387314266861681, + "tokens_seen": 2332688384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029818181818181815, + "loss": 2.7029, + "theoretical_loss": 3.3873065702735654, + "tokens_seen": 2332753920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981616161616162, + "loss": 2.662, + "theoretical_loss": 3.387298873962215, + "tokens_seen": 2332819456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981414141414142, + "loss": 2.6625, + "theoretical_loss": 3.3872911779276116, + "tokens_seen": 2332884992 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029812121212121213, + "loss": 2.3996, + "theoretical_loss": 3.387283482169738, + "tokens_seen": 2332950528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981010101010101, + "loss": 2.4591, + "theoretical_loss": 3.3872757866885763, + "tokens_seen": 2333016064 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029808080808080804, + "loss": 2.5562, + "theoretical_loss": 3.387268091484109, + "tokens_seen": 2333081600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980606060606061, + "loss": 2.3632, + "theoretical_loss": 3.387260396556318, + "tokens_seen": 2333147136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029804040404040407, + "loss": 2.4928, + "theoretical_loss": 3.3872527019051857, + "tokens_seen": 2333212672 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298020202020202, + "loss": 2.6419, + "theoretical_loss": 3.387245007530695, + "tokens_seen": 2333278208 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1287008, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1645236015319824, + "objective/train/theoretical_loss": 3.387237313432827, + "objective/train/tokens_used": 692202976, + "theoretical_loss": 3.387237313432827, + "tokens_seen": 2333343744 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298, + "loss": 2.4941, + "theoretical_loss": 3.387237313432827, + "tokens_seen": 2333343744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029797979797979794, + "loss": 2.8065, + "theoretical_loss": 3.387229619611565, + "tokens_seen": 2333409280 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297959595959596, + "loss": 2.7294, + "theoretical_loss": 3.3872219260668914, + "tokens_seen": 2333474816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029793939393939396, + "loss": 2.4559, + "theoretical_loss": 3.3872142327987875, + "tokens_seen": 2333540352 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979191919191919, + "loss": 2.5859, + "theoretical_loss": 3.3872065398072366, + "tokens_seen": 2333605888 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029789898989898987, + "loss": 2.6923, + "theoretical_loss": 3.38719884709222, + "tokens_seen": 2333671424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978787878787879, + "loss": 2.5763, + "theoretical_loss": 3.387191154653721, + "tokens_seen": 2333736960 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978585858585859, + "loss": 2.6255, + "theoretical_loss": 3.387183462491721, + "tokens_seen": 2333802496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029783838383838385, + "loss": 2.5329, + "theoretical_loss": 3.387175770606203, + "tokens_seen": 2333868032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978181818181818, + "loss": 2.6269, + "theoretical_loss": 3.3871680789971492, + "tokens_seen": 2333933568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029779797979797976, + "loss": 2.6989, + "theoretical_loss": 3.387160387664541, + "tokens_seen": 2333999104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977777777777778, + "loss": 2.8185, + "theoretical_loss": 3.3871526966083625, + "tokens_seen": 2334064640 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977575757575758, + "loss": 2.4832, + "theoretical_loss": 3.387145005828594, + "tokens_seen": 2334130176 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029773737373737374, + "loss": 2.6361, + "theoretical_loss": 3.3871373153252193, + "tokens_seen": 2334195712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977171717171717, + "loss": 2.3801, + "theoretical_loss": 3.3871296250982197, + "tokens_seen": 2334261248 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976969696969697, + "loss": 2.6512, + "theoretical_loss": 3.387121935147578, + "tokens_seen": 2334326784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029767676767676767, + "loss": 2.5558, + "theoretical_loss": 3.3871142454732768, + "tokens_seen": 2334392320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976565656565657, + "loss": 2.4155, + "theoretical_loss": 3.3871065560752975, + "tokens_seen": 2334457856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029763636363636364, + "loss": 2.6089, + "theoretical_loss": 3.3870988669536235, + "tokens_seen": 2334523392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976161616161616, + "loss": 2.4402, + "theoretical_loss": 3.3870911781082365, + "tokens_seen": 2334588928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975959595959596, + "loss": 2.5136, + "theoretical_loss": 3.3870834895391186, + "tokens_seen": 2334654464 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029757575757575756, + "loss": 2.5871, + "theoretical_loss": 3.3870758012462527, + "tokens_seen": 2334720000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029755555555555557, + "loss": 2.5719, + "theoretical_loss": 3.387068113229621, + "tokens_seen": 2334785536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029753535353535353, + "loss": 2.5404, + "theoretical_loss": 3.387060425489205, + "tokens_seen": 2334851072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029751515151515154, + "loss": 2.2632, + "theoretical_loss": 3.3870527380249884, + "tokens_seen": 2334916608 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1288435, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5754096508026123, + "objective/train/theoretical_loss": 3.3870450508369525, + "objective/train/tokens_used": 693841376, + "theoretical_loss": 3.3870450508369525, + "tokens_seen": 2334982144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974949494949495, + "loss": 2.5484, + "theoretical_loss": 3.3870450508369525, + "tokens_seen": 2334982144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029747474747474745, + "loss": 2.3657, + "theoretical_loss": 3.38703736392508, + "tokens_seen": 2335047680 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029745454545454546, + "loss": 2.688, + "theoretical_loss": 3.3870296772893536, + "tokens_seen": 2335113216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974343434343434, + "loss": 2.7214, + "theoretical_loss": 3.3870219909297545, + "tokens_seen": 2335178752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029741414141414143, + "loss": 2.6295, + "theoretical_loss": 3.387014304846266, + "tokens_seen": 2335244288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973939393939394, + "loss": 2.5466, + "theoretical_loss": 3.3870066190388703, + "tokens_seen": 2335309824 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973737373737374, + "loss": 2.4963, + "theoretical_loss": 3.3869989335075497, + "tokens_seen": 2335375360 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029735353535353536, + "loss": 2.5459, + "theoretical_loss": 3.3869912482522864, + "tokens_seen": 2335440896 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029733333333333337, + "loss": 2.6287, + "theoretical_loss": 3.3869835632730627, + "tokens_seen": 2335506432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973131313131313, + "loss": 2.4409, + "theoretical_loss": 3.386975878569861, + "tokens_seen": 2335571968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972929292929293, + "loss": 2.5834, + "theoretical_loss": 3.3869681941426633, + "tokens_seen": 2335637504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972727272727273, + "loss": 2.6672, + "theoretical_loss": 3.386960509991453, + "tokens_seen": 2335703040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029725252525252525, + "loss": 2.585, + "theoretical_loss": 3.386952826116211, + "tokens_seen": 2335768576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029723232323232326, + "loss": 2.6306, + "theoretical_loss": 3.386945142516921, + "tokens_seen": 2335834112 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972121212121212, + "loss": 2.5344, + "theoretical_loss": 3.3869374591935646, + "tokens_seen": 2335899648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971919191919192, + "loss": 2.5429, + "theoretical_loss": 3.386929776146124, + "tokens_seen": 2335965184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971717171717172, + "loss": 2.5843, + "theoretical_loss": 3.386922093374582, + "tokens_seen": 2336030720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971515151515152, + "loss": 2.568, + "theoretical_loss": 3.386914410878921, + "tokens_seen": 2336096256 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029713131313131315, + "loss": 2.637, + "theoretical_loss": 3.386906728659123, + "tokens_seen": 2336161792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971111111111111, + "loss": 2.3562, + "theoretical_loss": 3.3868990467151705, + "tokens_seen": 2336227328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029709090909090907, + "loss": 2.6356, + "theoretical_loss": 3.3868913650470462, + "tokens_seen": 2336292864 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970707070707071, + "loss": 2.6018, + "theoretical_loss": 3.386883683654732, + "tokens_seen": 2336358400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970505050505051, + "loss": 2.5028, + "theoretical_loss": 3.3868760025382096, + "tokens_seen": 2336423936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029703030303030305, + "loss": 2.4544, + "theoretical_loss": 3.386868321697463, + "tokens_seen": 2336489472 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297010101010101, + "loss": 2.4403, + "theoretical_loss": 3.3868606411324738, + "tokens_seen": 2336555008 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1288948, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5773682594299316, + "objective/train/theoretical_loss": 3.386852960843224, + "objective/train/tokens_used": 695479776, + "theoretical_loss": 3.386852960843224, + "tokens_seen": 2336620544 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029698989898989896, + "loss": 2.4461, + "theoretical_loss": 3.386852960843224, + "tokens_seen": 2336620544 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296969696969697, + "loss": 2.4447, + "theoretical_loss": 3.386845280829696, + "tokens_seen": 2336686080 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296949494949495, + "loss": 2.6317, + "theoretical_loss": 3.3868376010918726, + "tokens_seen": 2336751616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029692929292929294, + "loss": 2.6922, + "theoretical_loss": 3.3868299216297357, + "tokens_seen": 2336817152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969090909090909, + "loss": 2.5182, + "theoretical_loss": 3.3868222424432686, + "tokens_seen": 2336882688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029688888888888885, + "loss": 2.689, + "theoretical_loss": 3.3868145635324525, + "tokens_seen": 2336948224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968686868686869, + "loss": 2.5282, + "theoretical_loss": 3.3868068848972706, + "tokens_seen": 2337013760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968484848484849, + "loss": 2.5075, + "theoretical_loss": 3.3867992065377046, + "tokens_seen": 2337079296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029682828282828283, + "loss": 2.7941, + "theoretical_loss": 3.3867915284537378, + "tokens_seen": 2337144832 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968080808080808, + "loss": 2.8759, + "theoretical_loss": 3.386783850645352, + "tokens_seen": 2337210368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967878787878788, + "loss": 2.5257, + "theoretical_loss": 3.386776173112529, + "tokens_seen": 2337275904 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967676767676768, + "loss": 2.5613, + "theoretical_loss": 3.386768495855252, + "tokens_seen": 2337341440 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029674747474747477, + "loss": 2.5094, + "theoretical_loss": 3.386760818873504, + "tokens_seen": 2337406976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967272727272727, + "loss": 2.6228, + "theoretical_loss": 3.3867531421672656, + "tokens_seen": 2337472512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967070707070707, + "loss": 2.5084, + "theoretical_loss": 3.386745465736521, + "tokens_seen": 2337538048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966868686868687, + "loss": 2.4286, + "theoretical_loss": 3.3867377895812507, + "tokens_seen": 2337603584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966666666666667, + "loss": 2.5172, + "theoretical_loss": 3.386730113701439, + "tokens_seen": 2337669120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029664646464646466, + "loss": 2.5309, + "theoretical_loss": 3.386722438097067, + "tokens_seen": 2337734656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966262626262626, + "loss": 2.5358, + "theoretical_loss": 3.3867147627681176, + "tokens_seen": 2337800192 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966060606060606, + "loss": 2.5944, + "theoretical_loss": 3.3867070877145733, + "tokens_seen": 2337865728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965858585858586, + "loss": 2.6264, + "theoretical_loss": 3.3866994129364163, + "tokens_seen": 2337931264 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965656565656566, + "loss": 2.609, + "theoretical_loss": 3.3866917384336293, + "tokens_seen": 2337996800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029654545454545455, + "loss": 2.7484, + "theoretical_loss": 3.386684064206194, + "tokens_seen": 2338062336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965252525252525, + "loss": 2.5472, + "theoretical_loss": 3.3866763902540935, + "tokens_seen": 2338127872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965050505050505, + "loss": 2.788, + "theoretical_loss": 3.3866687165773097, + "tokens_seen": 2338193408 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1290000, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6659116744995117, + "objective/train/theoretical_loss": 3.3866610431758253, + "objective/train/tokens_used": 697118176, + "theoretical_loss": 3.3866610431758253, + "tokens_seen": 2338258944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964848484848485, + "loss": 2.5281, + "theoretical_loss": 3.3866610431758253, + "tokens_seen": 2338258944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964646464646465, + "loss": 2.613, + "theoretical_loss": 3.386653370049623, + "tokens_seen": 2338324480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029644444444444445, + "loss": 2.597, + "theoretical_loss": 3.3866456971986842, + "tokens_seen": 2338390016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964242424242424, + "loss": 2.4275, + "theoretical_loss": 3.3866380246229926, + "tokens_seen": 2338455552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964040404040404, + "loss": 2.4164, + "theoretical_loss": 3.3866303523225296, + "tokens_seen": 2338521088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029638383838383837, + "loss": 2.5275, + "theoretical_loss": 3.386622680297278, + "tokens_seen": 2338586624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963636363636364, + "loss": 2.5986, + "theoretical_loss": 3.3866150085472206, + "tokens_seen": 2338652160 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029634343434343434, + "loss": 2.5817, + "theoretical_loss": 3.3866073370723395, + "tokens_seen": 2338717696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029632323232323235, + "loss": 2.5196, + "theoretical_loss": 3.3865996658726165, + "tokens_seen": 2338783232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963030303030303, + "loss": 2.3676, + "theoretical_loss": 3.3865919949480348, + "tokens_seen": 2338848768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029628282828282826, + "loss": 2.3948, + "theoretical_loss": 3.3865843242985765, + "tokens_seen": 2338914304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962626262626263, + "loss": 2.5673, + "theoretical_loss": 3.3865766539242244, + "tokens_seen": 2338979840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029624242424242423, + "loss": 2.3985, + "theoretical_loss": 3.3865689838249606, + "tokens_seen": 2339045376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029622222222222224, + "loss": 2.7088, + "theoretical_loss": 3.3865613140007675, + "tokens_seen": 2339110912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962020202020202, + "loss": 2.6563, + "theoretical_loss": 3.3865536444516273, + "tokens_seen": 2339176448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029618181818181816, + "loss": 2.8682, + "theoretical_loss": 3.386545975177523, + "tokens_seen": 2339241984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029616161616161617, + "loss": 2.385, + "theoretical_loss": 3.386538306178437, + "tokens_seen": 2339307520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961414141414142, + "loss": 2.4802, + "theoretical_loss": 3.386530637454351, + "tokens_seen": 2339373056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029612121212121214, + "loss": 2.764, + "theoretical_loss": 3.3865229690052487, + "tokens_seen": 2339438592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961010101010101, + "loss": 2.6332, + "theoretical_loss": 3.3865153008311113, + "tokens_seen": 2339504128 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029608080808080805, + "loss": 2.3376, + "theoretical_loss": 3.3865076329319215, + "tokens_seen": 2339569664 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029606060606060606, + "loss": 2.5313, + "theoretical_loss": 3.386499965307662, + "tokens_seen": 2339635200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029604040404040407, + "loss": 2.5455, + "theoretical_loss": 3.3864922979583154, + "tokens_seen": 2339700736 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029602020202020203, + "loss": 2.4528, + "theoretical_loss": 3.386484630883864, + "tokens_seen": 2339766272 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296, + "loss": 2.5559, + "theoretical_loss": 3.3864769640842898, + "tokens_seen": 2339831808 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1290520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1252095699310303, + "objective/train/theoretical_loss": 3.386469297559576, + "objective/train/tokens_used": 698756576, + "theoretical_loss": 3.386469297559576, + "tokens_seen": 2339897344 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029597979797979794, + "loss": 2.5348, + "theoretical_loss": 3.386469297559576, + "tokens_seen": 2339897344 + }, + { + "epoch": 0.41, + "learning_rate": 0.000295959595959596, + "loss": 2.4725, + "theoretical_loss": 3.3864616313097047, + "tokens_seen": 2339962880 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029593939393939396, + "loss": 2.5402, + "theoretical_loss": 3.3864539653346584, + "tokens_seen": 2340028416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959191919191919, + "loss": 2.617, + "theoretical_loss": 3.386446299634419, + "tokens_seen": 2340093952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958989898989899, + "loss": 2.4474, + "theoretical_loss": 3.38643863420897, + "tokens_seen": 2340159488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029587878787878783, + "loss": 2.614, + "theoretical_loss": 3.386430969058293, + "tokens_seen": 2340225024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958585858585859, + "loss": 2.5714, + "theoretical_loss": 3.3864233041823706, + "tokens_seen": 2340290560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029583838383838386, + "loss": 2.5186, + "theoretical_loss": 3.3864156395811853, + "tokens_seen": 2340356096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958181818181818, + "loss": 2.6369, + "theoretical_loss": 3.38640797525472, + "tokens_seen": 2340421632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029579797979797977, + "loss": 2.3658, + "theoretical_loss": 3.386400311202957, + "tokens_seen": 2340487168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957777777777778, + "loss": 2.6491, + "theoretical_loss": 3.386392647425878, + "tokens_seen": 2340552704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957575757575758, + "loss": 2.6185, + "theoretical_loss": 3.3863849839234663, + "tokens_seen": 2340618240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029573737373737375, + "loss": 2.4718, + "theoretical_loss": 3.386377320695704, + "tokens_seen": 2340683776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957171717171717, + "loss": 2.4129, + "theoretical_loss": 3.386369657742574, + "tokens_seen": 2340749312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029569696969696966, + "loss": 2.5709, + "theoretical_loss": 3.386361995064058, + "tokens_seen": 2340814848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029567676767676773, + "loss": 2.7302, + "theoretical_loss": 3.3863543326601393, + "tokens_seen": 2340880384 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956565656565657, + "loss": 2.8111, + "theoretical_loss": 3.3863466705308, + "tokens_seen": 2340945920 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029563636363636364, + "loss": 2.5205, + "theoretical_loss": 3.386339008676022, + "tokens_seen": 2341011456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956161616161616, + "loss": 2.6271, + "theoretical_loss": 3.386331347095789, + "tokens_seen": 2341076992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955959595959596, + "loss": 2.5173, + "theoretical_loss": 3.3863236857900825, + "tokens_seen": 2341142528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955757575757576, + "loss": 2.766, + "theoretical_loss": 3.3863160247588855, + "tokens_seen": 2341208064 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955555555555556, + "loss": 2.6761, + "theoretical_loss": 3.38630836400218, + "tokens_seen": 2341273600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029553535353535353, + "loss": 2.6146, + "theoretical_loss": 3.3863007035199493, + "tokens_seen": 2341339136 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955151515151515, + "loss": 2.7249, + "theoretical_loss": 3.386293043312175, + "tokens_seen": 2341404672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002954949494949495, + "loss": 2.5817, + "theoretical_loss": 3.3862853833788398, + "tokens_seen": 2341470208 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1291680, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.936016082763672, + "objective/train/theoretical_loss": 3.3862777237199264, + "objective/train/tokens_used": 700394976, + "theoretical_loss": 3.3862777237199264, + "tokens_seen": 2341535744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002954747474747475, + "loss": 2.749, + "theoretical_loss": 3.3862777237199264, + "tokens_seen": 2341535744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029545454545454547, + "loss": 2.4722, + "theoretical_loss": 3.386270064335417, + "tokens_seen": 2341601280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029543434343434343, + "loss": 2.6946, + "theoretical_loss": 3.3862624052252945, + "tokens_seen": 2341666816 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029541414141414144, + "loss": 2.4421, + "theoretical_loss": 3.3862547463895414, + "tokens_seen": 2341732352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953939393939394, + "loss": 2.3572, + "theoretical_loss": 3.38624708782814, + "tokens_seen": 2341797888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953737373737374, + "loss": 2.7433, + "theoretical_loss": 3.3862394295410723, + "tokens_seen": 2341863424 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029535353535353536, + "loss": 2.5928, + "theoretical_loss": 3.3862317715283217, + "tokens_seen": 2341928960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953333333333333, + "loss": 2.4816, + "theoretical_loss": 3.38622411378987, + "tokens_seen": 2341994496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029531313131313133, + "loss": 2.6014, + "theoretical_loss": 3.3862164563257, + "tokens_seen": 2342060032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952929292929293, + "loss": 2.7071, + "theoretical_loss": 3.3862087991357943, + "tokens_seen": 2342125568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952727272727273, + "loss": 2.5742, + "theoretical_loss": 3.386201142220135, + "tokens_seen": 2342191104 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029525252525252526, + "loss": 2.4848, + "theoretical_loss": 3.3861934855787053, + "tokens_seen": 2342256640 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029523232323232327, + "loss": 2.3532, + "theoretical_loss": 3.386185829211487, + "tokens_seen": 2342322176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952121212121212, + "loss": 2.6362, + "theoretical_loss": 3.386178173118463, + "tokens_seen": 2342387712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951919191919192, + "loss": 2.875, + "theoretical_loss": 3.3861705172996155, + "tokens_seen": 2342453248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951717171717172, + "loss": 2.305, + "theoretical_loss": 3.3861628617549275, + "tokens_seen": 2342518784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029515151515151515, + "loss": 2.5675, + "theoretical_loss": 3.386155206484381, + "tokens_seen": 2342584320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029513131313131316, + "loss": 2.4903, + "theoretical_loss": 3.3861475514879587, + "tokens_seen": 2342649856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951111111111111, + "loss": 2.7167, + "theoretical_loss": 3.3861398967656435, + "tokens_seen": 2342715392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950909090909091, + "loss": 2.6, + "theoretical_loss": 3.386132242317417, + "tokens_seen": 2342780928 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950707070707071, + "loss": 2.5007, + "theoretical_loss": 3.3861245881432627, + "tokens_seen": 2342846464 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950505050505051, + "loss": 2.6889, + "theoretical_loss": 3.3861169342431627, + "tokens_seen": 2342912000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029503030303030305, + "loss": 2.5251, + "theoretical_loss": 3.3861092806170996, + "tokens_seen": 2342977536 + }, + { + "epoch": 0.42, + "learning_rate": 0.000295010101010101, + "loss": 2.6591, + "theoretical_loss": 3.386101627265056, + "tokens_seen": 2343043072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029498989898989897, + "loss": 2.8258, + "theoretical_loss": 3.3860939741870135, + "tokens_seen": 2343108608 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1292123, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4943063259124756, + "objective/train/theoretical_loss": 3.386086321382956, + "objective/train/tokens_used": 702033376, + "theoretical_loss": 3.386086321382956, + "tokens_seen": 2343174144 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294969696969697, + "loss": 2.5336, + "theoretical_loss": 3.386086321382956, + "tokens_seen": 2343174144 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294949494949495, + "loss": 2.6416, + "theoretical_loss": 3.3860786688528655, + "tokens_seen": 2343239680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029492929292929295, + "loss": 2.9048, + "theoretical_loss": 3.3860710165967243, + "tokens_seen": 2343305216 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949090909090909, + "loss": 2.4854, + "theoretical_loss": 3.386063364614515, + "tokens_seen": 2343370752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029488888888888886, + "loss": 2.59, + "theoretical_loss": 3.3860557129062205, + "tokens_seen": 2343436288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948686868686869, + "loss": 2.6525, + "theoretical_loss": 3.386048061471823, + "tokens_seen": 2343501824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948484848484849, + "loss": 2.88, + "theoretical_loss": 3.386040410311305, + "tokens_seen": 2343567360 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029482828282828284, + "loss": 2.6506, + "theoretical_loss": 3.386032759424649, + "tokens_seen": 2343632896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948080808080808, + "loss": 2.4685, + "theoretical_loss": 3.386025108811838, + "tokens_seen": 2343698432 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029478787878787875, + "loss": 2.5132, + "theoretical_loss": 3.386017458472854, + "tokens_seen": 2343763968 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947676767676768, + "loss": 2.7642, + "theoretical_loss": 3.38600980840768, + "tokens_seen": 2343829504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947474747474748, + "loss": 2.7564, + "theoretical_loss": 3.386002158616298, + "tokens_seen": 2343895040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029472727272727273, + "loss": 2.5789, + "theoretical_loss": 3.385994509098691, + "tokens_seen": 2343960576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947070707070707, + "loss": 2.6892, + "theoretical_loss": 3.3859868598548415, + "tokens_seen": 2344026112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029468686868686864, + "loss": 2.6213, + "theoretical_loss": 3.3859792108847317, + "tokens_seen": 2344091648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946666666666667, + "loss": 2.693, + "theoretical_loss": 3.3859715621883444, + "tokens_seen": 2344157184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029464646464646467, + "loss": 2.7018, + "theoretical_loss": 3.3859639137656625, + "tokens_seen": 2344222720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946262626262626, + "loss": 2.5896, + "theoretical_loss": 3.385956265616668, + "tokens_seen": 2344288256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946060606060606, + "loss": 2.4672, + "theoretical_loss": 3.385948617741344, + "tokens_seen": 2344353792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945858585858586, + "loss": 2.7143, + "theoretical_loss": 3.3859409701396723, + "tokens_seen": 2344419328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945656565656566, + "loss": 2.5054, + "theoretical_loss": 3.3859333228116357, + "tokens_seen": 2344484864 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029454545454545456, + "loss": 2.6772, + "theoretical_loss": 3.3859256757572176, + "tokens_seen": 2344550400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945252525252525, + "loss": 2.7907, + "theoretical_loss": 3.3859180289763993, + "tokens_seen": 2344615936 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029450505050505047, + "loss": 2.5474, + "theoretical_loss": 3.3859103824691643, + "tokens_seen": 2344681472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944848484848485, + "loss": 2.6514, + "theoretical_loss": 3.385902736235495, + "tokens_seen": 2344747008 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1293031, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.416686534881592, + "objective/train/theoretical_loss": 3.3858950902753735, + "objective/train/tokens_used": 703671776, + "theoretical_loss": 3.3858950902753735, + "tokens_seen": 2344812544 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944646464646465, + "loss": 2.6185, + "theoretical_loss": 3.3858950902753735, + "tokens_seen": 2344812544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029444444444444445, + "loss": 2.6763, + "theoretical_loss": 3.3858874445887825, + "tokens_seen": 2344878080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944242424242424, + "loss": 2.4738, + "theoretical_loss": 3.3858797991757053, + "tokens_seen": 2344943616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944040404040404, + "loss": 2.6323, + "theoretical_loss": 3.3858721540361234, + "tokens_seen": 2345009152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943838383838384, + "loss": 2.6377, + "theoretical_loss": 3.38586450917002, + "tokens_seen": 2345074688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943636363636364, + "loss": 2.4647, + "theoretical_loss": 3.3858568645773777, + "tokens_seen": 2345140224 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029434343434343434, + "loss": 2.5484, + "theoretical_loss": 3.385849220258179, + "tokens_seen": 2345205760 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943232323232323, + "loss": 2.6288, + "theoretical_loss": 3.3858415762124063, + "tokens_seen": 2345271296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943030303030303, + "loss": 2.5396, + "theoretical_loss": 3.3858339324400424, + "tokens_seen": 2345336832 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029428282828282827, + "loss": 2.4864, + "theoretical_loss": 3.3858262889410695, + "tokens_seen": 2345402368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942626262626263, + "loss": 2.5486, + "theoretical_loss": 3.3858186457154704, + "tokens_seen": 2345467904 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029424242424242424, + "loss": 2.6082, + "theoretical_loss": 3.385811002763228, + "tokens_seen": 2345533440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029422222222222225, + "loss": 2.3866, + "theoretical_loss": 3.3858033600843247, + "tokens_seen": 2345598976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942020202020202, + "loss": 2.429, + "theoretical_loss": 3.385795717678743, + "tokens_seen": 2345664512 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029418181818181816, + "loss": 2.7159, + "theoretical_loss": 3.3857880755464658, + "tokens_seen": 2345730048 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941616161616162, + "loss": 2.7147, + "theoretical_loss": 3.385780433687475, + "tokens_seen": 2345795584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029414141414141413, + "loss": 2.5166, + "theoretical_loss": 3.3857727921017533, + "tokens_seen": 2345861120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029412121212121214, + "loss": 2.3101, + "theoretical_loss": 3.385765150789284, + "tokens_seen": 2345926656 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941010101010101, + "loss": 2.6126, + "theoretical_loss": 3.385757509750049, + "tokens_seen": 2345992192 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029408080808080805, + "loss": 2.3296, + "theoretical_loss": 3.3857498689840315, + "tokens_seen": 2346057728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029406060606060607, + "loss": 2.4155, + "theoretical_loss": 3.385742228491214, + "tokens_seen": 2346123264 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940404040404041, + "loss": 2.655, + "theoretical_loss": 3.3857345882715784, + "tokens_seen": 2346188800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029402020202020203, + "loss": 2.5982, + "theoretical_loss": 3.385726948325108, + "tokens_seen": 2346254336 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294, + "loss": 2.7136, + "theoretical_loss": 3.385719308651785, + "tokens_seen": 2346319872 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029397979797979795, + "loss": 2.3272, + "theoretical_loss": 3.3857116692515925, + "tokens_seen": 2346385408 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1293973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.691797971725464, + "objective/train/theoretical_loss": 3.3857040301245123, + "objective/train/tokens_used": 705310176, + "theoretical_loss": 3.3857040301245123, + "tokens_seen": 2346450944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029395959595959596, + "loss": 2.5661, + "theoretical_loss": 3.3857040301245123, + "tokens_seen": 2346450944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029393939393939397, + "loss": 2.5383, + "theoretical_loss": 3.385696391270528, + "tokens_seen": 2346516480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939191919191919, + "loss": 2.3794, + "theoretical_loss": 3.3856887526896213, + "tokens_seen": 2346582016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938989898989899, + "loss": 2.6422, + "theoretical_loss": 3.3856811143817755, + "tokens_seen": 2346647552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938787878787879, + "loss": 2.5329, + "theoretical_loss": 3.3856734763469727, + "tokens_seen": 2346713088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938585858585859, + "loss": 2.5702, + "theoretical_loss": 3.385665838585196, + "tokens_seen": 2346778624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029383838383838386, + "loss": 2.3762, + "theoretical_loss": 3.385658201096428, + "tokens_seen": 2346844160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938181818181818, + "loss": 2.4848, + "theoretical_loss": 3.3856505638806507, + "tokens_seen": 2346909696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937979797979798, + "loss": 2.4811, + "theoretical_loss": 3.385642926937847, + "tokens_seen": 2346975232 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937777777777778, + "loss": 2.5428, + "theoretical_loss": 3.3856352902679996, + "tokens_seen": 2347040768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937575757575758, + "loss": 2.5246, + "theoretical_loss": 3.385627653871092, + "tokens_seen": 2347106304 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029373737373737375, + "loss": 2.4987, + "theoretical_loss": 3.385620017747105, + "tokens_seen": 2347171840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937171717171717, + "loss": 2.4716, + "theoretical_loss": 3.3856123818960224, + "tokens_seen": 2347237376 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029369696969696967, + "loss": 2.6963, + "theoretical_loss": 3.385604746317827, + "tokens_seen": 2347302912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029367676767676773, + "loss": 2.5826, + "theoretical_loss": 3.3855971110125007, + "tokens_seen": 2347368448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936565656565657, + "loss": 2.5608, + "theoretical_loss": 3.3855894759800265, + "tokens_seen": 2347433984 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029363636363636365, + "loss": 2.7137, + "theoretical_loss": 3.3855818412203873, + "tokens_seen": 2347499520 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936161616161616, + "loss": 2.3285, + "theoretical_loss": 3.3855742067335655, + "tokens_seen": 2347565056 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029359595959595956, + "loss": 2.7452, + "theoretical_loss": 3.3855665725195436, + "tokens_seen": 2347630592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935757575757576, + "loss": 2.5853, + "theoretical_loss": 3.385558938578304, + "tokens_seen": 2347696128 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935555555555556, + "loss": 2.5034, + "theoretical_loss": 3.3855513049098303, + "tokens_seen": 2347761664 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029353535353535354, + "loss": 2.543, + "theoretical_loss": 3.385543671514104, + "tokens_seen": 2347827200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935151515151515, + "loss": 2.3646, + "theoretical_loss": 3.385536038391108, + "tokens_seen": 2347892736 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934949494949495, + "loss": 2.4485, + "theoretical_loss": 3.3855284055408257, + "tokens_seen": 2347958272 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934747474747475, + "loss": 2.7168, + "theoretical_loss": 3.3855207729632393, + "tokens_seen": 2348023808 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1294468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1802330017089844, + "objective/train/theoretical_loss": 3.385513140658331, + "objective/train/tokens_used": 706948576, + "theoretical_loss": 3.385513140658331, + "tokens_seen": 2348089344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934545454545455, + "loss": 2.588, + "theoretical_loss": 3.385513140658331, + "tokens_seen": 2348089344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029343434343434343, + "loss": 2.6582, + "theoretical_loss": 3.385505508626084, + "tokens_seen": 2348154880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934141414141414, + "loss": 2.5386, + "theoretical_loss": 3.385497876866481, + "tokens_seen": 2348220416 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933939393939394, + "loss": 2.5273, + "theoretical_loss": 3.385490245379504, + "tokens_seen": 2348285952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933737373737374, + "loss": 2.475, + "theoretical_loss": 3.3854826141651366, + "tokens_seen": 2348351488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029335353535353537, + "loss": 2.6544, + "theoretical_loss": 3.385474983223361, + "tokens_seen": 2348417024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933333333333333, + "loss": 2.6035, + "theoretical_loss": 3.385467352554159, + "tokens_seen": 2348482560 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933131313131313, + "loss": 2.782, + "theoretical_loss": 3.3854597221575147, + "tokens_seen": 2348548096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932929292929293, + "loss": 2.4634, + "theoretical_loss": 3.38545209203341, + "tokens_seen": 2348613632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932727272727273, + "loss": 2.6999, + "theoretical_loss": 3.3854444621818276, + "tokens_seen": 2348679168 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029325252525252526, + "loss": 2.557, + "theoretical_loss": 3.38543683260275, + "tokens_seen": 2348744704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932323232323232, + "loss": 2.5582, + "theoretical_loss": 3.3854292032961606, + "tokens_seen": 2348810240 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029321212121212123, + "loss": 2.3718, + "theoretical_loss": 3.3854215742620415, + "tokens_seen": 2348875776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931919191919192, + "loss": 2.6344, + "theoretical_loss": 3.385413945500375, + "tokens_seen": 2348941312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931717171717172, + "loss": 2.8614, + "theoretical_loss": 3.3854063170111446, + "tokens_seen": 2349006848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029315151515151515, + "loss": 2.6338, + "theoretical_loss": 3.3853986887943326, + "tokens_seen": 2349072384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931313131313131, + "loss": 2.6859, + "theoretical_loss": 3.3853910608499214, + "tokens_seen": 2349137920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931111111111111, + "loss": 2.6705, + "theoretical_loss": 3.385383433177894, + "tokens_seen": 2349203456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930909090909091, + "loss": 2.5417, + "theoretical_loss": 3.3853758057782333, + "tokens_seen": 2349268992 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930707070707071, + "loss": 2.5322, + "theoretical_loss": 3.3853681786509213, + "tokens_seen": 2349334528 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029305050505050505, + "loss": 2.6303, + "theoretical_loss": 3.385360551795941, + "tokens_seen": 2349400064 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029303030303030306, + "loss": 2.734, + "theoretical_loss": 3.3853529252132755, + "tokens_seen": 2349465600 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293010101010101, + "loss": 2.5497, + "theoretical_loss": 3.385345298902907, + "tokens_seen": 2349531136 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029298989898989897, + "loss": 2.5689, + "theoretical_loss": 3.385337672864818, + "tokens_seen": 2349596672 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292969696969697, + "loss": 2.6474, + "theoretical_loss": 3.385330047098992, + "tokens_seen": 2349662208 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1295772, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4741108417510986, + "objective/train/theoretical_loss": 3.3853224216054105, + "objective/train/tokens_used": 708586976, + "theoretical_loss": 3.3853224216054105, + "tokens_seen": 2349727744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029294949494949494, + "loss": 2.5149, + "theoretical_loss": 3.3853224216054105, + "tokens_seen": 2349727744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029292929292929295, + "loss": 2.544, + "theoretical_loss": 3.3853147963840575, + "tokens_seen": 2349793280 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929090909090909, + "loss": 2.6175, + "theoretical_loss": 3.3853071714349143, + "tokens_seen": 2349858816 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029288888888888886, + "loss": 2.7578, + "theoretical_loss": 3.385299546757965, + "tokens_seen": 2349924352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928686868686869, + "loss": 2.6745, + "theoretical_loss": 3.3852919223531917, + "tokens_seen": 2349989888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928484848484849, + "loss": 2.3819, + "theoretical_loss": 3.3852842982205766, + "tokens_seen": 2350055424 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029282828282828284, + "loss": 2.3935, + "theoretical_loss": 3.385276674360103, + "tokens_seen": 2350120960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928080808080808, + "loss": 2.5262, + "theoretical_loss": 3.385269050771753, + "tokens_seen": 2350186496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029278787878787876, + "loss": 2.5877, + "theoretical_loss": 3.3852614274555104, + "tokens_seen": 2350252032 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029276767676767677, + "loss": 2.5712, + "theoretical_loss": 3.3852538044113567, + "tokens_seen": 2350317568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927474747474748, + "loss": 2.8019, + "theoretical_loss": 3.3852461816392756, + "tokens_seen": 2350383104 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029272727272727274, + "loss": 2.6033, + "theoretical_loss": 3.385238559139249, + "tokens_seen": 2350448640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927070707070707, + "loss": 2.5216, + "theoretical_loss": 3.3852309369112596, + "tokens_seen": 2350514176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029268686868686865, + "loss": 2.5175, + "theoretical_loss": 3.385223314955291, + "tokens_seen": 2350579712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926666666666667, + "loss": 2.4967, + "theoretical_loss": 3.3852156932713253, + "tokens_seen": 2350645248 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029264646464646467, + "loss": 2.5049, + "theoretical_loss": 3.385208071859345, + "tokens_seen": 2350710784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029262626262626263, + "loss": 2.6128, + "theoretical_loss": 3.385200450719333, + "tokens_seen": 2350776320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926060606060606, + "loss": 2.784, + "theoretical_loss": 3.3851928298512726, + "tokens_seen": 2350841856 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029258585858585854, + "loss": 2.6949, + "theoretical_loss": 3.3851852092551455, + "tokens_seen": 2350907392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925656565656566, + "loss": 2.6908, + "theoretical_loss": 3.3851775889309352, + "tokens_seen": 2350972928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029254545454545456, + "loss": 2.6029, + "theoretical_loss": 3.385169968878624, + "tokens_seen": 2351038464 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925252525252525, + "loss": 2.6881, + "theoretical_loss": 3.385162349098195, + "tokens_seen": 2351104000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925050505050505, + "loss": 2.5663, + "theoretical_loss": 3.38515472958963, + "tokens_seen": 2351169536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924848484848485, + "loss": 2.5862, + "theoretical_loss": 3.385147110352913, + "tokens_seen": 2351235072 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924646464646465, + "loss": 2.7534, + "theoretical_loss": 3.385139491388026, + "tokens_seen": 2351300608 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1296037, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.399495840072632, + "objective/train/theoretical_loss": 3.3851318726949513, + "objective/train/tokens_used": 710225376, + "theoretical_loss": 3.3851318726949513, + "tokens_seen": 2351366144 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029244444444444446, + "loss": 2.6963, + "theoretical_loss": 3.3851318726949513, + "tokens_seen": 2351366144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924242424242424, + "loss": 2.66, + "theoretical_loss": 3.385124254273673, + "tokens_seen": 2351431680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029240404040404037, + "loss": 2.7219, + "theoretical_loss": 3.3851166361241725, + "tokens_seen": 2351497216 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923838383838384, + "loss": 2.7103, + "theoretical_loss": 3.385109018246433, + "tokens_seen": 2351562752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923636363636364, + "loss": 2.6919, + "theoretical_loss": 3.385101400640438, + "tokens_seen": 2351628288 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029234343434343435, + "loss": 2.6424, + "theoretical_loss": 3.3850937833061687, + "tokens_seen": 2351693824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923232323232323, + "loss": 2.7473, + "theoretical_loss": 3.3850861662436085, + "tokens_seen": 2351759360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923030303030303, + "loss": 2.3818, + "theoretical_loss": 3.3850785494527407, + "tokens_seen": 2351824896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922828282828283, + "loss": 2.4927, + "theoretical_loss": 3.3850709329335475, + "tokens_seen": 2351890432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922626262626263, + "loss": 2.5495, + "theoretical_loss": 3.385063316686012, + "tokens_seen": 2351955968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029224242424242424, + "loss": 2.6525, + "theoretical_loss": 3.385055700710116, + "tokens_seen": 2352021504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922222222222222, + "loss": 2.6369, + "theoretical_loss": 3.3850480850058435, + "tokens_seen": 2352087040 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922020202020202, + "loss": 2.8024, + "theoretical_loss": 3.3850404695731764, + "tokens_seen": 2352152576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921818181818182, + "loss": 2.5016, + "theoretical_loss": 3.3850328544120982, + "tokens_seen": 2352218112 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921616161616162, + "loss": 2.933, + "theoretical_loss": 3.385025239522591, + "tokens_seen": 2352283648 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029214141414141414, + "loss": 2.5866, + "theoretical_loss": 3.385017624904637, + "tokens_seen": 2352349184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029212121212121215, + "loss": 2.5993, + "theoretical_loss": 3.3850100105582204, + "tokens_seen": 2352414720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921010101010101, + "loss": 2.7311, + "theoretical_loss": 3.3850023964833227, + "tokens_seen": 2352480256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920808080808081, + "loss": 2.6685, + "theoretical_loss": 3.384994782679928, + "tokens_seen": 2352545792 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029206060606060607, + "loss": 2.787, + "theoretical_loss": 3.3849871691480176, + "tokens_seen": 2352611328 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029204040404040403, + "loss": 2.5299, + "theoretical_loss": 3.384979555887575, + "tokens_seen": 2352676864 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029202020202020204, + "loss": 2.7781, + "theoretical_loss": 3.384971942898583, + "tokens_seen": 2352742400 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292, + "loss": 2.4366, + "theoretical_loss": 3.384964330181024, + "tokens_seen": 2352807936 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291979797979798, + "loss": 2.5819, + "theoretical_loss": 3.3849567177348807, + "tokens_seen": 2352873472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029195959595959596, + "loss": 2.4894, + "theoretical_loss": 3.3849491055601364, + "tokens_seen": 2352939008 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.863433837890625, + "objective/train/theoretical_loss": 3.3849414936567737, + "objective/train/tokens_used": 711863776, + "theoretical_loss": 3.3849414936567737, + "tokens_seen": 2353004544 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291939393939394, + "loss": 2.4362, + "theoretical_loss": 3.3849414936567737, + "tokens_seen": 2353004544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029191919191919193, + "loss": 2.6936, + "theoretical_loss": 3.3849338820247756, + "tokens_seen": 2353070080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918989898989899, + "loss": 2.624, + "theoretical_loss": 3.384926270664124, + "tokens_seen": 2353135616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918787878787879, + "loss": 2.7503, + "theoretical_loss": 3.3849186595748026, + "tokens_seen": 2353201152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029185858585858586, + "loss": 2.6667, + "theoretical_loss": 3.3849110487567935, + "tokens_seen": 2353266688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029183838383838387, + "loss": 2.565, + "theoretical_loss": 3.3849034382100798, + "tokens_seen": 2353332224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918181818181818, + "loss": 2.5301, + "theoretical_loss": 3.384895827934644, + "tokens_seen": 2353397760 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917979797979798, + "loss": 2.7628, + "theoretical_loss": 3.3848882179304693, + "tokens_seen": 2353463296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917777777777778, + "loss": 2.7541, + "theoretical_loss": 3.3848806081975384, + "tokens_seen": 2353528832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917575757575758, + "loss": 2.7078, + "theoretical_loss": 3.384872998735834, + "tokens_seen": 2353594368 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029173737373737376, + "loss": 2.6661, + "theoretical_loss": 3.3848653895453387, + "tokens_seen": 2353659904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917171717171717, + "loss": 2.5376, + "theoretical_loss": 3.3848577806260356, + "tokens_seen": 2353725440 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916969696969697, + "loss": 2.6752, + "theoretical_loss": 3.384850171977907, + "tokens_seen": 2353790976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916767676767677, + "loss": 2.4622, + "theoretical_loss": 3.3848425636009365, + "tokens_seen": 2353856512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916565656565657, + "loss": 2.4389, + "theoretical_loss": 3.384834955495106, + "tokens_seen": 2353922048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029163636363636365, + "loss": 2.777, + "theoretical_loss": 3.3848273476603987, + "tokens_seen": 2353987584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916161616161616, + "loss": 2.8391, + "theoretical_loss": 3.3848197400967974, + "tokens_seen": 2354053120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029159595959595957, + "loss": 2.8029, + "theoretical_loss": 3.3848121328042846, + "tokens_seen": 2354118656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029157575757575763, + "loss": 2.8413, + "theoretical_loss": 3.3848045257828434, + "tokens_seen": 2354184192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915555555555556, + "loss": 2.7559, + "theoretical_loss": 3.384796919032457, + "tokens_seen": 2354249728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029153535353535355, + "loss": 2.6729, + "theoretical_loss": 3.384789312553107, + "tokens_seen": 2354315264 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915151515151515, + "loss": 2.6606, + "theoretical_loss": 3.3847817063447776, + "tokens_seen": 2354380800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029149494949494946, + "loss": 2.5145, + "theoretical_loss": 3.3847741004074505, + "tokens_seen": 2354446336 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914747474747475, + "loss": 2.667, + "theoretical_loss": 3.3847664947411094, + "tokens_seen": 2354511872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914545454545455, + "loss": 2.6851, + "theoretical_loss": 3.3847588893457363, + "tokens_seen": 2354577408 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.831690549850464, + "objective/train/theoretical_loss": 3.3847512842213145, + "objective/train/tokens_used": 713502176, + "theoretical_loss": 3.3847512842213145, + "tokens_seen": 2354642944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029143434343434344, + "loss": 2.9613, + "theoretical_loss": 3.3847512842213145, + "tokens_seen": 2354642944 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914141414141414, + "loss": 2.83, + "theoretical_loss": 3.3847436793678263, + "tokens_seen": 2354708480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029139393939393935, + "loss": 2.8386, + "theoretical_loss": 3.384736074785255, + "tokens_seen": 2354774016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913737373737374, + "loss": 2.7046, + "theoretical_loss": 3.3847284704735836, + "tokens_seen": 2354839552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913535353535354, + "loss": 2.8229, + "theoretical_loss": 3.384720866432794, + "tokens_seen": 2354905088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029133333333333333, + "loss": 2.6687, + "theoretical_loss": 3.38471326266287, + "tokens_seen": 2354970624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913131313131313, + "loss": 2.781, + "theoretical_loss": 3.384705659163794, + "tokens_seen": 2355036160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912929292929293, + "loss": 2.7317, + "theoretical_loss": 3.384698055935549, + "tokens_seen": 2355101696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912727272727273, + "loss": 2.7084, + "theoretical_loss": 3.384690452978117, + "tokens_seen": 2355167232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029125252525252527, + "loss": 2.6948, + "theoretical_loss": 3.3846828502914823, + "tokens_seen": 2355232768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912323232323232, + "loss": 2.7676, + "theoretical_loss": 3.3846752478756263, + "tokens_seen": 2355298304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912121212121212, + "loss": 2.6817, + "theoretical_loss": 3.3846676457305325, + "tokens_seen": 2355363840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911919191919192, + "loss": 2.6406, + "theoretical_loss": 3.384660043856184, + "tokens_seen": 2355429376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911717171717172, + "loss": 2.6755, + "theoretical_loss": 3.3846524422525626, + "tokens_seen": 2355494912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029115151515151516, + "loss": 2.5943, + "theoretical_loss": 3.384644840919652, + "tokens_seen": 2355560448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911313131313131, + "loss": 2.7349, + "theoretical_loss": 3.384637239857435, + "tokens_seen": 2355625984 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029111111111111113, + "loss": 2.8644, + "theoretical_loss": 3.384629639065894, + "tokens_seen": 2355691520 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910909090909091, + "loss": 2.6618, + "theoretical_loss": 3.384622038545012, + "tokens_seen": 2355757056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910707070707071, + "loss": 2.7833, + "theoretical_loss": 3.384614438294772, + "tokens_seen": 2355822592 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029105050505050505, + "loss": 2.6828, + "theoretical_loss": 3.384606838315157, + "tokens_seen": 2355888128 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291030303030303, + "loss": 2.8611, + "theoretical_loss": 3.384599238606149, + "tokens_seen": 2355953664 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291010101010101, + "loss": 2.6833, + "theoretical_loss": 3.384591639167732, + "tokens_seen": 2356019200 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290989898989899, + "loss": 2.7436, + "theoretical_loss": 3.384584039999888, + "tokens_seen": 2356084736 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290969696969697, + "loss": 2.6357, + "theoretical_loss": 3.3845764411026, + "tokens_seen": 2356150272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029094949494949495, + "loss": 2.7971, + "theoretical_loss": 3.384568842475851, + "tokens_seen": 2356215808 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7116146087646484, + "objective/train/theoretical_loss": 3.384561244119624, + "objective/train/tokens_used": 715140576, + "theoretical_loss": 3.384561244119624, + "tokens_seen": 2356281344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029092929292929296, + "loss": 2.6878, + "theoretical_loss": 3.384561244119624, + "tokens_seen": 2356281344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909090909090909, + "loss": 2.7204, + "theoretical_loss": 3.3845536460339014, + "tokens_seen": 2356346880 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029088888888888887, + "loss": 2.9062, + "theoretical_loss": 3.384546048218666, + "tokens_seen": 2356412416 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908686868686869, + "loss": 2.7743, + "theoretical_loss": 3.384538450673901, + "tokens_seen": 2356477952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029084848484848484, + "loss": 2.8154, + "theoretical_loss": 3.3845308533995895, + "tokens_seen": 2356543488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029082828282828285, + "loss": 2.828, + "theoretical_loss": 3.3845232563957137, + "tokens_seen": 2356609024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908080808080808, + "loss": 2.8674, + "theoretical_loss": 3.3845156596622568, + "tokens_seen": 2356674560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029078787878787876, + "loss": 2.9225, + "theoretical_loss": 3.3845080631992017, + "tokens_seen": 2356740096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907676767676768, + "loss": 2.7783, + "theoretical_loss": 3.3845004670065313, + "tokens_seen": 2356805632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907474747474748, + "loss": 2.7414, + "theoretical_loss": 3.384492871084228, + "tokens_seen": 2356871168 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029072727272727274, + "loss": 2.7643, + "theoretical_loss": 3.3844852754322754, + "tokens_seen": 2356936704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907070707070707, + "loss": 2.8731, + "theoretical_loss": 3.3844776800506557, + "tokens_seen": 2357002240 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029068686868686866, + "loss": 2.4969, + "theoretical_loss": 3.384470084939352, + "tokens_seen": 2357067776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029066666666666667, + "loss": 2.6736, + "theoretical_loss": 3.3844624900983473, + "tokens_seen": 2357133312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906464646464647, + "loss": 2.802, + "theoretical_loss": 3.3844548955276244, + "tokens_seen": 2357198848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029062626262626263, + "loss": 2.6056, + "theoretical_loss": 3.3844473012271656, + "tokens_seen": 2357264384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906060606060606, + "loss": 2.7787, + "theoretical_loss": 3.3844397071969547, + "tokens_seen": 2357329920 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029058585858585855, + "loss": 2.8631, + "theoretical_loss": 3.3844321134369744, + "tokens_seen": 2357395456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905656565656566, + "loss": 2.7115, + "theoretical_loss": 3.384424519947207, + "tokens_seen": 2357460992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029054545454545457, + "loss": 2.7696, + "theoretical_loss": 3.384416926727636, + "tokens_seen": 2357526528 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029052525252525253, + "loss": 2.5302, + "theoretical_loss": 3.3844093337782435, + "tokens_seen": 2357592064 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905050505050505, + "loss": 2.7376, + "theoretical_loss": 3.384401741099013, + "tokens_seen": 2357657600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904848484848485, + "loss": 2.8724, + "theoretical_loss": 3.3843941486899274, + "tokens_seen": 2357723136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904646464646465, + "loss": 2.9169, + "theoretical_loss": 3.384386556550969, + "tokens_seen": 2357788672 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029044444444444446, + "loss": 2.7314, + "theoretical_loss": 3.3843789646821216, + "tokens_seen": 2357854208 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0510120391845703, + "objective/train/theoretical_loss": 3.384371373083367, + "objective/train/tokens_used": 716778976, + "theoretical_loss": 3.384371373083367, + "tokens_seen": 2357919744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904242424242424, + "loss": 2.6395, + "theoretical_loss": 3.384371373083367, + "tokens_seen": 2357919744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002904040404040404, + "loss": 2.6134, + "theoretical_loss": 3.3843637817546894, + "tokens_seen": 2357985280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029038383838383844, + "loss": 2.5383, + "theoretical_loss": 3.3843561906960704, + "tokens_seen": 2358050816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903636363636364, + "loss": 2.6797, + "theoretical_loss": 3.3843485999074936, + "tokens_seen": 2358116352 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029034343434343436, + "loss": 2.7628, + "theoretical_loss": 3.3843410093889417, + "tokens_seen": 2358181888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903232323232323, + "loss": 2.7176, + "theoretical_loss": 3.3843334191403978, + "tokens_seen": 2358247424 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029030303030303027, + "loss": 2.4503, + "theoretical_loss": 3.384325829161844, + "tokens_seen": 2358312960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029028282828282833, + "loss": 2.7259, + "theoretical_loss": 3.3843182394532643, + "tokens_seen": 2358378496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902626262626263, + "loss": 2.8343, + "theoretical_loss": 3.384310650014641, + "tokens_seen": 2358444032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029024242424242425, + "loss": 3.0389, + "theoretical_loss": 3.3843030608459572, + "tokens_seen": 2358509568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902222222222222, + "loss": 2.7251, + "theoretical_loss": 3.3842954719471954, + "tokens_seen": 2358575104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902020202020202, + "loss": 2.8515, + "theoretical_loss": 3.384287883318339, + "tokens_seen": 2358640640 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029018181818181823, + "loss": 2.7634, + "theoretical_loss": 3.3842802949593707, + "tokens_seen": 2358706176 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901616161616162, + "loss": 2.7505, + "theoretical_loss": 3.384272706870273, + "tokens_seen": 2358771712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029014141414141414, + "loss": 3.0544, + "theoretical_loss": 3.3842651190510296, + "tokens_seen": 2358837248 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901212121212121, + "loss": 2.9072, + "theoretical_loss": 3.384257531501623, + "tokens_seen": 2358902784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901010101010101, + "loss": 2.648, + "theoretical_loss": 3.384249944222036, + "tokens_seen": 2358968320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900808080808081, + "loss": 2.7514, + "theoretical_loss": 3.3842423572122513, + "tokens_seen": 2359033856 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900606060606061, + "loss": 2.8266, + "theoretical_loss": 3.384234770472253, + "tokens_seen": 2359099392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029004040404040403, + "loss": 2.9063, + "theoretical_loss": 3.384227184002022, + "tokens_seen": 2359164928 + }, + { + "epoch": 0.43, + "learning_rate": 0.000290020202020202, + "loss": 3.0149, + "theoretical_loss": 3.3842195978015432, + "tokens_seen": 2359230464 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029, + "loss": 2.9477, + "theoretical_loss": 3.3842120118707983, + "tokens_seen": 2359296000 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289979797979798, + "loss": 2.8882, + "theoretical_loss": 3.384204426209771, + "tokens_seen": 2359361536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028995959595959597, + "loss": 2.7126, + "theoretical_loss": 3.3841968408184435, + "tokens_seen": 2359427072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899393939393939, + "loss": 2.9074, + "theoretical_loss": 3.384189255696799, + "tokens_seen": 2359492608 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9561614990234375, + "objective/train/theoretical_loss": 3.3841816708448205, + "objective/train/tokens_used": 718417376, + "theoretical_loss": 3.3841816708448205, + "tokens_seen": 2359558144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028991919191919194, + "loss": 2.8289, + "theoretical_loss": 3.3841816708448205, + "tokens_seen": 2359558144 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898989898989899, + "loss": 2.7987, + "theoretical_loss": 3.3841740862624907, + "tokens_seen": 2359623680 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898787878787879, + "loss": 2.9507, + "theoretical_loss": 3.384166501949793, + "tokens_seen": 2359689216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028985858585858586, + "loss": 2.8454, + "theoretical_loss": 3.38415891790671, + "tokens_seen": 2359754752 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898383838383838, + "loss": 2.9303, + "theoretical_loss": 3.3841513341332243, + "tokens_seen": 2359820288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028981818181818183, + "loss": 2.6474, + "theoretical_loss": 3.3841437506293195, + "tokens_seen": 2359885824 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897979797979798, + "loss": 2.8907, + "theoretical_loss": 3.384136167394978, + "tokens_seen": 2359951360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897777777777778, + "loss": 2.8254, + "theoretical_loss": 3.384128584430183, + "tokens_seen": 2360016896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028975757575757576, + "loss": 2.7634, + "theoretical_loss": 3.3841210017349175, + "tokens_seen": 2360082432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028973737373737377, + "loss": 2.5879, + "theoretical_loss": 3.3841134193091644, + "tokens_seen": 2360147968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897171717171717, + "loss": 2.8118, + "theoretical_loss": 3.3841058371529065, + "tokens_seen": 2360213504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896969696969697, + "loss": 2.7495, + "theoretical_loss": 3.3840982552661263, + "tokens_seen": 2360279040 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896767676767677, + "loss": 2.8714, + "theoretical_loss": 3.384090673648808, + "tokens_seen": 2360344576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028965656565656565, + "loss": 2.8073, + "theoretical_loss": 3.3840830923009335, + "tokens_seen": 2360410112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028963636363636366, + "loss": 2.7274, + "theoretical_loss": 3.384075511222486, + "tokens_seen": 2360475648 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896161616161616, + "loss": 2.7599, + "theoretical_loss": 3.384067930413448, + "tokens_seen": 2360541184 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028959595959595957, + "loss": 2.9951, + "theoretical_loss": 3.3840603498738036, + "tokens_seen": 2360606720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895757575757576, + "loss": 2.8374, + "theoretical_loss": 3.3840527696035343, + "tokens_seen": 2360672256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895555555555556, + "loss": 2.7211, + "theoretical_loss": 3.3840451896026242, + "tokens_seen": 2360737792 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028953535353535355, + "loss": 2.9183, + "theoretical_loss": 3.384037609871056, + "tokens_seen": 2360803328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895151515151515, + "loss": 2.7325, + "theoretical_loss": 3.3840300304088125, + "tokens_seen": 2360868864 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028949494949494947, + "loss": 2.7306, + "theoretical_loss": 3.3840224512158765, + "tokens_seen": 2360934400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894747474747475, + "loss": 2.7594, + "theoretical_loss": 3.3840148722922314, + "tokens_seen": 2360999936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894545454545455, + "loss": 2.7942, + "theoretical_loss": 3.3840072936378593, + "tokens_seen": 2361065472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028943434343434344, + "loss": 2.8832, + "theoretical_loss": 3.3839997152527443, + "tokens_seen": 2361131008 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9772825241088867, + "objective/train/theoretical_loss": 3.3839921371368686, + "objective/train/tokens_used": 720055776, + "theoretical_loss": 3.3839921371368686, + "tokens_seen": 2361196544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894141414141414, + "loss": 2.8571, + "theoretical_loss": 3.3839921371368686, + "tokens_seen": 2361196544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028939393939393936, + "loss": 2.9166, + "theoretical_loss": 3.3839845592902154, + "tokens_seen": 2361262080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893737373737374, + "loss": 2.814, + "theoretical_loss": 3.3839769817127676, + "tokens_seen": 2361327616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893535353535354, + "loss": 2.7937, + "theoretical_loss": 3.383969404404508, + "tokens_seen": 2361393152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028933333333333334, + "loss": 2.8665, + "theoretical_loss": 3.38396182736542, + "tokens_seen": 2361458688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893131313131313, + "loss": 2.7818, + "theoretical_loss": 3.3839542505954863, + "tokens_seen": 2361524224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028929292929292925, + "loss": 2.5287, + "theoretical_loss": 3.38394667409469, + "tokens_seen": 2361589760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892727272727273, + "loss": 2.6725, + "theoretical_loss": 3.3839390978630135, + "tokens_seen": 2361655296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028925252525252527, + "loss": 2.7712, + "theoretical_loss": 3.3839315219004407, + "tokens_seen": 2361720832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028923232323232323, + "loss": 2.8722, + "theoretical_loss": 3.383923946206954, + "tokens_seen": 2361786368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892121212121212, + "loss": 2.8434, + "theoretical_loss": 3.3839163707825364, + "tokens_seen": 2361851904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891919191919192, + "loss": 2.8021, + "theoretical_loss": 3.383908795627171, + "tokens_seen": 2361917440 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891717171717172, + "loss": 2.8349, + "theoretical_loss": 3.383901220740841, + "tokens_seen": 2361982976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028915151515151517, + "loss": 2.7123, + "theoretical_loss": 3.383893646123529, + "tokens_seen": 2362048512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891313131313131, + "loss": 2.9454, + "theoretical_loss": 3.383886071775218, + "tokens_seen": 2362114048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891111111111111, + "loss": 2.8406, + "theoretical_loss": 3.383878497695891, + "tokens_seen": 2362179584 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890909090909091, + "loss": 2.7326, + "theoretical_loss": 3.383870923885531, + "tokens_seen": 2362245120 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890707070707071, + "loss": 2.854, + "theoretical_loss": 3.3838633503441216, + "tokens_seen": 2362310656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028905050505050506, + "loss": 2.9163, + "theoretical_loss": 3.383855777071645, + "tokens_seen": 2362376192 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289030303030303, + "loss": 2.7203, + "theoretical_loss": 3.383848204068084, + "tokens_seen": 2362441728 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289010101010101, + "loss": 2.844, + "theoretical_loss": 3.3838406313334226, + "tokens_seen": 2362507264 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288989898989899, + "loss": 2.8456, + "theoretical_loss": 3.383833058867643, + "tokens_seen": 2362572800 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288969696969697, + "loss": 2.7197, + "theoretical_loss": 3.3838254866707285, + "tokens_seen": 2362638336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028894949494949495, + "loss": 2.9643, + "theoretical_loss": 3.3838179147426617, + "tokens_seen": 2362703872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889292929292929, + "loss": 2.722, + "theoretical_loss": 3.383810343083426, + "tokens_seen": 2362769408 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1296887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9292616844177246, + "objective/train/theoretical_loss": 3.383802771693005, + "objective/train/tokens_used": 721694176, + "theoretical_loss": 3.383802771693005, + "tokens_seen": 2362834944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889090909090909, + "loss": 2.9646, + "theoretical_loss": 3.383802771693005, + "tokens_seen": 2362834944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888888888888889, + "loss": 2.825, + "theoretical_loss": 3.3837952005713805, + "tokens_seen": 2362900480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888686868686869, + "loss": 2.8537, + "theoretical_loss": 3.383787629718536, + "tokens_seen": 2362966016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028884848484848484, + "loss": 3.0183, + "theoretical_loss": 3.3837800591344545, + "tokens_seen": 2363031552 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028882828282828285, + "loss": 2.6299, + "theoretical_loss": 3.383772488819119, + "tokens_seen": 2363097088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888080808080808, + "loss": 2.886, + "theoretical_loss": 3.3837649187725125, + "tokens_seen": 2363162624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887878787878788, + "loss": 2.7028, + "theoretical_loss": 3.383757348994618, + "tokens_seen": 2363228160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887676767676768, + "loss": 2.8194, + "theoretical_loss": 3.383749779485419, + "tokens_seen": 2363293696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028874747474747474, + "loss": 2.7882, + "theoretical_loss": 3.3837422102448977, + "tokens_seen": 2363359232 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028872727272727275, + "loss": 2.8096, + "theoretical_loss": 3.383734641273038, + "tokens_seen": 2363424768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887070707070707, + "loss": 2.756, + "theoretical_loss": 3.3837270725698216, + "tokens_seen": 2363490304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886868686868687, + "loss": 2.6432, + "theoretical_loss": 3.383719504135233, + "tokens_seen": 2363555840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028866666666666667, + "loss": 2.9209, + "theoretical_loss": 3.383711935969254, + "tokens_seen": 2363621376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886464646464647, + "loss": 2.6537, + "theoretical_loss": 3.383704368071869, + "tokens_seen": 2363686912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028862626262626264, + "loss": 2.9342, + "theoretical_loss": 3.383696800443059, + "tokens_seen": 2363752448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886060606060606, + "loss": 2.9508, + "theoretical_loss": 3.383689233082809, + "tokens_seen": 2363817984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885858585858586, + "loss": 2.9502, + "theoretical_loss": 3.3836816659911007, + "tokens_seen": 2363883520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028856565656565656, + "loss": 2.7681, + "theoretical_loss": 3.383674099167918, + "tokens_seen": 2363949056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885454545454546, + "loss": 2.8636, + "theoretical_loss": 3.3836665326132436, + "tokens_seen": 2364014592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028852525252525253, + "loss": 2.8493, + "theoretical_loss": 3.38365896632706, + "tokens_seen": 2364080128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885050505050505, + "loss": 2.6714, + "theoretical_loss": 3.3836514003093514, + "tokens_seen": 2364145664 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884848484848485, + "loss": 2.8766, + "theoretical_loss": 3.3836438345601, + "tokens_seen": 2364211200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884646464646465, + "loss": 2.7063, + "theoretical_loss": 3.3836362690792887, + "tokens_seen": 2364276736 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028844444444444447, + "loss": 3.0039, + "theoretical_loss": 3.383628703866901, + "tokens_seen": 2364342272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884242424242424, + "loss": 2.6418, + "theoretical_loss": 3.38362113892292, + "tokens_seen": 2364407808 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1297770, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.729642629623413, + "objective/train/theoretical_loss": 3.3836135742473283, + "objective/train/tokens_used": 723332576, + "theoretical_loss": 3.3836135742473283, + "tokens_seen": 2364473344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884040404040404, + "loss": 2.6027, + "theoretical_loss": 3.3836135742473283, + "tokens_seen": 2364473344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883838383838384, + "loss": 2.8451, + "theoretical_loss": 3.3836060098401086, + "tokens_seen": 2364538880 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883636363636364, + "loss": 2.7172, + "theoretical_loss": 3.3835984457012453, + "tokens_seen": 2364604416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028834343434343436, + "loss": 2.9995, + "theoretical_loss": 3.3835908818307203, + "tokens_seen": 2364669952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883232323232323, + "loss": 2.7271, + "theoretical_loss": 3.383583318228517, + "tokens_seen": 2364735488 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883030303030303, + "loss": 2.701, + "theoretical_loss": 3.3835757548946184, + "tokens_seen": 2364801024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028828282828282834, + "loss": 2.955, + "theoretical_loss": 3.3835681918290077, + "tokens_seen": 2364866560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882626262626263, + "loss": 2.8081, + "theoretical_loss": 3.3835606290316678, + "tokens_seen": 2364932096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028824242424242425, + "loss": 2.6179, + "theoretical_loss": 3.3835530665025813, + "tokens_seen": 2364997632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882222222222222, + "loss": 2.6764, + "theoretical_loss": 3.383545504241732, + "tokens_seen": 2365063168 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028820202020202017, + "loss": 2.9193, + "theoretical_loss": 3.3835379422491028, + "tokens_seen": 2365128704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028818181818181823, + "loss": 2.6585, + "theoretical_loss": 3.3835303805246766, + "tokens_seen": 2365194240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881616161616162, + "loss": 2.7959, + "theoretical_loss": 3.3835228190684363, + "tokens_seen": 2365259776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028814141414141415, + "loss": 2.9671, + "theoretical_loss": 3.383515257880365, + "tokens_seen": 2365325312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881212121212121, + "loss": 2.6223, + "theoretical_loss": 3.3835076969604465, + "tokens_seen": 2365390848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028810101010101006, + "loss": 2.7847, + "theoretical_loss": 3.3835001363086628, + "tokens_seen": 2365456384 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880808080808081, + "loss": 2.6494, + "theoretical_loss": 3.383492575924998, + "tokens_seen": 2365521920 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880606060606061, + "loss": 2.6848, + "theoretical_loss": 3.383485015809434, + "tokens_seen": 2365587456 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028804040404040404, + "loss": 3.0481, + "theoretical_loss": 3.3834774559619545, + "tokens_seen": 2365652992 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288020202020202, + "loss": 2.7109, + "theoretical_loss": 3.3834698963825427, + "tokens_seen": 2365718528 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288, + "loss": 2.9154, + "theoretical_loss": 3.3834623370711814, + "tokens_seen": 2365784064 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287979797979798, + "loss": 2.8094, + "theoretical_loss": 3.3834547780278537, + "tokens_seen": 2365849600 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287959595959596, + "loss": 2.8296, + "theoretical_loss": 3.3834472192525427, + "tokens_seen": 2365915136 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028793939393939393, + "loss": 2.6589, + "theoretical_loss": 3.3834396607452315, + "tokens_seen": 2365980672 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879191919191919, + "loss": 2.7813, + "theoretical_loss": 3.3834321025059033, + "tokens_seen": 2366046208 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1298991, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9180285930633545, + "objective/train/theoretical_loss": 3.383424544534541, + "objective/train/tokens_used": 724970976, + "theoretical_loss": 3.383424544534541, + "tokens_seen": 2366111744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878989898989899, + "loss": 2.6105, + "theoretical_loss": 3.383424544534541, + "tokens_seen": 2366111744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878787878787879, + "loss": 2.8354, + "theoretical_loss": 3.383416986831128, + "tokens_seen": 2366177280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028785858585858587, + "loss": 2.6593, + "theoretical_loss": 3.383409429395647, + "tokens_seen": 2366242816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878383838383838, + "loss": 2.6004, + "theoretical_loss": 3.383401872228081, + "tokens_seen": 2366308352 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028781818181818184, + "loss": 2.8588, + "theoretical_loss": 3.383394315328413, + "tokens_seen": 2366373888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877979797979798, + "loss": 2.8283, + "theoretical_loss": 3.3833867586966266, + "tokens_seen": 2366439424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877777777777778, + "loss": 2.8935, + "theoretical_loss": 3.383379202332705, + "tokens_seen": 2366504960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028775757575757576, + "loss": 2.8037, + "theoretical_loss": 3.383371646236631, + "tokens_seen": 2366570496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877373737373737, + "loss": 2.7439, + "theoretical_loss": 3.383364090408387, + "tokens_seen": 2366636032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028771717171717173, + "loss": 2.7442, + "theoretical_loss": 3.383356534847957, + "tokens_seen": 2366701568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876969696969697, + "loss": 2.7006, + "theoretical_loss": 3.3833489795553238, + "tokens_seen": 2366767104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876767676767677, + "loss": 2.7843, + "theoretical_loss": 3.3833414245304705, + "tokens_seen": 2366832640 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028765656565656565, + "loss": 2.7358, + "theoretical_loss": 3.38333386977338, + "tokens_seen": 2366898176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028763636363636366, + "loss": 2.8552, + "theoretical_loss": 3.3833263152840356, + "tokens_seen": 2366963712 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876161616161616, + "loss": 2.833, + "theoretical_loss": 3.3833187610624207, + "tokens_seen": 2367029248 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875959595959596, + "loss": 2.8562, + "theoretical_loss": 3.383311207108518, + "tokens_seen": 2367094784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875757575757576, + "loss": 2.6936, + "theoretical_loss": 3.3833036534223107, + "tokens_seen": 2367160320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028755555555555555, + "loss": 2.8159, + "theoretical_loss": 3.383296100003782, + "tokens_seen": 2367225856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028753535353535356, + "loss": 2.8299, + "theoretical_loss": 3.3832885468529144, + "tokens_seen": 2367291392 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875151515151515, + "loss": 2.669, + "theoretical_loss": 3.383280993969692, + "tokens_seen": 2367356928 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028749494949494947, + "loss": 2.7604, + "theoretical_loss": 3.3832734413540972, + "tokens_seen": 2367422464 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874747474747475, + "loss": 2.6439, + "theoretical_loss": 3.3832658890061134, + "tokens_seen": 2367488000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874545454545455, + "loss": 2.8217, + "theoretical_loss": 3.3832583369257234, + "tokens_seen": 2367553536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028743434343434345, + "loss": 2.6788, + "theoretical_loss": 3.383250785112911, + "tokens_seen": 2367619072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874141414141414, + "loss": 2.7312, + "theoretical_loss": 3.3832432335676583, + "tokens_seen": 2367684608 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1299407, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.548238515853882, + "objective/train/theoretical_loss": 3.383235682289949, + "objective/train/tokens_used": 726609376, + "theoretical_loss": 3.383235682289949, + "tokens_seen": 2367750144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028739393939393936, + "loss": 2.7809, + "theoretical_loss": 3.383235682289949, + "tokens_seen": 2367750144 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873737373737374, + "loss": 2.8353, + "theoretical_loss": 3.3832281312797665, + "tokens_seen": 2367815680 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873535353535354, + "loss": 2.7917, + "theoretical_loss": 3.3832205805370936, + "tokens_seen": 2367881216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028733333333333334, + "loss": 2.59, + "theoretical_loss": 3.3832130300619134, + "tokens_seen": 2367946752 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873131313131313, + "loss": 2.7401, + "theoretical_loss": 3.383205479854209, + "tokens_seen": 2368012288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028729292929292926, + "loss": 2.7757, + "theoretical_loss": 3.3831979299139636, + "tokens_seen": 2368077824 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872727272727273, + "loss": 2.6085, + "theoretical_loss": 3.3831903802411603, + "tokens_seen": 2368143360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872525252525253, + "loss": 2.8826, + "theoretical_loss": 3.3831828308357816, + "tokens_seen": 2368208896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028723232323232324, + "loss": 2.6337, + "theoretical_loss": 3.3831752816978122, + "tokens_seen": 2368274432 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872121212121212, + "loss": 2.7368, + "theoretical_loss": 3.383167732827234, + "tokens_seen": 2368339968 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028719191919191915, + "loss": 2.7714, + "theoretical_loss": 3.38316018422403, + "tokens_seen": 2368405504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871717171717172, + "loss": 2.6164, + "theoretical_loss": 3.383152635888184, + "tokens_seen": 2368471040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028715151515151517, + "loss": 2.5985, + "theoretical_loss": 3.3831450878196785, + "tokens_seen": 2368536576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028713131313131313, + "loss": 2.9362, + "theoretical_loss": 3.3831375400184975, + "tokens_seen": 2368602112 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871111111111111, + "loss": 2.7855, + "theoretical_loss": 3.383129992484623, + "tokens_seen": 2368667648 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870909090909091, + "loss": 2.7867, + "theoretical_loss": 3.3831224452180395, + "tokens_seen": 2368733184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870707070707071, + "loss": 2.6093, + "theoretical_loss": 3.383114898218729, + "tokens_seen": 2368798720 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028705050505050506, + "loss": 2.6574, + "theoretical_loss": 3.383107351486675, + "tokens_seen": 2368864256 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287030303030303, + "loss": 2.7298, + "theoretical_loss": 3.3830998050218613, + "tokens_seen": 2368929792 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287010101010101, + "loss": 2.7609, + "theoretical_loss": 3.38309225882427, + "tokens_seen": 2368995328 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028698989898989904, + "loss": 2.935, + "theoretical_loss": 3.3830847128938846, + "tokens_seen": 2369060864 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286969696969697, + "loss": 2.5602, + "theoretical_loss": 3.3830771672306885, + "tokens_seen": 2369126400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028694949494949496, + "loss": 2.8036, + "theoretical_loss": 3.3830696218346645, + "tokens_seen": 2369191936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869292929292929, + "loss": 2.8502, + "theoretical_loss": 3.383062076705796, + "tokens_seen": 2369257472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869090909090909, + "loss": 2.8216, + "theoretical_loss": 3.383054531844066, + "tokens_seen": 2369323008 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1300592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9446256160736084, + "objective/train/theoretical_loss": 3.383046987249458, + "objective/train/tokens_used": 728247776, + "theoretical_loss": 3.383046987249458, + "tokens_seen": 2369388544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028688888888888894, + "loss": 2.8198, + "theoretical_loss": 3.383046987249458, + "tokens_seen": 2369388544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868686868686869, + "loss": 2.6907, + "theoretical_loss": 3.383039442921955, + "tokens_seen": 2369454080 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028684848484848485, + "loss": 2.8761, + "theoretical_loss": 3.3830318988615398, + "tokens_seen": 2369519616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868282828282828, + "loss": 2.7659, + "theoretical_loss": 3.3830243550681955, + "tokens_seen": 2369585152 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868080808080808, + "loss": 2.8049, + "theoretical_loss": 3.383016811541906, + "tokens_seen": 2369650688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028678787878787883, + "loss": 2.8566, + "theoretical_loss": 3.383009268282654, + "tokens_seen": 2369716224 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867676767676768, + "loss": 2.7565, + "theoretical_loss": 3.3830017252904225, + "tokens_seen": 2369781760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028674747474747474, + "loss": 2.7113, + "theoretical_loss": 3.382994182565195, + "tokens_seen": 2369847296 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867272727272727, + "loss": 2.7538, + "theoretical_loss": 3.382986640106954, + "tokens_seen": 2369912832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867070707070707, + "loss": 2.7925, + "theoretical_loss": 3.3829790979156837, + "tokens_seen": 2369978368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866868686868687, + "loss": 2.7005, + "theoretical_loss": 3.3829715559913667, + "tokens_seen": 2370043904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866666666666667, + "loss": 2.7673, + "theoretical_loss": 3.3829640143339863, + "tokens_seen": 2370109440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028664646464646463, + "loss": 2.6366, + "theoretical_loss": 3.3829564729435253, + "tokens_seen": 2370174976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028662626262626265, + "loss": 2.6965, + "theoretical_loss": 3.3829489318199673, + "tokens_seen": 2370240512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866060606060606, + "loss": 2.7347, + "theoretical_loss": 3.3829413909632953, + "tokens_seen": 2370306048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865858585858586, + "loss": 2.6849, + "theoretical_loss": 3.382933850373493, + "tokens_seen": 2370371584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028656565656565657, + "loss": 2.6003, + "theoretical_loss": 3.3829263100505425, + "tokens_seen": 2370437120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028654545454545453, + "loss": 2.6349, + "theoretical_loss": 3.3829187699944274, + "tokens_seen": 2370502656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028652525252525254, + "loss": 2.6287, + "theoretical_loss": 3.3829112302051314, + "tokens_seen": 2370568192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865050505050505, + "loss": 2.8806, + "theoretical_loss": 3.3829036906826375, + "tokens_seen": 2370633728 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864848484848485, + "loss": 2.7613, + "theoretical_loss": 3.3828961514269285, + "tokens_seen": 2370699264 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028646464646464646, + "loss": 2.7364, + "theoretical_loss": 3.3828886124379878, + "tokens_seen": 2370764800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864444444444445, + "loss": 2.653, + "theoretical_loss": 3.3828810737157986, + "tokens_seen": 2370830336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028642424242424243, + "loss": 2.7884, + "theoretical_loss": 3.3828735352603445, + "tokens_seen": 2370895872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864040404040404, + "loss": 2.7444, + "theoretical_loss": 3.3828659970716077, + "tokens_seen": 2370961408 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1301610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.708845376968384, + "objective/train/theoretical_loss": 3.3828584591495723, + "objective/train/tokens_used": 729886176, + "theoretical_loss": 3.3828584591495723, + "tokens_seen": 2371026944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863838383838384, + "loss": 2.6055, + "theoretical_loss": 3.3828584591495723, + "tokens_seen": 2371026944 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028636363636363636, + "loss": 2.6848, + "theoretical_loss": 3.382850921494221, + "tokens_seen": 2371092480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028634343434343437, + "loss": 2.9651, + "theoretical_loss": 3.382843384105537, + "tokens_seen": 2371158016 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863232323232323, + "loss": 2.7766, + "theoretical_loss": 3.382835846983504, + "tokens_seen": 2371223552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863030303030303, + "loss": 2.647, + "theoretical_loss": 3.3828283101281045, + "tokens_seen": 2371289088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862828282828283, + "loss": 2.7359, + "theoretical_loss": 3.382820773539322, + "tokens_seen": 2371354624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862626262626263, + "loss": 2.6404, + "theoretical_loss": 3.38281323721714, + "tokens_seen": 2371420160 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028624242424242426, + "loss": 2.7411, + "theoretical_loss": 3.382805701161541, + "tokens_seen": 2371485696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862222222222222, + "loss": 2.5558, + "theoretical_loss": 3.382798165372509, + "tokens_seen": 2371551232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862020202020202, + "loss": 2.7604, + "theoretical_loss": 3.382790629850027, + "tokens_seen": 2371616768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861818181818182, + "loss": 2.8462, + "theoretical_loss": 3.382783094594078, + "tokens_seen": 2371682304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861616161616162, + "loss": 2.6763, + "theoretical_loss": 3.382775559604645, + "tokens_seen": 2371747840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028614141414141415, + "loss": 2.7553, + "theoretical_loss": 3.382768024881712, + "tokens_seen": 2371813376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861212121212121, + "loss": 2.623, + "theoretical_loss": 3.382760490425261, + "tokens_seen": 2371878912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028610101010101007, + "loss": 2.6844, + "theoretical_loss": 3.3827529562352763, + "tokens_seen": 2371944448 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028608080808080813, + "loss": 2.804, + "theoretical_loss": 3.3827454223117406, + "tokens_seen": 2372009984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860606060606061, + "loss": 2.7884, + "theoretical_loss": 3.382737888654637, + "tokens_seen": 2372075520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028604040404040404, + "loss": 2.6557, + "theoretical_loss": 3.3827303552639494, + "tokens_seen": 2372141056 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286020202020202, + "loss": 2.851, + "theoretical_loss": 3.38272282213966, + "tokens_seen": 2372206592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028599999999999996, + "loss": 2.8257, + "theoretical_loss": 3.3827152892817534, + "tokens_seen": 2372272128 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285979797979798, + "loss": 2.8114, + "theoretical_loss": 3.3827077566902113, + "tokens_seen": 2372337664 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285959595959596, + "loss": 2.6481, + "theoretical_loss": 3.382700224365018, + "tokens_seen": 2372403200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028593939393939394, + "loss": 2.6321, + "theoretical_loss": 3.382692692306156, + "tokens_seen": 2372468736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859191919191919, + "loss": 2.8074, + "theoretical_loss": 3.382685160513609, + "tokens_seen": 2372534272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858989898989899, + "loss": 2.7955, + "theoretical_loss": 3.38267762898736, + "tokens_seen": 2372599808 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1302157, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.228212356567383, + "objective/train/theoretical_loss": 3.382670097727393, + "objective/train/tokens_used": 731524576, + "theoretical_loss": 3.382670097727393, + "tokens_seen": 2372665344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858787878787879, + "loss": 2.5333, + "theoretical_loss": 3.382670097727393, + "tokens_seen": 2372665344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858585858585859, + "loss": 2.7062, + "theoretical_loss": 3.38266256673369, + "tokens_seen": 2372730880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028583838383838383, + "loss": 2.8846, + "theoretical_loss": 3.3826550360062346, + "tokens_seen": 2372796416 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858181818181818, + "loss": 2.8929, + "theoretical_loss": 3.3826475055450107, + "tokens_seen": 2372861952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857979797979798, + "loss": 2.6605, + "theoretical_loss": 3.382639975350001, + "tokens_seen": 2372927488 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857777777777778, + "loss": 2.7888, + "theoretical_loss": 3.3826324454211885, + "tokens_seen": 2372993024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028575757575757577, + "loss": 2.6401, + "theoretical_loss": 3.382624915758557, + "tokens_seen": 2373058560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857373737373737, + "loss": 2.5738, + "theoretical_loss": 3.3826173863620896, + "tokens_seen": 2373124096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028571717171717173, + "loss": 2.5644, + "theoretical_loss": 3.382609857231769, + "tokens_seen": 2373189632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856969696969697, + "loss": 2.9077, + "theoretical_loss": 3.3826023283675792, + "tokens_seen": 2373255168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856767676767677, + "loss": 2.3755, + "theoretical_loss": 3.382594799769503, + "tokens_seen": 2373320704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028565656565656566, + "loss": 2.7103, + "theoretical_loss": 3.382587271437524, + "tokens_seen": 2373386240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856363636363636, + "loss": 2.6764, + "theoretical_loss": 3.382579743371625, + "tokens_seen": 2373451776 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856161616161616, + "loss": 2.6704, + "theoretical_loss": 3.3825722155717894, + "tokens_seen": 2373517312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855959595959596, + "loss": 2.4917, + "theoretical_loss": 3.3825646880380003, + "tokens_seen": 2373582848 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855757575757576, + "loss": 2.8492, + "theoretical_loss": 3.3825571607702414, + "tokens_seen": 2373648384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028555555555555555, + "loss": 2.8155, + "theoretical_loss": 3.3825496337684955, + "tokens_seen": 2373713920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028553535353535356, + "loss": 2.6375, + "theoretical_loss": 3.3825421070327466, + "tokens_seen": 2373779456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855151515151515, + "loss": 2.6984, + "theoretical_loss": 3.382534580562977, + "tokens_seen": 2373844992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854949494949495, + "loss": 2.7302, + "theoretical_loss": 3.3825270543591706, + "tokens_seen": 2373910528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854747474747475, + "loss": 2.6061, + "theoretical_loss": 3.3825195284213105, + "tokens_seen": 2373976064 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028545454545454544, + "loss": 2.6342, + "theoretical_loss": 3.3825120027493796, + "tokens_seen": 2374041600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028543434343434346, + "loss": 2.5622, + "theoretical_loss": 3.3825044773433617, + "tokens_seen": 2374107136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854141414141414, + "loss": 2.8012, + "theoretical_loss": 3.3824969522032395, + "tokens_seen": 2374172672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028539393939393937, + "loss": 2.8441, + "theoretical_loss": 3.3824894273289967, + "tokens_seen": 2374238208 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1303451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7277791500091553, + "objective/train/theoretical_loss": 3.3824819027206168, + "objective/train/tokens_used": 733162976, + "theoretical_loss": 3.3824819027206168, + "tokens_seen": 2374303744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002853737373737374, + "loss": 2.7293, + "theoretical_loss": 3.3824819027206168, + "tokens_seen": 2374303744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853535353535354, + "loss": 2.7882, + "theoretical_loss": 3.3824743783780824, + "tokens_seen": 2374369280 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028533333333333335, + "loss": 2.5895, + "theoretical_loss": 3.382466854301377, + "tokens_seen": 2374434816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853131313131313, + "loss": 2.6554, + "theoretical_loss": 3.3824593304904846, + "tokens_seen": 2374500352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852929292929293, + "loss": 2.5905, + "theoretical_loss": 3.382451806945387, + "tokens_seen": 2374565888 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028527272727272727, + "loss": 2.8454, + "theoretical_loss": 3.382444283666069, + "tokens_seen": 2374631424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852525252525253, + "loss": 2.6722, + "theoretical_loss": 3.3824367606525128, + "tokens_seen": 2374696960 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028523232323232324, + "loss": 2.6536, + "theoretical_loss": 3.3824292379047023, + "tokens_seen": 2374762496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852121212121212, + "loss": 2.7028, + "theoretical_loss": 3.3824217154226206, + "tokens_seen": 2374828032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851919191919192, + "loss": 2.7209, + "theoretical_loss": 3.382414193206251, + "tokens_seen": 2374893568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851717171717172, + "loss": 2.5259, + "theoretical_loss": 3.382406671255576, + "tokens_seen": 2374959104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851515151515152, + "loss": 2.5073, + "theoretical_loss": 3.3823991495705803, + "tokens_seen": 2375024640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028513131313131313, + "loss": 2.6183, + "theoretical_loss": 3.3823916281512463, + "tokens_seen": 2375090176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851111111111111, + "loss": 2.9363, + "theoretical_loss": 3.3823841069975575, + "tokens_seen": 2375155712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850909090909091, + "loss": 2.5514, + "theoretical_loss": 3.382376586109497, + "tokens_seen": 2375221248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850707070707071, + "loss": 2.7707, + "theoretical_loss": 3.3823690654870484, + "tokens_seen": 2375286784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028505050505050507, + "loss": 2.6421, + "theoretical_loss": 3.3823615451301947, + "tokens_seen": 2375352320 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285030303030303, + "loss": 2.7776, + "theoretical_loss": 3.3823540250389197, + "tokens_seen": 2375417856 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285010101010101, + "loss": 2.6807, + "theoretical_loss": 3.382346505213206, + "tokens_seen": 2375483392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028498989898989905, + "loss": 2.7964, + "theoretical_loss": 3.3823389856530373, + "tokens_seen": 2375548928 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284969696969697, + "loss": 2.7716, + "theoretical_loss": 3.382331466358397, + "tokens_seen": 2375614464 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028494949494949496, + "loss": 2.5232, + "theoretical_loss": 3.382323947329268, + "tokens_seen": 2375680000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849292929292929, + "loss": 2.6095, + "theoretical_loss": 3.382316428565634, + "tokens_seen": 2375745536 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849090909090909, + "loss": 2.7417, + "theoretical_loss": 3.382308910067478, + "tokens_seen": 2375811072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028488888888888894, + "loss": 2.5701, + "theoretical_loss": 3.3823013918347833, + "tokens_seen": 2375876608 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1303936, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6463329792022705, + "objective/train/theoretical_loss": 3.3822938738675337, + "objective/train/tokens_used": 734801376, + "theoretical_loss": 3.3822938738675337, + "tokens_seen": 2375942144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848686868686869, + "loss": 2.6626, + "theoretical_loss": 3.3822938738675337, + "tokens_seen": 2375942144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028484848484848485, + "loss": 2.6632, + "theoretical_loss": 3.3822863561657117, + "tokens_seen": 2376007680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848282828282828, + "loss": 2.6169, + "theoretical_loss": 3.3822788387293015, + "tokens_seen": 2376073216 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028480808080808077, + "loss": 2.8378, + "theoretical_loss": 3.382271321558286, + "tokens_seen": 2376138752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028478787878787883, + "loss": 2.7283, + "theoretical_loss": 3.382263804652648, + "tokens_seen": 2376204288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847676767676768, + "loss": 2.7262, + "theoretical_loss": 3.3822562880123717, + "tokens_seen": 2376269824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028474747474747475, + "loss": 2.6373, + "theoretical_loss": 3.3822487716374394, + "tokens_seen": 2376335360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847272727272727, + "loss": 2.4912, + "theoretical_loss": 3.382241255527836, + "tokens_seen": 2376400896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847070707070707, + "loss": 2.743, + "theoretical_loss": 3.3822337396835427, + "tokens_seen": 2376466432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846868686868687, + "loss": 2.855, + "theoretical_loss": 3.3822262241045444, + "tokens_seen": 2376531968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846666666666667, + "loss": 2.7858, + "theoretical_loss": 3.382218708790824, + "tokens_seen": 2376597504 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028464646464646464, + "loss": 2.4187, + "theoretical_loss": 3.382211193742365, + "tokens_seen": 2376663040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846262626262626, + "loss": 2.8033, + "theoretical_loss": 3.3822036789591503, + "tokens_seen": 2376728576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846060606060606, + "loss": 2.76, + "theoretical_loss": 3.3821961644411633, + "tokens_seen": 2376794112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845858585858586, + "loss": 2.7136, + "theoretical_loss": 3.3821886501883878, + "tokens_seen": 2376859648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845656565656566, + "loss": 2.6643, + "theoretical_loss": 3.3821811362008067, + "tokens_seen": 2376925184 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028454545454545453, + "loss": 2.569, + "theoretical_loss": 3.3821736224784034, + "tokens_seen": 2376990720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028452525252525254, + "loss": 2.8341, + "theoretical_loss": 3.382166109021161, + "tokens_seen": 2377056256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845050505050505, + "loss": 2.7549, + "theoretical_loss": 3.382158595829064, + "tokens_seen": 2377121792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844848484848485, + "loss": 2.6837, + "theoretical_loss": 3.3821510829020935, + "tokens_seen": 2377187328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028446464646464647, + "loss": 2.6413, + "theoretical_loss": 3.3821435702402347, + "tokens_seen": 2377252864 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844444444444444, + "loss": 2.5526, + "theoretical_loss": 3.3821360578434705, + "tokens_seen": 2377318400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028442424242424244, + "loss": 2.4521, + "theoretical_loss": 3.3821285457117845, + "tokens_seen": 2377383936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844040404040404, + "loss": 2.4732, + "theoretical_loss": 3.3821210338451593, + "tokens_seen": 2377449472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843838383838384, + "loss": 2.6388, + "theoretical_loss": 3.3821135222435785, + "tokens_seen": 2377515008 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1305174, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.373270273208618, + "objective/train/theoretical_loss": 3.382106010907026, + "objective/train/tokens_used": 736439776, + "theoretical_loss": 3.382106010907026, + "tokens_seen": 2377580544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028436363636363636, + "loss": 2.7005, + "theoretical_loss": 3.382106010907026, + "tokens_seen": 2377580544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028434343434343437, + "loss": 2.7245, + "theoretical_loss": 3.3820984998354837, + "tokens_seen": 2377646080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028432323232323233, + "loss": 2.7895, + "theoretical_loss": 3.382090989028937, + "tokens_seen": 2377711616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843030303030303, + "loss": 2.8304, + "theoretical_loss": 3.3820834784873677, + "tokens_seen": 2377777152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842828282828283, + "loss": 2.7219, + "theoretical_loss": 3.3820759682107595, + "tokens_seen": 2377842688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028426262626262625, + "loss": 2.6387, + "theoretical_loss": 3.382068458199096, + "tokens_seen": 2377908224 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028424242424242427, + "loss": 2.7945, + "theoretical_loss": 3.3820609484523603, + "tokens_seen": 2377973760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842222222222222, + "loss": 2.6654, + "theoretical_loss": 3.3820534389705363, + "tokens_seen": 2378039296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842020202020202, + "loss": 2.7555, + "theoretical_loss": 3.3820459297536067, + "tokens_seen": 2378104832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841818181818182, + "loss": 2.726, + "theoretical_loss": 3.382038420801555, + "tokens_seen": 2378170368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841616161616162, + "loss": 2.8324, + "theoretical_loss": 3.3820309121143652, + "tokens_seen": 2378235904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028414141414141416, + "loss": 2.5481, + "theoretical_loss": 3.3820234036920196, + "tokens_seen": 2378301440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841212121212121, + "loss": 2.657, + "theoretical_loss": 3.3820158955345025, + "tokens_seen": 2378366976 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028410101010101007, + "loss": 2.848, + "theoretical_loss": 3.3820083876417963, + "tokens_seen": 2378432512 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840808080808081, + "loss": 2.5993, + "theoretical_loss": 3.3820008800138854, + "tokens_seen": 2378498048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840606060606061, + "loss": 2.7084, + "theoretical_loss": 3.3819933726507525, + "tokens_seen": 2378563584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028404040404040405, + "loss": 2.4528, + "theoretical_loss": 3.381985865552381, + "tokens_seen": 2378629120 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284020202020202, + "loss": 2.5714, + "theoretical_loss": 3.3819783587187544, + "tokens_seen": 2378694656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028399999999999996, + "loss": 2.7692, + "theoretical_loss": 3.3819708521498564, + "tokens_seen": 2378760192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028397979797979803, + "loss": 2.4897, + "theoretical_loss": 3.3819633458456697, + "tokens_seen": 2378825728 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283959595959596, + "loss": 2.7357, + "theoretical_loss": 3.381955839806178, + "tokens_seen": 2378891264 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028393939393939394, + "loss": 2.7141, + "theoretical_loss": 3.3819483340313647, + "tokens_seen": 2378956800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839191919191919, + "loss": 2.6596, + "theoretical_loss": 3.3819408285212136, + "tokens_seen": 2379022336 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028389898989898986, + "loss": 2.6599, + "theoretical_loss": 3.3819333232757076, + "tokens_seen": 2379087872 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838787878787879, + "loss": 2.6759, + "theoretical_loss": 3.3819258182948295, + "tokens_seen": 2379153408 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1305886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3820316791534424, + "objective/train/theoretical_loss": 3.3819183135785638, + "objective/train/tokens_used": 738078176, + "theoretical_loss": 3.3819183135785638, + "tokens_seen": 2379218944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838585858585859, + "loss": 2.289, + "theoretical_loss": 3.3819183135785638, + "tokens_seen": 2379218944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028383838383838384, + "loss": 2.8343, + "theoretical_loss": 3.3819108091268935, + "tokens_seen": 2379284480 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838181818181818, + "loss": 2.7488, + "theoretical_loss": 3.3819033049398017, + "tokens_seen": 2379350016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837979797979798, + "loss": 2.7085, + "theoretical_loss": 3.3818958010172717, + "tokens_seen": 2379415552 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837777777777778, + "loss": 2.7585, + "theoretical_loss": 3.3818882973592874, + "tokens_seen": 2379481088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028375757575757577, + "loss": 2.7206, + "theoretical_loss": 3.381880793965832, + "tokens_seen": 2379546624 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028373737373737373, + "loss": 2.7653, + "theoretical_loss": 3.381873290836889, + "tokens_seen": 2379612160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837171717171717, + "loss": 2.6582, + "theoretical_loss": 3.3818657879724414, + "tokens_seen": 2379677696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836969696969697, + "loss": 2.658, + "theoretical_loss": 3.3818582853724726, + "tokens_seen": 2379743232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836767676767677, + "loss": 2.6887, + "theoretical_loss": 3.3818507830369664, + "tokens_seen": 2379808768 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028365656565656566, + "loss": 2.5464, + "theoretical_loss": 3.381843280965906, + "tokens_seen": 2379874304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836363636363636, + "loss": 2.6371, + "theoretical_loss": 3.3818357791592746, + "tokens_seen": 2379939840 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028361616161616163, + "loss": 2.5925, + "theoretical_loss": 3.381828277617056, + "tokens_seen": 2380005376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028359595959595964, + "loss": 2.7838, + "theoretical_loss": 3.381820776339233, + "tokens_seen": 2380070912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835757575757576, + "loss": 2.6511, + "theoretical_loss": 3.38181327532579, + "tokens_seen": 2380136448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028355555555555556, + "loss": 2.6926, + "theoretical_loss": 3.38180577457671, + "tokens_seen": 2380201984 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835353535353535, + "loss": 2.6903, + "theoretical_loss": 3.381798274091975, + "tokens_seen": 2380267520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835151515151515, + "loss": 2.6797, + "theoretical_loss": 3.381790773871571, + "tokens_seen": 2380333056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028349494949494954, + "loss": 2.6454, + "theoretical_loss": 3.381783273915479, + "tokens_seen": 2380398592 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834747474747475, + "loss": 2.5699, + "theoretical_loss": 3.381775774223684, + "tokens_seen": 2380464128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028345454545454545, + "loss": 2.5086, + "theoretical_loss": 3.3817682747961686, + "tokens_seen": 2380529664 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834343434343434, + "loss": 2.3691, + "theoretical_loss": 3.3817607756329164, + "tokens_seen": 2380595200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834141414141414, + "loss": 2.6804, + "theoretical_loss": 3.3817532767339107, + "tokens_seen": 2380660736 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028339393939393943, + "loss": 2.7132, + "theoretical_loss": 3.3817457780991353, + "tokens_seen": 2380726272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833737373737374, + "loss": 2.856, + "theoretical_loss": 3.3817382797285735, + "tokens_seen": 2380791808 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1307234, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6940321922302246, + "objective/train/theoretical_loss": 3.3817307816222084, + "objective/train/tokens_used": 739716576, + "theoretical_loss": 3.3817307816222084, + "tokens_seen": 2380857344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028335353535353534, + "loss": 2.6585, + "theoretical_loss": 3.3817307816222084, + "tokens_seen": 2380857344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028333333333333335, + "loss": 2.661, + "theoretical_loss": 3.381723283780024, + "tokens_seen": 2380922880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833131313131313, + "loss": 2.7875, + "theoretical_loss": 3.381715786202003, + "tokens_seen": 2380988416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832929292929293, + "loss": 2.7196, + "theoretical_loss": 3.381708288888129, + "tokens_seen": 2381053952 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832727272727273, + "loss": 2.5298, + "theoretical_loss": 3.381700791838386, + "tokens_seen": 2381119488 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028325252525252524, + "loss": 2.7766, + "theoretical_loss": 3.3816932950527567, + "tokens_seen": 2381185024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028323232323232325, + "loss": 2.7571, + "theoretical_loss": 3.381685798531225, + "tokens_seen": 2381250560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832121212121212, + "loss": 2.7308, + "theoretical_loss": 3.381678302273774, + "tokens_seen": 2381316096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831919191919192, + "loss": 2.4646, + "theoretical_loss": 3.3816708062803875, + "tokens_seen": 2381381632 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028317171717171717, + "loss": 2.6481, + "theoretical_loss": 3.381663310551049, + "tokens_seen": 2381447168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831515151515152, + "loss": 2.7243, + "theoretical_loss": 3.3816558150857414, + "tokens_seen": 2381512704 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028313131313131314, + "loss": 2.7129, + "theoretical_loss": 3.381648319884448, + "tokens_seen": 2381578240 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831111111111111, + "loss": 2.7205, + "theoretical_loss": 3.3816408249471532, + "tokens_seen": 2381643776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830909090909091, + "loss": 2.7871, + "theoretical_loss": 3.3816333302738397, + "tokens_seen": 2381709312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028307070707070706, + "loss": 2.6776, + "theoretical_loss": 3.381625835864491, + "tokens_seen": 2381774848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830505050505051, + "loss": 2.7531, + "theoretical_loss": 3.381618341719091, + "tokens_seen": 2381840384 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028303030303030303, + "loss": 2.6221, + "theoretical_loss": 3.3816108478376226, + "tokens_seen": 2381905920 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283010101010101, + "loss": 2.8476, + "theoretical_loss": 3.3816033542200694, + "tokens_seen": 2381971456 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282989898989899, + "loss": 2.5985, + "theoretical_loss": 3.3815958608664145, + "tokens_seen": 2382036992 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282969696969697, + "loss": 2.5843, + "theoretical_loss": 3.3815883677766423, + "tokens_seen": 2382102528 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028294949494949497, + "loss": 2.5626, + "theoretical_loss": 3.3815808749507354, + "tokens_seen": 2382168064 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829292929292929, + "loss": 2.611, + "theoretical_loss": 3.3815733823886776, + "tokens_seen": 2382233600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829090909090909, + "loss": 2.5989, + "theoretical_loss": 3.381565890090452, + "tokens_seen": 2382299136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828888888888889, + "loss": 2.6684, + "theoretical_loss": 3.381558398056043, + "tokens_seen": 2382364672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828686868686869, + "loss": 2.688, + "theoretical_loss": 3.3815509062854328, + "tokens_seen": 2382430208 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1308524, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5440561771392822, + "objective/train/theoretical_loss": 3.381543414778606, + "objective/train/tokens_used": 741354976, + "theoretical_loss": 3.381543414778606, + "tokens_seen": 2382495744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028284848484848486, + "loss": 2.5642, + "theoretical_loss": 3.381543414778606, + "tokens_seen": 2382495744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828282828282828, + "loss": 2.655, + "theoretical_loss": 3.381535923535545, + "tokens_seen": 2382561280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828080808080808, + "loss": 2.5311, + "theoretical_loss": 3.381528432556234, + "tokens_seen": 2382626816 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028278787878787884, + "loss": 2.3585, + "theoretical_loss": 3.3815209418406558, + "tokens_seen": 2382692352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827676767676768, + "loss": 2.6624, + "theoretical_loss": 3.3815134513887948, + "tokens_seen": 2382757888 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028274747474747475, + "loss": 2.7171, + "theoretical_loss": 3.3815059612006335, + "tokens_seen": 2382823424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827272727272727, + "loss": 2.6547, + "theoretical_loss": 3.381498471276156, + "tokens_seen": 2382888960 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028270707070707067, + "loss": 2.6185, + "theoretical_loss": 3.381490981615346, + "tokens_seen": 2382954496 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028268686868686873, + "loss": 2.588, + "theoretical_loss": 3.3814834922181856, + "tokens_seen": 2383020032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826666666666667, + "loss": 2.7493, + "theoretical_loss": 3.38147600308466, + "tokens_seen": 2383085568 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028264646464646465, + "loss": 2.6777, + "theoretical_loss": 3.3814685142147516, + "tokens_seen": 2383151104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826262626262626, + "loss": 2.7726, + "theoretical_loss": 3.381461025608444, + "tokens_seen": 2383216640 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826060606060606, + "loss": 2.5976, + "theoretical_loss": 3.381453537265721, + "tokens_seen": 2383282176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825858585858586, + "loss": 2.7502, + "theoretical_loss": 3.381446049186566, + "tokens_seen": 2383347712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825656565656566, + "loss": 2.7912, + "theoretical_loss": 3.381438561370962, + "tokens_seen": 2383413248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028254545454545454, + "loss": 2.4623, + "theoretical_loss": 3.3814310738188933, + "tokens_seen": 2383478784 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825252525252525, + "loss": 2.4759, + "theoretical_loss": 3.3814235865303424, + "tokens_seen": 2383544320 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825050505050505, + "loss": 2.722, + "theoretical_loss": 3.3814160995052935, + "tokens_seen": 2383609856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824848484848485, + "loss": 2.6941, + "theoretical_loss": 3.38140861274373, + "tokens_seen": 2383675392 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824646464646465, + "loss": 2.6101, + "theoretical_loss": 3.3814011262456356, + "tokens_seen": 2383740928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028244444444444443, + "loss": 2.5728, + "theoretical_loss": 3.3813936400109927, + "tokens_seen": 2383806464 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028242424242424244, + "loss": 2.6235, + "theoretical_loss": 3.3813861540397863, + "tokens_seen": 2383872000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824040404040404, + "loss": 2.5543, + "theoretical_loss": 3.381378668331999, + "tokens_seen": 2383937536 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823838383838384, + "loss": 2.6172, + "theoretical_loss": 3.381371182887614, + "tokens_seen": 2384003072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028236363636363637, + "loss": 2.5012, + "theoretical_loss": 3.3813636977066155, + "tokens_seen": 2384068608 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1309072, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.579392433166504, + "objective/train/theoretical_loss": 3.3813562127889867, + "objective/train/tokens_used": 742993376, + "theoretical_loss": 3.3813562127889867, + "tokens_seen": 2384134144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823434343434343, + "loss": 2.5352, + "theoretical_loss": 3.3813562127889867, + "tokens_seen": 2384134144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028232323232323233, + "loss": 2.5313, + "theoretical_loss": 3.3813487281347108, + "tokens_seen": 2384199680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823030303030303, + "loss": 2.5842, + "theoretical_loss": 3.3813412437437718, + "tokens_seen": 2384265216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822828282828283, + "loss": 2.4019, + "theoretical_loss": 3.381333759616153, + "tokens_seen": 2384330752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028226262626262626, + "loss": 2.5113, + "theoretical_loss": 3.381326275751838, + "tokens_seen": 2384396288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028224242424242427, + "loss": 2.7688, + "theoretical_loss": 3.38131879215081, + "tokens_seen": 2384461824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028222222222222223, + "loss": 2.5801, + "theoretical_loss": 3.3813113088130526, + "tokens_seen": 2384527360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822020202020202, + "loss": 2.5308, + "theoretical_loss": 3.38130382573855, + "tokens_seen": 2384592896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821818181818182, + "loss": 2.4669, + "theoretical_loss": 3.3812963429272846, + "tokens_seen": 2384658432 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028216161616161615, + "loss": 2.6314, + "theoretical_loss": 3.3812888603792404, + "tokens_seen": 2384723968 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028214141414141416, + "loss": 2.7238, + "theoretical_loss": 3.381281378094401, + "tokens_seen": 2384789504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821212121212121, + "loss": 2.5219, + "theoretical_loss": 3.3812738960727495, + "tokens_seen": 2384855040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821010101010101, + "loss": 2.562, + "theoretical_loss": 3.38126641431427, + "tokens_seen": 2384920576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820808080808081, + "loss": 2.6081, + "theoretical_loss": 3.381258932818946, + "tokens_seen": 2384986112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820606060606061, + "loss": 2.565, + "theoretical_loss": 3.3812514515867607, + "tokens_seen": 2385051648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028204040404040406, + "loss": 2.67, + "theoretical_loss": 3.3812439706176978, + "tokens_seen": 2385117184 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282020202020202, + "loss": 2.6214, + "theoretical_loss": 3.38123648991174, + "tokens_seen": 2385182720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028199999999999997, + "loss": 2.3948, + "theoretical_loss": 3.381229009468872, + "tokens_seen": 2385248256 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281979797979798, + "loss": 2.6144, + "theoretical_loss": 3.3812215292890766, + "tokens_seen": 2385313792 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281959595959596, + "loss": 2.2897, + "theoretical_loss": 3.3812140493723377, + "tokens_seen": 2385379328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028193939393939395, + "loss": 2.7042, + "theoretical_loss": 3.3812065697186386, + "tokens_seen": 2385444864 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819191919191919, + "loss": 2.4648, + "theoretical_loss": 3.381199090327963, + "tokens_seen": 2385510400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028189898989898986, + "loss": 2.6589, + "theoretical_loss": 3.381191611200294, + "tokens_seen": 2385575936 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028187878787878793, + "loss": 2.6317, + "theoretical_loss": 3.381184132335616, + "tokens_seen": 2385641472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818585858585859, + "loss": 2.7684, + "theoretical_loss": 3.381176653733912, + "tokens_seen": 2385707008 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1310224, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8640713691711426, + "objective/train/theoretical_loss": 3.381169175395165, + "objective/train/tokens_used": 744631776, + "theoretical_loss": 3.381169175395165, + "tokens_seen": 2385772544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028183838383838384, + "loss": 2.7092, + "theoretical_loss": 3.381169175395165, + "tokens_seen": 2385772544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818181818181818, + "loss": 2.6205, + "theoretical_loss": 3.3811616973193592, + "tokens_seen": 2385838080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817979797979798, + "loss": 2.4941, + "theoretical_loss": 3.381154219506478, + "tokens_seen": 2385903616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817777777777778, + "loss": 2.6066, + "theoretical_loss": 3.3811467419565044, + "tokens_seen": 2385969152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817575757575758, + "loss": 2.3554, + "theoretical_loss": 3.3811392646694234, + "tokens_seen": 2386034688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028173737373737373, + "loss": 2.7169, + "theoretical_loss": 3.381131787645217, + "tokens_seen": 2386100224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817171717171717, + "loss": 2.6337, + "theoretical_loss": 3.381124310883869, + "tokens_seen": 2386165760 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028169696969696976, + "loss": 2.6387, + "theoretical_loss": 3.381116834385364, + "tokens_seen": 2386231296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816767676767677, + "loss": 2.6259, + "theoretical_loss": 3.3811093581496845, + "tokens_seen": 2386296832 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028165656565656567, + "loss": 2.4768, + "theoretical_loss": 3.381101882176814, + "tokens_seen": 2386362368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816363636363636, + "loss": 2.6671, + "theoretical_loss": 3.3810944064667368, + "tokens_seen": 2386427904 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816161616161616, + "loss": 2.784, + "theoretical_loss": 3.3810869310194356, + "tokens_seen": 2386493440 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028159595959595965, + "loss": 2.5278, + "theoretical_loss": 3.381079455834895, + "tokens_seen": 2386558976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815757575757576, + "loss": 2.8345, + "theoretical_loss": 3.3810719809130974, + "tokens_seen": 2386624512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028155555555555556, + "loss": 2.738, + "theoretical_loss": 3.381064506254027, + "tokens_seen": 2386690048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815353535353535, + "loss": 2.8047, + "theoretical_loss": 3.381057031857667, + "tokens_seen": 2386755584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815151515151515, + "loss": 2.7453, + "theoretical_loss": 3.3810495577240016, + "tokens_seen": 2386821120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028149494949494954, + "loss": 2.7913, + "theoretical_loss": 3.3810420838530137, + "tokens_seen": 2386886656 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814747474747475, + "loss": 2.6015, + "theoretical_loss": 3.3810346102446873, + "tokens_seen": 2386952192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028145454545454546, + "loss": 2.4982, + "theoretical_loss": 3.3810271368990055, + "tokens_seen": 2387017728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814343434343434, + "loss": 2.8202, + "theoretical_loss": 3.381019663815952, + "tokens_seen": 2387083264 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814141414141414, + "loss": 2.6795, + "theoretical_loss": 3.3810121909955106, + "tokens_seen": 2387148800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028139393939393943, + "loss": 2.3849, + "theoretical_loss": 3.381004718437665, + "tokens_seen": 2387214336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813737373737374, + "loss": 2.5284, + "theoretical_loss": 3.3809972461423983, + "tokens_seen": 2387279872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028135353535353535, + "loss": 2.857, + "theoretical_loss": 3.380989774109694, + "tokens_seen": 2387345408 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1310846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.386476755142212, + "objective/train/theoretical_loss": 3.3809823023395364, + "objective/train/tokens_used": 746270176, + "theoretical_loss": 3.3809823023395364, + "tokens_seen": 2387410944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813333333333333, + "loss": 2.4214, + "theoretical_loss": 3.3809823023395364, + "tokens_seen": 2387410944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813131313131313, + "loss": 2.6282, + "theoretical_loss": 3.3809748308319083, + "tokens_seen": 2387476480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028129292929292933, + "loss": 2.579, + "theoretical_loss": 3.3809673595867933, + "tokens_seen": 2387542016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812727272727273, + "loss": 2.7025, + "theoretical_loss": 3.380959888604176, + "tokens_seen": 2387607552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028125252525252524, + "loss": 2.6733, + "theoretical_loss": 3.3809524178840387, + "tokens_seen": 2387673088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028123232323232325, + "loss": 2.4146, + "theoretical_loss": 3.3809449474263653, + "tokens_seen": 2387738624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812121212121212, + "loss": 2.6842, + "theoretical_loss": 3.38093747723114, + "tokens_seen": 2387804160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811919191919192, + "loss": 2.4407, + "theoretical_loss": 3.3809300072983453, + "tokens_seen": 2387869696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811717171717172, + "loss": 2.4228, + "theoretical_loss": 3.380922537627966, + "tokens_seen": 2387935232 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028115151515151513, + "loss": 2.7888, + "theoretical_loss": 3.380915068219985, + "tokens_seen": 2388000768 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028113131313131314, + "loss": 2.4777, + "theoretical_loss": 3.3809075990743858, + "tokens_seen": 2388066304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811111111111111, + "loss": 2.5646, + "theoretical_loss": 3.380900130191152, + "tokens_seen": 2388131840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810909090909091, + "loss": 2.5683, + "theoretical_loss": 3.3808926615702677, + "tokens_seen": 2388197376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028107070707070707, + "loss": 2.4524, + "theoretical_loss": 3.380885193211716, + "tokens_seen": 2388262912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810505050505051, + "loss": 2.6967, + "theoretical_loss": 3.380877725115481, + "tokens_seen": 2388328448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028103030303030304, + "loss": 2.6038, + "theoretical_loss": 3.380870257281545, + "tokens_seen": 2388393984 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281010101010101, + "loss": 2.4525, + "theoretical_loss": 3.380862789709893, + "tokens_seen": 2388459520 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280989898989899, + "loss": 2.5881, + "theoretical_loss": 3.3808553224005085, + "tokens_seen": 2388525056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028096969696969696, + "loss": 2.5306, + "theoretical_loss": 3.380847855353374, + "tokens_seen": 2388590592 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280949494949495, + "loss": 2.7317, + "theoretical_loss": 3.3808403885684744, + "tokens_seen": 2388656128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028092929292929293, + "loss": 2.4498, + "theoretical_loss": 3.380832922045792, + "tokens_seen": 2388721664 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002809090909090909, + "loss": 2.6253, + "theoretical_loss": 3.3808254557853115, + "tokens_seen": 2388787200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808888888888889, + "loss": 2.6037, + "theoretical_loss": 3.380817989787016, + "tokens_seen": 2388852736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808686868686869, + "loss": 2.8293, + "theoretical_loss": 3.3808105240508888, + "tokens_seen": 2388918272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028084848484848487, + "loss": 2.4457, + "theoretical_loss": 3.380803058576914, + "tokens_seen": 2388983808 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1312174, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4064464569091797, + "objective/train/theoretical_loss": 3.3807955933650753, + "objective/train/tokens_used": 747908576, + "theoretical_loss": 3.3807955933650753, + "tokens_seen": 2389049344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808282828282828, + "loss": 2.626, + "theoretical_loss": 3.3807955933650753, + "tokens_seen": 2389049344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808080808080808, + "loss": 2.7644, + "theoretical_loss": 3.3807881284153565, + "tokens_seen": 2389114880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807878787878788, + "loss": 2.7973, + "theoretical_loss": 3.38078066372774, + "tokens_seen": 2389180416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807676767676768, + "loss": 2.7315, + "theoretical_loss": 3.3807731993022108, + "tokens_seen": 2389245952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028074747474747476, + "loss": 2.5144, + "theoretical_loss": 3.3807657351387514, + "tokens_seen": 2389311488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807272727272727, + "loss": 2.5448, + "theoretical_loss": 3.380758271237346, + "tokens_seen": 2389377024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028070707070707067, + "loss": 2.5791, + "theoretical_loss": 3.380750807597978, + "tokens_seen": 2389442560 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028068686868686874, + "loss": 2.8976, + "theoretical_loss": 3.3807433442206314, + "tokens_seen": 2389508096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806666666666667, + "loss": 2.4212, + "theoretical_loss": 3.3807358811052897, + "tokens_seen": 2389573632 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028064646464646465, + "loss": 2.5482, + "theoretical_loss": 3.380728418251936, + "tokens_seen": 2389639168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806262626262626, + "loss": 2.4117, + "theoretical_loss": 3.380720955660555, + "tokens_seen": 2389704704 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028060606060606057, + "loss": 2.6246, + "theoretical_loss": 3.380713493331129, + "tokens_seen": 2389770240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028058585858585863, + "loss": 2.4588, + "theoretical_loss": 3.3807060312636423, + "tokens_seen": 2389835776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805656565656566, + "loss": 2.6998, + "theoretical_loss": 3.3806985694580782, + "tokens_seen": 2389901312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028054545454545454, + "loss": 2.4716, + "theoretical_loss": 3.380691107914421, + "tokens_seen": 2389966848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805252525252525, + "loss": 2.5811, + "theoretical_loss": 3.3806836466326535, + "tokens_seen": 2390032384 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805050505050505, + "loss": 2.7345, + "theoretical_loss": 3.38067618561276, + "tokens_seen": 2390097920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804848484848485, + "loss": 2.4443, + "theoretical_loss": 3.3806687248547242, + "tokens_seen": 2390163456 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804646464646465, + "loss": 2.6973, + "theoretical_loss": 3.3806612643585288, + "tokens_seen": 2390228992 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028044444444444444, + "loss": 2.5089, + "theoretical_loss": 3.380653804124158, + "tokens_seen": 2390294528 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804242424242424, + "loss": 2.5099, + "theoretical_loss": 3.380646344151596, + "tokens_seen": 2390360064 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804040404040404, + "loss": 2.7341, + "theoretical_loss": 3.3806388844408257, + "tokens_seen": 2390425600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002803838383838384, + "loss": 2.3642, + "theoretical_loss": 3.380631424991831, + "tokens_seen": 2390491136 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028036363636363637, + "loss": 2.7166, + "theoretical_loss": 3.380623965804595, + "tokens_seen": 2390556672 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028034343434343433, + "loss": 2.5862, + "theoretical_loss": 3.380616506879102, + "tokens_seen": 2390622208 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1312726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7473642826080322, + "objective/train/theoretical_loss": 3.3806090482153355, + "objective/train/tokens_used": 749546976, + "theoretical_loss": 3.3806090482153355, + "tokens_seen": 2390687744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803232323232323, + "loss": 2.7195, + "theoretical_loss": 3.3806090482153355, + "tokens_seen": 2390687744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803030303030303, + "loss": 2.6134, + "theoretical_loss": 3.3806015898132795, + "tokens_seen": 2390753280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802828282828283, + "loss": 2.4971, + "theoretical_loss": 3.3805941316729164, + "tokens_seen": 2390818816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028026262626262627, + "loss": 2.5792, + "theoretical_loss": 3.380586673794231, + "tokens_seen": 2390884352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802424242424242, + "loss": 2.7271, + "theoretical_loss": 3.380579216177207, + "tokens_seen": 2390949888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028022222222222223, + "loss": 2.3641, + "theoretical_loss": 3.3805717588218274, + "tokens_seen": 2391015424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802020202020202, + "loss": 2.4436, + "theoretical_loss": 3.380564301728076, + "tokens_seen": 2391080960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801818181818182, + "loss": 2.7457, + "theoretical_loss": 3.3805568448959367, + "tokens_seen": 2391146496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028016161616161616, + "loss": 2.599, + "theoretical_loss": 3.380549388325393, + "tokens_seen": 2391212032 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801414141414141, + "loss": 2.6285, + "theoretical_loss": 3.3805419320164285, + "tokens_seen": 2391277568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801212121212121, + "loss": 2.6415, + "theoretical_loss": 3.3805344759690272, + "tokens_seen": 2391343104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028010101010101014, + "loss": 2.6648, + "theoretical_loss": 3.3805270201831723, + "tokens_seen": 2391408640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002800808080808081, + "loss": 2.4228, + "theoretical_loss": 3.3805195646588473, + "tokens_seen": 2391474176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028006060606060605, + "loss": 2.7256, + "theoretical_loss": 3.3805121093960366, + "tokens_seen": 2391539712 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028004040404040406, + "loss": 2.6155, + "theoretical_loss": 3.3805046543947235, + "tokens_seen": 2391605248 + }, + { + "epoch": 0.45, + "learning_rate": 0.000280020202020202, + "loss": 2.7055, + "theoretical_loss": 3.3804971996548914, + "tokens_seen": 2391670784 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028000000000000003, + "loss": 2.764, + "theoretical_loss": 3.3804897451765243, + "tokens_seen": 2391736320 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279979797979798, + "loss": 2.4448, + "theoretical_loss": 3.3804822909596055, + "tokens_seen": 2391801856 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027995959595959594, + "loss": 2.5938, + "theoretical_loss": 3.3804748370041193, + "tokens_seen": 2391867392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027993939393939395, + "loss": 2.6107, + "theoretical_loss": 3.380467383310049, + "tokens_seen": 2391932928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799191919191919, + "loss": 2.6505, + "theoretical_loss": 3.380459929877378, + "tokens_seen": 2391998464 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798989898989899, + "loss": 2.4188, + "theoretical_loss": 3.3804524767060906, + "tokens_seen": 2392064000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798787878787879, + "loss": 2.6168, + "theoretical_loss": 3.3804450237961694, + "tokens_seen": 2392129536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798585858585859, + "loss": 2.6171, + "theoretical_loss": 3.3804375711475996, + "tokens_seen": 2392195072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027983838383838385, + "loss": 2.6437, + "theoretical_loss": 3.380430118760364, + "tokens_seen": 2392260608 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1313802, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4760336875915527, + "objective/train/theoretical_loss": 3.3804226666344457, + "objective/train/tokens_used": 751185376, + "theoretical_loss": 3.3804226666344457, + "tokens_seen": 2392326144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798181818181818, + "loss": 2.6366, + "theoretical_loss": 3.3804226666344457, + "tokens_seen": 2392326144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797979797979798, + "loss": 2.6789, + "theoretical_loss": 3.380415214769829, + "tokens_seen": 2392391680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027977777777777777, + "loss": 2.7024, + "theoretical_loss": 3.380407763166498, + "tokens_seen": 2392457216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797575757575758, + "loss": 2.6339, + "theoretical_loss": 3.380400311824436, + "tokens_seen": 2392522752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027973737373737374, + "loss": 2.4198, + "theoretical_loss": 3.3803928607436267, + "tokens_seen": 2392588288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797171717171717, + "loss": 2.5977, + "theoretical_loss": 3.3803854099240533, + "tokens_seen": 2392653824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796969696969697, + "loss": 2.6416, + "theoretical_loss": 3.3803779593657004, + "tokens_seen": 2392719360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796767676767677, + "loss": 2.7815, + "theoretical_loss": 3.380370509068551, + "tokens_seen": 2392784896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796565656565657, + "loss": 2.5654, + "theoretical_loss": 3.380363059032589, + "tokens_seen": 2392850432 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027963636363636363, + "loss": 2.4607, + "theoretical_loss": 3.3803556092577978, + "tokens_seen": 2392915968 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796161616161616, + "loss": 2.4767, + "theoretical_loss": 3.380348159744162, + "tokens_seen": 2392981504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795959595959596, + "loss": 2.342, + "theoretical_loss": 3.380340710491664, + "tokens_seen": 2393047040 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795757575757576, + "loss": 2.6491, + "theoretical_loss": 3.380333261500289, + "tokens_seen": 2393112576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027955555555555557, + "loss": 2.6328, + "theoretical_loss": 3.3803258127700193, + "tokens_seen": 2393178112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795353535353535, + "loss": 2.7062, + "theoretical_loss": 3.380318364300839, + "tokens_seen": 2393243648 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795151515151515, + "loss": 2.5557, + "theoretical_loss": 3.380310916092732, + "tokens_seen": 2393309184 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027949494949494955, + "loss": 2.7179, + "theoretical_loss": 3.3803034681456827, + "tokens_seen": 2393374720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794747474747475, + "loss": 2.6354, + "theoretical_loss": 3.3802960204596735, + "tokens_seen": 2393440256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027945454545454546, + "loss": 2.4896, + "theoretical_loss": 3.3802885730346888, + "tokens_seen": 2393505792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794343434343434, + "loss": 2.6934, + "theoretical_loss": 3.380281125870712, + "tokens_seen": 2393571328 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794141414141414, + "loss": 2.4742, + "theoretical_loss": 3.3802736789677272, + "tokens_seen": 2393636864 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027939393939393944, + "loss": 2.5481, + "theoretical_loss": 3.3802662323257175, + "tokens_seen": 2393702400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793737373737374, + "loss": 2.5392, + "theoretical_loss": 3.380258785944667, + "tokens_seen": 2393767936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027935353535353535, + "loss": 2.5711, + "theoretical_loss": 3.38025133982456, + "tokens_seen": 2393833472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793333333333333, + "loss": 2.4893, + "theoretical_loss": 3.380243893965379, + "tokens_seen": 2393899008 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1314506, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.881012201309204, + "objective/train/theoretical_loss": 3.380236448367109, + "objective/train/tokens_used": 752823776, + "theoretical_loss": 3.380236448367109, + "tokens_seen": 2393964544 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793131313131313, + "loss": 2.8527, + "theoretical_loss": 3.380236448367109, + "tokens_seen": 2393964544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027929292929292933, + "loss": 2.6398, + "theoretical_loss": 3.380229003029733, + "tokens_seen": 2394030080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792727272727273, + "loss": 2.637, + "theoretical_loss": 3.380221557953234, + "tokens_seen": 2394095616 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027925252525252525, + "loss": 2.7507, + "theoretical_loss": 3.380214113137597, + "tokens_seen": 2394161152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792323232323232, + "loss": 2.604, + "theoretical_loss": 3.3802066685828054, + "tokens_seen": 2394226688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792121212121212, + "loss": 2.6674, + "theoretical_loss": 3.3801992242888423, + "tokens_seen": 2394292224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791919191919192, + "loss": 2.6199, + "theoretical_loss": 3.380191780255692, + "tokens_seen": 2394357760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791717171717172, + "loss": 2.7008, + "theoretical_loss": 3.380184336483338, + "tokens_seen": 2394423296 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027915151515151514, + "loss": 2.6966, + "theoretical_loss": 3.380176892971764, + "tokens_seen": 2394488832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027913131313131315, + "loss": 2.7605, + "theoretical_loss": 3.3801694497209542, + "tokens_seen": 2394554368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791111111111111, + "loss": 2.4733, + "theoretical_loss": 3.3801620067308917, + "tokens_seen": 2394619904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790909090909091, + "loss": 2.5202, + "theoretical_loss": 3.380154564001561, + "tokens_seen": 2394685440 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790707070707071, + "loss": 2.455, + "theoretical_loss": 3.3801471215329446, + "tokens_seen": 2394750976 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027905050505050503, + "loss": 2.5015, + "theoretical_loss": 3.380139679325027, + "tokens_seen": 2394816512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027903030303030304, + "loss": 2.5401, + "theoretical_loss": 3.3801322373777927, + "tokens_seen": 2394882048 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279010101010101, + "loss": 2.5432, + "theoretical_loss": 3.380124795691224, + "tokens_seen": 2394947584 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278989898989899, + "loss": 2.5095, + "theoretical_loss": 3.3801173542653054, + "tokens_seen": 2395013120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027896969696969697, + "loss": 2.7084, + "theoretical_loss": 3.38010991310002, + "tokens_seen": 2395078656 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278949494949495, + "loss": 2.5357, + "theoretical_loss": 3.3801024721953525, + "tokens_seen": 2395144192 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027892929292929294, + "loss": 2.5665, + "theoretical_loss": 3.380095031551286, + "tokens_seen": 2395209728 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789090909090909, + "loss": 2.7815, + "theoretical_loss": 3.3800875911678046, + "tokens_seen": 2395275264 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788888888888889, + "loss": 2.6284, + "theoretical_loss": 3.3800801510448917, + "tokens_seen": 2395340800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027886868686868686, + "loss": 2.5418, + "theoretical_loss": 3.3800727111825317, + "tokens_seen": 2395406336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027884848484848487, + "loss": 2.5, + "theoretical_loss": 3.3800652715807074, + "tokens_seen": 2395471872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027882828282828283, + "loss": 2.41, + "theoretical_loss": 3.3800578322394026, + "tokens_seen": 2395537408 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1315670, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2574543952941895, + "objective/train/theoretical_loss": 3.3800503931586023, + "objective/train/tokens_used": 754462176, + "theoretical_loss": 3.3800503931586023, + "tokens_seen": 2395602944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788080808080808, + "loss": 2.4554, + "theoretical_loss": 3.3800503931586023, + "tokens_seen": 2395602944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787878787878788, + "loss": 2.547, + "theoretical_loss": 3.380042954338289, + "tokens_seen": 2395668480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787676767676768, + "loss": 2.5429, + "theoretical_loss": 3.380035515778447, + "tokens_seen": 2395734016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027874747474747476, + "loss": 2.8526, + "theoretical_loss": 3.3800280774790594, + "tokens_seen": 2395799552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787272727272727, + "loss": 2.556, + "theoretical_loss": 3.380020639440111, + "tokens_seen": 2395865088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787070707070707, + "loss": 2.6507, + "theoretical_loss": 3.380013201661585, + "tokens_seen": 2395930624 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786868686868687, + "loss": 2.5906, + "theoretical_loss": 3.380005764143465, + "tokens_seen": 2395996160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786666666666667, + "loss": 2.7161, + "theoretical_loss": 3.379998326885735, + "tokens_seen": 2396061696 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027864646464646466, + "loss": 2.4844, + "theoretical_loss": 3.379990889888379, + "tokens_seen": 2396127232 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786262626262626, + "loss": 2.4491, + "theoretical_loss": 3.3799834531513797, + "tokens_seen": 2396192768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027860606060606057, + "loss": 2.6456, + "theoretical_loss": 3.3799760166747226, + "tokens_seen": 2396258304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027858585858585864, + "loss": 2.5085, + "theoretical_loss": 3.37996858045839, + "tokens_seen": 2396323840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785656565656566, + "loss": 2.8677, + "theoretical_loss": 3.3799611445023663, + "tokens_seen": 2396389376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027854545454545455, + "loss": 2.5066, + "theoretical_loss": 3.379953708806635, + "tokens_seen": 2396454912 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785252525252525, + "loss": 2.4542, + "theoretical_loss": 3.3799462733711803, + "tokens_seen": 2396520448 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027850505050505046, + "loss": 2.5422, + "theoretical_loss": 3.379938838195985, + "tokens_seen": 2396585984 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027848484848484853, + "loss": 2.7111, + "theoretical_loss": 3.3799314032810344, + "tokens_seen": 2396651520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784646464646465, + "loss": 2.6067, + "theoretical_loss": 3.379923968626311, + "tokens_seen": 2396717056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027844444444444444, + "loss": 2.6897, + "theoretical_loss": 3.379916534231799, + "tokens_seen": 2396782592 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784242424242424, + "loss": 2.3453, + "theoretical_loss": 3.3799091000974824, + "tokens_seen": 2396848128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784040404040404, + "loss": 2.5504, + "theoretical_loss": 3.3799016662233448, + "tokens_seen": 2396913664 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783838383838384, + "loss": 2.7081, + "theoretical_loss": 3.3798942326093697, + "tokens_seen": 2396979200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783636363636364, + "loss": 2.5611, + "theoretical_loss": 3.3798867992555413, + "tokens_seen": 2397044736 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027834343434343434, + "loss": 2.7388, + "theoretical_loss": 3.379879366161843, + "tokens_seen": 2397110272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783232323232323, + "loss": 2.3034, + "theoretical_loss": 3.3798719333282587, + "tokens_seen": 2397175808 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1316257, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.401369094848633, + "objective/train/theoretical_loss": 3.3798645007547727, + "objective/train/tokens_used": 756100576, + "theoretical_loss": 3.3798645007547727, + "tokens_seen": 2397241344 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027830303030303036, + "loss": 2.619, + "theoretical_loss": 3.3798645007547727, + "tokens_seen": 2397241344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782828282828283, + "loss": 2.3024, + "theoretical_loss": 3.3798570684413685, + "tokens_seen": 2397306880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027826262626262627, + "loss": 2.5966, + "theoretical_loss": 3.3798496363880295, + "tokens_seen": 2397372416 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027824242424242423, + "loss": 2.6577, + "theoretical_loss": 3.3798422045947394, + "tokens_seen": 2397437952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782222222222222, + "loss": 2.6159, + "theoretical_loss": 3.3798347730614826, + "tokens_seen": 2397503488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027820202020202025, + "loss": 2.5999, + "theoretical_loss": 3.3798273417882427, + "tokens_seen": 2397569024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781818181818182, + "loss": 2.6694, + "theoretical_loss": 3.3798199107750038, + "tokens_seen": 2397634560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027816161616161616, + "loss": 2.616, + "theoretical_loss": 3.379812480021749, + "tokens_seen": 2397700096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781414141414141, + "loss": 2.5596, + "theoretical_loss": 3.3798050495284624, + "tokens_seen": 2397765632 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027812121212121213, + "loss": 2.3258, + "theoretical_loss": 3.3797976192951276, + "tokens_seen": 2397831168 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027810101010101014, + "loss": 2.6237, + "theoretical_loss": 3.379790189321729, + "tokens_seen": 2397896704 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780808080808081, + "loss": 2.5432, + "theoretical_loss": 3.3797827596082497, + "tokens_seen": 2397962240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027806060606060606, + "loss": 2.4558, + "theoretical_loss": 3.3797753301546742, + "tokens_seen": 2398027776 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278040404040404, + "loss": 2.4353, + "theoretical_loss": 3.379767900960986, + "tokens_seen": 2398093312 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278020202020202, + "loss": 2.6536, + "theoretical_loss": 3.3797604720271686, + "tokens_seen": 2398158848 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027800000000000004, + "loss": 2.6172, + "theoretical_loss": 3.3797530433532064, + "tokens_seen": 2398224384 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277979797979798, + "loss": 2.5712, + "theoretical_loss": 3.3797456149390825, + "tokens_seen": 2398289920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027795959595959595, + "loss": 2.4917, + "theoretical_loss": 3.379738186784781, + "tokens_seen": 2398355456 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027793939393939396, + "loss": 2.5065, + "theoretical_loss": 3.379730758890286, + "tokens_seen": 2398420992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779191919191919, + "loss": 2.4775, + "theoretical_loss": 3.379723331255581, + "tokens_seen": 2398486528 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027789898989898993, + "loss": 2.6639, + "theoretical_loss": 3.3797159038806504, + "tokens_seen": 2398552064 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778787878787879, + "loss": 2.4218, + "theoretical_loss": 3.379708476765477, + "tokens_seen": 2398617600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027785858585858584, + "loss": 2.5206, + "theoretical_loss": 3.3797010499100453, + "tokens_seen": 2398683136 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027783838383838385, + "loss": 2.6598, + "theoretical_loss": 3.379693623314339, + "tokens_seen": 2398748672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778181818181818, + "loss": 2.6251, + "theoretical_loss": 3.379686196978342, + "tokens_seen": 2398814208 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1317351, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.750336170196533, + "objective/train/theoretical_loss": 3.379678770902038, + "objective/train/tokens_used": 757738976, + "theoretical_loss": 3.379678770902038, + "tokens_seen": 2398879744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777979797979798, + "loss": 2.6726, + "theoretical_loss": 3.379678770902038, + "tokens_seen": 2398879744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777777777777778, + "loss": 2.4826, + "theoretical_loss": 3.3796713450854106, + "tokens_seen": 2398945280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777575757575758, + "loss": 2.5717, + "theoretical_loss": 3.3796639195284444, + "tokens_seen": 2399010816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027773737373737375, + "loss": 2.5577, + "theoretical_loss": 3.379656494231122, + "tokens_seen": 2399076352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777171717171717, + "loss": 2.1633, + "theoretical_loss": 3.3796490691934284, + "tokens_seen": 2399141888 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776969696969697, + "loss": 2.7594, + "theoretical_loss": 3.379641644415347, + "tokens_seen": 2399207424 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027767676767676767, + "loss": 2.5037, + "theoretical_loss": 3.3796342198968614, + "tokens_seen": 2399272960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776565656565657, + "loss": 2.4903, + "theoretical_loss": 3.3796267956379555, + "tokens_seen": 2399338496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027763636363636364, + "loss": 2.472, + "theoretical_loss": 3.3796193716386136, + "tokens_seen": 2399404032 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776161616161616, + "loss": 2.5285, + "theoretical_loss": 3.379611947898819, + "tokens_seen": 2399469568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775959595959596, + "loss": 2.6631, + "theoretical_loss": 3.379604524418556, + "tokens_seen": 2399535104 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775757575757576, + "loss": 2.6522, + "theoretical_loss": 3.379597101197808, + "tokens_seen": 2399600640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775555555555556, + "loss": 2.3629, + "theoretical_loss": 3.3795896782365586, + "tokens_seen": 2399666176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027753535353535353, + "loss": 2.5396, + "theoretical_loss": 3.3795822555347925, + "tokens_seen": 2399731712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775151515151515, + "loss": 2.5843, + "theoretical_loss": 3.3795748330924935, + "tokens_seen": 2399797248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774949494949495, + "loss": 2.4912, + "theoretical_loss": 3.3795674109096443, + "tokens_seen": 2399862784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774747474747475, + "loss": 2.5834, + "theoretical_loss": 3.37955998898623, + "tokens_seen": 2399928320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027745454545454547, + "loss": 2.5894, + "theoretical_loss": 3.3795525673222335, + "tokens_seen": 2399993856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774343434343434, + "loss": 2.6344, + "theoretical_loss": 3.3795451459176395, + "tokens_seen": 2400059392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774141414141414, + "loss": 2.5681, + "theoretical_loss": 3.3795377247724314, + "tokens_seen": 2400124928 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027739393939393945, + "loss": 2.5088, + "theoretical_loss": 3.3795303038865927, + "tokens_seen": 2400190464 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773737373737374, + "loss": 2.5978, + "theoretical_loss": 3.379522883260108, + "tokens_seen": 2400256000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027735353535353536, + "loss": 2.5245, + "theoretical_loss": 3.379515462892961, + "tokens_seen": 2400321536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773333333333333, + "loss": 2.4467, + "theoretical_loss": 3.379508042785135, + "tokens_seen": 2400387072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773131313131313, + "loss": 2.5966, + "theoretical_loss": 3.379500622936614, + "tokens_seen": 2400452608 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1318782, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.464829921722412, + "objective/train/theoretical_loss": 3.3794932033473826, + "objective/train/tokens_used": 759377376, + "theoretical_loss": 3.3794932033473826, + "tokens_seen": 2400518144 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027729292929292934, + "loss": 2.6287, + "theoretical_loss": 3.3794932033473826, + "tokens_seen": 2400518144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772727272727273, + "loss": 2.6682, + "theoretical_loss": 3.3794857840174233, + "tokens_seen": 2400583680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027725252525252525, + "loss": 2.4109, + "theoretical_loss": 3.3794783649467215, + "tokens_seen": 2400649216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772323232323232, + "loss": 2.691, + "theoretical_loss": 3.3794709461352603, + "tokens_seen": 2400714752 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772121212121212, + "loss": 2.5769, + "theoretical_loss": 3.379463527583024, + "tokens_seen": 2400780288 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027719191919191923, + "loss": 2.6515, + "theoretical_loss": 3.379456109289995, + "tokens_seen": 2400845824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771717171717172, + "loss": 2.5084, + "theoretical_loss": 3.379448691256159, + "tokens_seen": 2400911360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027715151515151514, + "loss": 2.5864, + "theoretical_loss": 3.379441273481499, + "tokens_seen": 2400976896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771313131313131, + "loss": 2.4552, + "theoretical_loss": 3.379433855965999, + "tokens_seen": 2401042432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771111111111111, + "loss": 2.5868, + "theoretical_loss": 3.3794264387096433, + "tokens_seen": 2401107968 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770909090909091, + "loss": 2.446, + "theoretical_loss": 3.379419021712415, + "tokens_seen": 2401173504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770707070707071, + "loss": 2.5476, + "theoretical_loss": 3.379411604974298, + "tokens_seen": 2401239040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027705050505050504, + "loss": 2.8137, + "theoretical_loss": 3.379404188495277, + "tokens_seen": 2401304576 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277030303030303, + "loss": 2.6532, + "theoretical_loss": 3.379396772275335, + "tokens_seen": 2401370112 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277010101010101, + "loss": 2.6834, + "theoretical_loss": 3.3793893563144564, + "tokens_seen": 2401435648 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276989898989899, + "loss": 2.4592, + "theoretical_loss": 3.3793819406126246, + "tokens_seen": 2401501184 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276969696969697, + "loss": 2.4117, + "theoretical_loss": 3.3793745251698244, + "tokens_seen": 2401566720 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027694949494949493, + "loss": 2.5192, + "theoretical_loss": 3.3793671099860387, + "tokens_seen": 2401632256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027692929292929294, + "loss": 2.6241, + "theoretical_loss": 3.379359695061252, + "tokens_seen": 2401697792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769090909090909, + "loss": 2.5624, + "theoretical_loss": 3.379352280395448, + "tokens_seen": 2401763328 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768888888888889, + "loss": 2.6287, + "theoretical_loss": 3.3793448659886103, + "tokens_seen": 2401828864 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027686868686868687, + "loss": 2.6455, + "theoretical_loss": 3.3793374518407235, + "tokens_seen": 2401894400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768484848484848, + "loss": 2.5204, + "theoretical_loss": 3.3793300379517706, + "tokens_seen": 2401959936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027682828282828283, + "loss": 2.5895, + "theoretical_loss": 3.379322624321736, + "tokens_seen": 2402025472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768080808080808, + "loss": 2.482, + "theoretical_loss": 3.3793152109506037, + "tokens_seen": 2402091008 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1319376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.870497226715088, + "objective/train/theoretical_loss": 3.379307797838357, + "objective/train/tokens_used": 761015776, + "theoretical_loss": 3.379307797838357, + "tokens_seen": 2402156544 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767878787878788, + "loss": 2.4212, + "theoretical_loss": 3.379307797838357, + "tokens_seen": 2402156544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027676767676767676, + "loss": 2.8379, + "theoretical_loss": 3.3793003849849805, + "tokens_seen": 2402222080 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027674747474747477, + "loss": 2.6259, + "theoretical_loss": 3.3792929723904583, + "tokens_seen": 2402287616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767272727272727, + "loss": 2.3961, + "theoretical_loss": 3.379285560054773, + "tokens_seen": 2402353152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027670707070707074, + "loss": 2.5514, + "theoretical_loss": 3.37927814797791, + "tokens_seen": 2402418688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766868686868687, + "loss": 2.4595, + "theoretical_loss": 3.379270736159852, + "tokens_seen": 2402484224 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027666666666666665, + "loss": 2.629, + "theoretical_loss": 3.3792633246005837, + "tokens_seen": 2402549760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027664646464646466, + "loss": 2.3439, + "theoretical_loss": 3.3792559133000886, + "tokens_seen": 2402615296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766262626262626, + "loss": 2.5871, + "theoretical_loss": 3.379248502258351, + "tokens_seen": 2402680832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027660606060606063, + "loss": 2.5348, + "theoretical_loss": 3.3792410914753543, + "tokens_seen": 2402746368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765858585858586, + "loss": 2.7453, + "theoretical_loss": 3.379233680951083, + "tokens_seen": 2402811904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765656565656566, + "loss": 2.3807, + "theoretical_loss": 3.37922627068552, + "tokens_seen": 2402877440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027654545454545456, + "loss": 2.3371, + "theoretical_loss": 3.37921886067865, + "tokens_seen": 2402942976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765252525252525, + "loss": 2.4897, + "theoretical_loss": 3.379211450930457, + "tokens_seen": 2403008512 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765050505050505, + "loss": 2.6656, + "theoretical_loss": 3.379204041440925, + "tokens_seen": 2403074048 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764848484848485, + "loss": 2.7301, + "theoretical_loss": 3.3791966322100375, + "tokens_seen": 2403139584 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764646464646465, + "loss": 2.6037, + "theoretical_loss": 3.3791892232377783, + "tokens_seen": 2403205120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027644444444444445, + "loss": 2.6713, + "theoretical_loss": 3.3791818145241317, + "tokens_seen": 2403270656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764242424242424, + "loss": 2.5729, + "theoretical_loss": 3.3791744060690814, + "tokens_seen": 2403336192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764040404040404, + "loss": 2.5159, + "theoretical_loss": 3.3791669978726113, + "tokens_seen": 2403401728 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763838383838384, + "loss": 2.4243, + "theoretical_loss": 3.379159589934706, + "tokens_seen": 2403467264 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763636363636364, + "loss": 2.46, + "theoretical_loss": 3.379152182255348, + "tokens_seen": 2403532800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027634343434343434, + "loss": 2.5554, + "theoretical_loss": 3.379144774834523, + "tokens_seen": 2403598336 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763232323232323, + "loss": 2.7471, + "theoretical_loss": 3.379137367672213, + "tokens_seen": 2403663872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763030303030303, + "loss": 2.431, + "theoretical_loss": 3.3791299607684038, + "tokens_seen": 2403729408 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1320625, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8724403381347656, + "objective/train/theoretical_loss": 3.3791225541230783, + "objective/train/tokens_used": 762654176, + "theoretical_loss": 3.3791225541230783, + "tokens_seen": 2403794944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762828282828283, + "loss": 2.8231, + "theoretical_loss": 3.3791225541230783, + "tokens_seen": 2403794944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762626262626263, + "loss": 2.6772, + "theoretical_loss": 3.3791151477362202, + "tokens_seen": 2403860480 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027624242424242423, + "loss": 2.6169, + "theoretical_loss": 3.379107741607814, + "tokens_seen": 2403926016 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762222222222222, + "loss": 2.5812, + "theoretical_loss": 3.3791003357378435, + "tokens_seen": 2403991552 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027620202020202026, + "loss": 2.0871, + "theoretical_loss": 3.379092930126293, + "tokens_seen": 2404057088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761818181818182, + "loss": 2.6355, + "theoretical_loss": 3.3790855247731457, + "tokens_seen": 2404122624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027616161616161617, + "loss": 2.5493, + "theoretical_loss": 3.379078119678386, + "tokens_seen": 2404188160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761414141414141, + "loss": 2.6729, + "theoretical_loss": 3.3790707148419976, + "tokens_seen": 2404253696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761212121212121, + "loss": 2.4889, + "theoretical_loss": 3.3790633102639647, + "tokens_seen": 2404319232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027610101010101015, + "loss": 2.5127, + "theoretical_loss": 3.379055905944271, + "tokens_seen": 2404384768 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760808080808081, + "loss": 2.6958, + "theoretical_loss": 3.3790485018829006, + "tokens_seen": 2404450304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027606060606060606, + "loss": 2.5552, + "theoretical_loss": 3.3790410980798375, + "tokens_seen": 2404515840 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276040404040404, + "loss": 2.64, + "theoretical_loss": 3.379033694535065, + "tokens_seen": 2404581376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027602020202020203, + "loss": 2.6541, + "theoretical_loss": 3.379026291248568, + "tokens_seen": 2404646912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027600000000000004, + "loss": 2.3945, + "theoretical_loss": 3.3790188882203305, + "tokens_seen": 2404712448 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275979797979798, + "loss": 2.486, + "theoretical_loss": 3.3790114854503357, + "tokens_seen": 2404777984 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027595959595959595, + "loss": 2.5952, + "theoretical_loss": 3.3790040829385677, + "tokens_seen": 2404843520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759393939393939, + "loss": 2.3296, + "theoretical_loss": 3.378996680685011, + "tokens_seen": 2404909056 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759191919191919, + "loss": 2.6235, + "theoretical_loss": 3.378989278689649, + "tokens_seen": 2404974592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027589898989898993, + "loss": 2.3744, + "theoretical_loss": 3.3789818769524658, + "tokens_seen": 2405040128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758787878787879, + "loss": 2.5835, + "theoretical_loss": 3.378974475473445, + "tokens_seen": 2405105664 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027585858585858585, + "loss": 2.5668, + "theoretical_loss": 3.3789670742525715, + "tokens_seen": 2405171200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027583838383838386, + "loss": 2.5913, + "theoretical_loss": 3.3789596732898284, + "tokens_seen": 2405236736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758181818181818, + "loss": 2.4548, + "theoretical_loss": 3.3789522725852, + "tokens_seen": 2405302272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757979797979798, + "loss": 2.6723, + "theoretical_loss": 3.3789448721386703, + "tokens_seen": 2405367808 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1321139, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0456953048706055, + "objective/train/theoretical_loss": 3.3789374719502234, + "objective/train/tokens_used": 764292576, + "theoretical_loss": 3.3789374719502234, + "tokens_seen": 2405433344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757777777777778, + "loss": 2.7444, + "theoretical_loss": 3.3789374719502234, + "tokens_seen": 2405433344 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027575757575757574, + "loss": 2.4849, + "theoretical_loss": 3.378930072019843, + "tokens_seen": 2405498880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027573737373737375, + "loss": 2.5797, + "theoretical_loss": 3.378922672347513, + "tokens_seen": 2405564416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757171717171717, + "loss": 2.4306, + "theoretical_loss": 3.378915272933218, + "tokens_seen": 2405629952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756969696969697, + "loss": 2.3946, + "theoretical_loss": 3.3789078737769414, + "tokens_seen": 2405695488 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756767676767677, + "loss": 2.5311, + "theoretical_loss": 3.378900474878667, + "tokens_seen": 2405761024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756565656565657, + "loss": 2.5221, + "theoretical_loss": 3.378893076238379, + "tokens_seen": 2405826560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027563636363636364, + "loss": 2.5755, + "theoretical_loss": 3.3788856778560614, + "tokens_seen": 2405892096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756161616161616, + "loss": 2.7056, + "theoretical_loss": 3.3788782797316985, + "tokens_seen": 2405957632 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755959595959596, + "loss": 2.4726, + "theoretical_loss": 3.3788708818652737, + "tokens_seen": 2406023168 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027557575757575757, + "loss": 2.5859, + "theoretical_loss": 3.3788634842567715, + "tokens_seen": 2406088704 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755555555555556, + "loss": 2.5748, + "theoretical_loss": 3.3788560869061754, + "tokens_seen": 2406154240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027553535353535354, + "loss": 2.4301, + "theoretical_loss": 3.37884868981347, + "tokens_seen": 2406219776 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755151515151515, + "loss": 2.4314, + "theoretical_loss": 3.3788412929786382, + "tokens_seen": 2406285312 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754949494949495, + "loss": 2.5033, + "theoretical_loss": 3.3788338964016655, + "tokens_seen": 2406350848 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754747474747475, + "loss": 2.5634, + "theoretical_loss": 3.3788265000825346, + "tokens_seen": 2406416384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027545454545454547, + "loss": 2.6706, + "theoretical_loss": 3.3788191040212303, + "tokens_seen": 2406481920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027543434343434343, + "loss": 2.5356, + "theoretical_loss": 3.3788117082177362, + "tokens_seen": 2406547456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754141414141414, + "loss": 2.4759, + "theoretical_loss": 3.3788043126720364, + "tokens_seen": 2406612992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753939393939394, + "loss": 2.2995, + "theoretical_loss": 3.378796917384115, + "tokens_seen": 2406678528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753737373737374, + "loss": 2.5573, + "theoretical_loss": 3.3787895223539555, + "tokens_seen": 2406744064 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027535353535353537, + "loss": 2.5764, + "theoretical_loss": 3.3787821275815424, + "tokens_seen": 2406809600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753333333333333, + "loss": 2.4525, + "theoretical_loss": 3.3787747330668596, + "tokens_seen": 2406875136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753131313131313, + "loss": 2.7627, + "theoretical_loss": 3.378767338809891, + "tokens_seen": 2406940672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027529292929292934, + "loss": 2.5667, + "theoretical_loss": 3.378759944810621, + "tokens_seen": 2407006208 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1322223, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9623414278030396, + "objective/train/theoretical_loss": 3.378752551069033, + "objective/train/tokens_used": 765930976, + "theoretical_loss": 3.378752551069033, + "tokens_seen": 2407071744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752727272727273, + "loss": 2.3891, + "theoretical_loss": 3.378752551069033, + "tokens_seen": 2407071744 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027525252525252526, + "loss": 2.5254, + "theoretical_loss": 3.3787451575851115, + "tokens_seen": 2407137280 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752323232323232, + "loss": 2.5511, + "theoretical_loss": 3.37873776435884, + "tokens_seen": 2407202816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027521212121212117, + "loss": 2.5441, + "theoretical_loss": 3.3787303713902026, + "tokens_seen": 2407268352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027519191919191924, + "loss": 2.5071, + "theoretical_loss": 3.3787229786791837, + "tokens_seen": 2407333888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751717171717172, + "loss": 2.6239, + "theoretical_loss": 3.378715586225767, + "tokens_seen": 2407399424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027515151515151515, + "loss": 2.5842, + "theoretical_loss": 3.3787081940299366, + "tokens_seen": 2407464960 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751313131313131, + "loss": 2.8004, + "theoretical_loss": 3.378700802091677, + "tokens_seen": 2407530496 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027511111111111106, + "loss": 2.638, + "theoretical_loss": 3.378693410410971, + "tokens_seen": 2407596032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027509090909090913, + "loss": 2.6841, + "theoretical_loss": 3.378686018987804, + "tokens_seen": 2407661568 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750707070707071, + "loss": 2.6142, + "theoretical_loss": 3.378678627822159, + "tokens_seen": 2407727104 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027505050505050504, + "loss": 2.6844, + "theoretical_loss": 3.3786712369140206, + "tokens_seen": 2407792640 + }, + { + "epoch": 0.46, + "learning_rate": 0.000275030303030303, + "loss": 2.6938, + "theoretical_loss": 3.3786638462633722, + "tokens_seen": 2407858176 + }, + { + "epoch": 0.46, + "learning_rate": 0.000275010101010101, + "loss": 2.3917, + "theoretical_loss": 3.378656455870199, + "tokens_seen": 2407923712 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274989898989899, + "loss": 2.4709, + "theoretical_loss": 3.3786490657344834, + "tokens_seen": 2407989248 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274969696969697, + "loss": 2.4547, + "theoretical_loss": 3.378641675856211, + "tokens_seen": 2408054784 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027494949494949494, + "loss": 2.4256, + "theoretical_loss": 3.378634286235364, + "tokens_seen": 2408120320 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749292929292929, + "loss": 2.5933, + "theoretical_loss": 3.378626896871929, + "tokens_seen": 2408185856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027490909090909096, + "loss": 2.398, + "theoretical_loss": 3.3786195077658876, + "tokens_seen": 2408251392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002748888888888889, + "loss": 2.586, + "theoretical_loss": 3.378612118917225, + "tokens_seen": 2408316928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027486868686868687, + "loss": 2.5633, + "theoretical_loss": 3.3786047303259252, + "tokens_seen": 2408382464 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027484848484848483, + "loss": 2.6146, + "theoretical_loss": 3.3785973419919717, + "tokens_seen": 2408448000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027482828282828284, + "loss": 2.4238, + "theoretical_loss": 3.378589953915349, + "tokens_seen": 2408513536 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027480808080808085, + "loss": 2.5799, + "theoretical_loss": 3.3785825660960414, + "tokens_seen": 2408579072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747878787878788, + "loss": 2.6225, + "theoretical_loss": 3.378575178534032, + "tokens_seen": 2408644608 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1322701, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3897078037261963, + "objective/train/theoretical_loss": 3.378567791229306, + "objective/train/tokens_used": 767569376, + "theoretical_loss": 3.378567791229306, + "tokens_seen": 2408710144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027476767676767676, + "loss": 2.4039, + "theoretical_loss": 3.378567791229306, + "tokens_seen": 2408710144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747474747474747, + "loss": 2.4518, + "theoretical_loss": 3.3785604041818464, + "tokens_seen": 2408775680 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027472727272727273, + "loss": 2.6513, + "theoretical_loss": 3.3785530173916376, + "tokens_seen": 2408841216 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027470707070707074, + "loss": 2.5527, + "theoretical_loss": 3.3785456308586643, + "tokens_seen": 2408906752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746868686868687, + "loss": 2.5835, + "theoretical_loss": 3.3785382445829093, + "tokens_seen": 2408972288 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027466666666666666, + "loss": 2.5775, + "theoretical_loss": 3.378530858564358, + "tokens_seen": 2409037824 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027464646464646467, + "loss": 2.6796, + "theoretical_loss": 3.378523472802993, + "tokens_seen": 2409103360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746262626262626, + "loss": 2.7412, + "theoretical_loss": 3.3785160872987996, + "tokens_seen": 2409168896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027460606060606064, + "loss": 2.4294, + "theoretical_loss": 3.3785087020517612, + "tokens_seen": 2409234432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745858585858586, + "loss": 2.4665, + "theoretical_loss": 3.378501317061862, + "tokens_seen": 2409299968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027456565656565655, + "loss": 2.6361, + "theoretical_loss": 3.378493932329086, + "tokens_seen": 2409365504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027454545454545456, + "loss": 2.5521, + "theoretical_loss": 3.3784865478534174, + "tokens_seen": 2409431040 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745252525252525, + "loss": 2.6366, + "theoretical_loss": 3.3784791636348404, + "tokens_seen": 2409496576 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027450505050505053, + "loss": 2.7747, + "theoretical_loss": 3.3784717796733386, + "tokens_seen": 2409562112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744848484848485, + "loss": 2.6529, + "theoretical_loss": 3.378464395968896, + "tokens_seen": 2409627648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744646464646465, + "loss": 2.6161, + "theoretical_loss": 3.3784570125214977, + "tokens_seen": 2409693184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027444444444444445, + "loss": 2.5866, + "theoretical_loss": 3.3784496293311266, + "tokens_seen": 2409758720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744242424242424, + "loss": 2.6144, + "theoretical_loss": 3.378442246397767, + "tokens_seen": 2409824256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744040404040404, + "loss": 2.5056, + "theoretical_loss": 3.3784348637214032, + "tokens_seen": 2409889792 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743838383838384, + "loss": 2.6486, + "theoretical_loss": 3.3784274813020194, + "tokens_seen": 2409955328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743636363636364, + "loss": 2.8171, + "theoretical_loss": 3.3784200991395994, + "tokens_seen": 2410020864 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027434343434343435, + "loss": 2.5236, + "theoretical_loss": 3.378412717234127, + "tokens_seen": 2410086400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743232323232323, + "loss": 2.6253, + "theoretical_loss": 3.3784053355855868, + "tokens_seen": 2410151936 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743030303030303, + "loss": 2.4956, + "theoretical_loss": 3.378397954193963, + "tokens_seen": 2410217472 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742828282828283, + "loss": 2.5192, + "theoretical_loss": 3.378390573059239, + "tokens_seen": 2410283008 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1324080, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.414530038833618, + "objective/train/theoretical_loss": 3.3783831921813996, + "objective/train/tokens_used": 769207776, + "theoretical_loss": 3.3783831921813996, + "tokens_seen": 2410348544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742626262626263, + "loss": 2.3967, + "theoretical_loss": 3.3783831921813996, + "tokens_seen": 2410348544 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027424242424242424, + "loss": 2.5257, + "theoretical_loss": 3.378375811560428, + "tokens_seen": 2410414080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742222222222222, + "loss": 2.6932, + "theoretical_loss": 3.378368431196309, + "tokens_seen": 2410479616 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742020202020202, + "loss": 2.4726, + "theoretical_loss": 3.3783610510890267, + "tokens_seen": 2410545152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741818181818182, + "loss": 2.5827, + "theoretical_loss": 3.3783536712385644, + "tokens_seen": 2410610688 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741616161616162, + "loss": 2.7406, + "theoretical_loss": 3.3783462916449074, + "tokens_seen": 2410676224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027414141414141413, + "loss": 2.5695, + "theoretical_loss": 3.3783389123080387, + "tokens_seen": 2410741760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741212121212121, + "loss": 2.68, + "theoretical_loss": 3.378331533227943, + "tokens_seen": 2410807296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027410101010101015, + "loss": 2.6111, + "theoretical_loss": 3.378324154404604, + "tokens_seen": 2410872832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740808080808081, + "loss": 2.5358, + "theoretical_loss": 3.3783167758380057, + "tokens_seen": 2410938368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027406060606060607, + "loss": 2.6948, + "theoretical_loss": 3.378309397528133, + "tokens_seen": 2411003904 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274040404040404, + "loss": 2.4084, + "theoretical_loss": 3.378302019474969, + "tokens_seen": 2411069440 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274020202020202, + "loss": 2.5457, + "theoretical_loss": 3.3782946416784982, + "tokens_seen": 2411134976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027400000000000005, + "loss": 2.4703, + "theoretical_loss": 3.378287264138705, + "tokens_seen": 2411200512 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273979797979798, + "loss": 2.5908, + "theoretical_loss": 3.378279886855573, + "tokens_seen": 2411266048 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027395959595959596, + "loss": 2.7809, + "theoretical_loss": 3.3782725098290864, + "tokens_seen": 2411331584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002739393939393939, + "loss": 2.6265, + "theoretical_loss": 3.3782651330592297, + "tokens_seen": 2411397120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027391919191919193, + "loss": 2.6918, + "theoretical_loss": 3.3782577565459864, + "tokens_seen": 2411462656 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027389898989898994, + "loss": 2.6622, + "theoretical_loss": 3.3782503802893413, + "tokens_seen": 2411528192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738787878787879, + "loss": 2.5508, + "theoretical_loss": 3.3782430042892777, + "tokens_seen": 2411593728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027385858585858585, + "loss": 2.4676, + "theoretical_loss": 3.3782356285457804, + "tokens_seen": 2411659264 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738383838383838, + "loss": 2.4347, + "theoretical_loss": 3.3782282530588335, + "tokens_seen": 2411724800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738181818181818, + "loss": 2.5036, + "theoretical_loss": 3.37822087782842, + "tokens_seen": 2411790336 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027379797979797983, + "loss": 2.7887, + "theoretical_loss": 3.3782135028545253, + "tokens_seen": 2411855872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737777777777778, + "loss": 2.6072, + "theoretical_loss": 3.378206128137133, + "tokens_seen": 2411921408 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1324705, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.521892786026001, + "objective/train/theoretical_loss": 3.3781987536762275, + "objective/train/tokens_used": 770846176, + "theoretical_loss": 3.3781987536762275, + "tokens_seen": 2411986944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027375757575757575, + "loss": 2.505, + "theoretical_loss": 3.3781987536762275, + "tokens_seen": 2411986944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737373737373737, + "loss": 2.5708, + "theoretical_loss": 3.378191379471792, + "tokens_seen": 2412052480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737171717171717, + "loss": 2.592, + "theoretical_loss": 3.3781840055238117, + "tokens_seen": 2412118016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736969696969697, + "loss": 2.5439, + "theoretical_loss": 3.3781766318322703, + "tokens_seen": 2412183552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736767676767677, + "loss": 2.3507, + "theoretical_loss": 3.3781692583971514, + "tokens_seen": 2412249088 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027365656565656564, + "loss": 2.5583, + "theoretical_loss": 3.3781618852184403, + "tokens_seen": 2412314624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027363636363636365, + "loss": 2.5696, + "theoretical_loss": 3.3781545122961205, + "tokens_seen": 2412380160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736161616161616, + "loss": 2.6648, + "theoretical_loss": 3.378147139630175, + "tokens_seen": 2412445696 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735959595959596, + "loss": 2.5808, + "theoretical_loss": 3.37813976722059, + "tokens_seen": 2412511232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735757575757576, + "loss": 2.451, + "theoretical_loss": 3.3781323950673485, + "tokens_seen": 2412576768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027355555555555553, + "loss": 2.6893, + "theoretical_loss": 3.3781250231704343, + "tokens_seen": 2412642304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027353535353535354, + "loss": 2.4823, + "theoretical_loss": 3.378117651529832, + "tokens_seen": 2412707840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735151515151515, + "loss": 2.4537, + "theoretical_loss": 3.378110280145526, + "tokens_seen": 2412773376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734949494949495, + "loss": 2.5139, + "theoretical_loss": 3.3781029090174997, + "tokens_seen": 2412838912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027347474747474747, + "loss": 2.7567, + "theoretical_loss": 3.378095538145738, + "tokens_seen": 2412904448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734545454545455, + "loss": 2.6232, + "theoretical_loss": 3.3780881675302243, + "tokens_seen": 2412969984 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027343434343434343, + "loss": 2.6075, + "theoretical_loss": 3.3780807971709432, + "tokens_seen": 2413035520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734141414141414, + "loss": 2.6903, + "theoretical_loss": 3.3780734270678785, + "tokens_seen": 2413101056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733939393939394, + "loss": 2.7068, + "theoretical_loss": 3.378066057221015, + "tokens_seen": 2413166592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027337373737373736, + "loss": 2.2726, + "theoretical_loss": 3.378058687630336, + "tokens_seen": 2413232128 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027335353535353537, + "loss": 2.4803, + "theoretical_loss": 3.3780513182958263, + "tokens_seen": 2413297664 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027333333333333333, + "loss": 2.3323, + "theoretical_loss": 3.3780439492174694, + "tokens_seen": 2413363200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733131313131313, + "loss": 2.7783, + "theoretical_loss": 3.3780365803952503, + "tokens_seen": 2413428736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732929292929293, + "loss": 2.5186, + "theoretical_loss": 3.378029211829152, + "tokens_seen": 2413494272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732727272727273, + "loss": 2.397, + "theoretical_loss": 3.3780218435191602, + "tokens_seen": 2413559808 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1325945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0629208087921143, + "objective/train/theoretical_loss": 3.3780144754652577, + "objective/train/tokens_used": 772484576, + "theoretical_loss": 3.3780144754652577, + "tokens_seen": 2413625344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027325252525252526, + "loss": 2.4717, + "theoretical_loss": 3.3780144754652577, + "tokens_seen": 2413625344 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732323232323232, + "loss": 2.7437, + "theoretical_loss": 3.378007107667429, + "tokens_seen": 2413690880 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027321212121212123, + "loss": 2.544, + "theoretical_loss": 3.3779997401256585, + "tokens_seen": 2413756416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731919191919192, + "loss": 2.8167, + "theoretical_loss": 3.37799237283993, + "tokens_seen": 2413821952 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731717171717172, + "loss": 2.6599, + "theoretical_loss": 3.3779850058102276, + "tokens_seen": 2413887488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027315151515151516, + "loss": 2.6438, + "theoretical_loss": 3.377977639036536, + "tokens_seen": 2413953024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731313131313131, + "loss": 2.7503, + "theoretical_loss": 3.377970272518839, + "tokens_seen": 2414018560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731111111111111, + "loss": 2.6223, + "theoretical_loss": 3.377962906257121, + "tokens_seen": 2414084096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027309090909090913, + "loss": 2.6295, + "theoretical_loss": 3.3779555402513655, + "tokens_seen": 2414149632 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002730707070707071, + "loss": 2.5036, + "theoretical_loss": 3.3779481745015576, + "tokens_seen": 2414215168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027305050505050505, + "loss": 2.7458, + "theoretical_loss": 3.3779408090076806, + "tokens_seen": 2414280704 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273030303030303, + "loss": 2.317, + "theoretical_loss": 3.3779334437697193, + "tokens_seen": 2414346240 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273010101010101, + "loss": 2.716, + "theoretical_loss": 3.3779260787876573, + "tokens_seen": 2414411776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027298989898989903, + "loss": 2.5639, + "theoretical_loss": 3.3779187140614795, + "tokens_seen": 2414477312 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272969696969697, + "loss": 2.8078, + "theoretical_loss": 3.377911349591169, + "tokens_seen": 2414542848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027294949494949494, + "loss": 2.3878, + "theoretical_loss": 3.3779039853767108, + "tokens_seen": 2414608384 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729292929292929, + "loss": 2.434, + "theoretical_loss": 3.377896621418089, + "tokens_seen": 2414673920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027290909090909096, + "loss": 2.5527, + "theoretical_loss": 3.377889257715288, + "tokens_seen": 2414739456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728888888888889, + "loss": 2.5444, + "theoretical_loss": 3.377881894268291, + "tokens_seen": 2414804992 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728686868686869, + "loss": 2.5772, + "theoretical_loss": 3.3778745310770826, + "tokens_seen": 2414870528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027284848484848483, + "loss": 2.498, + "theoretical_loss": 3.3778671681416474, + "tokens_seen": 2414936064 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728282828282828, + "loss": 2.4699, + "theoretical_loss": 3.3778598054619695, + "tokens_seen": 2415001600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027280808080808086, + "loss": 2.436, + "theoretical_loss": 3.377852443038033, + "tokens_seen": 2415067136 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727878787878788, + "loss": 2.4811, + "theoretical_loss": 3.3778450808698213, + "tokens_seen": 2415132672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027276767676767677, + "loss": 2.6489, + "theoretical_loss": 3.37783771895732, + "tokens_seen": 2415198208 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1326379, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.516770601272583, + "objective/train/theoretical_loss": 3.377830357300512, + "objective/train/tokens_used": 774122976, + "theoretical_loss": 3.377830357300512, + "tokens_seen": 2415263744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727474747474747, + "loss": 2.5568, + "theoretical_loss": 3.377830357300512, + "tokens_seen": 2415263744 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027272727272727274, + "loss": 2.5463, + "theoretical_loss": 3.377822995899382, + "tokens_seen": 2415329280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027270707070707075, + "loss": 2.7598, + "theoretical_loss": 3.3778156347539143, + "tokens_seen": 2415394816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002726868686868687, + "loss": 2.5923, + "theoretical_loss": 3.377808273864093, + "tokens_seen": 2415460352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027266666666666666, + "loss": 2.673, + "theoretical_loss": 3.377800913229902, + "tokens_seen": 2415525888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002726464646464646, + "loss": 2.3286, + "theoretical_loss": 3.377793552851326, + "tokens_seen": 2415591424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027262626262626263, + "loss": 2.6616, + "theoretical_loss": 3.3777861927283492, + "tokens_seen": 2415656960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027260606060606064, + "loss": 2.4835, + "theoretical_loss": 3.3777788328609555, + "tokens_seen": 2415722496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725858585858586, + "loss": 2.6633, + "theoretical_loss": 3.377771473249129, + "tokens_seen": 2415788032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027256565656565656, + "loss": 2.5684, + "theoretical_loss": 3.3777641138928534, + "tokens_seen": 2415853568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027254545454545457, + "loss": 2.5802, + "theoretical_loss": 3.377756754792114, + "tokens_seen": 2415919104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725252525252525, + "loss": 2.7372, + "theoretical_loss": 3.3777493959468945, + "tokens_seen": 2415984640 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027250505050505053, + "loss": 2.6714, + "theoretical_loss": 3.377742037357179, + "tokens_seen": 2416050176 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724848484848485, + "loss": 2.7078, + "theoretical_loss": 3.377734679022952, + "tokens_seen": 2416115712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027246464646464645, + "loss": 2.7154, + "theoretical_loss": 3.3777273209441976, + "tokens_seen": 2416181248 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027244444444444446, + "loss": 2.4861, + "theoretical_loss": 3.3777199631208994, + "tokens_seen": 2416246784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724242424242424, + "loss": 2.5344, + "theoretical_loss": 3.377712605553042, + "tokens_seen": 2416312320 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724040404040404, + "loss": 2.5755, + "theoretical_loss": 3.3777052482406105, + "tokens_seen": 2416377856 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723838383838384, + "loss": 2.4513, + "theoretical_loss": 3.3776978911835878, + "tokens_seen": 2416443392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723636363636364, + "loss": 2.4515, + "theoretical_loss": 3.3776905343819585, + "tokens_seen": 2416508928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027234343434343435, + "loss": 2.2856, + "theoretical_loss": 3.3776831778357073, + "tokens_seen": 2416574464 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723232323232323, + "loss": 2.4504, + "theoretical_loss": 3.377675821544818, + "tokens_seen": 2416640000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723030303030303, + "loss": 2.529, + "theoretical_loss": 3.3776684655092746, + "tokens_seen": 2416705536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722828282828283, + "loss": 2.4591, + "theoretical_loss": 3.377661109729061, + "tokens_seen": 2416771072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722626262626263, + "loss": 2.3674, + "theoretical_loss": 3.3776537542041627, + "tokens_seen": 2416836608 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1327590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.714198112487793, + "objective/train/theoretical_loss": 3.377646398934563, + "objective/train/tokens_used": 775761376, + "theoretical_loss": 3.377646398934563, + "tokens_seen": 2416902144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027224242424242424, + "loss": 2.4671, + "theoretical_loss": 3.377646398934563, + "tokens_seen": 2416902144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722222222222222, + "loss": 2.6425, + "theoretical_loss": 3.3776390439202464, + "tokens_seen": 2416967680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722020202020202, + "loss": 2.6495, + "theoretical_loss": 3.377631689161197, + "tokens_seen": 2417033216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721818181818182, + "loss": 2.6031, + "theoretical_loss": 3.377624334657399, + "tokens_seen": 2417098752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721616161616162, + "loss": 2.4543, + "theoretical_loss": 3.3776169804088365, + "tokens_seen": 2417164288 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027214141414141414, + "loss": 2.6633, + "theoretical_loss": 3.3776096264154942, + "tokens_seen": 2417229824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721212121212121, + "loss": 2.6781, + "theoretical_loss": 3.377602272677356, + "tokens_seen": 2417295360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721010101010101, + "loss": 2.4223, + "theoretical_loss": 3.3775949191944057, + "tokens_seen": 2417360896 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720808080808081, + "loss": 2.5522, + "theoretical_loss": 3.3775875659666283, + "tokens_seen": 2417426432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720606060606061, + "loss": 2.4789, + "theoretical_loss": 3.3775802129940073, + "tokens_seen": 2417491968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027204040404040403, + "loss": 2.4614, + "theoretical_loss": 3.377572860276528, + "tokens_seen": 2417557504 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272020202020202, + "loss": 2.7, + "theoretical_loss": 3.3775655078141735, + "tokens_seen": 2417623040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027200000000000005, + "loss": 2.5628, + "theoretical_loss": 3.3775581556069283, + "tokens_seen": 2417688576 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271979797979798, + "loss": 2.4734, + "theoretical_loss": 3.3775508036547772, + "tokens_seen": 2417754112 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027195959595959597, + "loss": 2.5551, + "theoretical_loss": 3.377543451957704, + "tokens_seen": 2417819648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719393939393939, + "loss": 2.7689, + "theoretical_loss": 3.3775361005156928, + "tokens_seen": 2417885184 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719191919191919, + "loss": 2.4865, + "theoretical_loss": 3.377528749328728, + "tokens_seen": 2417950720 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027189898989898994, + "loss": 2.5239, + "theoretical_loss": 3.377521398396794, + "tokens_seen": 2418016256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718787878787879, + "loss": 2.6182, + "theoretical_loss": 3.377514047719875, + "tokens_seen": 2418081792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027185858585858586, + "loss": 2.406, + "theoretical_loss": 3.377506697297955, + "tokens_seen": 2418147328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718383838383838, + "loss": 2.6016, + "theoretical_loss": 3.377499347131018, + "tokens_seen": 2418212864 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027181818181818177, + "loss": 2.4279, + "theoretical_loss": 3.377491997219049, + "tokens_seen": 2418278400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027179797979797984, + "loss": 2.2922, + "theoretical_loss": 3.377484647562032, + "tokens_seen": 2418343936 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717777777777778, + "loss": 2.5436, + "theoretical_loss": 3.377477298159951, + "tokens_seen": 2418409472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027175757575757575, + "loss": 2.4884, + "theoretical_loss": 3.37746994901279, + "tokens_seen": 2418475008 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1328863, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2463276386260986, + "objective/train/theoretical_loss": 3.377462600120534, + "objective/train/tokens_used": 777399776, + "theoretical_loss": 3.377462600120534, + "tokens_seen": 2418540544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717373737373737, + "loss": 2.4146, + "theoretical_loss": 3.377462600120534, + "tokens_seen": 2418540544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717171717171717, + "loss": 2.2153, + "theoretical_loss": 3.377455251483167, + "tokens_seen": 2418606080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027169696969696973, + "loss": 2.5368, + "theoretical_loss": 3.377447903100673, + "tokens_seen": 2418671616 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716767676767677, + "loss": 2.521, + "theoretical_loss": 3.3774405549730364, + "tokens_seen": 2418737152 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027165656565656564, + "loss": 2.5067, + "theoretical_loss": 3.3774332071002413, + "tokens_seen": 2418802688 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716363636363636, + "loss": 2.5375, + "theoretical_loss": 3.377425859482272, + "tokens_seen": 2418868224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716161616161616, + "loss": 2.7116, + "theoretical_loss": 3.3774185121191134, + "tokens_seen": 2418933760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715959595959596, + "loss": 2.4842, + "theoretical_loss": 3.3774111650107486, + "tokens_seen": 2418999296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715757575757576, + "loss": 2.491, + "theoretical_loss": 3.3774038181571626, + "tokens_seen": 2419064832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027155555555555554, + "loss": 2.6163, + "theoretical_loss": 3.37739647155834, + "tokens_seen": 2419130368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027153535353535355, + "loss": 2.7782, + "theoretical_loss": 3.3773891252142643, + "tokens_seen": 2419195904 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027151515151515156, + "loss": 2.5928, + "theoretical_loss": 3.37738177912492, + "tokens_seen": 2419261440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714949494949495, + "loss": 2.685, + "theoretical_loss": 3.3773744332902913, + "tokens_seen": 2419326976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027147474747474747, + "loss": 2.6787, + "theoretical_loss": 3.377367087710363, + "tokens_seen": 2419392512 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027145454545454543, + "loss": 2.4557, + "theoretical_loss": 3.3773597423851185, + "tokens_seen": 2419458048 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027143434343434344, + "loss": 2.5975, + "theoretical_loss": 3.3773523973145427, + "tokens_seen": 2419523584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027141414141414145, + "loss": 2.4962, + "theoretical_loss": 3.3773450524986197, + "tokens_seen": 2419589120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713939393939394, + "loss": 2.5028, + "theoretical_loss": 3.377337707937334, + "tokens_seen": 2419654656 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027137373737373737, + "loss": 2.6093, + "theoretical_loss": 3.3773303636306693, + "tokens_seen": 2419720192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713535353535354, + "loss": 2.7072, + "theoretical_loss": 3.377323019578611, + "tokens_seen": 2419785728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027133333333333333, + "loss": 2.6711, + "theoretical_loss": 3.377315675781142, + "tokens_seen": 2419851264 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027131313131313134, + "loss": 2.4018, + "theoretical_loss": 3.3773083322382473, + "tokens_seen": 2419916800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712929292929293, + "loss": 2.5922, + "theoretical_loss": 3.3773009889499113, + "tokens_seen": 2419982336 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027127272727272726, + "loss": 2.664, + "theoretical_loss": 3.377293645916118, + "tokens_seen": 2420047872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027125252525252527, + "loss": 2.4764, + "theoretical_loss": 3.3772863031368514, + "tokens_seen": 2420113408 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1329609, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3919801712036133, + "objective/train/theoretical_loss": 3.377278960612096, + "objective/train/tokens_used": 779038176, + "theoretical_loss": 3.377278960612096, + "tokens_seen": 2420178944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712323232323232, + "loss": 2.421, + "theoretical_loss": 3.377278960612096, + "tokens_seen": 2420178944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027121212121212124, + "loss": 2.6578, + "theoretical_loss": 3.3772716183418368, + "tokens_seen": 2420244480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711919191919192, + "loss": 2.7039, + "theoretical_loss": 3.3772642763260574, + "tokens_seen": 2420310016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711717171717172, + "loss": 2.6739, + "theoretical_loss": 3.377256934564742, + "tokens_seen": 2420375552 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027115151515151516, + "loss": 2.538, + "theoretical_loss": 3.377249593057875, + "tokens_seen": 2420441088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711313131313131, + "loss": 2.4294, + "theoretical_loss": 3.3772422518054412, + "tokens_seen": 2420506624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027111111111111113, + "loss": 2.7442, + "theoretical_loss": 3.377234910807424, + "tokens_seen": 2420572160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710909090909091, + "loss": 2.5176, + "theoretical_loss": 3.3772275700638086, + "tokens_seen": 2420637696 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710707070707071, + "loss": 2.6513, + "theoretical_loss": 3.3772202295745783, + "tokens_seen": 2420703232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027105050505050505, + "loss": 2.3484, + "theoretical_loss": 3.3772128893397184, + "tokens_seen": 2420768768 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271030303030303, + "loss": 2.4639, + "theoretical_loss": 3.377205549359213, + "tokens_seen": 2420834304 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271010101010101, + "loss": 2.6157, + "theoretical_loss": 3.377198209633046, + "tokens_seen": 2420899840 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027098989898989903, + "loss": 2.6086, + "theoretical_loss": 3.3771908701612015, + "tokens_seen": 2420965376 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270969696969697, + "loss": 2.4896, + "theoretical_loss": 3.3771835309436646, + "tokens_seen": 2421030912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027094949494949495, + "loss": 2.5821, + "theoretical_loss": 3.3771761919804186, + "tokens_seen": 2421096448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709292929292929, + "loss": 2.6494, + "theoretical_loss": 3.3771688532714492, + "tokens_seen": 2421161984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709090909090909, + "loss": 2.4321, + "theoretical_loss": 3.377161514816739, + "tokens_seen": 2421227520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708888888888889, + "loss": 2.736, + "theoretical_loss": 3.3771541766162736, + "tokens_seen": 2421293056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708686868686869, + "loss": 2.2884, + "theoretical_loss": 3.377146838670037, + "tokens_seen": 2421358592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027084848484848484, + "loss": 2.7259, + "theoretical_loss": 3.3771395009780134, + "tokens_seen": 2421424128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708282828282828, + "loss": 2.5679, + "theoretical_loss": 3.377132163540187, + "tokens_seen": 2421489664 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027080808080808086, + "loss": 2.6837, + "theoretical_loss": 3.3771248263565425, + "tokens_seen": 2421555200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707878787878788, + "loss": 2.702, + "theoretical_loss": 3.377117489427064, + "tokens_seen": 2421620736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707676767676768, + "loss": 2.6824, + "theoretical_loss": 3.3771101527517358, + "tokens_seen": 2421686272 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027074747474747473, + "loss": 2.6854, + "theoretical_loss": 3.377102816330542, + "tokens_seen": 2421751808 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1330720, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.00944447517395, + "objective/train/theoretical_loss": 3.377095480163467, + "objective/train/tokens_used": 780676576, + "theoretical_loss": 3.377095480163467, + "tokens_seen": 2421817344 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707272727272727, + "loss": 2.5717, + "theoretical_loss": 3.377095480163467, + "tokens_seen": 2421817344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027070707070707075, + "loss": 2.5181, + "theoretical_loss": 3.3770881442504956, + "tokens_seen": 2421882880 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706868686868687, + "loss": 2.5089, + "theoretical_loss": 3.3770808085916113, + "tokens_seen": 2421948416 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027066666666666667, + "loss": 2.1892, + "theoretical_loss": 3.3770734731867993, + "tokens_seen": 2422013952 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706464646464646, + "loss": 2.6041, + "theoretical_loss": 3.377066138036043, + "tokens_seen": 2422079488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027062626262626264, + "loss": 2.5648, + "theoretical_loss": 3.377058803139328, + "tokens_seen": 2422145024 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027060606060606065, + "loss": 2.5677, + "theoretical_loss": 3.3770514684966373, + "tokens_seen": 2422210560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705858585858586, + "loss": 2.5145, + "theoretical_loss": 3.377044134107956, + "tokens_seen": 2422276096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027056565656565656, + "loss": 2.6456, + "theoretical_loss": 3.3770367999732684, + "tokens_seen": 2422341632 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705454545454545, + "loss": 2.267, + "theoretical_loss": 3.377029466092558, + "tokens_seen": 2422407168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027052525252525253, + "loss": 2.7496, + "theoretical_loss": 3.3770221324658105, + "tokens_seen": 2422472704 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027050505050505054, + "loss": 2.5598, + "theoretical_loss": 3.3770147990930095, + "tokens_seen": 2422538240 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704848484848485, + "loss": 2.5307, + "theoretical_loss": 3.377007465974139, + "tokens_seen": 2422603776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027046464646464645, + "loss": 2.6812, + "theoretical_loss": 3.377000133109184, + "tokens_seen": 2422669312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704444444444444, + "loss": 2.6716, + "theoretical_loss": 3.3769928004981282, + "tokens_seen": 2422734848 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704242424242424, + "loss": 2.517, + "theoretical_loss": 3.3769854681409566, + "tokens_seen": 2422800384 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027040404040404043, + "loss": 2.4556, + "theoretical_loss": 3.3769781360376534, + "tokens_seen": 2422865920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703838383838384, + "loss": 2.7594, + "theoretical_loss": 3.3769708041882023, + "tokens_seen": 2422931456 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027036363636363635, + "loss": 2.501, + "theoretical_loss": 3.376963472592588, + "tokens_seen": 2422996992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027034343434343436, + "loss": 2.639, + "theoretical_loss": 3.376956141250796, + "tokens_seen": 2423062528 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703232323232323, + "loss": 2.5825, + "theoretical_loss": 3.3769488101628085, + "tokens_seen": 2423128064 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703030303030303, + "loss": 2.6151, + "theoretical_loss": 3.3769414793286114, + "tokens_seen": 2423193600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002702828282828283, + "loss": 2.5882, + "theoretical_loss": 3.3769341487481888, + "tokens_seen": 2423259136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027026262626262624, + "loss": 2.6113, + "theoretical_loss": 3.3769268184215244, + "tokens_seen": 2423324672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027024242424242425, + "loss": 2.4535, + "theoretical_loss": 3.3769194883486033, + "tokens_seen": 2423390208 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1331401, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.588832378387451, + "objective/train/theoretical_loss": 3.37691215852941, + "objective/train/tokens_used": 782314976, + "theoretical_loss": 3.37691215852941, + "tokens_seen": 2423455744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702222222222222, + "loss": 2.6272, + "theoretical_loss": 3.37691215852941, + "tokens_seen": 2423455744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702020202020202, + "loss": 2.7104, + "theoretical_loss": 3.376904828963928, + "tokens_seen": 2423521280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701818181818182, + "loss": 2.5889, + "theoretical_loss": 3.376897499652142, + "tokens_seen": 2423586816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701616161616162, + "loss": 2.3941, + "theoretical_loss": 3.376890170594036, + "tokens_seen": 2423652352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027014141414141414, + "loss": 2.5046, + "theoretical_loss": 3.376882841789596, + "tokens_seen": 2423717888 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701212121212121, + "loss": 2.5633, + "theoretical_loss": 3.3768755132388044, + "tokens_seen": 2423783424 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701010101010101, + "loss": 2.4163, + "theoretical_loss": 3.3768681849416464, + "tokens_seen": 2423848960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027008080808080807, + "loss": 2.5344, + "theoretical_loss": 3.3768608568981064, + "tokens_seen": 2423914496 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700606060606061, + "loss": 2.4975, + "theoretical_loss": 3.3768535291081685, + "tokens_seen": 2423980032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027004040404040404, + "loss": 2.7312, + "theoretical_loss": 3.3768462015718175, + "tokens_seen": 2424045568 + }, + { + "epoch": 0.47, + "learning_rate": 0.000270020202020202, + "loss": 2.3678, + "theoretical_loss": 3.3768388742890374, + "tokens_seen": 2424111104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027, + "loss": 2.6491, + "theoretical_loss": 3.3768315472598127, + "tokens_seen": 2424176640 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269979797979798, + "loss": 2.5082, + "theoretical_loss": 3.376824220484128, + "tokens_seen": 2424242176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026995959595959597, + "loss": 2.3295, + "theoretical_loss": 3.376816893961967, + "tokens_seen": 2424307712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026993939393939393, + "loss": 2.6033, + "theoretical_loss": 3.3768095676933143, + "tokens_seen": 2424373248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699191919191919, + "loss": 2.6762, + "theoretical_loss": 3.376802241678155, + "tokens_seen": 2424438784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698989898989899, + "loss": 2.5041, + "theoretical_loss": 3.3767949159164727, + "tokens_seen": 2424504320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698787878787879, + "loss": 2.4712, + "theoretical_loss": 3.3767875904082523, + "tokens_seen": 2424569856 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026985858585858586, + "loss": 2.3945, + "theoretical_loss": 3.3767802651534775, + "tokens_seen": 2424635392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698383838383838, + "loss": 2.7693, + "theoretical_loss": 3.376772940152133, + "tokens_seen": 2424700928 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026981818181818183, + "loss": 2.6028, + "theoretical_loss": 3.376765615404204, + "tokens_seen": 2424766464 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026979797979797984, + "loss": 2.8442, + "theoretical_loss": 3.376758290909673, + "tokens_seen": 2424832000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697777777777778, + "loss": 2.4963, + "theoretical_loss": 3.3767509666685265, + "tokens_seen": 2424897536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026975757575757576, + "loss": 2.7412, + "theoretical_loss": 3.3767436426807476, + "tokens_seen": 2424963072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697373737373737, + "loss": 2.6452, + "theoretical_loss": 3.376736318946321, + "tokens_seen": 2425028608 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1332594, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.556990146636963, + "objective/train/theoretical_loss": 3.376728995465231, + "objective/train/tokens_used": 783953376, + "theoretical_loss": 3.376728995465231, + "tokens_seen": 2425094144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697171717171717, + "loss": 2.4698, + "theoretical_loss": 3.376728995465231, + "tokens_seen": 2425094144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026969696969696974, + "loss": 2.4541, + "theoretical_loss": 3.376721672237462, + "tokens_seen": 2425159680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696767676767677, + "loss": 2.4361, + "theoretical_loss": 3.3767143492629987, + "tokens_seen": 2425225216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026965656565656565, + "loss": 2.3783, + "theoretical_loss": 3.3767070265418253, + "tokens_seen": 2425290752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696363636363636, + "loss": 2.6316, + "theoretical_loss": 3.3766997040739266, + "tokens_seen": 2425356288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026961616161616167, + "loss": 2.6758, + "theoretical_loss": 3.3766923818592858, + "tokens_seen": 2425421824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026959595959595963, + "loss": 2.5955, + "theoretical_loss": 3.3766850598978886, + "tokens_seen": 2425487360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695757575757576, + "loss": 2.5651, + "theoretical_loss": 3.3766777381897186, + "tokens_seen": 2425552896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026955555555555554, + "loss": 2.5277, + "theoretical_loss": 3.3766704167347603, + "tokens_seen": 2425618432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695353535353535, + "loss": 2.5461, + "theoretical_loss": 3.3766630955329986, + "tokens_seen": 2425683968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026951515151515156, + "loss": 2.4433, + "theoretical_loss": 3.3766557745844175, + "tokens_seen": 2425749504 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694949494949495, + "loss": 2.6348, + "theoretical_loss": 3.3766484538890014, + "tokens_seen": 2425815040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694747474747475, + "loss": 2.4347, + "theoretical_loss": 3.3766411334467348, + "tokens_seen": 2425880576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026945454545454543, + "loss": 2.7299, + "theoretical_loss": 3.376633813257602, + "tokens_seen": 2425946112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026943434343434345, + "loss": 2.5579, + "theoretical_loss": 3.376626493321588, + "tokens_seen": 2426011648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026941414141414146, + "loss": 2.7931, + "theoretical_loss": 3.376619173638676, + "tokens_seen": 2426077184 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693939393939394, + "loss": 2.6122, + "theoretical_loss": 3.3766118542088517, + "tokens_seen": 2426142720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026937373737373737, + "loss": 2.6723, + "theoretical_loss": 3.3766045350320986, + "tokens_seen": 2426208256 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026935353535353533, + "loss": 2.3997, + "theoretical_loss": 3.3765972161084012, + "tokens_seen": 2426273792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026933333333333334, + "loss": 2.5429, + "theoretical_loss": 3.3765898974377446, + "tokens_seen": 2426339328 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026931313131313135, + "loss": 2.6126, + "theoretical_loss": 3.3765825790201127, + "tokens_seen": 2426404864 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692929292929293, + "loss": 2.7448, + "theoretical_loss": 3.37657526085549, + "tokens_seen": 2426470400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026927272727272726, + "loss": 2.8911, + "theoretical_loss": 3.376567942943861, + "tokens_seen": 2426535936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692525252525253, + "loss": 2.558, + "theoretical_loss": 3.37656062528521, + "tokens_seen": 2426601472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026923232323232323, + "loss": 2.5752, + "theoretical_loss": 3.3765533078795213, + "tokens_seen": 2426667008 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1333076, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.361471652984619, + "objective/train/theoretical_loss": 3.3765459907267794, + "objective/train/tokens_used": 785591776, + "theoretical_loss": 3.3765459907267794, + "tokens_seen": 2426732544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026921212121212124, + "loss": 2.3541, + "theoretical_loss": 3.3765459907267794, + "tokens_seen": 2426732544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691919191919192, + "loss": 2.7144, + "theoretical_loss": 3.3765386738269694, + "tokens_seen": 2426798080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026917171717171716, + "loss": 2.3975, + "theoretical_loss": 3.3765313571800744, + "tokens_seen": 2426863616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026915151515151517, + "loss": 2.8372, + "theoretical_loss": 3.37652404078608, + "tokens_seen": 2426929152 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691313131313131, + "loss": 2.6128, + "theoretical_loss": 3.37651672464497, + "tokens_seen": 2426994688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026911111111111114, + "loss": 2.5108, + "theoretical_loss": 3.3765094087567293, + "tokens_seen": 2427060224 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690909090909091, + "loss": 2.6143, + "theoretical_loss": 3.3765020931213416, + "tokens_seen": 2427125760 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690707070707071, + "loss": 2.8087, + "theoretical_loss": 3.3764947777387926, + "tokens_seen": 2427191296 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026905050505050506, + "loss": 2.478, + "theoretical_loss": 3.3764874626090653, + "tokens_seen": 2427256832 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269030303030303, + "loss": 2.4421, + "theoretical_loss": 3.3764801477321447, + "tokens_seen": 2427322368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026901010101010103, + "loss": 2.4682, + "theoretical_loss": 3.3764728331080156, + "tokens_seen": 2427387904 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268989898989899, + "loss": 2.6778, + "theoretical_loss": 3.376465518736662, + "tokens_seen": 2427453440 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268969696969697, + "loss": 2.5871, + "theoretical_loss": 3.3764582046180687, + "tokens_seen": 2427518976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026894949494949495, + "loss": 2.7602, + "theoretical_loss": 3.3764508907522197, + "tokens_seen": 2427584512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689292929292929, + "loss": 2.5292, + "theoretical_loss": 3.3764435771390997, + "tokens_seen": 2427650048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689090909090909, + "loss": 2.4538, + "theoretical_loss": 3.376436263778693, + "tokens_seen": 2427715584 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026888888888888893, + "loss": 2.5796, + "theoretical_loss": 3.3764289506709844, + "tokens_seen": 2427781120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688686868686869, + "loss": 2.4144, + "theoretical_loss": 3.376421637815958, + "tokens_seen": 2427846656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026884848484848485, + "loss": 2.4935, + "theoretical_loss": 3.3764143252135983, + "tokens_seen": 2427912192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688282828282828, + "loss": 2.7623, + "theoretical_loss": 3.37640701286389, + "tokens_seen": 2427977728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688080808080808, + "loss": 2.3049, + "theoretical_loss": 3.376399700766817, + "tokens_seen": 2428043264 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687878787878788, + "loss": 2.4255, + "theoretical_loss": 3.3763923889223646, + "tokens_seen": 2428108800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687676767676768, + "loss": 2.4888, + "theoretical_loss": 3.376385077330516, + "tokens_seen": 2428174336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026874747474747474, + "loss": 2.6093, + "theoretical_loss": 3.3763777659912573, + "tokens_seen": 2428239872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687272727272727, + "loss": 2.5453, + "theoretical_loss": 3.376370454904572, + "tokens_seen": 2428305408 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1334211, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6763081550598145, + "objective/train/theoretical_loss": 3.376363144070444, + "objective/train/tokens_used": 787230176, + "theoretical_loss": 3.376363144070444, + "tokens_seen": 2428370944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026870707070707076, + "loss": 2.6711, + "theoretical_loss": 3.376363144070444, + "tokens_seen": 2428370944 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686868686868687, + "loss": 2.5968, + "theoretical_loss": 3.376355833488859, + "tokens_seen": 2428436480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686666666666667, + "loss": 2.4966, + "theoretical_loss": 3.3763485231598005, + "tokens_seen": 2428502016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026864646464646463, + "loss": 2.6842, + "theoretical_loss": 3.3763412130832537, + "tokens_seen": 2428567552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686262626262626, + "loss": 2.4217, + "theoretical_loss": 3.3763339032592024, + "tokens_seen": 2428633088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026860606060606065, + "loss": 2.5748, + "theoretical_loss": 3.376326593687631, + "tokens_seen": 2428698624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685858585858586, + "loss": 2.5352, + "theoretical_loss": 3.376319284368525, + "tokens_seen": 2428764160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026856565656565657, + "loss": 2.4552, + "theoretical_loss": 3.376311975301868, + "tokens_seen": 2428829696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685454545454545, + "loss": 2.4599, + "theoretical_loss": 3.3763046664876444, + "tokens_seen": 2428895232 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685252525252525, + "loss": 2.4581, + "theoretical_loss": 3.3762973579258393, + "tokens_seen": 2428960768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026850505050505055, + "loss": 2.6801, + "theoretical_loss": 3.3762900496164363, + "tokens_seen": 2429026304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684848484848485, + "loss": 2.6776, + "theoretical_loss": 3.3762827415594208, + "tokens_seen": 2429091840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026846464646464646, + "loss": 2.7273, + "theoretical_loss": 3.3762754337547767, + "tokens_seen": 2429157376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684444444444444, + "loss": 2.6648, + "theoretical_loss": 3.3762681262024885, + "tokens_seen": 2429222912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026842424242424243, + "loss": 2.5583, + "theoretical_loss": 3.3762608189025407, + "tokens_seen": 2429288448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026840404040404044, + "loss": 2.3741, + "theoretical_loss": 3.3762535118549186, + "tokens_seen": 2429353984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683838383838384, + "loss": 2.7823, + "theoretical_loss": 3.3762462050596054, + "tokens_seen": 2429419520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026836363636363635, + "loss": 2.7154, + "theoretical_loss": 3.3762388985165863, + "tokens_seen": 2429485056 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683434343434343, + "loss": 2.7091, + "theoretical_loss": 3.3762315922258455, + "tokens_seen": 2429550592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683232323232323, + "loss": 2.726, + "theoretical_loss": 3.3762242861873677, + "tokens_seen": 2429616128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026830303030303033, + "loss": 2.5146, + "theoretical_loss": 3.376216980401137, + "tokens_seen": 2429681664 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682828282828283, + "loss": 2.7387, + "theoretical_loss": 3.3762096748671384, + "tokens_seen": 2429747200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026826262626262624, + "loss": 2.441, + "theoretical_loss": 3.3762023695853562, + "tokens_seen": 2429812736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026824242424242426, + "loss": 2.805, + "theoretical_loss": 3.376195064555775, + "tokens_seen": 2429878272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682222222222222, + "loss": 2.6108, + "theoretical_loss": 3.376187759778379, + "tokens_seen": 2429943808 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1334672, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.394357204437256, + "objective/train/theoretical_loss": 3.376180455253153, + "objective/train/tokens_used": 788868576, + "theoretical_loss": 3.376180455253153, + "tokens_seen": 2430009344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682020202020202, + "loss": 2.4336, + "theoretical_loss": 3.376180455253153, + "tokens_seen": 2430009344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681818181818182, + "loss": 2.7224, + "theoretical_loss": 3.3761731509800814, + "tokens_seen": 2430074880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026816161616161614, + "loss": 2.5301, + "theoretical_loss": 3.3761658469591485, + "tokens_seen": 2430140416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026814141414141415, + "loss": 2.4406, + "theoretical_loss": 3.3761585431903387, + "tokens_seen": 2430205952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681212121212121, + "loss": 2.3457, + "theoretical_loss": 3.376151239673637, + "tokens_seen": 2430271488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681010101010101, + "loss": 2.439, + "theoretical_loss": 3.3761439364090275, + "tokens_seen": 2430337024 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680808080808081, + "loss": 2.4804, + "theoretical_loss": 3.376136633396495, + "tokens_seen": 2430402560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680606060606061, + "loss": 2.5079, + "theoretical_loss": 3.3761293306360236, + "tokens_seen": 2430468096 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026804040404040404, + "loss": 2.6971, + "theoretical_loss": 3.3761220281275985, + "tokens_seen": 2430533632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026802020202020205, + "loss": 2.7266, + "theoretical_loss": 3.3761147258712034, + "tokens_seen": 2430599168 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268, + "loss": 2.6872, + "theoretical_loss": 3.376107423866823, + "tokens_seen": 2430664704 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026797979797979797, + "loss": 2.6346, + "theoretical_loss": 3.376100122114442, + "tokens_seen": 2430730240 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267959595959596, + "loss": 2.5747, + "theoretical_loss": 3.376092820614045, + "tokens_seen": 2430795776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026793939393939393, + "loss": 2.552, + "theoretical_loss": 3.376085519365617, + "tokens_seen": 2430861312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026791919191919194, + "loss": 2.7006, + "theoretical_loss": 3.376078218369141, + "tokens_seen": 2430926848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678989898989899, + "loss": 2.5641, + "theoretical_loss": 3.376070917624603, + "tokens_seen": 2430992384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678787878787879, + "loss": 2.7543, + "theoretical_loss": 3.3760636171319867, + "tokens_seen": 2431057920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026785858585858587, + "loss": 2.7107, + "theoretical_loss": 3.3760563168912765, + "tokens_seen": 2431123456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678383838383838, + "loss": 2.6819, + "theoretical_loss": 3.3760490169024573, + "tokens_seen": 2431188992 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026781818181818184, + "loss": 2.5399, + "theoretical_loss": 3.376041717165514, + "tokens_seen": 2431254528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677979797979798, + "loss": 2.6909, + "theoretical_loss": 3.3760344176804304, + "tokens_seen": 2431320064 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677777777777778, + "loss": 2.6631, + "theoretical_loss": 3.3760271184471917, + "tokens_seen": 2431385600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026775757575757576, + "loss": 2.6917, + "theoretical_loss": 3.376019819465782, + "tokens_seen": 2431451136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677373737373737, + "loss": 2.4872, + "theoretical_loss": 3.376012520736185, + "tokens_seen": 2431516672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026771717171717173, + "loss": 2.5818, + "theoretical_loss": 3.376005222258387, + "tokens_seen": 2431582208 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1335677, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.379019260406494, + "objective/train/theoretical_loss": 3.3759979240323714, + "objective/train/tokens_used": 790506976, + "theoretical_loss": 3.3759979240323714, + "tokens_seen": 2431647744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026769696969696974, + "loss": 2.5296, + "theoretical_loss": 3.3759979240323714, + "tokens_seen": 2431647744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676767676767677, + "loss": 2.765, + "theoretical_loss": 3.3759906260581225, + "tokens_seen": 2431713280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026765656565656566, + "loss": 2.5465, + "theoretical_loss": 3.375983328335626, + "tokens_seen": 2431778816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676363636363636, + "loss": 2.3647, + "theoretical_loss": 3.375976030864865, + "tokens_seen": 2431844352 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676161616161616, + "loss": 2.6953, + "theoretical_loss": 3.3759687336458253, + "tokens_seen": 2431909888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026759595959595963, + "loss": 2.4947, + "theoretical_loss": 3.375961436678491, + "tokens_seen": 2431975424 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675757575757576, + "loss": 2.4316, + "theoretical_loss": 3.375954139962846, + "tokens_seen": 2432040960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026755555555555555, + "loss": 2.6234, + "theoretical_loss": 3.3759468434988755, + "tokens_seen": 2432106496 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675353535353535, + "loss": 2.5268, + "theoretical_loss": 3.375939547286564, + "tokens_seen": 2432172032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026751515151515157, + "loss": 2.735, + "theoretical_loss": 3.375932251325896, + "tokens_seen": 2432237568 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674949494949495, + "loss": 2.6445, + "theoretical_loss": 3.3759249556168553, + "tokens_seen": 2432303104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674747474747475, + "loss": 2.61, + "theoretical_loss": 3.375917660159428, + "tokens_seen": 2432368640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026745454545454544, + "loss": 2.4683, + "theoretical_loss": 3.3759103649535973, + "tokens_seen": 2432434176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674343434343434, + "loss": 2.5302, + "theoretical_loss": 3.375903069999348, + "tokens_seen": 2432499712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026741414141414146, + "loss": 2.7753, + "theoretical_loss": 3.375895775296665, + "tokens_seen": 2432565248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673939393939394, + "loss": 2.4033, + "theoretical_loss": 3.375888480845533, + "tokens_seen": 2432630784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673737373737374, + "loss": 2.6525, + "theoretical_loss": 3.375881186645936, + "tokens_seen": 2432696320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026735353535353533, + "loss": 2.5969, + "theoretical_loss": 3.3758738926978586, + "tokens_seen": 2432761856 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026733333333333334, + "loss": 2.6262, + "theoretical_loss": 3.375866599001286, + "tokens_seen": 2432827392 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026731313131313136, + "loss": 2.479, + "theoretical_loss": 3.375859305556202, + "tokens_seen": 2432892928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672929292929293, + "loss": 2.5408, + "theoretical_loss": 3.3758520123625915, + "tokens_seen": 2432958464 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026727272727272727, + "loss": 2.6221, + "theoretical_loss": 3.375844719420439, + "tokens_seen": 2433024000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672525252525252, + "loss": 2.5281, + "theoretical_loss": 3.3758374267297286, + "tokens_seen": 2433089536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026723232323232324, + "loss": 2.822, + "theoretical_loss": 3.375830134290446, + "tokens_seen": 2433155072 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026721212121212125, + "loss": 2.382, + "theoretical_loss": 3.3758228421025747, + "tokens_seen": 2433220608 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1336773, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.446194887161255, + "objective/train/theoretical_loss": 3.3758155501661, + "objective/train/tokens_used": 792145376, + "theoretical_loss": 3.3758155501661, + "tokens_seen": 2433286144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671919191919192, + "loss": 2.6136, + "theoretical_loss": 3.3758155501661, + "tokens_seen": 2433286144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026717171717171716, + "loss": 2.3919, + "theoretical_loss": 3.3758082584810056, + "tokens_seen": 2433351680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671515151515151, + "loss": 2.2964, + "theoretical_loss": 3.3758009670472773, + "tokens_seen": 2433417216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026713131313131313, + "loss": 2.4072, + "theoretical_loss": 3.375793675864898, + "tokens_seen": 2433482752 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026711111111111114, + "loss": 2.6743, + "theoretical_loss": 3.375786384933854, + "tokens_seen": 2433548288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002670909090909091, + "loss": 2.5573, + "theoretical_loss": 3.3757790942541286, + "tokens_seen": 2433613824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026707070707070705, + "loss": 2.6263, + "theoretical_loss": 3.375771803825707, + "tokens_seen": 2433679360 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026705050505050507, + "loss": 2.6231, + "theoretical_loss": 3.3757645136485737, + "tokens_seen": 2433744896 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267030303030303, + "loss": 2.4567, + "theoretical_loss": 3.375757223722713, + "tokens_seen": 2433810432 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026701010101010103, + "loss": 2.4794, + "theoretical_loss": 3.37574993404811, + "tokens_seen": 2433875968 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266989898989899, + "loss": 2.5598, + "theoretical_loss": 3.3757426446247485, + "tokens_seen": 2433941504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026696969696969695, + "loss": 2.4144, + "theoretical_loss": 3.3757353554526137, + "tokens_seen": 2434007040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026694949494949496, + "loss": 2.4774, + "theoretical_loss": 3.3757280665316896, + "tokens_seen": 2434072576 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669292929292929, + "loss": 2.6486, + "theoretical_loss": 3.375720777861962, + "tokens_seen": 2434138112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669090909090909, + "loss": 2.7292, + "theoretical_loss": 3.375713489443414, + "tokens_seen": 2434203648 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668888888888889, + "loss": 2.5281, + "theoretical_loss": 3.3757062012760306, + "tokens_seen": 2434269184 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668686868686869, + "loss": 2.4617, + "theoretical_loss": 3.375698913359797, + "tokens_seen": 2434334720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026684848484848485, + "loss": 2.5479, + "theoretical_loss": 3.3756916256946976, + "tokens_seen": 2434400256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668282828282828, + "loss": 2.8708, + "theoretical_loss": 3.375684338280716, + "tokens_seen": 2434465792 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668080808080808, + "loss": 2.6174, + "theoretical_loss": 3.3756770511178384, + "tokens_seen": 2434531328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667878787878788, + "loss": 2.584, + "theoretical_loss": 3.375669764206048, + "tokens_seen": 2434596864 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667676767676768, + "loss": 2.6701, + "theoretical_loss": 3.3756624775453306, + "tokens_seen": 2434662400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026674747474747474, + "loss": 2.5115, + "theoretical_loss": 3.3756551911356696, + "tokens_seen": 2434727936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667272727272727, + "loss": 2.608, + "theoretical_loss": 3.37564790497705, + "tokens_seen": 2434793472 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667070707070707, + "loss": 2.6837, + "theoretical_loss": 3.375640619069457, + "tokens_seen": 2434859008 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1337466, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9103434085845947, + "objective/train/theoretical_loss": 3.3756333334128743, + "objective/train/tokens_used": 793783776, + "theoretical_loss": 3.3756333334128743, + "tokens_seen": 2434924544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666868686868687, + "loss": 2.5773, + "theoretical_loss": 3.3756333334128743, + "tokens_seen": 2434924544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666666666666667, + "loss": 2.6097, + "theoretical_loss": 3.375626048007287, + "tokens_seen": 2434990080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026664646464646464, + "loss": 2.5248, + "theoretical_loss": 3.37561876285268, + "tokens_seen": 2435055616 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666262626262626, + "loss": 2.5488, + "theoretical_loss": 3.375611477949037, + "tokens_seen": 2435121152 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666060606060606, + "loss": 2.4839, + "theoretical_loss": 3.3756041932963434, + "tokens_seen": 2435186688 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665858585858586, + "loss": 2.5291, + "theoretical_loss": 3.3755969088945834, + "tokens_seen": 2435252224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026656565656565657, + "loss": 2.7532, + "theoretical_loss": 3.375589624743742, + "tokens_seen": 2435317760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026654545454545453, + "loss": 2.523, + "theoretical_loss": 3.375582340843803, + "tokens_seen": 2435383296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665252525252525, + "loss": 2.6294, + "theoretical_loss": 3.3755750571947516, + "tokens_seen": 2435448832 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026650505050505055, + "loss": 2.5958, + "theoretical_loss": 3.375567773796573, + "tokens_seen": 2435514368 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664848484848485, + "loss": 2.4305, + "theoretical_loss": 3.3755604906492502, + "tokens_seen": 2435579904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026646464646464646, + "loss": 2.6766, + "theoretical_loss": 3.3755532077527692, + "tokens_seen": 2435645440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664444444444444, + "loss": 2.5552, + "theoretical_loss": 3.3755459251071143, + "tokens_seen": 2435710976 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664242424242424, + "loss": 2.6694, + "theoretical_loss": 3.3755386427122698, + "tokens_seen": 2435776512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026640404040404044, + "loss": 2.5408, + "theoretical_loss": 3.3755313605682202, + "tokens_seen": 2435842048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663838383838384, + "loss": 2.4806, + "theoretical_loss": 3.375524078674951, + "tokens_seen": 2435907584 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026636363636363636, + "loss": 2.3821, + "theoretical_loss": 3.375516797032446, + "tokens_seen": 2435973120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663434343434343, + "loss": 2.6681, + "theoretical_loss": 3.3755095156406902, + "tokens_seen": 2436038656 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663232323232324, + "loss": 2.6113, + "theoretical_loss": 3.3755022344996677, + "tokens_seen": 2436104192 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026630303030303034, + "loss": 2.6237, + "theoretical_loss": 3.3754949536093637, + "tokens_seen": 2436169728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662828282828283, + "loss": 2.6986, + "theoretical_loss": 3.3754876729697627, + "tokens_seen": 2436235264 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026626262626262625, + "loss": 2.4305, + "theoretical_loss": 3.375480392580849, + "tokens_seen": 2436300800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662424242424242, + "loss": 2.5156, + "theoretical_loss": 3.3754731124426076, + "tokens_seen": 2436366336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026622222222222227, + "loss": 2.5138, + "theoretical_loss": 3.375465832555023, + "tokens_seen": 2436431872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026620202020202023, + "loss": 2.6387, + "theoretical_loss": 3.3754585529180794, + "tokens_seen": 2436497408 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1338189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9122860431671143, + "objective/train/theoretical_loss": 3.3754512735317626, + "objective/train/tokens_used": 795422176, + "theoretical_loss": 3.3754512735317626, + "tokens_seen": 2436562944 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661818181818182, + "loss": 2.6878, + "theoretical_loss": 3.3754512735317626, + "tokens_seen": 2436562944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026616161616161614, + "loss": 2.5895, + "theoretical_loss": 3.375443994396056, + "tokens_seen": 2436628480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026614141414141415, + "loss": 2.6319, + "theoretical_loss": 3.3754367155109444, + "tokens_seen": 2436694016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026612121212121217, + "loss": 2.5464, + "theoretical_loss": 3.3754294368764133, + "tokens_seen": 2436759552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661010101010101, + "loss": 2.6127, + "theoretical_loss": 3.3754221584924466, + "tokens_seen": 2436825088 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660808080808081, + "loss": 2.4074, + "theoretical_loss": 3.375414880359029, + "tokens_seen": 2436890624 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026606060606060604, + "loss": 2.6108, + "theoretical_loss": 3.3754076024761455, + "tokens_seen": 2436956160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026604040404040405, + "loss": 2.7403, + "theoretical_loss": 3.3754003248437803, + "tokens_seen": 2437021696 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026602020202020206, + "loss": 2.65, + "theoretical_loss": 3.375393047461918, + "tokens_seen": 2437087232 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266, + "loss": 2.4183, + "theoretical_loss": 3.375385770330544, + "tokens_seen": 2437152768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026597979797979797, + "loss": 2.7428, + "theoretical_loss": 3.3753784934496416, + "tokens_seen": 2437218304 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265959595959596, + "loss": 2.6406, + "theoretical_loss": 3.375371216819197, + "tokens_seen": 2437283840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026593939393939394, + "loss": 2.5311, + "theoretical_loss": 3.3753639404391937, + "tokens_seen": 2437349376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026591919191919195, + "loss": 2.5561, + "theoretical_loss": 3.375356664309617, + "tokens_seen": 2437414912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658989898989899, + "loss": 2.5476, + "theoretical_loss": 3.3753493884304513, + "tokens_seen": 2437480448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026587878787878786, + "loss": 2.73, + "theoretical_loss": 3.3753421128016807, + "tokens_seen": 2437545984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658585858585859, + "loss": 2.5517, + "theoretical_loss": 3.375334837423291, + "tokens_seen": 2437611520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026583838383838383, + "loss": 2.542, + "theoretical_loss": 3.375327562295266, + "tokens_seen": 2437677056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026581818181818184, + "loss": 2.5594, + "theoretical_loss": 3.3753202874175905, + "tokens_seen": 2437742592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657979797979798, + "loss": 2.635, + "theoretical_loss": 3.3753130127902495, + "tokens_seen": 2437808128 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657777777777778, + "loss": 2.6303, + "theoretical_loss": 3.3753057384132275, + "tokens_seen": 2437873664 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026575757575757577, + "loss": 2.6106, + "theoretical_loss": 3.3752984642865087, + "tokens_seen": 2437939200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657373737373737, + "loss": 2.8313, + "theoretical_loss": 3.3752911904100786, + "tokens_seen": 2438004736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026571717171717174, + "loss": 2.4608, + "theoretical_loss": 3.375283916783921, + "tokens_seen": 2438070272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656969696969697, + "loss": 2.8014, + "theoretical_loss": 3.375276643408021, + "tokens_seen": 2438135808 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1339345, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.939288377761841, + "objective/train/theoretical_loss": 3.3752693702823633, + "objective/train/tokens_used": 797060576, + "theoretical_loss": 3.3752693702823633, + "tokens_seen": 2438201344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656767676767677, + "loss": 2.6516, + "theoretical_loss": 3.3752693702823633, + "tokens_seen": 2438201344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026565656565656566, + "loss": 2.5418, + "theoretical_loss": 3.3752620974069325, + "tokens_seen": 2438266880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656363636363636, + "loss": 2.3661, + "theoretical_loss": 3.375254824781713, + "tokens_seen": 2438332416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026561616161616163, + "loss": 2.2171, + "theoretical_loss": 3.37524755240669, + "tokens_seen": 2438397952 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026559595959595964, + "loss": 2.7802, + "theoretical_loss": 3.3752402802818473, + "tokens_seen": 2438463488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655757575757576, + "loss": 2.5123, + "theoretical_loss": 3.3752330084071707, + "tokens_seen": 2438529024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026555555555555555, + "loss": 2.6926, + "theoretical_loss": 3.3752257367826446, + "tokens_seen": 2438594560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655353535353535, + "loss": 2.5719, + "theoretical_loss": 3.3752184654082527, + "tokens_seen": 2438660096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655151515151515, + "loss": 2.6028, + "theoretical_loss": 3.3752111942839806, + "tokens_seen": 2438725632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026549494949494953, + "loss": 2.551, + "theoretical_loss": 3.375203923409813, + "tokens_seen": 2438791168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654747474747475, + "loss": 2.5455, + "theoretical_loss": 3.375196652785734, + "tokens_seen": 2438856704 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026545454545454545, + "loss": 2.5723, + "theoretical_loss": 3.3751893824117287, + "tokens_seen": 2438922240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654343434343434, + "loss": 2.5632, + "theoretical_loss": 3.375182112287782, + "tokens_seen": 2438987776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026541414141414147, + "loss": 2.6393, + "theoretical_loss": 3.3751748424138777, + "tokens_seen": 2439053312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653939393939394, + "loss": 2.5782, + "theoretical_loss": 3.3751675727900015, + "tokens_seen": 2439118848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653737373737374, + "loss": 2.6618, + "theoretical_loss": 3.3751603034161373, + "tokens_seen": 2439184384 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026535353535353534, + "loss": 2.4814, + "theoretical_loss": 3.3751530342922704, + "tokens_seen": 2439249920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653333333333333, + "loss": 2.5982, + "theoretical_loss": 3.375145765418385, + "tokens_seen": 2439315456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026531313131313136, + "loss": 2.5569, + "theoretical_loss": 3.3751384967944658, + "tokens_seen": 2439380992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652929292929293, + "loss": 2.5505, + "theoretical_loss": 3.3751312284204977, + "tokens_seen": 2439446528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652727272727273, + "loss": 2.6894, + "theoretical_loss": 3.3751239602964658, + "tokens_seen": 2439512064 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026525252525252523, + "loss": 2.5101, + "theoretical_loss": 3.375116692422354, + "tokens_seen": 2439577600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652323232323232, + "loss": 2.6101, + "theoretical_loss": 3.3751094247981475, + "tokens_seen": 2439643136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026521212121212125, + "loss": 2.588, + "theoretical_loss": 3.3751021574238305, + "tokens_seen": 2439708672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002651919191919192, + "loss": 2.5619, + "theoretical_loss": 3.3750948902993883, + "tokens_seen": 2439774208 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1339900, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4708569049835205, + "objective/train/theoretical_loss": 3.3750876234248053, + "objective/train/tokens_used": 798698976, + "theoretical_loss": 3.3750876234248053, + "tokens_seen": 2439839744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026517171717171717, + "loss": 2.5385, + "theoretical_loss": 3.3750876234248053, + "tokens_seen": 2439839744 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651515151515151, + "loss": 2.54, + "theoretical_loss": 3.3750803568000665, + "tokens_seen": 2439905280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026513131313131314, + "loss": 2.5333, + "theoretical_loss": 3.375073090425156, + "tokens_seen": 2439970816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026511111111111115, + "loss": 2.611, + "theoretical_loss": 3.3750658243000586, + "tokens_seen": 2440036352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650909090909091, + "loss": 2.3361, + "theoretical_loss": 3.37505855842476, + "tokens_seen": 2440101888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026507070707070706, + "loss": 2.3275, + "theoretical_loss": 3.3750512927992435, + "tokens_seen": 2440167424 + }, + { + "epoch": 0.48, + "learning_rate": 0.000265050505050505, + "loss": 2.3733, + "theoretical_loss": 3.3750440274234945, + "tokens_seen": 2440232960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026503030303030303, + "loss": 2.6831, + "theoretical_loss": 3.3750367622974977, + "tokens_seen": 2440298496 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026501010101010104, + "loss": 2.2953, + "theoretical_loss": 3.375029497421238, + "tokens_seen": 2440364032 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264989898989899, + "loss": 2.5982, + "theoretical_loss": 3.3750222327946995, + "tokens_seen": 2440429568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026496969696969695, + "loss": 2.5708, + "theoretical_loss": 3.3750149684178674, + "tokens_seen": 2440495104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026494949494949496, + "loss": 2.4435, + "theoretical_loss": 3.3750077042907263, + "tokens_seen": 2440560640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002649292929292929, + "loss": 2.5218, + "theoretical_loss": 3.375000440413261, + "tokens_seen": 2440626176 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026490909090909093, + "loss": 2.4119, + "theoretical_loss": 3.374993176785456, + "tokens_seen": 2440691712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648888888888889, + "loss": 2.6621, + "theoretical_loss": 3.374985913407296, + "tokens_seen": 2440757248 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026486868686868685, + "loss": 2.6159, + "theoretical_loss": 3.374978650278766, + "tokens_seen": 2440822784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026484848484848486, + "loss": 2.7574, + "theoretical_loss": 3.374971387399851, + "tokens_seen": 2440888320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648282828282828, + "loss": 2.3123, + "theoretical_loss": 3.3749641247705346, + "tokens_seen": 2440953856 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648080808080808, + "loss": 2.5051, + "theoretical_loss": 3.3749568623908024, + "tokens_seen": 2441019392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647878787878788, + "loss": 2.6248, + "theoretical_loss": 3.374949600260639, + "tokens_seen": 2441084928 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647676767676768, + "loss": 2.6467, + "theoretical_loss": 3.3749423383800288, + "tokens_seen": 2441150464 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026474747474747475, + "loss": 2.652, + "theoretical_loss": 3.374935076748957, + "tokens_seen": 2441216000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647272727272727, + "loss": 2.4366, + "theoretical_loss": 3.374927815367408, + "tokens_seen": 2441281536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647070707070707, + "loss": 2.4901, + "theoretical_loss": 3.374920554235367, + "tokens_seen": 2441347072 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646868686868687, + "loss": 2.4452, + "theoretical_loss": 3.374913293352818, + "tokens_seen": 2441412608 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1341041, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8801026344299316, + "objective/train/theoretical_loss": 3.374906032719746, + "objective/train/tokens_used": 800337376, + "theoretical_loss": 3.374906032719746, + "tokens_seen": 2441478144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646666666666667, + "loss": 2.62, + "theoretical_loss": 3.374906032719746, + "tokens_seen": 2441478144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026464646464646464, + "loss": 2.5879, + "theoretical_loss": 3.3748987723361363, + "tokens_seen": 2441543680 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026462626262626265, + "loss": 2.2701, + "theoretical_loss": 3.3748915122019727, + "tokens_seen": 2441609216 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646060606060606, + "loss": 2.6169, + "theoretical_loss": 3.3748842523172407, + "tokens_seen": 2441674752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645858585858586, + "loss": 2.7176, + "theoretical_loss": 3.374876992681924, + "tokens_seen": 2441740288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645656565656566, + "loss": 2.6777, + "theoretical_loss": 3.3748697332960087, + "tokens_seen": 2441805824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026454545454545453, + "loss": 2.5114, + "theoretical_loss": 3.374862474159479, + "tokens_seen": 2441871360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026452525252525255, + "loss": 2.4141, + "theoretical_loss": 3.3748552152723192, + "tokens_seen": 2441936896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645050505050505, + "loss": 2.4707, + "theoretical_loss": 3.3748479566345146, + "tokens_seen": 2442002432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644848484848485, + "loss": 2.6373, + "theoretical_loss": 3.3748406982460497, + "tokens_seen": 2442067968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026446464646464647, + "loss": 2.733, + "theoretical_loss": 3.3748334401069093, + "tokens_seen": 2442133504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026444444444444443, + "loss": 2.6576, + "theoretical_loss": 3.374826182217078, + "tokens_seen": 2442199040 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026442424242424244, + "loss": 2.7022, + "theoretical_loss": 3.374818924576541, + "tokens_seen": 2442264576 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026440404040404045, + "loss": 2.5744, + "theoretical_loss": 3.3748116671852824, + "tokens_seen": 2442330112 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643838383838384, + "loss": 2.4534, + "theoretical_loss": 3.374804410043287, + "tokens_seen": 2442395648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026436363636363636, + "loss": 2.7643, + "theoretical_loss": 3.3747971531505403, + "tokens_seen": 2442461184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643434343434343, + "loss": 2.7687, + "theoretical_loss": 3.374789896507026, + "tokens_seen": 2442526720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026432323232323233, + "loss": 2.7009, + "theoretical_loss": 3.3747826401127297, + "tokens_seen": 2442592256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026430303030303034, + "loss": 2.4016, + "theoretical_loss": 3.374775383967636, + "tokens_seen": 2442657792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642828282828283, + "loss": 2.7504, + "theoretical_loss": 3.3747681280717297, + "tokens_seen": 2442723328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026426262626262626, + "loss": 2.7235, + "theoretical_loss": 3.3747608724249947, + "tokens_seen": 2442788864 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642424242424242, + "loss": 2.5554, + "theoretical_loss": 3.374753617027417, + "tokens_seen": 2442854400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642222222222223, + "loss": 2.7524, + "theoretical_loss": 3.374746361878981, + "tokens_seen": 2442919936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026420202020202023, + "loss": 2.5263, + "theoretical_loss": 3.374739106979671, + "tokens_seen": 2442985472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641818181818182, + "loss": 2.4002, + "theoretical_loss": 3.3747318523294716, + "tokens_seen": 2443051008 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1341769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2153728008270264, + "objective/train/theoretical_loss": 3.3747245979283687, + "objective/train/tokens_used": 801975776, + "theoretical_loss": 3.3747245979283687, + "tokens_seen": 2443116544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026416161616161615, + "loss": 2.4886, + "theoretical_loss": 3.3747245979283687, + "tokens_seen": 2443116544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641414141414141, + "loss": 2.5573, + "theoretical_loss": 3.3747173437763456, + "tokens_seen": 2443182080 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026412121212121217, + "loss": 2.5934, + "theoretical_loss": 3.374710089873388, + "tokens_seen": 2443247616 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026410101010101013, + "loss": 2.6106, + "theoretical_loss": 3.3747028362194813, + "tokens_seen": 2443313152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640808080808081, + "loss": 2.5297, + "theoretical_loss": 3.3746955828146086, + "tokens_seen": 2443378688 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026406060606060604, + "loss": 2.5335, + "theoretical_loss": 3.3746883296587558, + "tokens_seen": 2443444224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026404040404040405, + "loss": 2.6934, + "theoretical_loss": 3.3746810767519073, + "tokens_seen": 2443509760 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026402020202020206, + "loss": 2.6769, + "theoretical_loss": 3.374673824094048, + "tokens_seen": 2443575296 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264, + "loss": 2.6634, + "theoretical_loss": 3.374666571685163, + "tokens_seen": 2443640832 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263979797979798, + "loss": 2.5802, + "theoretical_loss": 3.3746593195252363, + "tokens_seen": 2443706368 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026395959595959593, + "loss": 2.7276, + "theoretical_loss": 3.3746520676142535, + "tokens_seen": 2443771904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026393939393939395, + "loss": 2.4221, + "theoretical_loss": 3.3746448159521987, + "tokens_seen": 2443837440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026391919191919196, + "loss": 2.6148, + "theoretical_loss": 3.374637564539057, + "tokens_seen": 2443902976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638989898989899, + "loss": 2.766, + "theoretical_loss": 3.374630313374813, + "tokens_seen": 2443968512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026387878787878787, + "loss": 2.5676, + "theoretical_loss": 3.374623062459452, + "tokens_seen": 2444034048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638585858585858, + "loss": 2.4868, + "theoretical_loss": 3.374615811792958, + "tokens_seen": 2444099584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026383838383838384, + "loss": 2.6143, + "theoretical_loss": 3.3746085613753163, + "tokens_seen": 2444165120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026381818181818185, + "loss": 2.4979, + "theoretical_loss": 3.374601311206512, + "tokens_seen": 2444230656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637979797979798, + "loss": 2.5034, + "theoretical_loss": 3.374594061286529, + "tokens_seen": 2444296192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026377777777777776, + "loss": 2.5198, + "theoretical_loss": 3.3745868116153526, + "tokens_seen": 2444361728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637575757575758, + "loss": 2.8063, + "theoretical_loss": 3.3745795621929675, + "tokens_seen": 2444427264 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026373737373737373, + "loss": 2.8533, + "theoretical_loss": 3.374572313019359, + "tokens_seen": 2444492800 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026371717171717174, + "loss": 2.494, + "theoretical_loss": 3.374565064094511, + "tokens_seen": 2444558336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636969696969697, + "loss": 2.6548, + "theoretical_loss": 3.374557815418409, + "tokens_seen": 2444623872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026367676767676766, + "loss": 2.6307, + "theoretical_loss": 3.3745505669910374, + "tokens_seen": 2444689408 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1342804, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6955645084381104, + "objective/train/theoretical_loss": 3.374543318812381, + "objective/train/tokens_used": 803614176, + "theoretical_loss": 3.374543318812381, + "tokens_seen": 2444754944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026365656565656567, + "loss": 2.6506, + "theoretical_loss": 3.374543318812381, + "tokens_seen": 2444754944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636363636363636, + "loss": 2.3067, + "theoretical_loss": 3.374536070882425, + "tokens_seen": 2444820480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026361616161616163, + "loss": 2.5639, + "theoretical_loss": 3.3745288232011537, + "tokens_seen": 2444886016 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635959595959596, + "loss": 2.4762, + "theoretical_loss": 3.3745215757685525, + "tokens_seen": 2444951552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635757575757576, + "loss": 2.6961, + "theoretical_loss": 3.3745143285846053, + "tokens_seen": 2445017088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026355555555555556, + "loss": 2.6124, + "theoretical_loss": 3.3745070816492975, + "tokens_seen": 2445082624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635353535353535, + "loss": 2.6841, + "theoretical_loss": 3.3744998349626143, + "tokens_seen": 2445148160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635151515151515, + "loss": 2.6576, + "theoretical_loss": 3.37449258852454, + "tokens_seen": 2445213696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634949494949495, + "loss": 2.6613, + "theoretical_loss": 3.374485342335059, + "tokens_seen": 2445279232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634747474747475, + "loss": 2.662, + "theoretical_loss": 3.374478096394157, + "tokens_seen": 2445344768 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026345454545454545, + "loss": 2.7094, + "theoretical_loss": 3.374470850701818, + "tokens_seen": 2445410304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634343434343434, + "loss": 2.6962, + "theoretical_loss": 3.3744636052580272, + "tokens_seen": 2445475840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634141414141414, + "loss": 2.6733, + "theoretical_loss": 3.3744563600627697, + "tokens_seen": 2445541376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026339393939393943, + "loss": 2.599, + "theoretical_loss": 3.37444911511603, + "tokens_seen": 2445606912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002633737373737374, + "loss": 2.4048, + "theoretical_loss": 3.3744418704177925, + "tokens_seen": 2445672448 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026335353535353534, + "loss": 2.6704, + "theoretical_loss": 3.3744346259680427, + "tokens_seen": 2445737984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002633333333333333, + "loss": 2.7065, + "theoretical_loss": 3.3744273817667656, + "tokens_seen": 2445803520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002633131313131313, + "loss": 2.696, + "theoretical_loss": 3.374420137813945, + "tokens_seen": 2445869056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632929292929293, + "loss": 2.6544, + "theoretical_loss": 3.3744128941095664, + "tokens_seen": 2445934592 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632727272727273, + "loss": 2.5253, + "theoretical_loss": 3.374405650653615, + "tokens_seen": 2446000128 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026325252525252524, + "loss": 2.58, + "theoretical_loss": 3.374398407446075, + "tokens_seen": 2446065664 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632323232323232, + "loss": 2.8209, + "theoretical_loss": 3.374391164486931, + "tokens_seen": 2446131200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026321212121212126, + "loss": 2.4928, + "theoretical_loss": 3.374383921776168, + "tokens_seen": 2446196736 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631919191919192, + "loss": 2.8239, + "theoretical_loss": 3.3743766793137717, + "tokens_seen": 2446262272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631717171717172, + "loss": 2.7722, + "theoretical_loss": 3.374369437099726, + "tokens_seen": 2446327808 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1343166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8937578201293945, + "objective/train/theoretical_loss": 3.374362195134016, + "objective/train/tokens_used": 805252576, + "theoretical_loss": 3.374362195134016, + "tokens_seen": 2446393344 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026315151515151513, + "loss": 2.7007, + "theoretical_loss": 3.374362195134016, + "tokens_seen": 2446393344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631313131313131, + "loss": 2.5962, + "theoretical_loss": 3.374354953416627, + "tokens_seen": 2446458880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026311111111111115, + "loss": 2.642, + "theoretical_loss": 3.374347711947543, + "tokens_seen": 2446524416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630909090909091, + "loss": 2.4228, + "theoretical_loss": 3.374340470726749, + "tokens_seen": 2446589952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026307070707070707, + "loss": 2.741, + "theoretical_loss": 3.37433322975423, + "tokens_seen": 2446655488 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263050505050505, + "loss": 2.7276, + "theoretical_loss": 3.374325989029971, + "tokens_seen": 2446721024 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026303030303030303, + "loss": 2.8696, + "theoretical_loss": 3.3743187485539567, + "tokens_seen": 2446786560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026301010101010104, + "loss": 2.4678, + "theoretical_loss": 3.3743115083261723, + "tokens_seen": 2446852096 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262989898989899, + "loss": 2.6835, + "theoretical_loss": 3.374304268346602, + "tokens_seen": 2446917632 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026296969696969696, + "loss": 2.7896, + "theoretical_loss": 3.374297028615231, + "tokens_seen": 2446983168 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629494949494949, + "loss": 2.7025, + "theoretical_loss": 3.374289789132044, + "tokens_seen": 2447048704 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262929292929293, + "loss": 2.6045, + "theoretical_loss": 3.3742825498970257, + "tokens_seen": 2447114240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026290909090909094, + "loss": 2.5814, + "theoretical_loss": 3.3742753109101615, + "tokens_seen": 2447179776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628888888888889, + "loss": 2.668, + "theoretical_loss": 3.3742680721714358, + "tokens_seen": 2447245312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026286868686868685, + "loss": 2.7779, + "theoretical_loss": 3.3742608336808337, + "tokens_seen": 2447310848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026284848484848486, + "loss": 2.6306, + "theoretical_loss": 3.37425359543834, + "tokens_seen": 2447376384 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628282828282829, + "loss": 2.6649, + "theoretical_loss": 3.3742463574439396, + "tokens_seen": 2447441920 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026280808080808083, + "loss": 2.5009, + "theoretical_loss": 3.374239119697617, + "tokens_seen": 2447507456 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627878787878788, + "loss": 2.7443, + "theoretical_loss": 3.374231882199357, + "tokens_seen": 2447572992 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026276767676767674, + "loss": 2.4669, + "theoretical_loss": 3.374224644949145, + "tokens_seen": 2447638528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026274747474747475, + "loss": 2.532, + "theoretical_loss": 3.374217407946966, + "tokens_seen": 2447704064 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026272727272727277, + "loss": 2.5868, + "theoretical_loss": 3.3742101711928036, + "tokens_seen": 2447769600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627070707070707, + "loss": 2.5711, + "theoretical_loss": 3.374202934686644, + "tokens_seen": 2447835136 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626868686868687, + "loss": 2.6259, + "theoretical_loss": 3.374195698428472, + "tokens_seen": 2447900672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626666666666667, + "loss": 2.6822, + "theoretical_loss": 3.3741884624182714, + "tokens_seen": 2447966208 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1344165, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.270679235458374, + "objective/train/theoretical_loss": 3.374181226656028, + "objective/train/tokens_used": 806890976, + "theoretical_loss": 3.374181226656028, + "tokens_seen": 2448031744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026264646464646465, + "loss": 2.9149, + "theoretical_loss": 3.374181226656028, + "tokens_seen": 2448031744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026262626262626266, + "loss": 2.7126, + "theoretical_loss": 3.3741739911417263, + "tokens_seen": 2448097280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626060606060606, + "loss": 2.4629, + "theoretical_loss": 3.374166755875351, + "tokens_seen": 2448162816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026258585858585857, + "loss": 2.7661, + "theoretical_loss": 3.3741595208568875, + "tokens_seen": 2448228352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625656565656566, + "loss": 2.512, + "theoretical_loss": 3.3741522860863205, + "tokens_seen": 2448293888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026254545454545454, + "loss": 2.7218, + "theoretical_loss": 3.3741450515636346, + "tokens_seen": 2448359424 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026252525252525255, + "loss": 2.7902, + "theoretical_loss": 3.3741378172888146, + "tokens_seen": 2448424960 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625050505050505, + "loss": 2.4496, + "theoretical_loss": 3.374130583261846, + "tokens_seen": 2448490496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624848484848485, + "loss": 2.4347, + "theoretical_loss": 3.374123349482713, + "tokens_seen": 2448556032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624646464646465, + "loss": 2.7818, + "theoretical_loss": 3.374116115951401, + "tokens_seen": 2448621568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026244444444444443, + "loss": 2.805, + "theoretical_loss": 3.374108882667894, + "tokens_seen": 2448687104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026242424242424244, + "loss": 2.7023, + "theoretical_loss": 3.374101649632178, + "tokens_seen": 2448752640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624040404040404, + "loss": 2.7081, + "theoretical_loss": 3.3740944168442373, + "tokens_seen": 2448818176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623838383838384, + "loss": 2.526, + "theoretical_loss": 3.3740871843040567, + "tokens_seen": 2448883712 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026236363636363637, + "loss": 2.5459, + "theoretical_loss": 3.3740799520116216, + "tokens_seen": 2448949248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623434343434343, + "loss": 2.6773, + "theoretical_loss": 3.3740727199669163, + "tokens_seen": 2449014784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026232323232323234, + "loss": 2.6045, + "theoretical_loss": 3.3740654881699257, + "tokens_seen": 2449080320 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026230303030303035, + "loss": 2.8309, + "theoretical_loss": 3.3740582566206356, + "tokens_seen": 2449145856 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622828282828283, + "loss": 2.687, + "theoretical_loss": 3.3740510253190297, + "tokens_seen": 2449211392 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026226262626262626, + "loss": 2.6226, + "theoretical_loss": 3.374043794265093, + "tokens_seen": 2449276928 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622424242424242, + "loss": 2.5032, + "theoretical_loss": 3.3740365634588114, + "tokens_seen": 2449342464 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026222222222222223, + "loss": 2.5379, + "theoretical_loss": 3.374029332900169, + "tokens_seen": 2449408000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026220202020202024, + "loss": 2.4041, + "theoretical_loss": 3.3740221025891506, + "tokens_seen": 2449473536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621818181818182, + "loss": 2.6704, + "theoretical_loss": 3.3740148725257417, + "tokens_seen": 2449539072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026216161616161615, + "loss": 2.5828, + "theoretical_loss": 3.374007642709927, + "tokens_seen": 2449604608 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1344624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0327699184417725, + "objective/train/theoretical_loss": 3.3740004131416907, + "objective/train/tokens_used": 808529376, + "theoretical_loss": 3.3740004131416907, + "tokens_seen": 2449670144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621414141414141, + "loss": 2.7585, + "theoretical_loss": 3.3740004131416907, + "tokens_seen": 2449670144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621212121212122, + "loss": 2.5421, + "theoretical_loss": 3.3739931838210184, + "tokens_seen": 2449735680 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026210101010101013, + "loss": 2.6363, + "theoretical_loss": 3.373985954747895, + "tokens_seen": 2449801216 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620808080808081, + "loss": 2.7274, + "theoretical_loss": 3.3739787259223055, + "tokens_seen": 2449866752 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026206060606060605, + "loss": 2.5467, + "theoretical_loss": 3.3739714973442343, + "tokens_seen": 2449932288 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262040404040404, + "loss": 2.404, + "theoretical_loss": 3.3739642690136664, + "tokens_seen": 2449997824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026202020202020207, + "loss": 2.7878, + "theoretical_loss": 3.373957040930587, + "tokens_seen": 2450063360 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262, + "loss": 2.5061, + "theoretical_loss": 3.3739498130949808, + "tokens_seen": 2450128896 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261979797979798, + "loss": 2.6148, + "theoretical_loss": 3.373942585506833, + "tokens_seen": 2450194432 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026195959595959594, + "loss": 2.6403, + "theoretical_loss": 3.3739353581661278, + "tokens_seen": 2450259968 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619393939393939, + "loss": 2.9545, + "theoretical_loss": 3.3739281310728506, + "tokens_seen": 2450325504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026191919191919196, + "loss": 2.5943, + "theoretical_loss": 3.3739209042269867, + "tokens_seen": 2450391040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618989898989899, + "loss": 2.8379, + "theoretical_loss": 3.3739136776285203, + "tokens_seen": 2450456576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618787878787879, + "loss": 2.627, + "theoretical_loss": 3.373906451277437, + "tokens_seen": 2450522112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026185858585858583, + "loss": 2.5092, + "theoretical_loss": 3.373899225173721, + "tokens_seen": 2450587648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026183838383838384, + "loss": 2.4616, + "theoretical_loss": 3.373891999317358, + "tokens_seen": 2450653184 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026181818181818185, + "loss": 2.5581, + "theoretical_loss": 3.3738847737083324, + "tokens_seen": 2450718720 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617979797979798, + "loss": 2.6438, + "theoretical_loss": 3.373877548346629, + "tokens_seen": 2450784256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026177777777777777, + "loss": 2.5025, + "theoretical_loss": 3.3738703232322327, + "tokens_seen": 2450849792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617575757575757, + "loss": 2.6267, + "theoretical_loss": 3.373863098365129, + "tokens_seen": 2450915328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026173737373737374, + "loss": 2.5287, + "theoretical_loss": 3.373855873745302, + "tokens_seen": 2450980864 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026171717171717175, + "loss": 2.6813, + "theoretical_loss": 3.3738486493727375, + "tokens_seen": 2451046400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616969696969697, + "loss": 2.6658, + "theoretical_loss": 3.37384142524742, + "tokens_seen": 2451111936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026167676767676766, + "loss": 2.7852, + "theoretical_loss": 3.3738342013693345, + "tokens_seen": 2451177472 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026165656565656567, + "loss": 2.7797, + "theoretical_loss": 3.373826977738466, + "tokens_seen": 2451243008 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1344624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.707127571105957, + "objective/train/theoretical_loss": 3.373819754354799, + "objective/train/tokens_used": 810167776, + "theoretical_loss": 3.373819754354799, + "tokens_seen": 2451308544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026163636363636363, + "loss": 2.7378, + "theoretical_loss": 3.373819754354799, + "tokens_seen": 2451308544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026161616161616164, + "loss": 2.727, + "theoretical_loss": 3.3738125312183187, + "tokens_seen": 2451374080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615959595959596, + "loss": 2.7233, + "theoretical_loss": 3.3738053083290103, + "tokens_seen": 2451439616 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026157575757575755, + "loss": 2.7302, + "theoretical_loss": 3.3737980856868584, + "tokens_seen": 2451505152 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026155555555555556, + "loss": 2.6963, + "theoretical_loss": 3.3737908632918483, + "tokens_seen": 2451570688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615353535353535, + "loss": 2.624, + "theoretical_loss": 3.3737836411439646, + "tokens_seen": 2451636224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026151515151515153, + "loss": 2.7915, + "theoretical_loss": 3.373776419243192, + "tokens_seen": 2451701760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614949494949495, + "loss": 2.5758, + "theoretical_loss": 3.3737691975895157, + "tokens_seen": 2451767296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614747474747475, + "loss": 2.6416, + "theoretical_loss": 3.373761976182921, + "tokens_seen": 2451832832 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026145454545454546, + "loss": 2.7037, + "theoretical_loss": 3.3737547550233926, + "tokens_seen": 2451898368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614343434343434, + "loss": 2.6136, + "theoretical_loss": 3.3737475341109153, + "tokens_seen": 2451963904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614141414141414, + "loss": 2.748, + "theoretical_loss": 3.373740313445474, + "tokens_seen": 2452029440 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613939393939394, + "loss": 2.91, + "theoretical_loss": 3.373733093027054, + "tokens_seen": 2452094976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613737373737374, + "loss": 2.6432, + "theoretical_loss": 3.37372587285564, + "tokens_seen": 2452160512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026135353535353535, + "loss": 2.8583, + "theoretical_loss": 3.373718652931217, + "tokens_seen": 2452226048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613333333333333, + "loss": 2.7154, + "theoretical_loss": 3.37371143325377, + "tokens_seen": 2452291584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613131313131313, + "loss": 2.7039, + "theoretical_loss": 3.3737042138232836, + "tokens_seen": 2452357120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026129292929292933, + "loss": 2.8158, + "theoretical_loss": 3.373696994639743, + "tokens_seen": 2452422656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612727272727273, + "loss": 3.048, + "theoretical_loss": 3.3736897757031334, + "tokens_seen": 2452488192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026125252525252524, + "loss": 2.6986, + "theoretical_loss": 3.3736825570134394, + "tokens_seen": 2452553728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612323232323232, + "loss": 2.5484, + "theoretical_loss": 3.373675338570646, + "tokens_seen": 2452619264 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612121212121212, + "loss": 2.9467, + "theoretical_loss": 3.3736681203747385, + "tokens_seen": 2452684800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611919191919192, + "loss": 2.9616, + "theoretical_loss": 3.3736609024257014, + "tokens_seen": 2452750336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611717171717172, + "loss": 2.6523, + "theoretical_loss": 3.37365368472352, + "tokens_seen": 2452815872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026115151515151514, + "loss": 2.6857, + "theoretical_loss": 3.373646467268179, + "tokens_seen": 2452881408 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1345199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.841261625289917, + "objective/train/theoretical_loss": 3.373639250059664, + "objective/train/tokens_used": 811806176, + "theoretical_loss": 3.373639250059664, + "tokens_seen": 2452946944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026113131313131315, + "loss": 2.7445, + "theoretical_loss": 3.373639250059664, + "tokens_seen": 2452946944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026111111111111116, + "loss": 2.7205, + "theoretical_loss": 3.373632033097959, + "tokens_seen": 2453012480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610909090909091, + "loss": 2.6992, + "theoretical_loss": 3.3736248163830496, + "tokens_seen": 2453078016 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026107070707070707, + "loss": 2.9203, + "theoretical_loss": 3.3736175999149203, + "tokens_seen": 2453143552 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026105050505050503, + "loss": 2.6908, + "theoretical_loss": 3.3736103836935567, + "tokens_seen": 2453209088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026103030303030304, + "loss": 2.8833, + "theoretical_loss": 3.3736031677189433, + "tokens_seen": 2453274624 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026101010101010105, + "loss": 2.7665, + "theoretical_loss": 3.3735959519910654, + "tokens_seen": 2453340160 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260989898989899, + "loss": 2.8848, + "theoretical_loss": 3.3735887365099075, + "tokens_seen": 2453405696 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026096969696969696, + "loss": 2.7735, + "theoretical_loss": 3.373581521275455, + "tokens_seen": 2453471232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002609494949494949, + "loss": 2.6306, + "theoretical_loss": 3.3735743062876926, + "tokens_seen": 2453536768 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260929292929293, + "loss": 2.8885, + "theoretical_loss": 3.3735670915466054, + "tokens_seen": 2453602304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026090909090909094, + "loss": 2.7228, + "theoretical_loss": 3.3735598770521786, + "tokens_seen": 2453667840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608888888888889, + "loss": 2.6217, + "theoretical_loss": 3.3735526628043973, + "tokens_seen": 2453733376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026086868686868686, + "loss": 2.766, + "theoretical_loss": 3.3735454488032457, + "tokens_seen": 2453798912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608484848484848, + "loss": 2.8502, + "theoretical_loss": 3.3735382350487093, + "tokens_seen": 2453864448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608282828282829, + "loss": 2.6791, + "theoretical_loss": 3.3735310215407726, + "tokens_seen": 2453929984 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026080808080808084, + "loss": 3.0647, + "theoretical_loss": 3.3735238082794217, + "tokens_seen": 2453995520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607878787878788, + "loss": 2.9451, + "theoretical_loss": 3.37351659526464, + "tokens_seen": 2454061056 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026076767676767675, + "loss": 2.8323, + "theoretical_loss": 3.3735093824964144, + "tokens_seen": 2454126592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026074747474747476, + "loss": 2.836, + "theoretical_loss": 3.3735021699747283, + "tokens_seen": 2454192128 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026072727272727277, + "loss": 2.8259, + "theoretical_loss": 3.373494957699567, + "tokens_seen": 2454257664 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026070707070707073, + "loss": 2.6386, + "theoretical_loss": 3.3734877456709165, + "tokens_seen": 2454323200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606868686868687, + "loss": 2.8226, + "theoretical_loss": 3.3734805338887606, + "tokens_seen": 2454388736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026066666666666664, + "loss": 2.8312, + "theoretical_loss": 3.3734733223530844, + "tokens_seen": 2454454272 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026064646464646465, + "loss": 2.6148, + "theoretical_loss": 3.3734661110638737, + "tokens_seen": 2454519808 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1346301, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8723297119140625, + "objective/train/theoretical_loss": 3.373458900021113, + "objective/train/tokens_used": 813444576, + "theoretical_loss": 3.373458900021113, + "tokens_seen": 2454585344 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026062626262626266, + "loss": 2.6411, + "theoretical_loss": 3.373458900021113, + "tokens_seen": 2454585344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606060606060606, + "loss": 2.7596, + "theoretical_loss": 3.3734516892247868, + "tokens_seen": 2454650880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605858585858586, + "loss": 2.7109, + "theoretical_loss": 3.373444478674881, + "tokens_seen": 2454716416 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026056565656565653, + "loss": 2.668, + "theoretical_loss": 3.3734372683713802, + "tokens_seen": 2454781952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026054545454545455, + "loss": 2.763, + "theoretical_loss": 3.37343005831427, + "tokens_seen": 2454847488 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026052525252525256, + "loss": 2.84, + "theoretical_loss": 3.373422848503534, + "tokens_seen": 2454913024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605050505050505, + "loss": 2.8758, + "theoretical_loss": 3.373415638939158, + "tokens_seen": 2454978560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026048484848484847, + "loss": 2.6563, + "theoretical_loss": 3.3734084296211275, + "tokens_seen": 2455044096 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604646464646465, + "loss": 2.7618, + "theoretical_loss": 3.3734012205494266, + "tokens_seen": 2455109632 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026044444444444444, + "loss": 2.6284, + "theoretical_loss": 3.373394011724041, + "tokens_seen": 2455175168 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026042424242424245, + "loss": 2.6634, + "theoretical_loss": 3.3733868031449554, + "tokens_seen": 2455240704 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604040404040404, + "loss": 2.6675, + "theoretical_loss": 3.373379594812155, + "tokens_seen": 2455306240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026038383838383836, + "loss": 2.7761, + "theoretical_loss": 3.3733723867256247, + "tokens_seen": 2455371776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603636363636364, + "loss": 2.6757, + "theoretical_loss": 3.373365178885349, + "tokens_seen": 2455437312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026034343434343433, + "loss": 2.9078, + "theoretical_loss": 3.373357971291314, + "tokens_seen": 2455502848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026032323232323234, + "loss": 2.8445, + "theoretical_loss": 3.373350763943504, + "tokens_seen": 2455568384 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603030303030303, + "loss": 2.8749, + "theoretical_loss": 3.373343556841904, + "tokens_seen": 2455633920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602828282828283, + "loss": 2.9043, + "theoretical_loss": 3.373336349986499, + "tokens_seen": 2455699456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026026262626262627, + "loss": 2.8745, + "theoretical_loss": 3.3733291433772745, + "tokens_seen": 2455764992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602424242424242, + "loss": 2.7883, + "theoretical_loss": 3.373321937014215, + "tokens_seen": 2455830528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026022222222222223, + "loss": 2.9081, + "theoretical_loss": 3.3733147308973055, + "tokens_seen": 2455896064 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602020202020202, + "loss": 2.5358, + "theoretical_loss": 3.3733075250265316, + "tokens_seen": 2455961600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002601818181818182, + "loss": 2.7218, + "theoretical_loss": 3.373300319401878, + "tokens_seen": 2456027136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026016161616161616, + "loss": 2.5663, + "theoretical_loss": 3.3732931140233293, + "tokens_seen": 2456092672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002601414141414141, + "loss": 2.7825, + "theoretical_loss": 3.373285908890871, + "tokens_seen": 2456158208 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1346849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8223564624786377, + "objective/train/theoretical_loss": 3.3732787040044885, + "objective/train/tokens_used": 815082976, + "theoretical_loss": 3.3732787040044885, + "tokens_seen": 2456223744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026012121212121213, + "loss": 2.7546, + "theoretical_loss": 3.3732787040044885, + "tokens_seen": 2456223744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026010101010101014, + "loss": 2.6832, + "theoretical_loss": 3.373271499364166, + "tokens_seen": 2456289280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600808080808081, + "loss": 2.9912, + "theoretical_loss": 3.373264294969889, + "tokens_seen": 2456354816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026006060606060605, + "loss": 2.7522, + "theoretical_loss": 3.3732570908216424, + "tokens_seen": 2456420352 + }, + { + "epoch": 0.49, + "learning_rate": 0.000260040404040404, + "loss": 2.8512, + "theoretical_loss": 3.3732498869194116, + "tokens_seen": 2456485888 + }, + { + "epoch": 0.49, + "learning_rate": 0.000260020202020202, + "loss": 2.6009, + "theoretical_loss": 3.373242683263181, + "tokens_seen": 2456551424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026000000000000003, + "loss": 2.6549, + "theoretical_loss": 3.373235479852936, + "tokens_seen": 2456616960 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259979797979798, + "loss": 2.7297, + "theoretical_loss": 3.3732282766886614, + "tokens_seen": 2456682496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025995959595959595, + "loss": 2.655, + "theoretical_loss": 3.3732210737703427, + "tokens_seen": 2456748032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599393939393939, + "loss": 2.6077, + "theoretical_loss": 3.3732138710979647, + "tokens_seen": 2456813568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025991919191919197, + "loss": 2.7402, + "theoretical_loss": 3.3732066686715125, + "tokens_seen": 2456879104 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598989898989899, + "loss": 2.8478, + "theoretical_loss": 3.373199466490971, + "tokens_seen": 2456944640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598787878787879, + "loss": 2.8656, + "theoretical_loss": 3.373192264556325, + "tokens_seen": 2457010176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025985858585858584, + "loss": 2.7574, + "theoretical_loss": 3.3731850628675604, + "tokens_seen": 2457075712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598383838383838, + "loss": 2.4306, + "theoretical_loss": 3.373177861424661, + "tokens_seen": 2457141248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025981818181818186, + "loss": 2.7078, + "theoretical_loss": 3.373170660227613, + "tokens_seen": 2457206784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597979797979798, + "loss": 2.9213, + "theoretical_loss": 3.373163459276401, + "tokens_seen": 2457272320 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597777777777778, + "loss": 2.7051, + "theoretical_loss": 3.3731562585710098, + "tokens_seen": 2457337856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025975757575757573, + "loss": 2.7108, + "theoretical_loss": 3.3731490581114247, + "tokens_seen": 2457403392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025973737373737374, + "loss": 2.8107, + "theoretical_loss": 3.373141857897631, + "tokens_seen": 2457468928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025971717171717175, + "loss": 2.8593, + "theoretical_loss": 3.3731346579296133, + "tokens_seen": 2457534464 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002596969696969697, + "loss": 2.4869, + "theoretical_loss": 3.373127458207357, + "tokens_seen": 2457600000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025967676767676767, + "loss": 2.6961, + "theoretical_loss": 3.3731202587308475, + "tokens_seen": 2457665536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002596565656565656, + "loss": 2.8116, + "theoretical_loss": 3.373113059500069, + "tokens_seen": 2457731072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025963636363636363, + "loss": 2.7153, + "theoretical_loss": 3.3731058605150066, + "tokens_seen": 2457796608 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1347964, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6259384155273438, + "objective/train/theoretical_loss": 3.373098661775646, + "objective/train/tokens_used": 816721376, + "theoretical_loss": 3.373098661775646, + "tokens_seen": 2457862144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025961616161616165, + "loss": 2.6826, + "theoretical_loss": 3.373098661775646, + "tokens_seen": 2457862144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595959595959596, + "loss": 2.3772, + "theoretical_loss": 3.3730914632819715, + "tokens_seen": 2457927680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025957575757575756, + "loss": 2.6872, + "theoretical_loss": 3.3730842650339694, + "tokens_seen": 2457993216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025955555555555557, + "loss": 2.7928, + "theoretical_loss": 3.3730770670316237, + "tokens_seen": 2458058752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595353535353535, + "loss": 2.7637, + "theoretical_loss": 3.3730698692749197, + "tokens_seen": 2458124288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025951515151515154, + "loss": 2.8141, + "theoretical_loss": 3.3730626717638423, + "tokens_seen": 2458189824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594949494949495, + "loss": 2.5939, + "theoretical_loss": 3.3730554744983774, + "tokens_seen": 2458255360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025947474747474745, + "loss": 2.7473, + "theoretical_loss": 3.373048277478509, + "tokens_seen": 2458320896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025945454545454546, + "loss": 2.7952, + "theoretical_loss": 3.3730410807042226, + "tokens_seen": 2458386432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594343434343435, + "loss": 2.7902, + "theoretical_loss": 3.373033884175504, + "tokens_seen": 2458451968 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025941414141414143, + "loss": 2.6969, + "theoretical_loss": 3.3730266878923367, + "tokens_seen": 2458517504 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593939393939394, + "loss": 2.8438, + "theoretical_loss": 3.3730194918547074, + "tokens_seen": 2458583040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593737373737374, + "loss": 2.776, + "theoretical_loss": 3.3730122960626, + "tokens_seen": 2458648576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025935353535353536, + "loss": 2.7972, + "theoretical_loss": 3.373005100516, + "tokens_seen": 2458714112 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025933333333333337, + "loss": 2.5417, + "theoretical_loss": 3.372997905214893, + "tokens_seen": 2458779648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593131313131313, + "loss": 2.757, + "theoretical_loss": 3.372990710159263, + "tokens_seen": 2458845184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592929292929293, + "loss": 2.6848, + "theoretical_loss": 3.3729835153490964, + "tokens_seen": 2458910720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592727272727273, + "loss": 2.4838, + "theoretical_loss": 3.372976320784377, + "tokens_seen": 2458976256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025925252525252525, + "loss": 2.542, + "theoretical_loss": 3.3729691264650903, + "tokens_seen": 2459041792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025923232323232326, + "loss": 2.7042, + "theoretical_loss": 3.3729619323912217, + "tokens_seen": 2459107328 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592121212121212, + "loss": 2.6411, + "theoretical_loss": 3.3729547385627563, + "tokens_seen": 2459172864 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025919191919191923, + "loss": 2.6033, + "theoretical_loss": 3.372947544979679, + "tokens_seen": 2459238400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591717171717172, + "loss": 2.7508, + "theoretical_loss": 3.372940351641975, + "tokens_seen": 2459303936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025915151515151514, + "loss": 2.7737, + "theoretical_loss": 3.372933158549629, + "tokens_seen": 2459369472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025913131313131315, + "loss": 2.7519, + "theoretical_loss": 3.3729259657026263, + "tokens_seen": 2459435008 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1348509, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.897902011871338, + "objective/train/theoretical_loss": 3.3729187731009524, + "objective/train/tokens_used": 818359776, + "theoretical_loss": 3.3729187731009524, + "tokens_seen": 2459500544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591111111111111, + "loss": 2.856, + "theoretical_loss": 3.3729187731009524, + "tokens_seen": 2459500544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590909090909091, + "loss": 2.8222, + "theoretical_loss": 3.372911580744592, + "tokens_seen": 2459566080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590707070707071, + "loss": 2.819, + "theoretical_loss": 3.37290438863353, + "tokens_seen": 2459631616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025905050505050503, + "loss": 2.7431, + "theoretical_loss": 3.3728971967677523, + "tokens_seen": 2459697152 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025903030303030304, + "loss": 2.622, + "theoretical_loss": 3.372890005147243, + "tokens_seen": 2459762688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025901010101010106, + "loss": 2.5188, + "theoretical_loss": 3.3728828137719877, + "tokens_seen": 2459828224 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258989898989899, + "loss": 2.5795, + "theoretical_loss": 3.372875622641972, + "tokens_seen": 2459893760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025896969696969697, + "loss": 2.7337, + "theoretical_loss": 3.3728684317571798, + "tokens_seen": 2459959296 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589494949494949, + "loss": 2.8569, + "theoretical_loss": 3.3728612411175973, + "tokens_seen": 2460024832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025892929292929294, + "loss": 2.7593, + "theoretical_loss": 3.372854050723209, + "tokens_seen": 2460090368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025890909090909095, + "loss": 2.5794, + "theoretical_loss": 3.372846860574, + "tokens_seen": 2460155904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588888888888889, + "loss": 2.6916, + "theoretical_loss": 3.372839670669956, + "tokens_seen": 2460221440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025886868686868686, + "loss": 2.5212, + "theoretical_loss": 3.3728324810110615, + "tokens_seen": 2460286976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588484848484848, + "loss": 2.6511, + "theoretical_loss": 3.372825291597302, + "tokens_seen": 2460352512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025882828282828283, + "loss": 2.7294, + "theoretical_loss": 3.3728181024286625, + "tokens_seen": 2460418048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025880808080808084, + "loss": 2.3928, + "theoretical_loss": 3.372810913505128, + "tokens_seen": 2460483584 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587878787878788, + "loss": 2.6117, + "theoretical_loss": 3.3728037248266833, + "tokens_seen": 2460549120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025876767676767675, + "loss": 2.5043, + "theoretical_loss": 3.372796536393314, + "tokens_seen": 2460614656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587474747474747, + "loss": 2.6559, + "theoretical_loss": 3.3727893482050053, + "tokens_seen": 2460680192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587272727272728, + "loss": 2.7152, + "theoretical_loss": 3.372782160261742, + "tokens_seen": 2460745728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025870707070707073, + "loss": 2.7525, + "theoretical_loss": 3.3727749725635094, + "tokens_seen": 2460811264 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586868686868687, + "loss": 2.8658, + "theoretical_loss": 3.372767785110293, + "tokens_seen": 2460876800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025866666666666665, + "loss": 2.7361, + "theoretical_loss": 3.3727605979020767, + "tokens_seen": 2460942336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586464646464646, + "loss": 2.695, + "theoretical_loss": 3.372753410938847, + "tokens_seen": 2461007872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025862626262626267, + "loss": 2.86, + "theoretical_loss": 3.372746224220588, + "tokens_seen": 2461073408 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1349876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.583599328994751, + "objective/train/theoretical_loss": 3.3727390377472855, + "objective/train/tokens_used": 819998176, + "theoretical_loss": 3.3727390377472855, + "tokens_seen": 2461138944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586060606060606, + "loss": 2.6283, + "theoretical_loss": 3.3727390377472855, + "tokens_seen": 2461138944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585858585858586, + "loss": 2.5535, + "theoretical_loss": 3.372731851518924, + "tokens_seen": 2461204480 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025856565656565654, + "loss": 2.4328, + "theoretical_loss": 3.3727246655354897, + "tokens_seen": 2461270016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025854545454545455, + "loss": 2.9047, + "theoretical_loss": 3.3727174797969663, + "tokens_seen": 2461335552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025852525252525256, + "loss": 2.6543, + "theoretical_loss": 3.3727102943033405, + "tokens_seen": 2461401088 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585050505050505, + "loss": 2.6586, + "theoretical_loss": 3.372703109054596, + "tokens_seen": 2461466624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584848484848485, + "loss": 2.4107, + "theoretical_loss": 3.372695924050719, + "tokens_seen": 2461532160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025846464646464643, + "loss": 2.6501, + "theoretical_loss": 3.3726887392916938, + "tokens_seen": 2461597696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025844444444444444, + "loss": 2.708, + "theoretical_loss": 3.372681554777506, + "tokens_seen": 2461663232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025842424242424246, + "loss": 2.7723, + "theoretical_loss": 3.372674370508141, + "tokens_seen": 2461728768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584040404040404, + "loss": 2.7073, + "theoretical_loss": 3.3726671864835835, + "tokens_seen": 2461794304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025838383838383837, + "loss": 2.6628, + "theoretical_loss": 3.3726600027038183, + "tokens_seen": 2461859840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583636363636364, + "loss": 2.61, + "theoretical_loss": 3.372652819168832, + "tokens_seen": 2461925376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025834343434343434, + "loss": 2.5823, + "theoretical_loss": 3.3726456358786074, + "tokens_seen": 2461990912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025832323232323235, + "loss": 2.755, + "theoretical_loss": 3.3726384528331317, + "tokens_seen": 2462056448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583030303030303, + "loss": 2.5989, + "theoretical_loss": 3.3726312700323895, + "tokens_seen": 2462121984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025828282828282826, + "loss": 2.54, + "theoretical_loss": 3.3726240874763658, + "tokens_seen": 2462187520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025826262626262627, + "loss": 2.929, + "theoretical_loss": 3.372616905165045, + "tokens_seen": 2462253056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025824242424242423, + "loss": 2.5968, + "theoretical_loss": 3.372609723098414, + "tokens_seen": 2462318592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025822222222222224, + "loss": 2.661, + "theoretical_loss": 3.3726025412764558, + "tokens_seen": 2462384128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582020202020202, + "loss": 2.6983, + "theoretical_loss": 3.372595359699157, + "tokens_seen": 2462449664 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581818181818182, + "loss": 2.6407, + "theoretical_loss": 3.372588178366503, + "tokens_seen": 2462515200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025816161616161617, + "loss": 2.5894, + "theoretical_loss": 3.372580997278478, + "tokens_seen": 2462580736 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581414141414141, + "loss": 2.5867, + "theoretical_loss": 3.3725738164350676, + "tokens_seen": 2462646272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025812121212121213, + "loss": 2.8174, + "theoretical_loss": 3.372566635836257, + "tokens_seen": 2462711808 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1350584, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0239999294281006, + "objective/train/theoretical_loss": 3.372559455482031, + "objective/train/tokens_used": 821636576, + "theoretical_loss": 3.372559455482031, + "tokens_seen": 2462777344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581010101010101, + "loss": 2.8393, + "theoretical_loss": 3.372559455482031, + "tokens_seen": 2462777344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002580808080808081, + "loss": 2.7599, + "theoretical_loss": 3.3725522753723753, + "tokens_seen": 2462842880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025806060606060606, + "loss": 2.9517, + "theoretical_loss": 3.3725450955072747, + "tokens_seen": 2462908416 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258040404040404, + "loss": 2.6314, + "theoretical_loss": 3.3725379158867144, + "tokens_seen": 2462973952 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258020202020202, + "loss": 2.4321, + "theoretical_loss": 3.3725307365106794, + "tokens_seen": 2463039488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025800000000000004, + "loss": 2.7111, + "theoretical_loss": 3.3725235573791554, + "tokens_seen": 2463105024 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257979797979798, + "loss": 2.5913, + "theoretical_loss": 3.3725163784921275, + "tokens_seen": 2463170560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025795959595959595, + "loss": 2.7813, + "theoretical_loss": 3.37250919984958, + "tokens_seen": 2463236096 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579393939393939, + "loss": 2.6053, + "theoretical_loss": 3.372502021451499, + "tokens_seen": 2463301632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579191919191919, + "loss": 2.7379, + "theoretical_loss": 3.3724948432978694, + "tokens_seen": 2463367168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025789898989898993, + "loss": 2.5913, + "theoretical_loss": 3.372487665388676, + "tokens_seen": 2463432704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578787878787879, + "loss": 2.8309, + "theoretical_loss": 3.3724804877239047, + "tokens_seen": 2463498240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025785858585858584, + "loss": 2.5279, + "theoretical_loss": 3.37247331030354, + "tokens_seen": 2463563776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578383838383838, + "loss": 2.7626, + "theoretical_loss": 3.372466133127568, + "tokens_seen": 2463629312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025781818181818187, + "loss": 2.6182, + "theoretical_loss": 3.372458956195972, + "tokens_seen": 2463694848 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577979797979798, + "loss": 2.5245, + "theoretical_loss": 3.372451779508739, + "tokens_seen": 2463760384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577777777777778, + "loss": 2.7016, + "theoretical_loss": 3.372444603065854, + "tokens_seen": 2463825920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025775757575757574, + "loss": 2.661, + "theoretical_loss": 3.3724374268673016, + "tokens_seen": 2463891456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025773737373737375, + "loss": 2.4547, + "theoretical_loss": 3.372430250913067, + "tokens_seen": 2463956992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025771717171717176, + "loss": 2.6454, + "theoretical_loss": 3.3724230752031357, + "tokens_seen": 2464022528 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576969696969697, + "loss": 2.5513, + "theoretical_loss": 3.3724158997374922, + "tokens_seen": 2464088064 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025767676767676767, + "loss": 2.755, + "theoretical_loss": 3.3724087245161227, + "tokens_seen": 2464153600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025765656565656563, + "loss": 2.6897, + "theoretical_loss": 3.3724015495390116, + "tokens_seen": 2464219136 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576363636363637, + "loss": 2.7595, + "theoretical_loss": 3.3723943748061447, + "tokens_seen": 2464284672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025761616161616165, + "loss": 2.3972, + "theoretical_loss": 3.372387200317507, + "tokens_seen": 2464350208 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1351934, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.799548864364624, + "objective/train/theoretical_loss": 3.372380026073083, + "objective/train/tokens_used": 823274976, + "theoretical_loss": 3.372380026073083, + "tokens_seen": 2464415744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575959595959596, + "loss": 2.624, + "theoretical_loss": 3.372380026073083, + "tokens_seen": 2464415744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025757575757575756, + "loss": 2.8706, + "theoretical_loss": 3.372372852072859, + "tokens_seen": 2464481280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575555555555555, + "loss": 2.7484, + "theoretical_loss": 3.3723656783168194, + "tokens_seen": 2464546816 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575353535353536, + "loss": 2.5338, + "theoretical_loss": 3.3723585048049496, + "tokens_seen": 2464612352 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025751515151515154, + "loss": 2.51, + "theoretical_loss": 3.372351331537235, + "tokens_seen": 2464677888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574949494949495, + "loss": 2.6639, + "theoretical_loss": 3.3723441585136604, + "tokens_seen": 2464743424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025747474747474746, + "loss": 2.4146, + "theoretical_loss": 3.3723369857342114, + "tokens_seen": 2464808960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574545454545454, + "loss": 2.7182, + "theoretical_loss": 3.3723298131988733, + "tokens_seen": 2464874496 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574343434343435, + "loss": 2.8449, + "theoretical_loss": 3.3723226409076306, + "tokens_seen": 2464940032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025741414141414144, + "loss": 2.6352, + "theoretical_loss": 3.3723154688604695, + "tokens_seen": 2465005568 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573939393939394, + "loss": 2.6361, + "theoretical_loss": 3.372308297057374, + "tokens_seen": 2465071104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025737373737373735, + "loss": 2.4359, + "theoretical_loss": 3.3723011254983306, + "tokens_seen": 2465136640 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025735353535353536, + "loss": 2.5939, + "theoretical_loss": 3.3722939541833235, + "tokens_seen": 2465202176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025733333333333337, + "loss": 2.6743, + "theoretical_loss": 3.3722867831123384, + "tokens_seen": 2465267712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025731313131313133, + "loss": 2.7261, + "theoretical_loss": 3.3722796122853604, + "tokens_seen": 2465333248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572929292929293, + "loss": 2.5865, + "theoretical_loss": 3.372272441702375, + "tokens_seen": 2465398784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025727272727272724, + "loss": 2.7297, + "theoretical_loss": 3.3722652713633665, + "tokens_seen": 2465464320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025725252525252525, + "loss": 2.5624, + "theoretical_loss": 3.372258101268321, + "tokens_seen": 2465529856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025723232323232326, + "loss": 2.7513, + "theoretical_loss": 3.372250931417223, + "tokens_seen": 2465595392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572121212121212, + "loss": 2.6505, + "theoretical_loss": 3.3722437618100587, + "tokens_seen": 2465660928 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571919191919192, + "loss": 2.7273, + "theoretical_loss": 3.3722365924468125, + "tokens_seen": 2465726464 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571717171717172, + "loss": 2.6835, + "theoretical_loss": 3.3722294233274703, + "tokens_seen": 2465792000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025715151515151515, + "loss": 2.893, + "theoretical_loss": 3.3722222544520166, + "tokens_seen": 2465857536 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025713131313131316, + "loss": 2.457, + "theoretical_loss": 3.372215085820437, + "tokens_seen": 2465923072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571111111111111, + "loss": 2.6738, + "theoretical_loss": 3.372207917432717, + "tokens_seen": 2465988608 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1353245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.568368434906006, + "objective/train/theoretical_loss": 3.372200749288841, + "objective/train/tokens_used": 824913376, + "theoretical_loss": 3.372200749288841, + "tokens_seen": 2466054144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025709090909090907, + "loss": 2.7485, + "theoretical_loss": 3.372200749288841, + "tokens_seen": 2466054144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570707070707071, + "loss": 2.6928, + "theoretical_loss": 3.372193581388795, + "tokens_seen": 2466119680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025705050505050504, + "loss": 2.7153, + "theoretical_loss": 3.372186413732564, + "tokens_seen": 2466185216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025703030303030305, + "loss": 2.6976, + "theoretical_loss": 3.3721792463201328, + "tokens_seen": 2466250752 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257010101010101, + "loss": 2.6583, + "theoretical_loss": 3.372172079151487, + "tokens_seen": 2466316288 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256989898989899, + "loss": 2.3912, + "theoretical_loss": 3.372164912226612, + "tokens_seen": 2466381824 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256969696969697, + "loss": 2.4446, + "theoretical_loss": 3.3721577455454925, + "tokens_seen": 2466447360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025694949494949493, + "loss": 2.5305, + "theoretical_loss": 3.3721505791081148, + "tokens_seen": 2466512896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025692929292929294, + "loss": 2.7362, + "theoretical_loss": 3.372143412914463, + "tokens_seen": 2466578432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569090909090909, + "loss": 2.5031, + "theoretical_loss": 3.372136246964523, + "tokens_seen": 2466643968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568888888888889, + "loss": 2.6468, + "theoretical_loss": 3.3721290812582794, + "tokens_seen": 2466709504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025686868686868687, + "loss": 2.6966, + "theoretical_loss": 3.372121915795718, + "tokens_seen": 2466775040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568484848484848, + "loss": 2.5698, + "theoretical_loss": 3.3721147505768236, + "tokens_seen": 2466840576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025682828282828284, + "loss": 2.6447, + "theoretical_loss": 3.372107585601582, + "tokens_seen": 2466906112 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025680808080808085, + "loss": 2.5622, + "theoretical_loss": 3.3721004208699785, + "tokens_seen": 2466971648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567878787878788, + "loss": 2.7665, + "theoretical_loss": 3.372093256381998, + "tokens_seen": 2467037184 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025676767676767676, + "loss": 2.4958, + "theoretical_loss": 3.372086092137625, + "tokens_seen": 2467102720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567474747474747, + "loss": 2.7717, + "theoretical_loss": 3.3720789281368457, + "tokens_seen": 2467168256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025672727272727273, + "loss": 2.5001, + "theoretical_loss": 3.3720717643796454, + "tokens_seen": 2467233792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025670707070707074, + "loss": 2.7851, + "theoretical_loss": 3.3720646008660093, + "tokens_seen": 2467299328 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566868686868687, + "loss": 2.6388, + "theoretical_loss": 3.372057437595922, + "tokens_seen": 2467364864 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025666666666666665, + "loss": 2.5015, + "theoretical_loss": 3.3720502745693697, + "tokens_seen": 2467430400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566464646464646, + "loss": 2.4632, + "theoretical_loss": 3.3720431117863363, + "tokens_seen": 2467495936 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566262626262627, + "loss": 2.6969, + "theoretical_loss": 3.3720359492468086, + "tokens_seen": 2467561472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025660606060606063, + "loss": 2.6717, + "theoretical_loss": 3.3720287869507706, + "tokens_seen": 2467627008 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1353826, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4293365478515625, + "objective/train/theoretical_loss": 3.372021624898209, + "objective/train/tokens_used": 826551776, + "theoretical_loss": 3.372021624898209, + "tokens_seen": 2467692544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565858585858586, + "loss": 2.7575, + "theoretical_loss": 3.372021624898209, + "tokens_seen": 2467692544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025656565656565655, + "loss": 2.7289, + "theoretical_loss": 3.3720144630891076, + "tokens_seen": 2467758080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565454545454545, + "loss": 2.7331, + "theoretical_loss": 3.372007301523452, + "tokens_seen": 2467823616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025652525252525257, + "loss": 2.6931, + "theoretical_loss": 3.372000140201228, + "tokens_seen": 2467889152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565050505050505, + "loss": 2.6272, + "theoretical_loss": 3.37199297912242, + "tokens_seen": 2467954688 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564848484848485, + "loss": 2.5799, + "theoretical_loss": 3.3719858182870146, + "tokens_seen": 2468020224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025646464646464644, + "loss": 2.6323, + "theoretical_loss": 3.3719786576949957, + "tokens_seen": 2468085760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025644444444444445, + "loss": 2.6646, + "theoretical_loss": 3.3719714973463493, + "tokens_seen": 2468151296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025642424242424246, + "loss": 2.5733, + "theoretical_loss": 3.3719643372410606, + "tokens_seen": 2468216832 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564040404040404, + "loss": 2.7042, + "theoretical_loss": 3.3719571773791146, + "tokens_seen": 2468282368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563838383838384, + "loss": 2.5914, + "theoretical_loss": 3.371950017760497, + "tokens_seen": 2468347904 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025636363636363633, + "loss": 2.7883, + "theoretical_loss": 3.3719428583851925, + "tokens_seen": 2468413440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025634343434343434, + "loss": 2.7139, + "theoretical_loss": 3.3719356992531866, + "tokens_seen": 2468478976 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025632323232323235, + "loss": 2.6229, + "theoretical_loss": 3.3719285403644648, + "tokens_seen": 2468544512 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563030303030303, + "loss": 2.6614, + "theoretical_loss": 3.3719213817190123, + "tokens_seen": 2468610048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025628282828282827, + "loss": 2.5427, + "theoretical_loss": 3.3719142233168142, + "tokens_seen": 2468675584 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562626262626263, + "loss": 2.7659, + "theoretical_loss": 3.371907065157856, + "tokens_seen": 2468741120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025624242424242424, + "loss": 2.5301, + "theoretical_loss": 3.371899907242123, + "tokens_seen": 2468806656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025622222222222225, + "loss": 2.6349, + "theoretical_loss": 3.3718927495696, + "tokens_seen": 2468872192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562020202020202, + "loss": 2.6135, + "theoretical_loss": 3.371885592140273, + "tokens_seen": 2468937728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025618181818181816, + "loss": 2.4874, + "theoretical_loss": 3.3718784349541266, + "tokens_seen": 2469003264 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025616161616161617, + "loss": 2.7144, + "theoretical_loss": 3.371871278011146, + "tokens_seen": 2469068800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025614141414141413, + "loss": 2.7758, + "theoretical_loss": 3.371864121311318, + "tokens_seen": 2469134336 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025612121212121214, + "loss": 2.6126, + "theoretical_loss": 3.3718569648546257, + "tokens_seen": 2469199872 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561010101010101, + "loss": 2.5413, + "theoretical_loss": 3.371849808641056, + "tokens_seen": 2469265408 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1355023, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1884193420410156, + "objective/train/theoretical_loss": 3.371842652670593, + "objective/train/tokens_used": 828190176, + "theoretical_loss": 3.371842652670593, + "tokens_seen": 2469330944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560808080808081, + "loss": 2.3851, + "theoretical_loss": 3.371842652670593, + "tokens_seen": 2469330944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025606060606060606, + "loss": 2.5171, + "theoretical_loss": 3.3718354969432234, + "tokens_seen": 2469396480 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256040404040404, + "loss": 2.4676, + "theoretical_loss": 3.371828341458931, + "tokens_seen": 2469462016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025602020202020203, + "loss": 2.6147, + "theoretical_loss": 3.3718211862177023, + "tokens_seen": 2469527552 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256, + "loss": 2.6157, + "theoretical_loss": 3.371814031219522, + "tokens_seen": 2469593088 + }, + { + "epoch": 0.49, + "learning_rate": 0.000255979797979798, + "loss": 2.5725, + "theoretical_loss": 3.371806876464375, + "tokens_seen": 2469658624 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025595959595959596, + "loss": 2.7654, + "theoretical_loss": 3.371799721952248, + "tokens_seen": 2469724160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025593939393939397, + "loss": 2.5189, + "theoretical_loss": 3.3717925676831246, + "tokens_seen": 2469789696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559191919191919, + "loss": 2.6325, + "theoretical_loss": 3.3717854136569914, + "tokens_seen": 2469855232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025589898989898994, + "loss": 2.7308, + "theoretical_loss": 3.3717782598738326, + "tokens_seen": 2469920768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558787878787879, + "loss": 2.6253, + "theoretical_loss": 3.3717711063336346, + "tokens_seen": 2469986304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025585858585858585, + "loss": 2.5337, + "theoretical_loss": 3.371763953036382, + "tokens_seen": 2470051840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025583838383838386, + "loss": 2.4684, + "theoretical_loss": 3.37175679998206, + "tokens_seen": 2470117376 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558181818181818, + "loss": 2.8133, + "theoretical_loss": 3.3717496471706543, + "tokens_seen": 2470182912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025579797979797983, + "loss": 2.5437, + "theoretical_loss": 3.3717424946021506, + "tokens_seen": 2470248448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557777777777778, + "loss": 2.5857, + "theoretical_loss": 3.3717353422765335, + "tokens_seen": 2470313984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025575757575757574, + "loss": 2.5808, + "theoretical_loss": 3.371728190193788, + "tokens_seen": 2470379520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025573737373737375, + "loss": 2.5107, + "theoretical_loss": 3.3717210383539005, + "tokens_seen": 2470445056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025571717171717176, + "loss": 3.0126, + "theoretical_loss": 3.3717138867568552, + "tokens_seen": 2470510592 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556969696969697, + "loss": 2.9824, + "theoretical_loss": 3.3717067354026384, + "tokens_seen": 2470576128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556767676767677, + "loss": 2.8434, + "theoretical_loss": 3.371699584291235, + "tokens_seen": 2470641664 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025565656565656563, + "loss": 2.5783, + "theoretical_loss": 3.37169243342263, + "tokens_seen": 2470707200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025563636363636365, + "loss": 2.655, + "theoretical_loss": 3.371685282796809, + "tokens_seen": 2470772736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025561616161616166, + "loss": 2.8134, + "theoretical_loss": 3.3716781324137575, + "tokens_seen": 2470838272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555959595959596, + "loss": 2.4752, + "theoretical_loss": 3.3716709822734603, + "tokens_seen": 2470903808 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1355628, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6723058223724365, + "objective/train/theoretical_loss": 3.371663832375903, + "objective/train/tokens_used": 829828576, + "theoretical_loss": 3.371663832375903, + "tokens_seen": 2470969344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025557575757575757, + "loss": 2.6169, + "theoretical_loss": 3.371663832375903, + "tokens_seen": 2470969344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025555555555555553, + "loss": 2.7038, + "theoretical_loss": 3.3716566827210714, + "tokens_seen": 2471034880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025553535353535354, + "loss": 2.8198, + "theoretical_loss": 3.3716495333089505, + "tokens_seen": 2471100416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025551515151515155, + "loss": 2.7255, + "theoretical_loss": 3.371642384139525, + "tokens_seen": 2471165952 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554949494949495, + "loss": 2.7225, + "theoretical_loss": 3.371635235212781, + "tokens_seen": 2471231488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025547474747474746, + "loss": 2.6788, + "theoretical_loss": 3.3716280865287036, + "tokens_seen": 2471297024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554545454545454, + "loss": 2.5661, + "theoretical_loss": 3.371620938087278, + "tokens_seen": 2471362560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554343434343435, + "loss": 2.3911, + "theoretical_loss": 3.3716137898884897, + "tokens_seen": 2471428096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025541414141414144, + "loss": 2.4517, + "theoretical_loss": 3.371606641932324, + "tokens_seen": 2471493632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553939393939394, + "loss": 2.693, + "theoretical_loss": 3.371599494218766, + "tokens_seen": 2471559168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025537373737373736, + "loss": 2.6192, + "theoretical_loss": 3.3715923467478013, + "tokens_seen": 2471624704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553535353535353, + "loss": 2.586, + "theoretical_loss": 3.3715851995194153, + "tokens_seen": 2471690240 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553333333333334, + "loss": 2.5344, + "theoretical_loss": 3.371578052533593, + "tokens_seen": 2471755776 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025531313131313133, + "loss": 2.7059, + "theoretical_loss": 3.3715709057903203, + "tokens_seen": 2471821312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552929292929293, + "loss": 2.556, + "theoretical_loss": 3.3715637592895815, + "tokens_seen": 2471886848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025527272727272725, + "loss": 2.6224, + "theoretical_loss": 3.371556613031363, + "tokens_seen": 2471952384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025525252525252526, + "loss": 2.6685, + "theoretical_loss": 3.3715494670156496, + "tokens_seen": 2472017920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025523232323232327, + "loss": 2.5177, + "theoretical_loss": 3.371542321242427, + "tokens_seen": 2472083456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025521212121212123, + "loss": 2.5755, + "theoretical_loss": 3.3715351757116805, + "tokens_seen": 2472148992 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551919191919192, + "loss": 2.6052, + "theoretical_loss": 3.371528030423395, + "tokens_seen": 2472214528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025517171717171714, + "loss": 2.5406, + "theoretical_loss": 3.3715208853775556, + "tokens_seen": 2472280064 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025515151515151515, + "loss": 2.6819, + "theoretical_loss": 3.371513740574149, + "tokens_seen": 2472345600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025513131313131316, + "loss": 2.7209, + "theoretical_loss": 3.3715065960131594, + "tokens_seen": 2472411136 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551111111111111, + "loss": 2.6593, + "theoretical_loss": 3.3714994516945724, + "tokens_seen": 2472476672 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002550909090909091, + "loss": 2.7952, + "theoretical_loss": 3.3714923076183734, + "tokens_seen": 2472542208 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1356720, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1001343727111816, + "objective/train/theoretical_loss": 3.371485163784548, + "objective/train/tokens_used": 831466976, + "theoretical_loss": 3.371485163784548, + "tokens_seen": 2472607744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002550707070707071, + "loss": 2.6932, + "theoretical_loss": 3.371485163784548, + "tokens_seen": 2472607744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025505050505050504, + "loss": 2.438, + "theoretical_loss": 3.371478020193081, + "tokens_seen": 2472673280 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025503030303030306, + "loss": 2.7169, + "theoretical_loss": 3.3714708768439583, + "tokens_seen": 2472738816 + }, + { + "epoch": 0.5, + "learning_rate": 0.000255010101010101, + "loss": 2.5973, + "theoretical_loss": 3.371463733737165, + "tokens_seen": 2472804352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025498989898989897, + "loss": 2.5285, + "theoretical_loss": 3.3714565908726866, + "tokens_seen": 2472869888 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254969696969697, + "loss": 2.835, + "theoretical_loss": 3.371449448250508, + "tokens_seen": 2472935424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025494949494949494, + "loss": 2.6019, + "theoretical_loss": 3.371442305870615, + "tokens_seen": 2473000960 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025492929292929295, + "loss": 2.5988, + "theoretical_loss": 3.3714351637329933, + "tokens_seen": 2473066496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549090909090909, + "loss": 2.3961, + "theoretical_loss": 3.3714280218376276, + "tokens_seen": 2473132032 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548888888888889, + "loss": 2.8445, + "theoretical_loss": 3.371420880184503, + "tokens_seen": 2473197568 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548686868686869, + "loss": 2.6405, + "theoretical_loss": 3.371413738773606, + "tokens_seen": 2473263104 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025484848484848483, + "loss": 2.6328, + "theoretical_loss": 3.371406597604921, + "tokens_seen": 2473328640 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025482828282828284, + "loss": 2.5317, + "theoretical_loss": 3.371399456678434, + "tokens_seen": 2473394176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548080808080808, + "loss": 2.7039, + "theoretical_loss": 3.3713923159941297, + "tokens_seen": 2473459712 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547878787878788, + "loss": 2.6338, + "theoretical_loss": 3.371385175551994, + "tokens_seen": 2473525248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025476767676767677, + "loss": 2.7724, + "theoretical_loss": 3.371378035352012, + "tokens_seen": 2473590784 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547474747474747, + "loss": 2.642, + "theoretical_loss": 3.3713708953941692, + "tokens_seen": 2473656320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025472727272727273, + "loss": 2.4035, + "theoretical_loss": 3.371363755678451, + "tokens_seen": 2473721856 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025470707070707075, + "loss": 2.6874, + "theoretical_loss": 3.3713566162048427, + "tokens_seen": 2473787392 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546868686868687, + "loss": 2.4818, + "theoretical_loss": 3.3713494769733297, + "tokens_seen": 2473852928 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025466666666666666, + "loss": 2.5782, + "theoretical_loss": 3.3713423379838976, + "tokens_seen": 2473918464 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546464646464646, + "loss": 2.4878, + "theoretical_loss": 3.3713351992365315, + "tokens_seen": 2473984000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546262626262626, + "loss": 2.7492, + "theoretical_loss": 3.3713280607312166, + "tokens_seen": 2474049536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025460606060606064, + "loss": 2.7206, + "theoretical_loss": 3.3713209224679384, + "tokens_seen": 2474115072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545858585858586, + "loss": 2.7274, + "theoretical_loss": 3.3713137844466825, + "tokens_seen": 2474180608 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1357343, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6152400970458984, + "objective/train/theoretical_loss": 3.3713066466674344, + "objective/train/tokens_used": 833105376, + "theoretical_loss": 3.3713066466674344, + "tokens_seen": 2474246144 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025456565656565655, + "loss": 2.5409, + "theoretical_loss": 3.3713066466674344, + "tokens_seen": 2474246144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545454545454545, + "loss": 2.3603, + "theoretical_loss": 3.3712995091301794, + "tokens_seen": 2474311680 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545252525252526, + "loss": 2.6706, + "theoretical_loss": 3.3712923718349024, + "tokens_seen": 2474377216 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025450505050505053, + "loss": 2.6257, + "theoretical_loss": 3.371285234781589, + "tokens_seen": 2474442752 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544848484848485, + "loss": 2.6054, + "theoretical_loss": 3.371278097970225, + "tokens_seen": 2474508288 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025446464646464644, + "loss": 2.608, + "theoretical_loss": 3.371270961400796, + "tokens_seen": 2474573824 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544444444444444, + "loss": 2.4753, + "theoretical_loss": 3.371263825073286, + "tokens_seen": 2474639360 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025442424242424247, + "loss": 2.6152, + "theoretical_loss": 3.371256688987682, + "tokens_seen": 2474704896 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544040404040404, + "loss": 2.5844, + "theoretical_loss": 3.3712495531439686, + "tokens_seen": 2474770432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543838383838384, + "loss": 2.6685, + "theoretical_loss": 3.371242417542131, + "tokens_seen": 2474835968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025436363636363634, + "loss": 2.5292, + "theoretical_loss": 3.371235282182155, + "tokens_seen": 2474901504 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025434343434343435, + "loss": 2.0997, + "theoretical_loss": 3.3712281470640257, + "tokens_seen": 2474967040 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025432323232323236, + "loss": 2.6647, + "theoretical_loss": 3.3712210121877293, + "tokens_seen": 2475032576 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543030303030303, + "loss": 2.5317, + "theoretical_loss": 3.37121387755325, + "tokens_seen": 2475098112 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025428282828282827, + "loss": 2.5788, + "theoretical_loss": 3.371206743160574, + "tokens_seen": 2475163648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025426262626262623, + "loss": 2.6145, + "theoretical_loss": 3.3711996090096865, + "tokens_seen": 2475229184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542424242424243, + "loss": 2.5927, + "theoretical_loss": 3.371192475100573, + "tokens_seen": 2475294720 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025422222222222225, + "loss": 2.6617, + "theoretical_loss": 3.3711853414332187, + "tokens_seen": 2475360256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542020202020202, + "loss": 2.6614, + "theoretical_loss": 3.371178208007609, + "tokens_seen": 2475425792 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025418181818181817, + "loss": 2.6295, + "theoretical_loss": 3.3711710748237294, + "tokens_seen": 2475491328 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541616161616161, + "loss": 2.727, + "theoretical_loss": 3.3711639418815658, + "tokens_seen": 2475556864 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541414141414142, + "loss": 2.3632, + "theoretical_loss": 3.3711568091811026, + "tokens_seen": 2475622400 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025412121212121214, + "loss": 2.4289, + "theoretical_loss": 3.371149676722326, + "tokens_seen": 2475687936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541010101010101, + "loss": 2.6175, + "theoretical_loss": 3.371142544505221, + "tokens_seen": 2475753472 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025408080808080806, + "loss": 2.6898, + "theoretical_loss": 3.3711354125297732, + "tokens_seen": 2475819008 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1358722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.468827486038208, + "objective/train/theoretical_loss": 3.3711282807959684, + "objective/train/tokens_used": 834743776, + "theoretical_loss": 3.3711282807959684, + "tokens_seen": 2475884544 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025406060606060607, + "loss": 2.6217, + "theoretical_loss": 3.3711282807959684, + "tokens_seen": 2475884544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540404040404041, + "loss": 2.8655, + "theoretical_loss": 3.371121149303791, + "tokens_seen": 2475950080 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025402020202020204, + "loss": 2.7529, + "theoretical_loss": 3.3711140180532273, + "tokens_seen": 2476015616 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254, + "loss": 2.5568, + "theoretical_loss": 3.3711068870442626, + "tokens_seen": 2476081152 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025397979797979795, + "loss": 2.661, + "theoretical_loss": 3.371099756276882, + "tokens_seen": 2476146688 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025395959595959596, + "loss": 2.7211, + "theoretical_loss": 3.3710926257510714, + "tokens_seen": 2476212224 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253939393939394, + "loss": 2.4742, + "theoretical_loss": 3.3710854954668155, + "tokens_seen": 2476277760 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025391919191919193, + "loss": 2.6528, + "theoretical_loss": 3.3710783654241, + "tokens_seen": 2476343296 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538989898989899, + "loss": 2.4986, + "theoretical_loss": 3.371071235622911, + "tokens_seen": 2476408832 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538787878787879, + "loss": 2.8291, + "theoretical_loss": 3.371064106063233, + "tokens_seen": 2476474368 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025385858585858585, + "loss": 2.7524, + "theoretical_loss": 3.3710569767450522, + "tokens_seen": 2476539904 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025383838383838387, + "loss": 2.4977, + "theoretical_loss": 3.3710498476683535, + "tokens_seen": 2476605440 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538181818181818, + "loss": 2.6181, + "theoretical_loss": 3.371042718833122, + "tokens_seen": 2476670976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537979797979798, + "loss": 2.7013, + "theoretical_loss": 3.371035590239344, + "tokens_seen": 2476736512 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537777777777778, + "loss": 2.6835, + "theoretical_loss": 3.3710284618870046, + "tokens_seen": 2476802048 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025375757575757575, + "loss": 2.4376, + "theoretical_loss": 3.3710213337760893, + "tokens_seen": 2476867584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025373737373737376, + "loss": 2.6812, + "theoretical_loss": 3.371014205906583, + "tokens_seen": 2476933120 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537171717171717, + "loss": 2.5216, + "theoretical_loss": 3.371007078278472, + "tokens_seen": 2476998656 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536969696969697, + "loss": 2.9095, + "theoretical_loss": 3.3709999508917408, + "tokens_seen": 2477064192 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536767676767677, + "loss": 2.4722, + "theoretical_loss": 3.3709928237463758, + "tokens_seen": 2477129728 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025365656565656564, + "loss": 2.3031, + "theoretical_loss": 3.3709856968423617, + "tokens_seen": 2477195264 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025363636363636365, + "loss": 2.7561, + "theoretical_loss": 3.3709785701796844, + "tokens_seen": 2477260800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536161616161616, + "loss": 2.3269, + "theoretical_loss": 3.370971443758329, + "tokens_seen": 2477326336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535959595959596, + "loss": 2.5325, + "theoretical_loss": 3.3709643175782813, + "tokens_seen": 2477391872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535757575757576, + "loss": 2.8376, + "theoretical_loss": 3.3709571916395262, + "tokens_seen": 2477457408 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1359325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4937326908111572, + "objective/train/theoretical_loss": 3.3709500659420497, + "objective/train/tokens_used": 836382176, + "theoretical_loss": 3.3709500659420497, + "tokens_seen": 2477522944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025355555555555553, + "loss": 2.4791, + "theoretical_loss": 3.3709500659420497, + "tokens_seen": 2477522944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025353535353535354, + "loss": 2.5437, + "theoretical_loss": 3.370942940485837, + "tokens_seen": 2477588480 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025351515151515155, + "loss": 2.6954, + "theoretical_loss": 3.3709358152708737, + "tokens_seen": 2477654016 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534949494949495, + "loss": 2.673, + "theoretical_loss": 3.3709286902971454, + "tokens_seen": 2477719552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025347474747474747, + "loss": 2.6018, + "theoretical_loss": 3.370921565564637, + "tokens_seen": 2477785088 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534545454545454, + "loss": 2.4899, + "theoretical_loss": 3.3709144410733343, + "tokens_seen": 2477850624 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025343434343434344, + "loss": 2.6202, + "theoretical_loss": 3.3709073168232226, + "tokens_seen": 2477916160 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025341414141414145, + "loss": 2.4644, + "theoretical_loss": 3.3709001928142874, + "tokens_seen": 2477981696 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533939393939394, + "loss": 2.613, + "theoretical_loss": 3.3708930690465144, + "tokens_seen": 2478047232 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025337373737373736, + "loss": 2.6285, + "theoretical_loss": 3.3708859455198885, + "tokens_seen": 2478112768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533535353535353, + "loss": 2.6962, + "theoretical_loss": 3.370878822234396, + "tokens_seen": 2478178304 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533333333333334, + "loss": 2.5714, + "theoretical_loss": 3.3708716991900216, + "tokens_seen": 2478243840 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025331313131313134, + "loss": 2.7378, + "theoretical_loss": 3.3708645763867513, + "tokens_seen": 2478309376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532929292929293, + "loss": 2.7543, + "theoretical_loss": 3.3708574538245704, + "tokens_seen": 2478374912 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025327272727272725, + "loss": 2.6731, + "theoretical_loss": 3.370850331503464, + "tokens_seen": 2478440448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532525252525252, + "loss": 2.6831, + "theoretical_loss": 3.370843209423418, + "tokens_seen": 2478505984 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532323232323233, + "loss": 2.5356, + "theoretical_loss": 3.3708360875844177, + "tokens_seen": 2478571520 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025321212121212123, + "loss": 2.7289, + "theoretical_loss": 3.3708289659864485, + "tokens_seen": 2478637056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002531919191919192, + "loss": 2.7892, + "theoretical_loss": 3.3708218446294964, + "tokens_seen": 2478702592 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025317171717171715, + "loss": 2.7162, + "theoretical_loss": 3.3708147235135457, + "tokens_seen": 2478768128 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025315151515151516, + "loss": 2.5791, + "theoretical_loss": 3.370807602638583, + "tokens_seen": 2478833664 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025313131313131317, + "loss": 2.394, + "theoretical_loss": 3.3708004820045936, + "tokens_seen": 2478899200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002531111111111111, + "loss": 2.4905, + "theoretical_loss": 3.3707933616115624, + "tokens_seen": 2478964736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530909090909091, + "loss": 2.5083, + "theoretical_loss": 3.3707862414594754, + "tokens_seen": 2479030272 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025307070707070704, + "loss": 2.6632, + "theoretical_loss": 3.370779121548318, + "tokens_seen": 2479095808 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1360622, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.839552640914917, + "objective/train/theoretical_loss": 3.370772001878075, + "objective/train/tokens_used": 838020576, + "theoretical_loss": 3.370772001878075, + "tokens_seen": 2479161344 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025305050505050505, + "loss": 2.5514, + "theoretical_loss": 3.370772001878075, + "tokens_seen": 2479161344 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025303030303030306, + "loss": 2.5342, + "theoretical_loss": 3.370764882448733, + "tokens_seen": 2479226880 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253010101010101, + "loss": 2.3905, + "theoretical_loss": 3.370757763260277, + "tokens_seen": 2479292416 + }, + { + "epoch": 0.5, + "learning_rate": 0.000252989898989899, + "loss": 2.6789, + "theoretical_loss": 3.3707506443126922, + "tokens_seen": 2479357952 + }, + { + "epoch": 0.5, + "learning_rate": 0.000252969696969697, + "loss": 2.4022, + "theoretical_loss": 3.3707435256059646, + "tokens_seen": 2479423488 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025294949494949494, + "loss": 2.3827, + "theoretical_loss": 3.370736407140079, + "tokens_seen": 2479489024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025292929292929295, + "loss": 2.5959, + "theoretical_loss": 3.3707292889150215, + "tokens_seen": 2479554560 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529090909090909, + "loss": 2.5813, + "theoretical_loss": 3.3707221709307773, + "tokens_seen": 2479620096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025288888888888887, + "loss": 2.7431, + "theoretical_loss": 3.370715053187332, + "tokens_seen": 2479685632 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528686868686869, + "loss": 2.6039, + "theoretical_loss": 3.370707935684671, + "tokens_seen": 2479751168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025284848484848484, + "loss": 2.596, + "theoretical_loss": 3.37070081842278, + "tokens_seen": 2479816704 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025282828282828285, + "loss": 2.7236, + "theoretical_loss": 3.3706937014016445, + "tokens_seen": 2479882240 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528080808080808, + "loss": 2.3529, + "theoretical_loss": 3.3706865846212493, + "tokens_seen": 2479947776 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527878787878788, + "loss": 2.4214, + "theoretical_loss": 3.3706794680815806, + "tokens_seen": 2480013312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025276767676767677, + "loss": 2.4925, + "theoretical_loss": 3.3706723517826243, + "tokens_seen": 2480078848 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025274747474747473, + "loss": 2.8759, + "theoretical_loss": 3.3706652357243643, + "tokens_seen": 2480144384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025272727272727274, + "loss": 2.7379, + "theoretical_loss": 3.3706581199067878, + "tokens_seen": 2480209920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527070707070707, + "loss": 2.568, + "theoretical_loss": 3.3706510043298796, + "tokens_seen": 2480275456 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002526868686868687, + "loss": 2.6996, + "theoretical_loss": 3.370643888993625, + "tokens_seen": 2480340992 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025266666666666666, + "loss": 2.6869, + "theoretical_loss": 3.37063677389801, + "tokens_seen": 2480406528 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002526464646464646, + "loss": 2.5509, + "theoretical_loss": 3.3706296590430194, + "tokens_seen": 2480472064 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025262626262626263, + "loss": 2.3146, + "theoretical_loss": 3.3706225444286395, + "tokens_seen": 2480537600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025260606060606064, + "loss": 2.5581, + "theoretical_loss": 3.370615430054855, + "tokens_seen": 2480603136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002525858585858586, + "loss": 2.3669, + "theoretical_loss": 3.370608315921652, + "tokens_seen": 2480668672 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025256565656565656, + "loss": 2.5651, + "theoretical_loss": 3.3706012020290164, + "tokens_seen": 2480734208 + }, + { + "debugging/Self-BLEU-5": 0.3554135683424447, + "debugging/distinct-1-grams": 0.8323901098901099, + "debugging/distinct-2-grams": 0.9677981864682632, + "debugging/entropy-1-grams": 5.1378514335831085, + "debugging/entropy-2-grams": 5.562533421675665, + "debugging/length": 516.75, + "debugging/num_segments": 4, + "epoch": 0.5, + "objective/train/docs_used": 1361770, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.56823992729187, + "objective/train/theoretical_loss": 3.370594088376933, + "objective/train/tokens_used": 839658976, + "theoretical_loss": 3.370594088376933, + "tokens_seen": 2480799744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025254545454545457, + "loss": 2.726, + "theoretical_loss": 3.370594088376933, + "tokens_seen": 2480799744 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002525252525252525, + "loss": 2.3474, + "theoretical_loss": 3.370586974965387, + "tokens_seen": 2480865280 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025250505050505054, + "loss": 2.647, + "theoretical_loss": 3.370579861794365, + "tokens_seen": 2480930816 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002524848484848485, + "loss": 2.3668, + "theoretical_loss": 3.370572748863851, + "tokens_seen": 2480996352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025246464646464645, + "loss": 2.6206, + "theoretical_loss": 3.3705656361738323, + "tokens_seen": 2481061888 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025244444444444446, + "loss": 2.5581, + "theoretical_loss": 3.370558523724293, + "tokens_seen": 2481127424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025242424242424247, + "loss": 2.5056, + "theoretical_loss": 3.370551411515219, + "tokens_seen": 2481192960 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025240404040404043, + "loss": 2.7892, + "theoretical_loss": 3.3705442995465966, + "tokens_seen": 2481258496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523838383838384, + "loss": 2.4186, + "theoretical_loss": 3.3705371878184103, + "tokens_seen": 2481324032 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025236363636363634, + "loss": 2.7285, + "theoretical_loss": 3.3705300763306463, + "tokens_seen": 2481389568 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025234343434343435, + "loss": 2.7742, + "theoretical_loss": 3.3705229650832895, + "tokens_seen": 2481455104 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025232323232323236, + "loss": 2.548, + "theoretical_loss": 3.3705158540763263, + "tokens_seen": 2481520640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523030303030303, + "loss": 2.5098, + "theoretical_loss": 3.370508743309741, + "tokens_seen": 2481586176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002522828282828283, + "loss": 2.6916, + "theoretical_loss": 3.3705016327835198, + "tokens_seen": 2481651712 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025226262626262624, + "loss": 2.4806, + "theoretical_loss": 3.370494522497649, + "tokens_seen": 2481717248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025224242424242425, + "loss": 2.6428, + "theoretical_loss": 3.3704874124521127, + "tokens_seen": 2481782784 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025222222222222226, + "loss": 2.2933, + "theoretical_loss": 3.3704803026468975, + "tokens_seen": 2481848320 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002522020202020202, + "loss": 2.8177, + "theoretical_loss": 3.3704731930819882, + "tokens_seen": 2481913856 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025218181818181817, + "loss": 2.6725, + "theoretical_loss": 3.370466083757371, + "tokens_seen": 2481979392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025216161616161613, + "loss": 2.7135, + "theoretical_loss": 3.3704589746730305, + "tokens_seen": 2482044928 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002521414141414142, + "loss": 2.4809, + "theoretical_loss": 3.3704518658289535, + "tokens_seen": 2482110464 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025212121212121215, + "loss": 2.5915, + "theoretical_loss": 3.3704447572251244, + "tokens_seen": 2482176000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002521010101010101, + "loss": 2.4467, + "theoretical_loss": 3.3704376488615293, + "tokens_seen": 2482241536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025208080808080806, + "loss": 2.6161, + "theoretical_loss": 3.3704305407381536, + "tokens_seen": 2482307072 + }, + { + "epoch": 0.5, + "learning_rate": 0.000252060606060606, + "loss": 2.7558, + "theoretical_loss": 3.370423432854983, + "tokens_seen": 2482372608 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1362312, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5898001194000244, + "objective/train/theoretical_loss": 3.370416325212003, + "objective/train/tokens_used": 841297376, + "theoretical_loss": 3.370416325212003, + "tokens_seen": 2482438144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002520404040404041, + "loss": 2.5715, + "theoretical_loss": 3.370416325212003, + "tokens_seen": 2482438144 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025202020202020204, + "loss": 2.6876, + "theoretical_loss": 3.370409217809199, + "tokens_seen": 2482503680 + }, + { + "epoch": 0.5, + "learning_rate": 0.000252, + "loss": 2.4408, + "theoretical_loss": 3.3704021106465567, + "tokens_seen": 2482569216 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025197979797979796, + "loss": 2.6548, + "theoretical_loss": 3.3703950037240613, + "tokens_seen": 2482634752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025195959595959597, + "loss": 2.4615, + "theoretical_loss": 3.370387897041699, + "tokens_seen": 2482700288 + }, + { + "epoch": 0.5, + "learning_rate": 0.000251939393939394, + "loss": 2.7206, + "theoretical_loss": 3.3703807905994547, + "tokens_seen": 2482765824 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025191919191919194, + "loss": 2.6135, + "theoretical_loss": 3.370373684397314, + "tokens_seen": 2482831360 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002518989898989899, + "loss": 2.3843, + "theoretical_loss": 3.370366578435263, + "tokens_seen": 2482896896 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025187878787878785, + "loss": 2.5069, + "theoretical_loss": 3.370359472713287, + "tokens_seen": 2482962432 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025185858585858586, + "loss": 2.6309, + "theoretical_loss": 3.370352367231371, + "tokens_seen": 2483027968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025183838383838387, + "loss": 2.5311, + "theoretical_loss": 3.3703452619895016, + "tokens_seen": 2483093504 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025181818181818183, + "loss": 2.6555, + "theoretical_loss": 3.370338156987663, + "tokens_seen": 2483159040 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517979797979798, + "loss": 2.6517, + "theoretical_loss": 3.370331052225842, + "tokens_seen": 2483224576 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517777777777778, + "loss": 2.77, + "theoretical_loss": 3.3703239477040237, + "tokens_seen": 2483290112 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025175757575757575, + "loss": 2.4177, + "theoretical_loss": 3.3703168434221937, + "tokens_seen": 2483355648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025173737373737376, + "loss": 2.5524, + "theoretical_loss": 3.370309739380337, + "tokens_seen": 2483421184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517171717171717, + "loss": 2.6492, + "theoretical_loss": 3.3703026355784402, + "tokens_seen": 2483486720 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516969696969697, + "loss": 2.5522, + "theoretical_loss": 3.3702955320164882, + "tokens_seen": 2483552256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516767676767677, + "loss": 2.4527, + "theoretical_loss": 3.370288428694467, + "tokens_seen": 2483617792 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025165656565656565, + "loss": 2.5929, + "theoretical_loss": 3.3702813256123614, + "tokens_seen": 2483683328 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025163636363636366, + "loss": 2.6087, + "theoretical_loss": 3.370274222770157, + "tokens_seen": 2483748864 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516161616161616, + "loss": 2.519, + "theoretical_loss": 3.3702671201678407, + "tokens_seen": 2483814400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515959595959596, + "loss": 2.545, + "theoretical_loss": 3.3702600178053967, + "tokens_seen": 2483879936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515757575757576, + "loss": 2.6979, + "theoretical_loss": 3.370252915682811, + "tokens_seen": 2483945472 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025155555555555554, + "loss": 2.5112, + "theoretical_loss": 3.3702458138000693, + "tokens_seen": 2484011008 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1363163, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.647827625274658, + "objective/train/theoretical_loss": 3.370238712157157, + "objective/train/tokens_used": 842935776, + "theoretical_loss": 3.370238712157157, + "tokens_seen": 2484076544 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025153535353535355, + "loss": 2.5919, + "theoretical_loss": 3.370238712157157, + "tokens_seen": 2484076544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515151515151515, + "loss": 2.5548, + "theoretical_loss": 3.3702316107540597, + "tokens_seen": 2484142080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514949494949495, + "loss": 2.6185, + "theoretical_loss": 3.3702245095907633, + "tokens_seen": 2484207616 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514747474747475, + "loss": 2.3203, + "theoretical_loss": 3.3702174086672527, + "tokens_seen": 2484273152 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025145454545454543, + "loss": 2.4668, + "theoretical_loss": 3.3702103079835144, + "tokens_seen": 2484338688 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025143434343434344, + "loss": 2.4689, + "theoretical_loss": 3.3702032075395327, + "tokens_seen": 2484404224 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025141414141414145, + "loss": 2.4303, + "theoretical_loss": 3.3701961073352944, + "tokens_seen": 2484469760 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002513939393939394, + "loss": 2.6945, + "theoretical_loss": 3.3701890073707848, + "tokens_seen": 2484535296 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025137373737373737, + "loss": 2.5638, + "theoretical_loss": 3.370181907645989, + "tokens_seen": 2484600832 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002513535353535353, + "loss": 2.5574, + "theoretical_loss": 3.370174808160893, + "tokens_seen": 2484666368 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025133333333333333, + "loss": 2.4983, + "theoretical_loss": 3.3701677089154822, + "tokens_seen": 2484731904 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025131313131313135, + "loss": 2.5216, + "theoretical_loss": 3.370160609909742, + "tokens_seen": 2484797440 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002512929292929293, + "loss": 2.6948, + "theoretical_loss": 3.3701535111436587, + "tokens_seen": 2484862976 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025127272727272726, + "loss": 2.3537, + "theoretical_loss": 3.3701464126172174, + "tokens_seen": 2484928512 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002512525252525252, + "loss": 2.7894, + "theoretical_loss": 3.3701393143304035, + "tokens_seen": 2484994048 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002512323232323233, + "loss": 2.7231, + "theoretical_loss": 3.370132216283203, + "tokens_seen": 2485059584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025121212121212124, + "loss": 2.5109, + "theoretical_loss": 3.370125118475601, + "tokens_seen": 2485125120 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002511919191919192, + "loss": 2.531, + "theoretical_loss": 3.370118020907584, + "tokens_seen": 2485190656 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025117171717171715, + "loss": 2.4603, + "theoretical_loss": 3.370110923579136, + "tokens_seen": 2485256192 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002511515151515151, + "loss": 2.4416, + "theoretical_loss": 3.3701038264902445, + "tokens_seen": 2485321728 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002511313131313132, + "loss": 2.6198, + "theoretical_loss": 3.3700967296408937, + "tokens_seen": 2485387264 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025111111111111113, + "loss": 2.5903, + "theoretical_loss": 3.37008963303107, + "tokens_seen": 2485452800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002510909090909091, + "loss": 2.5919, + "theoretical_loss": 3.3700825366607585, + "tokens_seen": 2485518336 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025107070707070704, + "loss": 2.5818, + "theoretical_loss": 3.370075440529945, + "tokens_seen": 2485583872 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025105050505050506, + "loss": 2.5391, + "theoretical_loss": 3.370068344638615, + "tokens_seen": 2485649408 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1363769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.790882110595703, + "objective/train/theoretical_loss": 3.370061248986754, + "objective/train/tokens_used": 844574176, + "theoretical_loss": 3.370061248986754, + "tokens_seen": 2485714944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025103030303030307, + "loss": 2.7666, + "theoretical_loss": 3.370061248986754, + "tokens_seen": 2485714944 + }, + { + "epoch": 0.5, + "learning_rate": 0.000251010101010101, + "loss": 2.7097, + "theoretical_loss": 3.3700541535743485, + "tokens_seen": 2485780480 + }, + { + "epoch": 0.5, + "learning_rate": 0.000250989898989899, + "loss": 2.366, + "theoretical_loss": 3.370047058401383, + "tokens_seen": 2485846016 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025096969696969694, + "loss": 2.7431, + "theoretical_loss": 3.3700399634678435, + "tokens_seen": 2485911552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025094949494949495, + "loss": 2.4257, + "theoretical_loss": 3.3700328687737153, + "tokens_seen": 2485977088 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025092929292929296, + "loss": 2.5628, + "theoretical_loss": 3.370025774318985, + "tokens_seen": 2486042624 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002509090909090909, + "loss": 2.6971, + "theoretical_loss": 3.3700186801036374, + "tokens_seen": 2486108160 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508888888888889, + "loss": 2.6251, + "theoretical_loss": 3.370011586127658, + "tokens_seen": 2486173696 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025086868686868683, + "loss": 2.5609, + "theoretical_loss": 3.3700044923910326, + "tokens_seen": 2486239232 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508484848484849, + "loss": 2.6603, + "theoretical_loss": 3.369997398893747, + "tokens_seen": 2486304768 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025082828282828285, + "loss": 2.6501, + "theoretical_loss": 3.3699903056357865, + "tokens_seen": 2486370304 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508080808080808, + "loss": 2.6643, + "theoretical_loss": 3.3699832126171376, + "tokens_seen": 2486435840 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025078787878787877, + "loss": 2.6763, + "theoretical_loss": 3.3699761198377844, + "tokens_seen": 2486501376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507676767676768, + "loss": 2.5123, + "theoretical_loss": 3.369969027297714, + "tokens_seen": 2486566912 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507474747474748, + "loss": 2.4762, + "theoretical_loss": 3.369961934996911, + "tokens_seen": 2486632448 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025072727272727275, + "loss": 2.817, + "theoretical_loss": 3.369954842935362, + "tokens_seen": 2486697984 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507070707070707, + "loss": 2.5725, + "theoretical_loss": 3.3699477511130516, + "tokens_seen": 2486763520 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025068686868686866, + "loss": 2.3673, + "theoretical_loss": 3.3699406595299655, + "tokens_seen": 2486829056 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025066666666666667, + "loss": 2.4388, + "theoretical_loss": 3.36993356818609, + "tokens_seen": 2486894592 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002506464646464647, + "loss": 2.4689, + "theoretical_loss": 3.3699264770814104, + "tokens_seen": 2486960128 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025062626262626264, + "loss": 2.6646, + "theoretical_loss": 3.3699193862159125, + "tokens_seen": 2487025664 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002506060606060606, + "loss": 2.5553, + "theoretical_loss": 3.3699122955895815, + "tokens_seen": 2487091200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002505858585858586, + "loss": 2.8432, + "theoretical_loss": 3.3699052052024037, + "tokens_seen": 2487156736 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025056565656565656, + "loss": 2.72, + "theoretical_loss": 3.369898115054364, + "tokens_seen": 2487222272 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002505454545454546, + "loss": 2.6203, + "theoretical_loss": 3.3698910251454484, + "tokens_seen": 2487287808 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1365266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5479819774627686, + "objective/train/theoretical_loss": 3.3698839354756425, + "objective/train/tokens_used": 846212576, + "theoretical_loss": 3.3698839354756425, + "tokens_seen": 2487353344 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025052525252525253, + "loss": 2.4434, + "theoretical_loss": 3.3698839354756425, + "tokens_seen": 2487353344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002505050505050505, + "loss": 2.6013, + "theoretical_loss": 3.369876846044932, + "tokens_seen": 2487418880 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002504848484848485, + "loss": 2.4104, + "theoretical_loss": 3.3698697568533023, + "tokens_seen": 2487484416 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025046464646464646, + "loss": 2.7333, + "theoretical_loss": 3.3698626679007395, + "tokens_seen": 2487549952 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025044444444444447, + "loss": 2.6047, + "theoretical_loss": 3.3698555791872287, + "tokens_seen": 2487615488 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002504242424242424, + "loss": 2.4058, + "theoretical_loss": 3.369848490712756, + "tokens_seen": 2487681024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025040404040404043, + "loss": 2.5335, + "theoretical_loss": 3.3698414024773062, + "tokens_seen": 2487746560 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002503838383838384, + "loss": 2.5476, + "theoretical_loss": 3.3698343144808662, + "tokens_seen": 2487812096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025036363636363635, + "loss": 2.4196, + "theoretical_loss": 3.369827226723421, + "tokens_seen": 2487877632 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025034343434343436, + "loss": 2.4556, + "theoretical_loss": 3.3698201392049563, + "tokens_seen": 2487943168 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002503232323232323, + "loss": 2.5638, + "theoretical_loss": 3.3698130519254574, + "tokens_seen": 2488008704 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002503030303030303, + "loss": 2.5288, + "theoretical_loss": 3.3698059648849106, + "tokens_seen": 2488074240 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002502828282828283, + "loss": 2.4758, + "theoretical_loss": 3.369798878083301, + "tokens_seen": 2488139776 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025026262626262624, + "loss": 2.7406, + "theoretical_loss": 3.3697917915206146, + "tokens_seen": 2488205312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025024242424242425, + "loss": 2.5244, + "theoretical_loss": 3.3697847051968366, + "tokens_seen": 2488270848 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025022222222222226, + "loss": 2.3793, + "theoretical_loss": 3.3697776191119537, + "tokens_seen": 2488336384 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002502020202020202, + "loss": 2.5258, + "theoretical_loss": 3.36977053326595, + "tokens_seen": 2488401920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501818181818182, + "loss": 2.3719, + "theoretical_loss": 3.3697634476588125, + "tokens_seen": 2488467456 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025016161616161613, + "loss": 2.5511, + "theoretical_loss": 3.369756362290526, + "tokens_seen": 2488532992 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025014141414141414, + "loss": 2.4933, + "theoretical_loss": 3.369749277161077, + "tokens_seen": 2488598528 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025012121212121216, + "loss": 2.6113, + "theoretical_loss": 3.3697421922704502, + "tokens_seen": 2488664064 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501010101010101, + "loss": 2.4248, + "theoretical_loss": 3.3697351076186317, + "tokens_seen": 2488729600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025008080808080807, + "loss": 2.4509, + "theoretical_loss": 3.3697280232056075, + "tokens_seen": 2488795136 + }, + { + "epoch": 0.5, + "learning_rate": 0.000250060606060606, + "loss": 2.5798, + "theoretical_loss": 3.369720939031363, + "tokens_seen": 2488860672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002500404040404041, + "loss": 2.6715, + "theoretical_loss": 3.3697138550958834, + "tokens_seen": 2488926208 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1366649, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3071372509002686, + "objective/train/theoretical_loss": 3.369706771399155, + "objective/train/tokens_used": 847850976, + "theoretical_loss": 3.369706771399155, + "tokens_seen": 2488991744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00025002020202020205, + "loss": 2.6215, + "theoretical_loss": 3.369706771399155, + "tokens_seen": 2488991744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00025, + "loss": 2.6834, + "theoretical_loss": 3.3696996879411634, + "tokens_seen": 2489057280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024997979797979796, + "loss": 2.6139, + "theoretical_loss": 3.3696926047218936, + "tokens_seen": 2489122816 + }, + { + "epoch": 0.51, + "learning_rate": 0.000249959595959596, + "loss": 2.5778, + "theoretical_loss": 3.3696855217413324, + "tokens_seen": 2489188352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024993939393939393, + "loss": 2.294, + "theoretical_loss": 3.3696784389994643, + "tokens_seen": 2489253888 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024991919191919194, + "loss": 2.5554, + "theoretical_loss": 3.369671356496276, + "tokens_seen": 2489319424 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002498989898989899, + "loss": 2.7245, + "theoretical_loss": 3.369664274231752, + "tokens_seen": 2489384960 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024987878787878785, + "loss": 2.5064, + "theoretical_loss": 3.36965719220588, + "tokens_seen": 2489450496 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024985858585858587, + "loss": 2.6125, + "theoretical_loss": 3.369650110418643, + "tokens_seen": 2489516032 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002498383838383838, + "loss": 2.333, + "theoretical_loss": 3.3696430288700285, + "tokens_seen": 2489581568 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024981818181818183, + "loss": 2.4657, + "theoretical_loss": 3.369635947560022, + "tokens_seen": 2489647104 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497979797979798, + "loss": 2.4729, + "theoretical_loss": 3.3696288664886085, + "tokens_seen": 2489712640 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024977777777777775, + "loss": 2.5708, + "theoretical_loss": 3.369621785655774, + "tokens_seen": 2489778176 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024975757575757576, + "loss": 2.447, + "theoretical_loss": 3.3696147050615046, + "tokens_seen": 2489843712 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497373737373737, + "loss": 2.6113, + "theoretical_loss": 3.3696076247057856, + "tokens_seen": 2489909248 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497171717171717, + "loss": 2.6729, + "theoretical_loss": 3.369600544588603, + "tokens_seen": 2489974784 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496969696969697, + "loss": 2.5747, + "theoretical_loss": 3.3695934647099413, + "tokens_seen": 2490040320 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496767676767677, + "loss": 2.4099, + "theoretical_loss": 3.3695863850697876, + "tokens_seen": 2490105856 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024965656565656565, + "loss": 2.6807, + "theoretical_loss": 3.369579305668127, + "tokens_seen": 2490171392 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496363636363636, + "loss": 2.4302, + "theoretical_loss": 3.3695722265049457, + "tokens_seen": 2490236928 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496161616161616, + "loss": 2.5694, + "theoretical_loss": 3.3695651475802286, + "tokens_seen": 2490302464 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495959595959596, + "loss": 2.4473, + "theoretical_loss": 3.3695580688939617, + "tokens_seen": 2490368000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495757575757576, + "loss": 2.4847, + "theoretical_loss": 3.3695509904461307, + "tokens_seen": 2490433536 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024955555555555554, + "loss": 2.8896, + "theoretical_loss": 3.3695439122367215, + "tokens_seen": 2490499072 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024953535353535355, + "loss": 2.5302, + "theoretical_loss": 3.3695368342657197, + "tokens_seen": 2490564608 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1367045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7037811279296875, + "objective/train/theoretical_loss": 3.3695297565331104, + "objective/train/tokens_used": 849489376, + "theoretical_loss": 3.3695297565331104, + "tokens_seen": 2490630144 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495151515151515, + "loss": 2.4598, + "theoretical_loss": 3.3695297565331104, + "tokens_seen": 2490630144 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002494949494949495, + "loss": 2.6655, + "theoretical_loss": 3.36952267903888, + "tokens_seen": 2490695680 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002494747474747475, + "loss": 2.6655, + "theoretical_loss": 3.3695156017830143, + "tokens_seen": 2490761216 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024945454545454544, + "loss": 2.6021, + "theoretical_loss": 3.3695085247654992, + "tokens_seen": 2490826752 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024943434343434345, + "loss": 2.4376, + "theoretical_loss": 3.3695014479863192, + "tokens_seen": 2490892288 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002494141414141414, + "loss": 2.6308, + "theoretical_loss": 3.369494371445461, + "tokens_seen": 2490957824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002493939393939394, + "loss": 2.6892, + "theoretical_loss": 3.36948729514291, + "tokens_seen": 2491023360 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024937373737373737, + "loss": 2.4959, + "theoretical_loss": 3.369480219078652, + "tokens_seen": 2491088896 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002493535353535354, + "loss": 2.5588, + "theoretical_loss": 3.3694731432526726, + "tokens_seen": 2491154432 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024933333333333334, + "loss": 2.5366, + "theoretical_loss": 3.3694660676649573, + "tokens_seen": 2491219968 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024931313131313135, + "loss": 2.6207, + "theoretical_loss": 3.3694589923154923, + "tokens_seen": 2491285504 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492929292929293, + "loss": 2.6988, + "theoretical_loss": 3.369451917204263, + "tokens_seen": 2491351040 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024927272727272727, + "loss": 2.6747, + "theoretical_loss": 3.3694448423312555, + "tokens_seen": 2491416576 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492525252525253, + "loss": 2.5209, + "theoretical_loss": 3.369437767696455, + "tokens_seen": 2491482112 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024923232323232323, + "loss": 2.5245, + "theoretical_loss": 3.3694306932998472, + "tokens_seen": 2491547648 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024921212121212124, + "loss": 2.5096, + "theoretical_loss": 3.3694236191414184, + "tokens_seen": 2491613184 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491919191919192, + "loss": 2.5984, + "theoretical_loss": 3.3694165452211537, + "tokens_seen": 2491678720 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491717171717172, + "loss": 2.4263, + "theoretical_loss": 3.369409471539039, + "tokens_seen": 2491744256 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024915151515151517, + "loss": 2.5756, + "theoretical_loss": 3.36940239809506, + "tokens_seen": 2491809792 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491313131313131, + "loss": 2.6384, + "theoretical_loss": 3.369395324889203, + "tokens_seen": 2491875328 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024911111111111114, + "loss": 2.6184, + "theoretical_loss": 3.369388251921453, + "tokens_seen": 2491940864 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490909090909091, + "loss": 2.6661, + "theoretical_loss": 3.369381179191796, + "tokens_seen": 2492006400 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490707070707071, + "loss": 2.6451, + "theoretical_loss": 3.369374106700217, + "tokens_seen": 2492071936 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024905050505050506, + "loss": 2.5322, + "theoretical_loss": 3.3693670344467033, + "tokens_seen": 2492137472 + }, + { + "epoch": 0.51, + "learning_rate": 0.000249030303030303, + "loss": 2.5058, + "theoretical_loss": 3.3693599624312394, + "tokens_seen": 2492203008 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1368139, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.515371799468994, + "objective/train/theoretical_loss": 3.3693528906538113, + "objective/train/tokens_used": 851127776, + "theoretical_loss": 3.3693528906538113, + "tokens_seen": 2492268544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024901010101010103, + "loss": 2.6865, + "theoretical_loss": 3.3693528906538113, + "tokens_seen": 2492268544 + }, + { + "epoch": 0.51, + "learning_rate": 0.000248989898989899, + "loss": 2.3844, + "theoretical_loss": 3.369345819114405, + "tokens_seen": 2492334080 + }, + { + "epoch": 0.51, + "learning_rate": 0.000248969696969697, + "loss": 2.5789, + "theoretical_loss": 3.3693387478130057, + "tokens_seen": 2492399616 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024894949494949495, + "loss": 2.5623, + "theoretical_loss": 3.3693316767495993, + "tokens_seen": 2492465152 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002489292929292929, + "loss": 2.757, + "theoretical_loss": 3.3693246059241724, + "tokens_seen": 2492530688 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002489090909090909, + "loss": 2.8262, + "theoretical_loss": 3.3693175353367093, + "tokens_seen": 2492596224 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488888888888889, + "loss": 2.673, + "theoretical_loss": 3.3693104649871968, + "tokens_seen": 2492661760 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488686868686869, + "loss": 2.5363, + "theoretical_loss": 3.36930339487562, + "tokens_seen": 2492727296 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024884848484848485, + "loss": 2.4885, + "theoretical_loss": 3.369296325001965, + "tokens_seen": 2492792832 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488282828282828, + "loss": 2.7316, + "theoretical_loss": 3.369289255366218, + "tokens_seen": 2492858368 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488080808080808, + "loss": 2.5233, + "theoretical_loss": 3.369282185968364, + "tokens_seen": 2492923904 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024878787878787877, + "loss": 2.7775, + "theoretical_loss": 3.3692751168083883, + "tokens_seen": 2492989440 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487676767676768, + "loss": 2.6249, + "theoretical_loss": 3.369268047886278, + "tokens_seen": 2493054976 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024874747474747474, + "loss": 2.6697, + "theoretical_loss": 3.369260979202018, + "tokens_seen": 2493120512 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487272727272727, + "loss": 2.7337, + "theoretical_loss": 3.3692539107555937, + "tokens_seen": 2493186048 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487070707070707, + "loss": 2.3319, + "theoretical_loss": 3.3692468425469917, + "tokens_seen": 2493251584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024868686868686866, + "loss": 2.4852, + "theoretical_loss": 3.3692397745761973, + "tokens_seen": 2493317120 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486666666666667, + "loss": 2.6736, + "theoretical_loss": 3.369232706843196, + "tokens_seen": 2493382656 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024864646464646463, + "loss": 2.4356, + "theoretical_loss": 3.3692256393479747, + "tokens_seen": 2493448192 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024862626262626264, + "loss": 2.637, + "theoretical_loss": 3.3692185720905172, + "tokens_seen": 2493513728 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486060606060606, + "loss": 2.476, + "theoretical_loss": 3.369211505070811, + "tokens_seen": 2493579264 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024858585858585856, + "loss": 2.3298, + "theoretical_loss": 3.3692044382888415, + "tokens_seen": 2493644800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024856565656565657, + "loss": 2.4841, + "theoretical_loss": 3.3691973717445935, + "tokens_seen": 2493710336 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002485454545454545, + "loss": 2.5156, + "theoretical_loss": 3.369190305438054, + "tokens_seen": 2493775872 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024852525252525254, + "loss": 2.6971, + "theoretical_loss": 3.369183239369208, + "tokens_seen": 2493841408 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1368726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9163858890533447, + "objective/train/theoretical_loss": 3.369176173538041, + "objective/train/tokens_used": 852766176, + "theoretical_loss": 3.369176173538041, + "tokens_seen": 2493906944 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002485050505050505, + "loss": 2.6727, + "theoretical_loss": 3.369176173538041, + "tokens_seen": 2493906944 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002484848484848485, + "loss": 2.7429, + "theoretical_loss": 3.36916910794454, + "tokens_seen": 2493972480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024846464646464646, + "loss": 2.5531, + "theoretical_loss": 3.3691620425886897, + "tokens_seen": 2494038016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024844444444444447, + "loss": 2.6035, + "theoretical_loss": 3.369154977470476, + "tokens_seen": 2494103552 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024842424242424243, + "loss": 2.3908, + "theoretical_loss": 3.3691479125898844, + "tokens_seen": 2494169088 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002484040404040404, + "loss": 2.5897, + "theoretical_loss": 3.369140847946902, + "tokens_seen": 2494234624 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002483838383838384, + "loss": 2.5264, + "theoretical_loss": 3.3691337835415127, + "tokens_seen": 2494300160 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024836363636363635, + "loss": 2.6173, + "theoretical_loss": 3.369126719373704, + "tokens_seen": 2494365696 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024834343434343436, + "loss": 2.6316, + "theoretical_loss": 3.36911965544346, + "tokens_seen": 2494431232 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002483232323232323, + "loss": 2.4897, + "theoretical_loss": 3.369112591750768, + "tokens_seen": 2494496768 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024830303030303033, + "loss": 2.5915, + "theoretical_loss": 3.369105528295613, + "tokens_seen": 2494562304 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002482828282828283, + "loss": 2.668, + "theoretical_loss": 3.3690984650779807, + "tokens_seen": 2494627840 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024826262626262625, + "loss": 2.4532, + "theoretical_loss": 3.369091402097857, + "tokens_seen": 2494693376 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024824242424242426, + "loss": 2.3615, + "theoretical_loss": 3.3690843393552274, + "tokens_seen": 2494758912 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002482222222222222, + "loss": 2.5497, + "theoretical_loss": 3.3690772768500787, + "tokens_seen": 2494824448 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002482020202020202, + "loss": 2.7915, + "theoretical_loss": 3.3690702145823956, + "tokens_seen": 2494889984 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481818181818182, + "loss": 2.3947, + "theoretical_loss": 3.3690631525521644, + "tokens_seen": 2494955520 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481616161616162, + "loss": 2.5586, + "theoretical_loss": 3.3690560907593707, + "tokens_seen": 2495021056 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024814141414141415, + "loss": 2.6466, + "theoretical_loss": 3.369049029204, + "tokens_seen": 2495086592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024812121212121216, + "loss": 2.6203, + "theoretical_loss": 3.369041967886039, + "tokens_seen": 2495152128 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481010101010101, + "loss": 2.4112, + "theoretical_loss": 3.3690349068054726, + "tokens_seen": 2495217664 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002480808080808081, + "loss": 2.5501, + "theoretical_loss": 3.3690278459622864, + "tokens_seen": 2495283200 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002480606060606061, + "loss": 2.6416, + "theoretical_loss": 3.3690207853564673, + "tokens_seen": 2495348736 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024804040404040404, + "loss": 2.7519, + "theoretical_loss": 3.369013724988, + "tokens_seen": 2495414272 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024802020202020205, + "loss": 2.4194, + "theoretical_loss": 3.369006664856871, + "tokens_seen": 2495479808 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1369317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.740476608276367, + "objective/train/theoretical_loss": 3.368999604963066, + "objective/train/tokens_used": 854404576, + "theoretical_loss": 3.368999604963066, + "tokens_seen": 2495545344 + }, + { + "epoch": 0.51, + "learning_rate": 0.000248, + "loss": 2.5074, + "theoretical_loss": 3.368999604963066, + "tokens_seen": 2495545344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024797979797979797, + "loss": 2.5753, + "theoretical_loss": 3.36899254530657, + "tokens_seen": 2495610880 + }, + { + "epoch": 0.51, + "learning_rate": 0.000247959595959596, + "loss": 2.6143, + "theoretical_loss": 3.36898548588737, + "tokens_seen": 2495676416 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024793939393939394, + "loss": 2.3874, + "theoretical_loss": 3.3689784267054503, + "tokens_seen": 2495741952 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024791919191919195, + "loss": 2.6073, + "theoretical_loss": 3.3689713677607984, + "tokens_seen": 2495807488 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002478989898989899, + "loss": 2.6966, + "theoretical_loss": 3.368964309053399, + "tokens_seen": 2495873024 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024787878787878786, + "loss": 2.6409, + "theoretical_loss": 3.368957250583238, + "tokens_seen": 2495938560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024785858585858587, + "loss": 2.5733, + "theoretical_loss": 3.368950192350302, + "tokens_seen": 2496004096 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024783838383838383, + "loss": 2.941, + "theoretical_loss": 3.3689431343545753, + "tokens_seen": 2496069632 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024781818181818184, + "loss": 2.7823, + "theoretical_loss": 3.368936076596045, + "tokens_seen": 2496135168 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477979797979798, + "loss": 2.7744, + "theoretical_loss": 3.3689290190746966, + "tokens_seen": 2496200704 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477777777777778, + "loss": 2.7765, + "theoretical_loss": 3.3689219617905155, + "tokens_seen": 2496266240 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024775757575757576, + "loss": 2.5725, + "theoretical_loss": 3.368914904743488, + "tokens_seen": 2496331776 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477373737373737, + "loss": 2.6258, + "theoretical_loss": 3.3689078479335994, + "tokens_seen": 2496397312 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024771717171717173, + "loss": 2.5606, + "theoretical_loss": 3.368900791360836, + "tokens_seen": 2496462848 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476969696969697, + "loss": 2.5343, + "theoretical_loss": 3.3688937350251833, + "tokens_seen": 2496528384 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476767676767677, + "loss": 2.7174, + "theoretical_loss": 3.368886678926627, + "tokens_seen": 2496593920 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024765656565656566, + "loss": 2.4547, + "theoretical_loss": 3.3688796230651534, + "tokens_seen": 2496659456 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476363636363636, + "loss": 2.4884, + "theoretical_loss": 3.3688725674407483, + "tokens_seen": 2496724992 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476161616161616, + "loss": 2.6397, + "theoretical_loss": 3.3688655120533966, + "tokens_seen": 2496790528 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475959595959596, + "loss": 2.7922, + "theoretical_loss": 3.3688584569030855, + "tokens_seen": 2496856064 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475757575757576, + "loss": 2.62, + "theoretical_loss": 3.368851401989799, + "tokens_seen": 2496921600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024755555555555555, + "loss": 2.5287, + "theoretical_loss": 3.368844347313525, + "tokens_seen": 2496987136 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475353535353535, + "loss": 2.6028, + "theoretical_loss": 3.3688372928742476, + "tokens_seen": 2497052672 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475151515151515, + "loss": 2.4231, + "theoretical_loss": 3.3688302386719537, + "tokens_seen": 2497118208 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1369981, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4275753498077393, + "objective/train/theoretical_loss": 3.3688231847066286, + "objective/train/tokens_used": 856042976, + "theoretical_loss": 3.3688231847066286, + "tokens_seen": 2497183744 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474949494949495, + "loss": 2.4614, + "theoretical_loss": 3.3688231847066286, + "tokens_seen": 2497183744 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474747474747475, + "loss": 2.4062, + "theoretical_loss": 3.3688161309782583, + "tokens_seen": 2497249280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024745454545454544, + "loss": 2.5543, + "theoretical_loss": 3.3688090774868287, + "tokens_seen": 2497314816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024743434343434345, + "loss": 2.6275, + "theoretical_loss": 3.368802024232325, + "tokens_seen": 2497380352 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474141414141414, + "loss": 2.4845, + "theoretical_loss": 3.3687949712147343, + "tokens_seen": 2497445888 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024739393939393937, + "loss": 2.608, + "theoretical_loss": 3.368787918434041, + "tokens_seen": 2497511424 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002473737373737374, + "loss": 2.6319, + "theoretical_loss": 3.3687808658902316, + "tokens_seen": 2497576960 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024735353535353533, + "loss": 2.5962, + "theoretical_loss": 3.3687738135832923, + "tokens_seen": 2497642496 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024733333333333335, + "loss": 2.4208, + "theoretical_loss": 3.3687667615132084, + "tokens_seen": 2497708032 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002473131313131313, + "loss": 2.6172, + "theoretical_loss": 3.368759709679966, + "tokens_seen": 2497773568 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472929292929293, + "loss": 2.5665, + "theoretical_loss": 3.3687526580835505, + "tokens_seen": 2497839104 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024727272727272727, + "loss": 2.4846, + "theoretical_loss": 3.368745606723948, + "tokens_seen": 2497904640 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472525252525253, + "loss": 2.5455, + "theoretical_loss": 3.3687385556011447, + "tokens_seen": 2497970176 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024723232323232324, + "loss": 2.822, + "theoretical_loss": 3.368731504715126, + "tokens_seen": 2498035712 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472121212121212, + "loss": 2.6857, + "theoretical_loss": 3.3687244540658776, + "tokens_seen": 2498101248 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002471919191919192, + "loss": 2.5812, + "theoretical_loss": 3.3687174036533856, + "tokens_seen": 2498166784 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024717171717171716, + "loss": 2.5216, + "theoretical_loss": 3.368710353477636, + "tokens_seen": 2498232320 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002471515151515152, + "loss": 2.6513, + "theoretical_loss": 3.3687033035386147, + "tokens_seen": 2498297856 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024713131313131313, + "loss": 2.5227, + "theoretical_loss": 3.3686962538363066, + "tokens_seen": 2498363392 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024711111111111114, + "loss": 2.525, + "theoretical_loss": 3.368689204370699, + "tokens_seen": 2498428928 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002470909090909091, + "loss": 2.5978, + "theoretical_loss": 3.3686821551417765, + "tokens_seen": 2498494464 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002470707070707071, + "loss": 2.6067, + "theoretical_loss": 3.3686751061495253, + "tokens_seen": 2498560000 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024705050505050507, + "loss": 2.6635, + "theoretical_loss": 3.368668057393932, + "tokens_seen": 2498625536 + }, + { + "epoch": 0.51, + "learning_rate": 0.000247030303030303, + "loss": 2.4435, + "theoretical_loss": 3.368661008874981, + "tokens_seen": 2498691072 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024701010101010104, + "loss": 2.6276, + "theoretical_loss": 3.3686539605926598, + "tokens_seen": 2498756608 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1371264, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.608736515045166, + "objective/train/theoretical_loss": 3.3686469125469527, + "objective/train/tokens_used": 857681376, + "theoretical_loss": 3.3686469125469527, + "tokens_seen": 2498822144 + }, + { + "epoch": 0.51, + "learning_rate": 0.000246989898989899, + "loss": 2.5918, + "theoretical_loss": 3.3686469125469527, + "tokens_seen": 2498822144 + }, + { + "epoch": 0.51, + "learning_rate": 0.000246969696969697, + "loss": 2.5816, + "theoretical_loss": 3.3686398647378466, + "tokens_seen": 2498887680 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024694949494949496, + "loss": 2.4516, + "theoretical_loss": 3.3686328171653273, + "tokens_seen": 2498953216 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024692929292929297, + "loss": 2.7305, + "theoretical_loss": 3.36862576982938, + "tokens_seen": 2499018752 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024690909090909093, + "loss": 2.4957, + "theoretical_loss": 3.368618722729991, + "tokens_seen": 2499084288 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002468888888888889, + "loss": 2.6394, + "theoretical_loss": 3.3686116758671463, + "tokens_seen": 2499149824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002468686868686869, + "loss": 2.6072, + "theoretical_loss": 3.3686046292408314, + "tokens_seen": 2499215360 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024684848484848485, + "loss": 2.779, + "theoretical_loss": 3.368597582851032, + "tokens_seen": 2499280896 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024682828282828286, + "loss": 2.6426, + "theoretical_loss": 3.368590536697735, + "tokens_seen": 2499346432 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002468080808080808, + "loss": 2.7482, + "theoretical_loss": 3.368583490780925, + "tokens_seen": 2499411968 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467878787878788, + "loss": 2.4946, + "theoretical_loss": 3.3685764451005884, + "tokens_seen": 2499477504 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467676767676768, + "loss": 2.7292, + "theoretical_loss": 3.368569399656711, + "tokens_seen": 2499543040 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024674747474747475, + "loss": 2.4935, + "theoretical_loss": 3.368562354449279, + "tokens_seen": 2499608576 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024672727272727276, + "loss": 2.5991, + "theoretical_loss": 3.368555309478278, + "tokens_seen": 2499674112 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467070707070707, + "loss": 2.7613, + "theoretical_loss": 3.3685482647436937, + "tokens_seen": 2499739648 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024668686868686867, + "loss": 2.5811, + "theoretical_loss": 3.368541220245512, + "tokens_seen": 2499805184 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002466666666666667, + "loss": 2.6333, + "theoretical_loss": 3.368534175983719, + "tokens_seen": 2499870720 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024664646464646464, + "loss": 2.5164, + "theoretical_loss": 3.3685271319583006, + "tokens_seen": 2499936256 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024662626262626265, + "loss": 2.6044, + "theoretical_loss": 3.3685200881692423, + "tokens_seen": 2500001792 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002466060606060606, + "loss": 2.5888, + "theoretical_loss": 3.36851304461653, + "tokens_seen": 2500067328 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024658585858585856, + "loss": 2.4215, + "theoretical_loss": 3.36850600130015, + "tokens_seen": 2500132864 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002465656565656566, + "loss": 2.6991, + "theoretical_loss": 3.3684989582200884, + "tokens_seen": 2500198400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024654545454545453, + "loss": 2.8688, + "theoretical_loss": 3.3684919153763304, + "tokens_seen": 2500263936 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024652525252525254, + "loss": 2.5872, + "theoretical_loss": 3.368484872768862, + "tokens_seen": 2500329472 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002465050505050505, + "loss": 2.6992, + "theoretical_loss": 3.3684778303976692, + "tokens_seen": 2500395008 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1372421, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.858076333999634, + "objective/train/theoretical_loss": 3.3684707882627376, + "objective/train/tokens_used": 859319776, + "theoretical_loss": 3.3684707882627376, + "tokens_seen": 2500460544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024648484848484846, + "loss": 2.5673, + "theoretical_loss": 3.3684707882627376, + "tokens_seen": 2500460544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024646464646464647, + "loss": 2.5925, + "theoretical_loss": 3.3684637463640534, + "tokens_seen": 2500526080 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002464444444444444, + "loss": 2.6613, + "theoretical_loss": 3.368456704701603, + "tokens_seen": 2500591616 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024642424242424243, + "loss": 2.6533, + "theoretical_loss": 3.3684496632753715, + "tokens_seen": 2500657152 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002464040404040404, + "loss": 2.541, + "theoretical_loss": 3.3684426220853445, + "tokens_seen": 2500722688 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002463838383838384, + "loss": 2.5539, + "theoretical_loss": 3.368435581131509, + "tokens_seen": 2500788224 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024636363636363636, + "loss": 2.4263, + "theoretical_loss": 3.36842854041385, + "tokens_seen": 2500853760 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002463434343434343, + "loss": 2.3381, + "theoretical_loss": 3.368421499932354, + "tokens_seen": 2500919296 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024632323232323233, + "loss": 2.5069, + "theoretical_loss": 3.368414459687006, + "tokens_seen": 2500984832 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002463030303030303, + "loss": 2.641, + "theoretical_loss": 3.3684074196777924, + "tokens_seen": 2501050368 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462828282828283, + "loss": 2.6466, + "theoretical_loss": 3.3684003799047, + "tokens_seen": 2501115904 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024626262626262625, + "loss": 2.6269, + "theoretical_loss": 3.368393340367713, + "tokens_seen": 2501181440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024624242424242426, + "loss": 2.4895, + "theoretical_loss": 3.3683863010668187, + "tokens_seen": 2501246976 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462222222222222, + "loss": 2.6638, + "theoretical_loss": 3.368379262002002, + "tokens_seen": 2501312512 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024620202020202023, + "loss": 2.5963, + "theoretical_loss": 3.3683722231732496, + "tokens_seen": 2501378048 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461818181818182, + "loss": 2.6803, + "theoretical_loss": 3.3683651845805467, + "tokens_seen": 2501443584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024616161616161614, + "loss": 2.5318, + "theoretical_loss": 3.3683581462238794, + "tokens_seen": 2501509120 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024614141414141416, + "loss": 2.5654, + "theoretical_loss": 3.368351108103234, + "tokens_seen": 2501574656 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461212121212121, + "loss": 2.638, + "theoretical_loss": 3.3683440702185963, + "tokens_seen": 2501640192 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461010101010101, + "loss": 2.6747, + "theoretical_loss": 3.3683370325699515, + "tokens_seen": 2501705728 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002460808080808081, + "loss": 2.4621, + "theoretical_loss": 3.3683299951572865, + "tokens_seen": 2501771264 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002460606060606061, + "loss": 2.8799, + "theoretical_loss": 3.3683229579805865, + "tokens_seen": 2501836800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024604040404040405, + "loss": 2.5774, + "theoretical_loss": 3.368315921039838, + "tokens_seen": 2501902336 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024602020202020206, + "loss": 2.6532, + "theoretical_loss": 3.3683088843350264, + "tokens_seen": 2501967872 + }, + { + "epoch": 0.51, + "learning_rate": 0.000246, + "loss": 2.5827, + "theoretical_loss": 3.3683018478661375, + "tokens_seen": 2502033408 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1372950, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.237164258956909, + "objective/train/theoretical_loss": 3.3682948116331577, + "objective/train/tokens_used": 860958176, + "theoretical_loss": 3.3682948116331577, + "tokens_seen": 2502098944 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245979797979798, + "loss": 2.4457, + "theoretical_loss": 3.3682948116331577, + "tokens_seen": 2502098944 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245959595959596, + "loss": 2.6712, + "theoretical_loss": 3.3682877756360727, + "tokens_seen": 2502164480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024593939393939394, + "loss": 2.7075, + "theoretical_loss": 3.3682807398748684, + "tokens_seen": 2502230016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024591919191919195, + "loss": 2.7458, + "theoretical_loss": 3.3682737043495306, + "tokens_seen": 2502295552 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458989898989899, + "loss": 2.5921, + "theoretical_loss": 3.3682666690600453, + "tokens_seen": 2502361088 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458787878787879, + "loss": 2.6946, + "theoretical_loss": 3.3682596340063986, + "tokens_seen": 2502426624 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458585858585859, + "loss": 2.5563, + "theoretical_loss": 3.368252599188576, + "tokens_seen": 2502492160 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024583838383838383, + "loss": 2.5767, + "theoretical_loss": 3.368245564606564, + "tokens_seen": 2502557696 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024581818181818184, + "loss": 2.4948, + "theoretical_loss": 3.3682385302603484, + "tokens_seen": 2502623232 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457979797979798, + "loss": 2.7634, + "theoretical_loss": 3.3682314961499147, + "tokens_seen": 2502688768 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457777777777778, + "loss": 2.7744, + "theoretical_loss": 3.368224462275249, + "tokens_seen": 2502754304 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024575757575757577, + "loss": 2.4339, + "theoretical_loss": 3.368217428636337, + "tokens_seen": 2502819840 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457373737373737, + "loss": 2.5071, + "theoretical_loss": 3.3682103952331652, + "tokens_seen": 2502885376 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024571717171717174, + "loss": 2.3734, + "theoretical_loss": 3.3682033620657195, + "tokens_seen": 2502950912 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002456969696969697, + "loss": 2.6853, + "theoretical_loss": 3.368196329133985, + "tokens_seen": 2503016448 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002456767676767677, + "loss": 2.3629, + "theoretical_loss": 3.368189296437949, + "tokens_seen": 2503081984 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024565656565656566, + "loss": 2.5044, + "theoretical_loss": 3.368182263977596, + "tokens_seen": 2503147520 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002456363636363636, + "loss": 2.4397, + "theoretical_loss": 3.3681752317529123, + "tokens_seen": 2503213056 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024561616161616163, + "loss": 2.7891, + "theoretical_loss": 3.3681681997638844, + "tokens_seen": 2503278592 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455959595959596, + "loss": 2.5209, + "theoretical_loss": 3.368161168010498, + "tokens_seen": 2503344128 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455757575757576, + "loss": 2.6058, + "theoretical_loss": 3.368154136492739, + "tokens_seen": 2503409664 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024555555555555556, + "loss": 2.6038, + "theoretical_loss": 3.368147105210593, + "tokens_seen": 2503475200 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455353535353535, + "loss": 2.7226, + "theoretical_loss": 3.368140074164047, + "tokens_seen": 2503540736 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455151515151515, + "loss": 2.7272, + "theoretical_loss": 3.3681330433530854, + "tokens_seen": 2503606272 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454949494949495, + "loss": 2.658, + "theoretical_loss": 3.3681260127776946, + "tokens_seen": 2503671808 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1374275, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5912024974823, + "objective/train/theoretical_loss": 3.3681189824378617, + "objective/train/tokens_used": 862596576, + "theoretical_loss": 3.3681189824378617, + "tokens_seen": 2503737344 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454747474747475, + "loss": 2.703, + "theoretical_loss": 3.3681189824378617, + "tokens_seen": 2503737344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024545454545454545, + "loss": 2.7464, + "theoretical_loss": 3.3681119523335714, + "tokens_seen": 2503802880 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454343434343434, + "loss": 2.4792, + "theoretical_loss": 3.36810492246481, + "tokens_seen": 2503868416 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454141414141414, + "loss": 2.6586, + "theoretical_loss": 3.3680978928315635, + "tokens_seen": 2503933952 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024539393939393937, + "loss": 2.4613, + "theoretical_loss": 3.368090863433818, + "tokens_seen": 2503999488 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002453737373737374, + "loss": 2.6626, + "theoretical_loss": 3.368083834271559, + "tokens_seen": 2504065024 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024535353535353534, + "loss": 2.5153, + "theoretical_loss": 3.3680768053447725, + "tokens_seen": 2504130560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024533333333333335, + "loss": 2.3254, + "theoretical_loss": 3.3680697766534453, + "tokens_seen": 2504196096 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002453131313131313, + "loss": 2.541, + "theoretical_loss": 3.3680627481975622, + "tokens_seen": 2504261632 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024529292929292927, + "loss": 2.6514, + "theoretical_loss": 3.36805571997711, + "tokens_seen": 2504327168 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002452727272727273, + "loss": 2.5615, + "theoretical_loss": 3.3680486919920742, + "tokens_seen": 2504392704 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024525252525252523, + "loss": 2.7122, + "theoretical_loss": 3.368041664242441, + "tokens_seen": 2504458240 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024523232323232324, + "loss": 2.5342, + "theoretical_loss": 3.3680346367281957, + "tokens_seen": 2504523776 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002452121212121212, + "loss": 2.496, + "theoretical_loss": 3.3680276094493253, + "tokens_seen": 2504589312 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002451919191919192, + "loss": 2.4943, + "theoretical_loss": 3.368020582405815, + "tokens_seen": 2504654848 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024517171717171717, + "loss": 2.3659, + "theoretical_loss": 3.3680135555976514, + "tokens_seen": 2504720384 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002451515151515152, + "loss": 2.6034, + "theoretical_loss": 3.36800652902482, + "tokens_seen": 2504785920 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024513131313131314, + "loss": 2.6833, + "theoretical_loss": 3.367999502687306, + "tokens_seen": 2504851456 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002451111111111111, + "loss": 2.476, + "theoretical_loss": 3.367992476585097, + "tokens_seen": 2504916992 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002450909090909091, + "loss": 2.4239, + "theoretical_loss": 3.367985450718178, + "tokens_seen": 2504982528 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024507070707070706, + "loss": 2.5158, + "theoretical_loss": 3.367978425086535, + "tokens_seen": 2505048064 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024505050505050507, + "loss": 2.4461, + "theoretical_loss": 3.367971399690154, + "tokens_seen": 2505113600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024503030303030303, + "loss": 2.622, + "theoretical_loss": 3.3679643745290213, + "tokens_seen": 2505179136 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024501010101010104, + "loss": 2.5459, + "theoretical_loss": 3.367957349603123, + "tokens_seen": 2505244672 + }, + { + "epoch": 0.51, + "learning_rate": 0.000244989898989899, + "loss": 2.5114, + "theoretical_loss": 3.367950324912444, + "tokens_seen": 2505310208 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1374858, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.38922381401062, + "objective/train/theoretical_loss": 3.367943300456971, + "objective/train/tokens_used": 864234976, + "theoretical_loss": 3.367943300456971, + "tokens_seen": 2505375744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024496969696969695, + "loss": 2.6752, + "theoretical_loss": 3.367943300456971, + "tokens_seen": 2505375744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024494949494949497, + "loss": 2.4714, + "theoretical_loss": 3.3679362762366902, + "tokens_seen": 2505441280 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002449292929292929, + "loss": 2.7833, + "theoretical_loss": 3.3679292522515873, + "tokens_seen": 2505506816 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024490909090909093, + "loss": 2.6454, + "theoretical_loss": 3.3679222285016484, + "tokens_seen": 2505572352 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448888888888889, + "loss": 2.6881, + "theoretical_loss": 3.367915204986859, + "tokens_seen": 2505637888 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448686868686869, + "loss": 2.4393, + "theoretical_loss": 3.3679081817072056, + "tokens_seen": 2505703424 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024484848484848486, + "loss": 2.6554, + "theoretical_loss": 3.367901158662674, + "tokens_seen": 2505768960 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024482828282828287, + "loss": 2.607, + "theoretical_loss": 3.36789413585325, + "tokens_seen": 2505834496 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448080808080808, + "loss": 2.3392, + "theoretical_loss": 3.36788711327892, + "tokens_seen": 2505900032 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002447878787878788, + "loss": 2.3996, + "theoretical_loss": 3.36788009093967, + "tokens_seen": 2505965568 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002447676767676768, + "loss": 2.4207, + "theoretical_loss": 3.367873068835485, + "tokens_seen": 2506031104 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024474747474747475, + "loss": 2.493, + "theoretical_loss": 3.367866046966353, + "tokens_seen": 2506096640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024472727272727276, + "loss": 2.6139, + "theoretical_loss": 3.367859025332258, + "tokens_seen": 2506162176 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002447070707070707, + "loss": 2.3547, + "theoretical_loss": 3.3678520039331863, + "tokens_seen": 2506227712 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446868686868687, + "loss": 2.5617, + "theoretical_loss": 3.3678449827691246, + "tokens_seen": 2506293248 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446666666666667, + "loss": 2.6531, + "theoretical_loss": 3.3678379618400585, + "tokens_seen": 2506358784 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024464646464646464, + "loss": 2.5635, + "theoretical_loss": 3.367830941145974, + "tokens_seen": 2506424320 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024462626262626265, + "loss": 2.5852, + "theoretical_loss": 3.3678239206868574, + "tokens_seen": 2506489856 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446060606060606, + "loss": 2.6026, + "theoretical_loss": 3.3678169004626946, + "tokens_seen": 2506555392 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024458585858585857, + "loss": 2.6732, + "theoretical_loss": 3.3678098804734713, + "tokens_seen": 2506620928 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445656565656566, + "loss": 2.6017, + "theoretical_loss": 3.3678028607191735, + "tokens_seen": 2506686464 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024454545454545454, + "loss": 2.5659, + "theoretical_loss": 3.3677958411997873, + "tokens_seen": 2506752000 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024452525252525255, + "loss": 2.4344, + "theoretical_loss": 3.3677888219152985, + "tokens_seen": 2506817536 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445050505050505, + "loss": 2.5002, + "theoretical_loss": 3.367781802865694, + "tokens_seen": 2506883072 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024448484848484846, + "loss": 2.5845, + "theoretical_loss": 3.3677747840509586, + "tokens_seen": 2506948608 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1376185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2249252796173096, + "objective/train/theoretical_loss": 3.367767765471079, + "objective/train/tokens_used": 865873376, + "theoretical_loss": 3.367767765471079, + "tokens_seen": 2507014144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024446464646464647, + "loss": 2.4663, + "theoretical_loss": 3.367767765471079, + "tokens_seen": 2507014144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024444444444444443, + "loss": 2.4525, + "theoretical_loss": 3.367760747126041, + "tokens_seen": 2507079680 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024442424242424244, + "loss": 2.6758, + "theoretical_loss": 3.367753729015831, + "tokens_seen": 2507145216 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002444040404040404, + "loss": 2.4877, + "theoretical_loss": 3.367746711140434, + "tokens_seen": 2507210752 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443838383838384, + "loss": 2.6594, + "theoretical_loss": 3.3677396934998374, + "tokens_seen": 2507276288 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024436363636363636, + "loss": 2.5969, + "theoretical_loss": 3.367732676094026, + "tokens_seen": 2507341824 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443434343434343, + "loss": 2.6704, + "theoretical_loss": 3.367725658922986, + "tokens_seen": 2507407360 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024432323232323233, + "loss": 2.6806, + "theoretical_loss": 3.3677186419867042, + "tokens_seen": 2507472896 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443030303030303, + "loss": 2.3309, + "theoretical_loss": 3.3677116252851658, + "tokens_seen": 2507538432 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442828282828283, + "loss": 2.4701, + "theoretical_loss": 3.3677046088183573, + "tokens_seen": 2507603968 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024426262626262626, + "loss": 2.5221, + "theoretical_loss": 3.367697592586264, + "tokens_seen": 2507669504 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442424242424242, + "loss": 2.6411, + "theoretical_loss": 3.367690576588873, + "tokens_seen": 2507735040 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442222222222222, + "loss": 2.5891, + "theoretical_loss": 3.3676835608261695, + "tokens_seen": 2507800576 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442020202020202, + "loss": 2.617, + "theoretical_loss": 3.36767654529814, + "tokens_seen": 2507866112 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002441818181818182, + "loss": 2.6677, + "theoretical_loss": 3.36766953000477, + "tokens_seen": 2507931648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024416161616161615, + "loss": 2.6607, + "theoretical_loss": 3.367662514946046, + "tokens_seen": 2507997184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024414141414141416, + "loss": 2.4414, + "theoretical_loss": 3.3676555001219537, + "tokens_seen": 2508062720 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024412121212121215, + "loss": 2.5747, + "theoretical_loss": 3.367648485532479, + "tokens_seen": 2508128256 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002441010101010101, + "loss": 2.7421, + "theoretical_loss": 3.367641471177609, + "tokens_seen": 2508193792 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024408080808080809, + "loss": 2.4892, + "theoretical_loss": 3.367634457057328, + "tokens_seen": 2508259328 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024406060606060607, + "loss": 2.4781, + "theoretical_loss": 3.367627443171623, + "tokens_seen": 2508324864 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024404040404040403, + "loss": 2.4819, + "theoretical_loss": 3.36762042952048, + "tokens_seen": 2508390400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024402020202020204, + "loss": 2.4511, + "theoretical_loss": 3.367613416103885, + "tokens_seen": 2508455936 + }, + { + "epoch": 0.52, + "learning_rate": 0.000244, + "loss": 2.6446, + "theoretical_loss": 3.3676064029218242, + "tokens_seen": 2508521472 + }, + { + "epoch": 0.52, + "learning_rate": 0.000243979797979798, + "loss": 2.5491, + "theoretical_loss": 3.3675993899742833, + "tokens_seen": 2508587008 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1376516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9129412174224854, + "objective/train/theoretical_loss": 3.3675923772612486, + "objective/train/tokens_used": 867511776, + "theoretical_loss": 3.3675923772612486, + "tokens_seen": 2508652544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024395959595959596, + "loss": 2.923, + "theoretical_loss": 3.3675923772612486, + "tokens_seen": 2508652544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024393939393939392, + "loss": 2.5342, + "theoretical_loss": 3.367585364782706, + "tokens_seen": 2508718080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024391919191919193, + "loss": 2.5773, + "theoretical_loss": 3.3675783525386414, + "tokens_seen": 2508783616 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002438989898989899, + "loss": 2.7633, + "theoretical_loss": 3.367571340529041, + "tokens_seen": 2508849152 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002438787878787879, + "loss": 2.6471, + "theoretical_loss": 3.3675643287538906, + "tokens_seen": 2508914688 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024385858585858586, + "loss": 2.6584, + "theoretical_loss": 3.3675573172131763, + "tokens_seen": 2508980224 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024383838383838384, + "loss": 2.7547, + "theoretical_loss": 3.3675503059068848, + "tokens_seen": 2509045760 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024381818181818182, + "loss": 2.3317, + "theoretical_loss": 3.3675432948350013, + "tokens_seen": 2509111296 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002437979797979798, + "loss": 2.6839, + "theoretical_loss": 3.367536283997512, + "tokens_seen": 2509176832 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002437777777777778, + "loss": 2.5136, + "theoretical_loss": 3.3675292733944033, + "tokens_seen": 2509242368 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024375757575757575, + "loss": 2.5828, + "theoretical_loss": 3.367522263025661, + "tokens_seen": 2509307904 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024373737373737373, + "loss": 2.7007, + "theoretical_loss": 3.3675152528912715, + "tokens_seen": 2509373440 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024371717171717172, + "loss": 2.6331, + "theoretical_loss": 3.36750824299122, + "tokens_seen": 2509438976 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002436969696969697, + "loss": 2.7396, + "theoretical_loss": 3.367501233325493, + "tokens_seen": 2509504512 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024367676767676768, + "loss": 2.6129, + "theoretical_loss": 3.3674942238940773, + "tokens_seen": 2509570048 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024365656565656567, + "loss": 2.6242, + "theoretical_loss": 3.3674872146969577, + "tokens_seen": 2509635584 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024363636363636362, + "loss": 2.593, + "theoretical_loss": 3.367480205734121, + "tokens_seen": 2509701120 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024361616161616164, + "loss": 2.6288, + "theoretical_loss": 3.367473197005553, + "tokens_seen": 2509766656 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002435959595959596, + "loss": 2.4618, + "theoretical_loss": 3.36746618851124, + "tokens_seen": 2509832192 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024357575757575758, + "loss": 2.5529, + "theoretical_loss": 3.3674591802511675, + "tokens_seen": 2509897728 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024355555555555556, + "loss": 2.6104, + "theoretical_loss": 3.367452172225322, + "tokens_seen": 2509963264 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024353535353535352, + "loss": 2.4727, + "theoretical_loss": 3.36744516443369, + "tokens_seen": 2510028800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024351515151515153, + "loss": 2.5098, + "theoretical_loss": 3.367438156876257, + "tokens_seen": 2510094336 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024349494949494949, + "loss": 2.6105, + "theoretical_loss": 3.3674311495530085, + "tokens_seen": 2510159872 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002434747474747475, + "loss": 2.6655, + "theoretical_loss": 3.3674241424639315, + "tokens_seen": 2510225408 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1376516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8044562339782715, + "objective/train/theoretical_loss": 3.3674171356090117, + "objective/train/tokens_used": 869150176, + "theoretical_loss": 3.3674171356090117, + "tokens_seen": 2510290944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024345454545454545, + "loss": 2.6998, + "theoretical_loss": 3.3674171356090117, + "tokens_seen": 2510290944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024343434343434346, + "loss": 2.6286, + "theoretical_loss": 3.367410128988235, + "tokens_seen": 2510356480 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024341414141414142, + "loss": 2.6773, + "theoretical_loss": 3.367403122601588, + "tokens_seen": 2510422016 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024339393939393938, + "loss": 2.9049, + "theoretical_loss": 3.367396116449056, + "tokens_seen": 2510487552 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002433737373737374, + "loss": 2.597, + "theoretical_loss": 3.3673891105306257, + "tokens_seen": 2510553088 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024335353535353535, + "loss": 2.7125, + "theoretical_loss": 3.3673821048462833, + "tokens_seen": 2510618624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024333333333333336, + "loss": 2.7389, + "theoretical_loss": 3.367375099396014, + "tokens_seen": 2510684160 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024331313131313131, + "loss": 2.5175, + "theoretical_loss": 3.3673680941798048, + "tokens_seen": 2510749696 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002432929292929293, + "loss": 2.6552, + "theoretical_loss": 3.367361089197641, + "tokens_seen": 2510815232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024327272727272728, + "loss": 2.5474, + "theoretical_loss": 3.3673540844495093, + "tokens_seen": 2510880768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024325252525252527, + "loss": 2.5989, + "theoretical_loss": 3.3673470799353953, + "tokens_seen": 2510946304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024323232323232325, + "loss": 2.7037, + "theoretical_loss": 3.367340075655285, + "tokens_seen": 2511011840 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002432121212121212, + "loss": 2.6618, + "theoretical_loss": 3.367333071609165, + "tokens_seen": 2511077376 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002431919191919192, + "loss": 2.7728, + "theoretical_loss": 3.3673260677970216, + "tokens_seen": 2511142912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024317171717171717, + "loss": 2.5152, + "theoretical_loss": 3.36731906421884, + "tokens_seen": 2511208448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024315151515151516, + "loss": 2.6549, + "theoretical_loss": 3.3673120608746068, + "tokens_seen": 2511273984 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024313131313131314, + "loss": 2.7633, + "theoretical_loss": 3.3673050577643076, + "tokens_seen": 2511339520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024311111111111113, + "loss": 2.4972, + "theoretical_loss": 3.367298054887929, + "tokens_seen": 2511405056 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024309090909090908, + "loss": 2.5136, + "theoretical_loss": 3.3672910522454575, + "tokens_seen": 2511470592 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002430707070707071, + "loss": 2.8358, + "theoretical_loss": 3.367284049836878, + "tokens_seen": 2511536128 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024305050505050505, + "loss": 2.5822, + "theoretical_loss": 3.367277047662177, + "tokens_seen": 2511601664 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024303030303030304, + "loss": 2.8525, + "theoretical_loss": 3.3672700457213414, + "tokens_seen": 2511667200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024301010101010102, + "loss": 2.6291, + "theoretical_loss": 3.3672630440143565, + "tokens_seen": 2511732736 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024298989898989898, + "loss": 2.4815, + "theoretical_loss": 3.367256042541208, + "tokens_seen": 2511798272 + }, + { + "epoch": 0.52, + "learning_rate": 0.000242969696969697, + "loss": 2.6621, + "theoretical_loss": 3.3672490413018834, + "tokens_seen": 2511863808 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1377404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.514570951461792, + "objective/train/theoretical_loss": 3.3672420402963676, + "objective/train/tokens_used": 870788576, + "theoretical_loss": 3.3672420402963676, + "tokens_seen": 2511929344 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024294949494949494, + "loss": 2.7543, + "theoretical_loss": 3.3672420402963676, + "tokens_seen": 2511929344 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024292929292929296, + "loss": 2.642, + "theoretical_loss": 3.367235039524647, + "tokens_seen": 2511994880 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002429090909090909, + "loss": 2.6038, + "theoretical_loss": 3.3672280389867075, + "tokens_seen": 2512060416 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024288888888888887, + "loss": 2.6941, + "theoretical_loss": 3.3672210386825356, + "tokens_seen": 2512125952 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024286868686868688, + "loss": 2.6416, + "theoretical_loss": 3.3672140386121177, + "tokens_seen": 2512191488 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024284848484848484, + "loss": 2.6449, + "theoretical_loss": 3.3672070387754385, + "tokens_seen": 2512257024 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024282828282828285, + "loss": 2.6641, + "theoretical_loss": 3.3672000391724857, + "tokens_seen": 2512322560 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002428080808080808, + "loss": 2.8028, + "theoretical_loss": 3.367193039803244, + "tokens_seen": 2512388096 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002427878787878788, + "loss": 2.6898, + "theoretical_loss": 3.367186040667701, + "tokens_seen": 2512453632 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024276767676767677, + "loss": 2.5409, + "theoretical_loss": 3.3671790417658416, + "tokens_seen": 2512519168 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024274747474747476, + "loss": 2.5854, + "theoretical_loss": 3.3671720430976526, + "tokens_seen": 2512584704 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024272727272727274, + "loss": 2.5927, + "theoretical_loss": 3.3671650446631194, + "tokens_seen": 2512650240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002427070707070707, + "loss": 2.6868, + "theoretical_loss": 3.367158046462229, + "tokens_seen": 2512715776 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024268686868686868, + "loss": 2.5709, + "theoretical_loss": 3.367151048494967, + "tokens_seen": 2512781312 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024266666666666667, + "loss": 2.6336, + "theoretical_loss": 3.367144050761319, + "tokens_seen": 2512846848 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024264646464646465, + "loss": 2.7417, + "theoretical_loss": 3.367137053261272, + "tokens_seen": 2512912384 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024262626262626263, + "loss": 2.6892, + "theoretical_loss": 3.367130055994812, + "tokens_seen": 2512977920 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024260606060606062, + "loss": 2.532, + "theoretical_loss": 3.367123058961924, + "tokens_seen": 2513043456 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002425858585858586, + "loss": 2.52, + "theoretical_loss": 3.3671160621625957, + "tokens_seen": 2513108992 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024256565656565659, + "loss": 2.6054, + "theoretical_loss": 3.3671090655968126, + "tokens_seen": 2513174528 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024254545454545454, + "loss": 2.4477, + "theoretical_loss": 3.3671020692645603, + "tokens_seen": 2513240064 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024252525252525253, + "loss": 2.7192, + "theoretical_loss": 3.3670950731658253, + "tokens_seen": 2513305600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002425050505050505, + "loss": 2.6857, + "theoretical_loss": 3.3670880773005942, + "tokens_seen": 2513371136 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002424848484848485, + "loss": 2.5658, + "theoretical_loss": 3.3670810816688523, + "tokens_seen": 2513436672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024246464646464648, + "loss": 2.6159, + "theoretical_loss": 3.367074086270586, + "tokens_seen": 2513502208 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1378058, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6538960933685303, + "objective/train/theoretical_loss": 3.367067091105782, + "objective/train/tokens_used": 872426976, + "theoretical_loss": 3.367067091105782, + "tokens_seen": 2513567744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024244444444444443, + "loss": 2.7245, + "theoretical_loss": 3.367067091105782, + "tokens_seen": 2513567744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024242424242424245, + "loss": 2.6674, + "theoretical_loss": 3.3670600961744253, + "tokens_seen": 2513633280 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002424040404040404, + "loss": 2.708, + "theoretical_loss": 3.367053101476503, + "tokens_seen": 2513698816 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024238383838383841, + "loss": 2.6521, + "theoretical_loss": 3.367046107012001, + "tokens_seen": 2513764352 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024236363636363637, + "loss": 2.6415, + "theoretical_loss": 3.3670391127809047, + "tokens_seen": 2513829888 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024234343434343433, + "loss": 2.7366, + "theoretical_loss": 3.3670321187832015, + "tokens_seen": 2513895424 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024232323232323234, + "loss": 2.6968, + "theoretical_loss": 3.3670251250188765, + "tokens_seen": 2513960960 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002423030303030303, + "loss": 2.6258, + "theoretical_loss": 3.3670181314879164, + "tokens_seen": 2514026496 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002422828282828283, + "loss": 2.6262, + "theoretical_loss": 3.3670111381903065, + "tokens_seen": 2514092032 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024226262626262626, + "loss": 2.663, + "theoretical_loss": 3.3670041451260344, + "tokens_seen": 2514157568 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024224242424242425, + "loss": 2.7351, + "theoretical_loss": 3.3669971522950846, + "tokens_seen": 2514223104 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024222222222222223, + "loss": 2.6203, + "theoretical_loss": 3.3669901596974445, + "tokens_seen": 2514288640 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002422020202020202, + "loss": 2.6981, + "theoretical_loss": 3.3669831673330997, + "tokens_seen": 2514354176 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002421818181818182, + "loss": 2.4737, + "theoretical_loss": 3.366976175202036, + "tokens_seen": 2514419712 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024216161616161616, + "loss": 2.7378, + "theoretical_loss": 3.3669691833042403, + "tokens_seen": 2514485248 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024214141414141414, + "loss": 2.784, + "theoretical_loss": 3.3669621916396983, + "tokens_seen": 2514550784 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024212121212121212, + "loss": 2.5699, + "theoretical_loss": 3.366955200208396, + "tokens_seen": 2514616320 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002421010101010101, + "loss": 2.8178, + "theoretical_loss": 3.36694820901032, + "tokens_seen": 2514681856 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002420808080808081, + "loss": 2.5103, + "theoretical_loss": 3.366941218045456, + "tokens_seen": 2514747392 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024206060606060608, + "loss": 2.6782, + "theoretical_loss": 3.36693422731379, + "tokens_seen": 2514812928 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024204040404040403, + "loss": 2.6132, + "theoretical_loss": 3.366927236815309, + "tokens_seen": 2514878464 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024202020202020202, + "loss": 2.7149, + "theoretical_loss": 3.3669202465499986, + "tokens_seen": 2514944000 + }, + { + "epoch": 0.52, + "learning_rate": 0.000242, + "loss": 2.565, + "theoretical_loss": 3.3669132565178446, + "tokens_seen": 2515009536 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024197979797979798, + "loss": 2.6122, + "theoretical_loss": 3.366906266718834, + "tokens_seen": 2515075072 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024195959595959597, + "loss": 2.4542, + "theoretical_loss": 3.366899277152952, + "tokens_seen": 2515140608 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1379276, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5083673000335693, + "objective/train/theoretical_loss": 3.366892287820185, + "objective/train/tokens_used": 874065376, + "theoretical_loss": 3.366892287820185, + "tokens_seen": 2515206144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024193939393939393, + "loss": 2.5853, + "theoretical_loss": 3.366892287820185, + "tokens_seen": 2515206144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024191919191919194, + "loss": 2.6708, + "theoretical_loss": 3.36688529872052, + "tokens_seen": 2515271680 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002418989898989899, + "loss": 2.604, + "theoretical_loss": 3.366878309853942, + "tokens_seen": 2515337216 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002418787878787879, + "loss": 2.7872, + "theoretical_loss": 3.3668713212204375, + "tokens_seen": 2515402752 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024185858585858586, + "loss": 2.8209, + "theoretical_loss": 3.3668643328199934, + "tokens_seen": 2515468288 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024183838383838382, + "loss": 2.7967, + "theoretical_loss": 3.366857344652595, + "tokens_seen": 2515533824 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024181818181818183, + "loss": 2.5932, + "theoretical_loss": 3.3668503567182286, + "tokens_seen": 2515599360 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024179797979797979, + "loss": 2.5671, + "theoretical_loss": 3.3668433690168804, + "tokens_seen": 2515664896 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002417777777777778, + "loss": 2.6938, + "theoretical_loss": 3.366836381548537, + "tokens_seen": 2515730432 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024175757575757575, + "loss": 2.6772, + "theoretical_loss": 3.366829394313184, + "tokens_seen": 2515795968 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024173737373737376, + "loss": 2.4069, + "theoretical_loss": 3.3668224073108077, + "tokens_seen": 2515861504 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024171717171717172, + "loss": 2.7365, + "theoretical_loss": 3.3668154205413945, + "tokens_seen": 2515927040 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002416969696969697, + "loss": 2.8026, + "theoretical_loss": 3.3668084340049305, + "tokens_seen": 2515992576 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002416767676767677, + "loss": 2.5881, + "theoretical_loss": 3.366801447701402, + "tokens_seen": 2516058112 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024165656565656565, + "loss": 2.4471, + "theoretical_loss": 3.366794461630794, + "tokens_seen": 2516123648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024163636363636366, + "loss": 2.461, + "theoretical_loss": 3.3667874757930942, + "tokens_seen": 2516189184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024161616161616161, + "loss": 2.6363, + "theoretical_loss": 3.366780490188288, + "tokens_seen": 2516254720 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002415959595959596, + "loss": 2.6407, + "theoretical_loss": 3.3667735048163623, + "tokens_seen": 2516320256 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024157575757575758, + "loss": 2.6058, + "theoretical_loss": 3.366766519677302, + "tokens_seen": 2516385792 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024155555555555557, + "loss": 2.7431, + "theoretical_loss": 3.366759534771094, + "tokens_seen": 2516451328 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024153535353535355, + "loss": 2.5965, + "theoretical_loss": 3.3667525500977247, + "tokens_seen": 2516516864 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024151515151515153, + "loss": 2.6952, + "theoretical_loss": 3.36674556565718, + "tokens_seen": 2516582400 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002414949494949495, + "loss": 2.5901, + "theoretical_loss": 3.366738581449446, + "tokens_seen": 2516647936 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024147474747474748, + "loss": 2.4266, + "theoretical_loss": 3.3667315974745096, + "tokens_seen": 2516713472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024145454545454546, + "loss": 2.8792, + "theoretical_loss": 3.3667246137323557, + "tokens_seen": 2516779008 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1380577, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4944279193878174, + "objective/train/theoretical_loss": 3.366717630222971, + "objective/train/tokens_used": 875703776, + "theoretical_loss": 3.366717630222971, + "tokens_seen": 2516844544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024143434343434344, + "loss": 2.5337, + "theoretical_loss": 3.366717630222971, + "tokens_seen": 2516844544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024141414141414143, + "loss": 2.8, + "theoretical_loss": 3.366710646946342, + "tokens_seen": 2516910080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024139393939393938, + "loss": 2.6108, + "theoretical_loss": 3.366703663902455, + "tokens_seen": 2516975616 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002413737373737374, + "loss": 2.5537, + "theoretical_loss": 3.366696681091296, + "tokens_seen": 2517041152 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024135353535353535, + "loss": 2.6381, + "theoretical_loss": 3.3666896985128503, + "tokens_seen": 2517106688 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024133333333333334, + "loss": 2.7723, + "theoretical_loss": 3.3666827161671056, + "tokens_seen": 2517172224 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024131313131313132, + "loss": 2.7055, + "theoretical_loss": 3.366675734054047, + "tokens_seen": 2517237760 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024129292929292928, + "loss": 2.5981, + "theoretical_loss": 3.366668752173661, + "tokens_seen": 2517303296 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002412727272727273, + "loss": 2.6816, + "theoretical_loss": 3.3666617705259343, + "tokens_seen": 2517368832 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024125252525252524, + "loss": 2.6118, + "theoretical_loss": 3.366654789110852, + "tokens_seen": 2517434368 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024123232323232326, + "loss": 2.6029, + "theoretical_loss": 3.366647807928401, + "tokens_seen": 2517499904 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002412121212121212, + "loss": 2.5859, + "theoretical_loss": 3.3666408269785677, + "tokens_seen": 2517565440 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002411919191919192, + "loss": 2.737, + "theoretical_loss": 3.3666338462613377, + "tokens_seen": 2517630976 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024117171717171718, + "loss": 2.638, + "theoretical_loss": 3.3666268657766976, + "tokens_seen": 2517696512 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024115151515151514, + "loss": 2.5856, + "theoretical_loss": 3.3666198855246336, + "tokens_seen": 2517762048 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024113131313131315, + "loss": 2.6656, + "theoretical_loss": 3.3666129055051317, + "tokens_seen": 2517827584 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002411111111111111, + "loss": 2.5916, + "theoretical_loss": 3.3666059257181784, + "tokens_seen": 2517893120 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002410909090909091, + "loss": 2.4775, + "theoretical_loss": 3.366598946163759, + "tokens_seen": 2517958656 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024107070707070707, + "loss": 2.5556, + "theoretical_loss": 3.366591966841861, + "tokens_seen": 2518024192 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024105050505050506, + "loss": 2.4534, + "theoretical_loss": 3.3665849877524696, + "tokens_seen": 2518089728 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024103030303030304, + "loss": 2.6444, + "theoretical_loss": 3.366578008895572, + "tokens_seen": 2518155264 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024101010101010102, + "loss": 2.366, + "theoretical_loss": 3.366571030271153, + "tokens_seen": 2518220800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024098989898989898, + "loss": 2.5635, + "theoretical_loss": 3.3665640518792, + "tokens_seen": 2518286336 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024096969696969697, + "loss": 2.5745, + "theoretical_loss": 3.366557073719699, + "tokens_seen": 2518351872 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024094949494949495, + "loss": 2.6506, + "theoretical_loss": 3.3665500957926358, + "tokens_seen": 2518417408 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1381008, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.737955093383789, + "objective/train/theoretical_loss": 3.366543118097997, + "objective/train/tokens_used": 877342176, + "theoretical_loss": 3.366543118097997, + "tokens_seen": 2518482944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024092929292929293, + "loss": 2.5444, + "theoretical_loss": 3.366543118097997, + "tokens_seen": 2518482944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024090909090909092, + "loss": 2.7326, + "theoretical_loss": 3.366536140635768, + "tokens_seen": 2518548480 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002408888888888889, + "loss": 2.537, + "theoretical_loss": 3.366529163405936, + "tokens_seen": 2518614016 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024086868686868689, + "loss": 2.7069, + "theoretical_loss": 3.366522186408487, + "tokens_seen": 2518679552 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024084848484848484, + "loss": 2.7154, + "theoretical_loss": 3.3665152096434072, + "tokens_seen": 2518745088 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024082828282828285, + "loss": 2.546, + "theoretical_loss": 3.3665082331106824, + "tokens_seen": 2518810624 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002408080808080808, + "loss": 2.546, + "theoretical_loss": 3.366501256810299, + "tokens_seen": 2518876160 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002407878787878788, + "loss": 2.4728, + "theoretical_loss": 3.3664942807422436, + "tokens_seen": 2518941696 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024076767676767678, + "loss": 2.4573, + "theoretical_loss": 3.366487304906502, + "tokens_seen": 2519007232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024074747474747474, + "loss": 2.6196, + "theoretical_loss": 3.3664803293030605, + "tokens_seen": 2519072768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024072727272727275, + "loss": 2.4906, + "theoretical_loss": 3.366473353931905, + "tokens_seen": 2519138304 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002407070707070707, + "loss": 2.6368, + "theoretical_loss": 3.366466378793023, + "tokens_seen": 2519203840 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024068686868686871, + "loss": 2.7033, + "theoretical_loss": 3.3664594038863993, + "tokens_seen": 2519269376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024066666666666667, + "loss": 2.4871, + "theoretical_loss": 3.366452429212021, + "tokens_seen": 2519334912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024064646464646463, + "loss": 2.6816, + "theoretical_loss": 3.3664454547698734, + "tokens_seen": 2519400448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024062626262626264, + "loss": 2.4434, + "theoretical_loss": 3.366438480559944, + "tokens_seen": 2519465984 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002406060606060606, + "loss": 2.8236, + "theoretical_loss": 3.3664315065822175, + "tokens_seen": 2519531520 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002405858585858586, + "loss": 2.7616, + "theoretical_loss": 3.3664245328366817, + "tokens_seen": 2519597056 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024056565656565656, + "loss": 2.523, + "theoretical_loss": 3.3664175593233217, + "tokens_seen": 2519662592 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024054545454545455, + "loss": 2.498, + "theoretical_loss": 3.366410586042124, + "tokens_seen": 2519728128 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024052525252525253, + "loss": 2.5989, + "theoretical_loss": 3.366403612993075, + "tokens_seen": 2519793664 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024050505050505052, + "loss": 2.6097, + "theoretical_loss": 3.3663966401761614, + "tokens_seen": 2519859200 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002404848484848485, + "loss": 2.5093, + "theoretical_loss": 3.3663896675913683, + "tokens_seen": 2519924736 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024046464646464646, + "loss": 2.7254, + "theoretical_loss": 3.366382695238683, + "tokens_seen": 2519990272 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024044444444444444, + "loss": 2.468, + "theoretical_loss": 3.366375723118091, + "tokens_seen": 2520055808 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1382202, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.422396421432495, + "objective/train/theoretical_loss": 3.366368751229579, + "objective/train/tokens_used": 878980576, + "theoretical_loss": 3.366368751229579, + "tokens_seen": 2520121344 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024042424242424242, + "loss": 2.5207, + "theoretical_loss": 3.366368751229579, + "tokens_seen": 2520121344 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002404040404040404, + "loss": 2.5935, + "theoretical_loss": 3.3663617795731327, + "tokens_seen": 2520186880 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002403838383838384, + "loss": 2.5413, + "theoretical_loss": 3.3663548081487393, + "tokens_seen": 2520252416 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024036363636363638, + "loss": 2.5449, + "theoretical_loss": 3.366347836956384, + "tokens_seen": 2520317952 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024034343434343433, + "loss": 2.5786, + "theoretical_loss": 3.3663408659960536, + "tokens_seen": 2520383488 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024032323232323234, + "loss": 2.5854, + "theoretical_loss": 3.3663338952677346, + "tokens_seen": 2520449024 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002403030303030303, + "loss": 2.5456, + "theoretical_loss": 3.3663269247714123, + "tokens_seen": 2520514560 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024028282828282828, + "loss": 2.5977, + "theoretical_loss": 3.366319954507074, + "tokens_seen": 2520580096 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024026262626262627, + "loss": 2.6083, + "theoretical_loss": 3.366312984474705, + "tokens_seen": 2520645632 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024024242424242423, + "loss": 2.5057, + "theoretical_loss": 3.3663060146742927, + "tokens_seen": 2520711168 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024022222222222224, + "loss": 2.5763, + "theoretical_loss": 3.3662990451058223, + "tokens_seen": 2520776704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002402020202020202, + "loss": 2.6627, + "theoretical_loss": 3.3662920757692802, + "tokens_seen": 2520842240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002401818181818182, + "loss": 2.611, + "theoretical_loss": 3.3662851066646535, + "tokens_seen": 2520907776 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024016161616161616, + "loss": 2.6441, + "theoretical_loss": 3.3662781377919275, + "tokens_seen": 2520973312 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024014141414141415, + "loss": 2.5646, + "theoretical_loss": 3.366271169151089, + "tokens_seen": 2521038848 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024012121212121213, + "loss": 2.5792, + "theoretical_loss": 3.3662642007421235, + "tokens_seen": 2521104384 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024010101010101009, + "loss": 2.5278, + "theoretical_loss": 3.366257232565018, + "tokens_seen": 2521169920 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002400808080808081, + "loss": 2.6671, + "theoretical_loss": 3.366250264619759, + "tokens_seen": 2521235456 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024006060606060605, + "loss": 2.6492, + "theoretical_loss": 3.366243296906332, + "tokens_seen": 2521300992 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024004040404040407, + "loss": 2.681, + "theoretical_loss": 3.3662363294247237, + "tokens_seen": 2521366528 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024002020202020202, + "loss": 2.4801, + "theoretical_loss": 3.36622936217492, + "tokens_seen": 2521432064 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024, + "loss": 2.83, + "theoretical_loss": 3.3662223951569077, + "tokens_seen": 2521497600 + }, + { + "epoch": 0.52, + "learning_rate": 0.000239979797979798, + "loss": 2.3534, + "theoretical_loss": 3.3662154283706727, + "tokens_seen": 2521563136 + }, + { + "epoch": 0.52, + "learning_rate": 0.00023995959595959597, + "loss": 2.3948, + "theoretical_loss": 3.3662084618162016, + "tokens_seen": 2521628672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00023993939393939396, + "loss": 2.6524, + "theoretical_loss": 3.3662014954934802, + "tokens_seen": 2521694208 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1382657, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4263954162597656, + "objective/train/theoretical_loss": 3.366194529402495, + "objective/train/tokens_used": 880618976, + "theoretical_loss": 3.366194529402495, + "tokens_seen": 2521759744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023991919191919191, + "loss": 2.647, + "theoretical_loss": 3.366194529402495, + "tokens_seen": 2521759744 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002398989898989899, + "loss": 2.604, + "theoretical_loss": 3.366187563543232, + "tokens_seen": 2521825280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023987878787878788, + "loss": 2.9063, + "theoretical_loss": 3.366180597915678, + "tokens_seen": 2521890816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023985858585858587, + "loss": 2.4861, + "theoretical_loss": 3.3661736325198186, + "tokens_seen": 2521956352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023983838383838385, + "loss": 2.7311, + "theoretical_loss": 3.366166667355641, + "tokens_seen": 2522021888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023981818181818183, + "loss": 2.6749, + "theoretical_loss": 3.366159702423131, + "tokens_seen": 2522087424 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002397979797979798, + "loss": 2.8589, + "theoretical_loss": 3.366152737722274, + "tokens_seen": 2522152960 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002397777777777778, + "loss": 2.6195, + "theoretical_loss": 3.366145773253058, + "tokens_seen": 2522218496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023975757575757576, + "loss": 2.7144, + "theoretical_loss": 3.366138809015468, + "tokens_seen": 2522284032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023973737373737374, + "loss": 2.6001, + "theoretical_loss": 3.3661318450094906, + "tokens_seen": 2522349568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023971717171717173, + "loss": 2.5175, + "theoretical_loss": 3.366124881235112, + "tokens_seen": 2522415104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023969696969696968, + "loss": 2.5426, + "theoretical_loss": 3.3661179176923186, + "tokens_seen": 2522480640 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002396767676767677, + "loss": 2.3801, + "theoretical_loss": 3.3661109543810968, + "tokens_seen": 2522546176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023965656565656565, + "loss": 2.5913, + "theoretical_loss": 3.3661039913014332, + "tokens_seen": 2522611712 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023963636363636366, + "loss": 2.4643, + "theoretical_loss": 3.3660970284533134, + "tokens_seen": 2522677248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023961616161616162, + "loss": 2.512, + "theoretical_loss": 3.3660900658367234, + "tokens_seen": 2522742784 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023959595959595958, + "loss": 2.5461, + "theoretical_loss": 3.3660831034516505, + "tokens_seen": 2522808320 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002395757575757576, + "loss": 2.6507, + "theoretical_loss": 3.3660761412980804, + "tokens_seen": 2522873856 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023955555555555554, + "loss": 2.7701, + "theoretical_loss": 3.366069179376, + "tokens_seen": 2522939392 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023953535353535356, + "loss": 2.6033, + "theoretical_loss": 3.3660622176853945, + "tokens_seen": 2523004928 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002395151515151515, + "loss": 2.5006, + "theoretical_loss": 3.366055256226251, + "tokens_seen": 2523070464 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002394949494949495, + "loss": 2.5732, + "theoretical_loss": 3.3660482949985555, + "tokens_seen": 2523136000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023947474747474748, + "loss": 2.6068, + "theoretical_loss": 3.3660413340022943, + "tokens_seen": 2523201536 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023945454545454546, + "loss": 2.6709, + "theoretical_loss": 3.366034373237454, + "tokens_seen": 2523267072 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023943434343434345, + "loss": 2.6736, + "theoretical_loss": 3.36602741270402, + "tokens_seen": 2523332608 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1383478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6999809741973877, + "objective/train/theoretical_loss": 3.36602045240198, + "objective/train/tokens_used": 882257376, + "theoretical_loss": 3.36602045240198, + "tokens_seen": 2523398144 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002394141414141414, + "loss": 2.7498, + "theoretical_loss": 3.36602045240198, + "tokens_seen": 2523398144 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002393939393939394, + "loss": 2.786, + "theoretical_loss": 3.3660134923313194, + "tokens_seen": 2523463680 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023937373737373737, + "loss": 2.6101, + "theoretical_loss": 3.3660065324920247, + "tokens_seen": 2523529216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023935353535353536, + "loss": 2.5801, + "theoretical_loss": 3.365999572884082, + "tokens_seen": 2523594752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023933333333333334, + "loss": 2.5639, + "theoretical_loss": 3.365992613507478, + "tokens_seen": 2523660288 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023931313131313133, + "loss": 2.5456, + "theoretical_loss": 3.3659856543621984, + "tokens_seen": 2523725824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023929292929292928, + "loss": 2.5165, + "theoretical_loss": 3.3659786954482303, + "tokens_seen": 2523791360 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002392727272727273, + "loss": 2.672, + "theoretical_loss": 3.3659717367655593, + "tokens_seen": 2523856896 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023925252525252525, + "loss": 2.703, + "theoretical_loss": 3.3659647783141717, + "tokens_seen": 2523922432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023923232323232323, + "loss": 2.7265, + "theoretical_loss": 3.3659578200940548, + "tokens_seen": 2523987968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023921212121212122, + "loss": 2.7405, + "theoretical_loss": 3.3659508621051937, + "tokens_seen": 2524053504 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002391919191919192, + "loss": 2.7003, + "theoretical_loss": 3.365943904347575, + "tokens_seen": 2524119040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023917171717171719, + "loss": 2.7965, + "theoretical_loss": 3.3659369468211855, + "tokens_seen": 2524184576 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023915151515151514, + "loss": 2.6441, + "theoretical_loss": 3.3659299895260113, + "tokens_seen": 2524250112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023913131313131315, + "loss": 2.5258, + "theoretical_loss": 3.365923032462039, + "tokens_seen": 2524315648 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002391111111111111, + "loss": 2.6987, + "theoretical_loss": 3.365916075629254, + "tokens_seen": 2524381184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023909090909090912, + "loss": 2.6635, + "theoretical_loss": 3.365909119027643, + "tokens_seen": 2524446720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023907070707070708, + "loss": 2.6035, + "theoretical_loss": 3.3659021626571928, + "tokens_seen": 2524512256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023905050505050504, + "loss": 2.4848, + "theoretical_loss": 3.36589520651789, + "tokens_seen": 2524577792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023903030303030305, + "loss": 2.4361, + "theoretical_loss": 3.3658882506097196, + "tokens_seen": 2524643328 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239010101010101, + "loss": 2.7254, + "theoretical_loss": 3.3658812949326684, + "tokens_seen": 2524708864 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023898989898989901, + "loss": 2.662, + "theoretical_loss": 3.365874339486724, + "tokens_seen": 2524774400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023896969696969697, + "loss": 2.7091, + "theoretical_loss": 3.365867384271871, + "tokens_seen": 2524839936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023894949494949496, + "loss": 2.4473, + "theoretical_loss": 3.3658604292880963, + "tokens_seen": 2524905472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023892929292929294, + "loss": 2.6125, + "theoretical_loss": 3.3658534745353865, + "tokens_seen": 2524971008 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1384021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5116851329803467, + "objective/train/theoretical_loss": 3.3658465200137275, + "objective/train/tokens_used": 883895776, + "theoretical_loss": 3.3658465200137275, + "tokens_seen": 2525036544 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002389090909090909, + "loss": 2.7281, + "theoretical_loss": 3.3658465200137275, + "tokens_seen": 2525036544 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002388888888888889, + "loss": 2.4563, + "theoretical_loss": 3.3658395657231064, + "tokens_seen": 2525102080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023886868686868686, + "loss": 2.8797, + "theoretical_loss": 3.365832611663509, + "tokens_seen": 2525167616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023884848484848485, + "loss": 2.6666, + "theoretical_loss": 3.3658256578349213, + "tokens_seen": 2525233152 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023882828282828283, + "loss": 2.603, + "theoretical_loss": 3.36581870423733, + "tokens_seen": 2525298688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023880808080808082, + "loss": 2.7394, + "theoretical_loss": 3.365811750870722, + "tokens_seen": 2525364224 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002387878787878788, + "loss": 2.6197, + "theoretical_loss": 3.3658047977350822, + "tokens_seen": 2525429760 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023876767676767678, + "loss": 2.6502, + "theoretical_loss": 3.3657978448303982, + "tokens_seen": 2525495296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023874747474747474, + "loss": 2.6006, + "theoretical_loss": 3.365790892156656, + "tokens_seen": 2525560832 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023872727272727272, + "loss": 2.5446, + "theoretical_loss": 3.365783939713842, + "tokens_seen": 2525626368 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002387070707070707, + "loss": 2.5886, + "theoretical_loss": 3.365776987501942, + "tokens_seen": 2525691904 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002386868686868687, + "loss": 2.6032, + "theoretical_loss": 3.365770035520943, + "tokens_seen": 2525757440 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023866666666666668, + "loss": 2.5466, + "theoretical_loss": 3.365763083770831, + "tokens_seen": 2525822976 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023864646464646463, + "loss": 2.7014, + "theoretical_loss": 3.3657561322515925, + "tokens_seen": 2525888512 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023862626262626264, + "loss": 2.7514, + "theoretical_loss": 3.3657491809632134, + "tokens_seen": 2525954048 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002386060606060606, + "loss": 2.5629, + "theoretical_loss": 3.365742229905681, + "tokens_seen": 2526019584 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002385858585858586, + "loss": 2.7119, + "theoretical_loss": 3.3657352790789807, + "tokens_seen": 2526085120 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023856565656565657, + "loss": 2.6874, + "theoretical_loss": 3.365728328483099, + "tokens_seen": 2526150656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023854545454545453, + "loss": 2.5777, + "theoretical_loss": 3.365721378118023, + "tokens_seen": 2526216192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023852525252525254, + "loss": 2.6121, + "theoretical_loss": 3.365714427983738, + "tokens_seen": 2526281728 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002385050505050505, + "loss": 2.5176, + "theoretical_loss": 3.365707478080231, + "tokens_seen": 2526347264 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002384848484848485, + "loss": 2.6428, + "theoretical_loss": 3.3657005284074883, + "tokens_seen": 2526412800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023846464646464646, + "loss": 2.6427, + "theoretical_loss": 3.3656935789654963, + "tokens_seen": 2526478336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023844444444444445, + "loss": 2.5281, + "theoretical_loss": 3.3656866297542405, + "tokens_seen": 2526543872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023842424242424243, + "loss": 2.5407, + "theoretical_loss": 3.3656796807737086, + "tokens_seen": 2526609408 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1385247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5841684341430664, + "objective/train/theoretical_loss": 3.365672732023886, + "objective/train/tokens_used": 885534176, + "theoretical_loss": 3.365672732023886, + "tokens_seen": 2526674944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023840404040404041, + "loss": 2.5472, + "theoretical_loss": 3.365672732023886, + "tokens_seen": 2526674944 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002383838383838384, + "loss": 2.4669, + "theoretical_loss": 3.3656657835047596, + "tokens_seen": 2526740480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023836363636363635, + "loss": 2.6163, + "theoretical_loss": 3.3656588352163155, + "tokens_seen": 2526806016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023834343434343437, + "loss": 2.5835, + "theoretical_loss": 3.3656518871585397, + "tokens_seen": 2526871552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023832323232323232, + "loss": 2.4678, + "theoretical_loss": 3.3656449393314194, + "tokens_seen": 2526937088 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002383030303030303, + "loss": 2.7687, + "theoretical_loss": 3.36563799173494, + "tokens_seen": 2527002624 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002382828282828283, + "loss": 2.7377, + "theoretical_loss": 3.365631044369089, + "tokens_seen": 2527068160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023826262626262627, + "loss": 2.4071, + "theoretical_loss": 3.3656240972338516, + "tokens_seen": 2527133696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023824242424242426, + "loss": 2.517, + "theoretical_loss": 3.3656171503292147, + "tokens_seen": 2527199232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023822222222222224, + "loss": 2.6606, + "theoretical_loss": 3.365610203655165, + "tokens_seen": 2527264768 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002382020202020202, + "loss": 2.5508, + "theoretical_loss": 3.3656032572116885, + "tokens_seen": 2527330304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023818181818181818, + "loss": 2.5295, + "theoretical_loss": 3.365596310998771, + "tokens_seen": 2527395840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023816161616161617, + "loss": 2.6017, + "theoretical_loss": 3.3655893650164, + "tokens_seen": 2527461376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023814141414141415, + "loss": 2.3504, + "theoretical_loss": 3.3655824192645616, + "tokens_seen": 2527526912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023812121212121214, + "loss": 2.5375, + "theoretical_loss": 3.3655754737432413, + "tokens_seen": 2527592448 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002381010101010101, + "loss": 2.4184, + "theoretical_loss": 3.3655685284524264, + "tokens_seen": 2527657984 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002380808080808081, + "loss": 2.6848, + "theoretical_loss": 3.3655615833921027, + "tokens_seen": 2527723520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023806060606060606, + "loss": 2.5189, + "theoretical_loss": 3.365554638562257, + "tokens_seen": 2527789056 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023804040404040404, + "loss": 2.7523, + "theoretical_loss": 3.3655476939628755, + "tokens_seen": 2527854592 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023802020202020203, + "loss": 2.7487, + "theoretical_loss": 3.3655407495939444, + "tokens_seen": 2527920128 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023799999999999998, + "loss": 2.6504, + "theoretical_loss": 3.3655338054554504, + "tokens_seen": 2527985664 + }, + { + "epoch": 0.53, + "learning_rate": 0.000237979797979798, + "loss": 2.6846, + "theoretical_loss": 3.36552686154738, + "tokens_seen": 2528051200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023795959595959595, + "loss": 2.6029, + "theoretical_loss": 3.365519917869719, + "tokens_seen": 2528116736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023793939393939396, + "loss": 2.4627, + "theoretical_loss": 3.365512974422454, + "tokens_seen": 2528182272 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023791919191919192, + "loss": 2.6922, + "theoretical_loss": 3.365506031205572, + "tokens_seen": 2528247808 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1385945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7589683532714844, + "objective/train/theoretical_loss": 3.3654990882190585, + "objective/train/tokens_used": 887172576, + "theoretical_loss": 3.3654990882190585, + "tokens_seen": 2528313344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378989898989899, + "loss": 2.3046, + "theoretical_loss": 3.3654990882190585, + "tokens_seen": 2528313344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378787878787879, + "loss": 2.5149, + "theoretical_loss": 3.3654921454629, + "tokens_seen": 2528378880 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023785858585858585, + "loss": 2.5721, + "theoretical_loss": 3.3654852029370836, + "tokens_seen": 2528444416 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023783838383838386, + "loss": 2.8291, + "theoretical_loss": 3.365478260641595, + "tokens_seen": 2528509952 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378181818181818, + "loss": 2.3108, + "theoretical_loss": 3.365471318576421, + "tokens_seen": 2528575488 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002377979797979798, + "loss": 2.6363, + "theoretical_loss": 3.3654643767415475, + "tokens_seen": 2528641024 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023777777777777778, + "loss": 2.5172, + "theoretical_loss": 3.3654574351369617, + "tokens_seen": 2528706560 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023775757575757577, + "loss": 2.4522, + "theoretical_loss": 3.365450493762649, + "tokens_seen": 2528772096 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023773737373737375, + "loss": 2.5228, + "theoretical_loss": 3.365443552618596, + "tokens_seen": 2528837632 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023771717171717173, + "loss": 2.5641, + "theoretical_loss": 3.3654366117047902, + "tokens_seen": 2528903168 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002376969696969697, + "loss": 2.526, + "theoretical_loss": 3.365429671021217, + "tokens_seen": 2528968704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023767676767676767, + "loss": 2.3843, + "theoretical_loss": 3.365422730567863, + "tokens_seen": 2529034240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023765656565656566, + "loss": 2.517, + "theoretical_loss": 3.365415790344714, + "tokens_seen": 2529099776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023763636363636364, + "loss": 2.6491, + "theoretical_loss": 3.3654088503517574, + "tokens_seen": 2529165312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023761616161616163, + "loss": 2.5191, + "theoretical_loss": 3.365401910588979, + "tokens_seen": 2529230848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023759595959595958, + "loss": 2.6534, + "theoretical_loss": 3.3653949710563653, + "tokens_seen": 2529296384 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002375757575757576, + "loss": 2.557, + "theoretical_loss": 3.365388031753903, + "tokens_seen": 2529361920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023755555555555555, + "loss": 2.5502, + "theoretical_loss": 3.365381092681578, + "tokens_seen": 2529427456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023753535353535356, + "loss": 2.5511, + "theoretical_loss": 3.3653741538393778, + "tokens_seen": 2529492992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023751515151515152, + "loss": 2.6639, + "theoretical_loss": 3.365367215227287, + "tokens_seen": 2529558528 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002374949494949495, + "loss": 2.7265, + "theoretical_loss": 3.3653602768452937, + "tokens_seen": 2529624064 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023747474747474749, + "loss": 2.4818, + "theoretical_loss": 3.3653533386933834, + "tokens_seen": 2529689600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023745454545454544, + "loss": 2.5552, + "theoretical_loss": 3.3653464007715423, + "tokens_seen": 2529755136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023743434343434345, + "loss": 2.5522, + "theoretical_loss": 3.3653394630797577, + "tokens_seen": 2529820672 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002374141414141414, + "loss": 2.6629, + "theoretical_loss": 3.3653325256180153, + "tokens_seen": 2529886208 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1387296, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6340339183807373, + "objective/train/theoretical_loss": 3.365325588386302, + "objective/train/tokens_used": 888810976, + "theoretical_loss": 3.365325588386302, + "tokens_seen": 2529951744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023739393939393942, + "loss": 2.5533, + "theoretical_loss": 3.365325588386302, + "tokens_seen": 2529951744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023737373737373738, + "loss": 2.4233, + "theoretical_loss": 3.3653186513846034, + "tokens_seen": 2530017280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023735353535353534, + "loss": 2.5152, + "theoretical_loss": 3.365311714612907, + "tokens_seen": 2530082816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023733333333333335, + "loss": 2.6877, + "theoretical_loss": 3.3653047780711987, + "tokens_seen": 2530148352 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002373131313131313, + "loss": 2.588, + "theoretical_loss": 3.365297841759465, + "tokens_seen": 2530213888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023729292929292931, + "loss": 2.6193, + "theoretical_loss": 3.365290905677692, + "tokens_seen": 2530279424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023727272727272727, + "loss": 2.5231, + "theoretical_loss": 3.365283969825866, + "tokens_seen": 2530344960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023725252525252526, + "loss": 2.6264, + "theoretical_loss": 3.365277034203974, + "tokens_seen": 2530410496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023723232323232324, + "loss": 2.7149, + "theoretical_loss": 3.365270098812003, + "tokens_seen": 2530476032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023721212121212122, + "loss": 2.6112, + "theoretical_loss": 3.3652631636499377, + "tokens_seen": 2530541568 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002371919191919192, + "loss": 2.4873, + "theoretical_loss": 3.3652562287177656, + "tokens_seen": 2530607104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023717171717171716, + "loss": 2.5812, + "theoretical_loss": 3.3652492940154732, + "tokens_seen": 2530672640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023715151515151515, + "loss": 2.4426, + "theoretical_loss": 3.3652423595430467, + "tokens_seen": 2530738176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023713131313131313, + "loss": 2.5431, + "theoretical_loss": 3.3652354253004724, + "tokens_seen": 2530803712 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023711111111111112, + "loss": 2.5856, + "theoretical_loss": 3.365228491287737, + "tokens_seen": 2530869248 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002370909090909091, + "loss": 2.5204, + "theoretical_loss": 3.3652215575048268, + "tokens_seen": 2530934784 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023707070707070708, + "loss": 2.4103, + "theoretical_loss": 3.365214623951728, + "tokens_seen": 2531000320 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023705050505050504, + "loss": 2.552, + "theoretical_loss": 3.365207690628427, + "tokens_seen": 2531065856 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023703030303030305, + "loss": 2.6296, + "theoretical_loss": 3.365200757534911, + "tokens_seen": 2531131392 + }, + { + "epoch": 0.53, + "learning_rate": 0.000237010101010101, + "loss": 2.6525, + "theoretical_loss": 3.365193824671166, + "tokens_seen": 2531196928 + }, + { + "epoch": 0.53, + "learning_rate": 0.000236989898989899, + "loss": 2.5798, + "theoretical_loss": 3.365186892037178, + "tokens_seen": 2531262464 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023696969696969698, + "loss": 2.7886, + "theoretical_loss": 3.3651799596329344, + "tokens_seen": 2531328000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023694949494949493, + "loss": 2.634, + "theoretical_loss": 3.3651730274584204, + "tokens_seen": 2531393536 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023692929292929294, + "loss": 2.5433, + "theoretical_loss": 3.365166095513623, + "tokens_seen": 2531459072 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002369090909090909, + "loss": 2.6011, + "theoretical_loss": 3.365159163798529, + "tokens_seen": 2531524608 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1387695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.649080514907837, + "objective/train/theoretical_loss": 3.3651522323131244, + "objective/train/tokens_used": 890449376, + "theoretical_loss": 3.3651522323131244, + "tokens_seen": 2531590144 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002368888888888889, + "loss": 2.4917, + "theoretical_loss": 3.3651522323131244, + "tokens_seen": 2531590144 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023686868686868687, + "loss": 2.7032, + "theoretical_loss": 3.3651453010573964, + "tokens_seen": 2531655680 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023684848484848485, + "loss": 2.5129, + "theoretical_loss": 3.36513837003133, + "tokens_seen": 2531721216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023682828282828284, + "loss": 2.5074, + "theoretical_loss": 3.365131439234913, + "tokens_seen": 2531786752 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002368080808080808, + "loss": 2.8184, + "theoretical_loss": 3.3651245086681314, + "tokens_seen": 2531852288 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002367878787878788, + "loss": 2.6483, + "theoretical_loss": 3.3651175783309712, + "tokens_seen": 2531917824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023676767676767676, + "loss": 2.5097, + "theoretical_loss": 3.3651106482234194, + "tokens_seen": 2531983360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023674747474747475, + "loss": 2.7178, + "theoretical_loss": 3.3651037183454626, + "tokens_seen": 2532048896 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023672727272727273, + "loss": 2.9447, + "theoretical_loss": 3.3650967886970866, + "tokens_seen": 2532114432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023670707070707071, + "loss": 2.7031, + "theoretical_loss": 3.365089859278278, + "tokens_seen": 2532179968 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002366868686868687, + "loss": 2.4232, + "theoretical_loss": 3.365082930089024, + "tokens_seen": 2532245504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023666666666666668, + "loss": 2.7103, + "theoretical_loss": 3.36507600112931, + "tokens_seen": 2532311040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023664646464646464, + "loss": 2.4652, + "theoretical_loss": 3.3650690723991232, + "tokens_seen": 2532376576 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023662626262626262, + "loss": 2.4705, + "theoretical_loss": 3.36506214389845, + "tokens_seen": 2532442112 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002366060606060606, + "loss": 2.4316, + "theoretical_loss": 3.3650552156272764, + "tokens_seen": 2532507648 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002365858585858586, + "loss": 2.5798, + "theoretical_loss": 3.3650482875855894, + "tokens_seen": 2532573184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023656565656565657, + "loss": 2.7893, + "theoretical_loss": 3.365041359773375, + "tokens_seen": 2532638720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023654545454545456, + "loss": 2.6254, + "theoretical_loss": 3.36503443219062, + "tokens_seen": 2532704256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023652525252525254, + "loss": 2.7588, + "theoretical_loss": 3.36502750483731, + "tokens_seen": 2532769792 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002365050505050505, + "loss": 2.5603, + "theoretical_loss": 3.365020577713433, + "tokens_seen": 2532835328 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002364848484848485, + "loss": 2.5845, + "theoretical_loss": 3.365013650818974, + "tokens_seen": 2532900864 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023646464646464647, + "loss": 2.533, + "theoretical_loss": 3.3650067241539205, + "tokens_seen": 2532966400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023644444444444445, + "loss": 2.5883, + "theoretical_loss": 3.3649997977182586, + "tokens_seen": 2533031936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023642424242424244, + "loss": 2.7452, + "theoretical_loss": 3.3649928715119746, + "tokens_seen": 2533097472 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002364040404040404, + "loss": 2.6755, + "theoretical_loss": 3.3649859455350555, + "tokens_seen": 2533163008 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1388555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5344579219818115, + "objective/train/theoretical_loss": 3.364979019787487, + "objective/train/tokens_used": 892087776, + "theoretical_loss": 3.364979019787487, + "tokens_seen": 2533228544 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002363838383838384, + "loss": 2.6651, + "theoretical_loss": 3.364979019787487, + "tokens_seen": 2533228544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023636363636363636, + "loss": 2.5549, + "theoretical_loss": 3.3649720942692563, + "tokens_seen": 2533294080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023634343434343437, + "loss": 2.4487, + "theoretical_loss": 3.364965168980349, + "tokens_seen": 2533359616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023632323232323233, + "loss": 2.6822, + "theoretical_loss": 3.3649582439207526, + "tokens_seen": 2533425152 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023630303030303028, + "loss": 2.7001, + "theoretical_loss": 3.3649513190904528, + "tokens_seen": 2533490688 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002362828282828283, + "loss": 2.6022, + "theoretical_loss": 3.364944394489436, + "tokens_seen": 2533556224 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023626262626262625, + "loss": 2.5296, + "theoretical_loss": 3.36493747011769, + "tokens_seen": 2533621760 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023624242424242426, + "loss": 2.5767, + "theoretical_loss": 3.3649305459751995, + "tokens_seen": 2533687296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023622222222222222, + "loss": 2.7167, + "theoretical_loss": 3.3649236220619523, + "tokens_seen": 2533752832 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002362020202020202, + "loss": 2.5037, + "theoretical_loss": 3.364916698377934, + "tokens_seen": 2533818368 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002361818181818182, + "loss": 2.6725, + "theoretical_loss": 3.364909774923132, + "tokens_seen": 2533883904 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023616161616161617, + "loss": 2.4854, + "theoretical_loss": 3.3649028516975314, + "tokens_seen": 2533949440 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023614141414141416, + "loss": 2.5974, + "theoretical_loss": 3.36489592870112, + "tokens_seen": 2534014976 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002361212121212121, + "loss": 2.7305, + "theoretical_loss": 3.364889005933884, + "tokens_seen": 2534080512 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002361010101010101, + "loss": 2.5157, + "theoretical_loss": 3.364882083395809, + "tokens_seen": 2534146048 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023608080808080808, + "loss": 2.4699, + "theoretical_loss": 3.364875161086883, + "tokens_seen": 2534211584 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023606060606060607, + "loss": 2.561, + "theoretical_loss": 3.364868239007091, + "tokens_seen": 2534277120 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023604040404040405, + "loss": 2.7101, + "theoretical_loss": 3.3648613171564206, + "tokens_seen": 2534342656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023602020202020203, + "loss": 2.3929, + "theoretical_loss": 3.364854395534858, + "tokens_seen": 2534408192 + }, + { + "epoch": 0.53, + "learning_rate": 0.000236, + "loss": 2.8132, + "theoretical_loss": 3.3648474741423895, + "tokens_seen": 2534473728 + }, + { + "epoch": 0.53, + "learning_rate": 0.000235979797979798, + "loss": 2.3725, + "theoretical_loss": 3.3648405529790013, + "tokens_seen": 2534539264 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023595959595959596, + "loss": 2.4755, + "theoretical_loss": 3.36483363204468, + "tokens_seen": 2534604800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023593939393939394, + "loss": 2.5545, + "theoretical_loss": 3.364826711339413, + "tokens_seen": 2534670336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023591919191919193, + "loss": 2.3091, + "theoretical_loss": 3.364819790863186, + "tokens_seen": 2534735872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023589898989898988, + "loss": 2.6303, + "theoretical_loss": 3.364812870615985, + "tokens_seen": 2534801408 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1389642, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.644278049468994, + "objective/train/theoretical_loss": 3.364805950597798, + "objective/train/tokens_used": 893726176, + "theoretical_loss": 3.364805950597798, + "tokens_seen": 2534866944 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002358787878787879, + "loss": 2.6679, + "theoretical_loss": 3.364805950597798, + "tokens_seen": 2534866944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023585858585858585, + "loss": 2.5396, + "theoretical_loss": 3.3647990308086104, + "tokens_seen": 2534932480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023583838383838386, + "loss": 2.66, + "theoretical_loss": 3.3647921112484087, + "tokens_seen": 2534998016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023581818181818182, + "loss": 2.5682, + "theoretical_loss": 3.36478519191718, + "tokens_seen": 2535063552 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357979797979798, + "loss": 2.6106, + "theoretical_loss": 3.36477827281491, + "tokens_seen": 2535129088 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357777777777778, + "loss": 2.5681, + "theoretical_loss": 3.3647713539415856, + "tokens_seen": 2535194624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023575757575757574, + "loss": 2.5381, + "theoretical_loss": 3.364764435297194, + "tokens_seen": 2535260160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023573737373737375, + "loss": 2.5639, + "theoretical_loss": 3.3647575168817205, + "tokens_seen": 2535325696 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357171717171717, + "loss": 2.4618, + "theoretical_loss": 3.364750598695152, + "tokens_seen": 2535391232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023569696969696972, + "loss": 2.6874, + "theoretical_loss": 3.3647436807374755, + "tokens_seen": 2535456768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023567676767676768, + "loss": 2.4307, + "theoretical_loss": 3.3647367630086773, + "tokens_seen": 2535522304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023565656565656566, + "loss": 2.8406, + "theoretical_loss": 3.3647298455087435, + "tokens_seen": 2535587840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023563636363636365, + "loss": 2.7256, + "theoretical_loss": 3.364722928237661, + "tokens_seen": 2535653376 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002356161616161616, + "loss": 2.6546, + "theoretical_loss": 3.3647160111954166, + "tokens_seen": 2535718912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023559595959595962, + "loss": 2.6839, + "theoretical_loss": 3.364709094381996, + "tokens_seen": 2535784448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023557575757575757, + "loss": 2.7654, + "theoretical_loss": 3.364702177797386, + "tokens_seen": 2535849984 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023555555555555556, + "loss": 2.534, + "theoretical_loss": 3.364695261441574, + "tokens_seen": 2535915520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023553535353535354, + "loss": 2.4804, + "theoretical_loss": 3.364688345314545, + "tokens_seen": 2535981056 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023551515151515152, + "loss": 2.7425, + "theoretical_loss": 3.364681429416287, + "tokens_seen": 2536046592 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002354949494949495, + "loss": 2.6467, + "theoretical_loss": 3.3646745137467855, + "tokens_seen": 2536112128 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002354747474747475, + "loss": 2.546, + "theoretical_loss": 3.3646675983060272, + "tokens_seen": 2536177664 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023545454545454545, + "loss": 2.7171, + "theoretical_loss": 3.364660683093999, + "tokens_seen": 2536243200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023543434343434343, + "loss": 2.55, + "theoretical_loss": 3.364653768110687, + "tokens_seen": 2536308736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023541414141414142, + "loss": 2.4196, + "theoretical_loss": 3.3646468533560787, + "tokens_seen": 2536374272 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002353939393939394, + "loss": 2.7091, + "theoretical_loss": 3.364639938830159, + "tokens_seen": 2536439808 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1390171, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7468748092651367, + "objective/train/theoretical_loss": 3.3646330245329157, + "objective/train/tokens_used": 895364576, + "theoretical_loss": 3.3646330245329157, + "tokens_seen": 2536505344 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023537373737373738, + "loss": 2.6023, + "theoretical_loss": 3.3646330245329157, + "tokens_seen": 2536505344 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023535353535353534, + "loss": 2.8589, + "theoretical_loss": 3.364626110464335, + "tokens_seen": 2536570880 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023533333333333335, + "loss": 2.5933, + "theoretical_loss": 3.364619196624403, + "tokens_seen": 2536636416 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002353131313131313, + "loss": 2.6814, + "theoretical_loss": 3.3646122830131064, + "tokens_seen": 2536701952 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023529292929292932, + "loss": 2.6057, + "theoretical_loss": 3.3646053696304326, + "tokens_seen": 2536767488 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023527272727272728, + "loss": 2.6537, + "theoretical_loss": 3.364598456476367, + "tokens_seen": 2536833024 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023525252525252523, + "loss": 2.5661, + "theoretical_loss": 3.3645915435508966, + "tokens_seen": 2536898560 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023523232323232325, + "loss": 2.6555, + "theoretical_loss": 3.364584630854008, + "tokens_seen": 2536964096 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002352121212121212, + "loss": 2.6126, + "theoretical_loss": 3.3645777183856875, + "tokens_seen": 2537029632 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002351919191919192, + "loss": 2.7495, + "theoretical_loss": 3.364570806145922, + "tokens_seen": 2537095168 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023517171717171717, + "loss": 2.2758, + "theoretical_loss": 3.364563894134698, + "tokens_seen": 2537160704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023515151515151515, + "loss": 2.4934, + "theoretical_loss": 3.3645569823520014, + "tokens_seen": 2537226240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023513131313131314, + "loss": 2.5878, + "theoretical_loss": 3.3645500707978195, + "tokens_seen": 2537291776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023511111111111112, + "loss": 2.6998, + "theoretical_loss": 3.3645431594721384, + "tokens_seen": 2537357312 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002350909090909091, + "loss": 2.7549, + "theoretical_loss": 3.364536248374945, + "tokens_seen": 2537422848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023507070707070706, + "loss": 2.6067, + "theoretical_loss": 3.3645293375062253, + "tokens_seen": 2537488384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023505050505050505, + "loss": 2.4318, + "theoretical_loss": 3.3645224268659666, + "tokens_seen": 2537553920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023503030303030303, + "loss": 2.494, + "theoretical_loss": 3.364515516454155, + "tokens_seen": 2537619456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023501010101010101, + "loss": 2.6901, + "theoretical_loss": 3.364508606270777, + "tokens_seen": 2537684992 + }, + { + "epoch": 0.53, + "learning_rate": 0.000234989898989899, + "loss": 2.904, + "theoretical_loss": 3.364501696315819, + "tokens_seen": 2537750528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023496969696969698, + "loss": 2.6135, + "theoretical_loss": 3.364494786589268, + "tokens_seen": 2537816064 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023494949494949494, + "loss": 2.5393, + "theoretical_loss": 3.3644878770911104, + "tokens_seen": 2537881600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023492929292929295, + "loss": 2.6888, + "theoretical_loss": 3.3644809678213328, + "tokens_seen": 2537947136 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002349090909090909, + "loss": 2.6243, + "theoretical_loss": 3.364474058779922, + "tokens_seen": 2538012672 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002348888888888889, + "loss": 2.6966, + "theoretical_loss": 3.3644671499668632, + "tokens_seen": 2538078208 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1391171, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3983640670776367, + "objective/train/theoretical_loss": 3.3644602413821447, + "objective/train/tokens_used": 897002976, + "theoretical_loss": 3.3644602413821447, + "tokens_seen": 2538143744 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023486868686868688, + "loss": 2.3945, + "theoretical_loss": 3.3644602413821447, + "tokens_seen": 2538143744 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023484848484848486, + "loss": 2.8113, + "theoretical_loss": 3.364453333025752, + "tokens_seen": 2538209280 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023482828282828284, + "loss": 2.576, + "theoretical_loss": 3.3644464248976718, + "tokens_seen": 2538274816 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002348080808080808, + "loss": 2.6328, + "theoretical_loss": 3.364439516997891, + "tokens_seen": 2538340352 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002347878787878788, + "loss": 2.6638, + "theoretical_loss": 3.3644326093263963, + "tokens_seen": 2538405888 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023476767676767677, + "loss": 2.5303, + "theoretical_loss": 3.3644257018831736, + "tokens_seen": 2538471424 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023474747474747475, + "loss": 2.5935, + "theoretical_loss": 3.36441879466821, + "tokens_seen": 2538536960 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023472727272727274, + "loss": 2.8733, + "theoretical_loss": 3.3644118876814924, + "tokens_seen": 2538602496 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002347070707070707, + "loss": 2.5653, + "theoretical_loss": 3.364404980923006, + "tokens_seen": 2538668032 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002346868686868687, + "loss": 2.5322, + "theoretical_loss": 3.3643980743927386, + "tokens_seen": 2538733568 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023466666666666666, + "loss": 2.6673, + "theoretical_loss": 3.3643911680906764, + "tokens_seen": 2538799104 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023464646464646467, + "loss": 2.4581, + "theoretical_loss": 3.364384262016806, + "tokens_seen": 2538864640 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023462626262626263, + "loss": 2.4531, + "theoretical_loss": 3.3643773561711137, + "tokens_seen": 2538930176 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002346060606060606, + "loss": 2.5356, + "theoretical_loss": 3.3643704505535865, + "tokens_seen": 2538995712 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002345858585858586, + "loss": 2.3763, + "theoretical_loss": 3.3643635451642107, + "tokens_seen": 2539061248 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023456565656565655, + "loss": 2.5074, + "theoretical_loss": 3.364356640002973, + "tokens_seen": 2539126784 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023454545454545456, + "loss": 2.5965, + "theoretical_loss": 3.36434973506986, + "tokens_seen": 2539192320 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023452525252525252, + "loss": 2.608, + "theoretical_loss": 3.364342830364858, + "tokens_seen": 2539257856 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002345050505050505, + "loss": 2.6128, + "theoretical_loss": 3.364335925887954, + "tokens_seen": 2539323392 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002344848484848485, + "loss": 2.8063, + "theoretical_loss": 3.364329021639134, + "tokens_seen": 2539388928 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023446464646464647, + "loss": 2.6855, + "theoretical_loss": 3.3643221176183853, + "tokens_seen": 2539454464 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023444444444444446, + "loss": 2.5181, + "theoretical_loss": 3.364315213825694, + "tokens_seen": 2539520000 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023442424242424244, + "loss": 2.7349, + "theoretical_loss": 3.364308310261047, + "tokens_seen": 2539585536 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002344040404040404, + "loss": 2.6431, + "theoretical_loss": 3.3643014069244304, + "tokens_seen": 2539651072 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023438383838383838, + "loss": 2.5689, + "theoretical_loss": 3.364294503815831, + "tokens_seen": 2539716608 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1391800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2752268314361572, + "objective/train/theoretical_loss": 3.3642876009352354, + "objective/train/tokens_used": 898641376, + "theoretical_loss": 3.3642876009352354, + "tokens_seen": 2539782144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023436363636363637, + "loss": 2.566, + "theoretical_loss": 3.3642876009352354, + "tokens_seen": 2539782144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023434343434343435, + "loss": 2.4836, + "theoretical_loss": 3.3642806982826308, + "tokens_seen": 2539847680 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023432323232323233, + "loss": 2.7731, + "theoretical_loss": 3.3642737958580025, + "tokens_seen": 2539913216 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002343030303030303, + "loss": 2.4425, + "theoretical_loss": 3.364266893661338, + "tokens_seen": 2539978752 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002342828282828283, + "loss": 2.6432, + "theoretical_loss": 3.3642599916926237, + "tokens_seen": 2540044288 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023426262626262626, + "loss": 2.5975, + "theoretical_loss": 3.3642530899518466, + "tokens_seen": 2540109824 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023424242424242427, + "loss": 2.5968, + "theoretical_loss": 3.3642461884389925, + "tokens_seen": 2540175360 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023422222222222223, + "loss": 2.5379, + "theoretical_loss": 3.3642392871540485, + "tokens_seen": 2540240896 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023420202020202018, + "loss": 2.574, + "theoretical_loss": 3.364232386097001, + "tokens_seen": 2540306432 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002341818181818182, + "loss": 2.6247, + "theoretical_loss": 3.364225485267837, + "tokens_seen": 2540371968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023416161616161615, + "loss": 2.6482, + "theoretical_loss": 3.364218584666542, + "tokens_seen": 2540437504 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023414141414141416, + "loss": 2.7569, + "theoretical_loss": 3.364211684293104, + "tokens_seen": 2540503040 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023412121212121212, + "loss": 2.6188, + "theoretical_loss": 3.3642047841475087, + "tokens_seen": 2540568576 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002341010101010101, + "loss": 2.5926, + "theoretical_loss": 3.364197884229743, + "tokens_seen": 2540634112 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002340808080808081, + "loss": 2.6181, + "theoretical_loss": 3.3641909845397935, + "tokens_seen": 2540699648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023406060606060604, + "loss": 2.5699, + "theoretical_loss": 3.364184085077647, + "tokens_seen": 2540765184 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023404040404040405, + "loss": 2.471, + "theoretical_loss": 3.3641771858432894, + "tokens_seen": 2540830720 + }, + { + "epoch": 0.54, + "learning_rate": 0.000234020202020202, + "loss": 2.5748, + "theoretical_loss": 3.364170286836708, + "tokens_seen": 2540896256 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023400000000000002, + "loss": 2.5245, + "theoretical_loss": 3.3641633880578894, + "tokens_seen": 2540961792 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023397979797979798, + "loss": 2.7463, + "theoretical_loss": 3.3641564895068194, + "tokens_seen": 2541027328 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023395959595959596, + "loss": 2.4495, + "theoretical_loss": 3.3641495911834856, + "tokens_seen": 2541092864 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023393939393939395, + "loss": 2.62, + "theoretical_loss": 3.3641426930878744, + "tokens_seen": 2541158400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023391919191919193, + "loss": 2.5062, + "theoretical_loss": 3.3641357952199717, + "tokens_seen": 2541223936 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023389898989898992, + "loss": 2.3945, + "theoretical_loss": 3.3641288975797647, + "tokens_seen": 2541289472 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023387878787878787, + "loss": 2.5318, + "theoretical_loss": 3.36412200016724, + "tokens_seen": 2541355008 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1392945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5443549156188965, + "objective/train/theoretical_loss": 3.3641151029823844, + "objective/train/tokens_used": 900279776, + "theoretical_loss": 3.3641151029823844, + "tokens_seen": 2541420544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023385858585858586, + "loss": 2.7348, + "theoretical_loss": 3.3641151029823844, + "tokens_seen": 2541420544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023383838383838384, + "loss": 2.5947, + "theoretical_loss": 3.364108206025184, + "tokens_seen": 2541486080 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023381818181818182, + "loss": 2.533, + "theoretical_loss": 3.3641013092956253, + "tokens_seen": 2541551616 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002337979797979798, + "loss": 2.5768, + "theoretical_loss": 3.364094412793696, + "tokens_seen": 2541617152 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002337777777777778, + "loss": 2.6141, + "theoretical_loss": 3.3640875165193815, + "tokens_seen": 2541682688 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023375757575757575, + "loss": 2.5833, + "theoretical_loss": 3.364080620472669, + "tokens_seen": 2541748224 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023373737373737376, + "loss": 2.7621, + "theoretical_loss": 3.3640737246535455, + "tokens_seen": 2541813760 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023371717171717172, + "loss": 2.7913, + "theoretical_loss": 3.364066829061997, + "tokens_seen": 2541879296 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002336969696969697, + "loss": 2.5465, + "theoretical_loss": 3.36405993369801, + "tokens_seen": 2541944832 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023367676767676768, + "loss": 2.4993, + "theoretical_loss": 3.3640530385615715, + "tokens_seen": 2542010368 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023365656565656564, + "loss": 2.4185, + "theoretical_loss": 3.364046143652668, + "tokens_seen": 2542075904 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023363636363636365, + "loss": 2.6962, + "theoretical_loss": 3.3640392489712863, + "tokens_seen": 2542141440 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002336161616161616, + "loss": 2.6121, + "theoretical_loss": 3.3640323545174127, + "tokens_seen": 2542206976 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023359595959595962, + "loss": 2.6012, + "theoretical_loss": 3.3640254602910344, + "tokens_seen": 2542272512 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023357575757575758, + "loss": 2.5755, + "theoretical_loss": 3.3640185662921374, + "tokens_seen": 2542338048 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023355555555555556, + "loss": 2.4853, + "theoretical_loss": 3.3640116725207085, + "tokens_seen": 2542403584 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023353535353535355, + "loss": 2.3537, + "theoretical_loss": 3.3640047789767342, + "tokens_seen": 2542469120 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002335151515151515, + "loss": 2.7106, + "theoretical_loss": 3.363997885660202, + "tokens_seen": 2542534656 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002334949494949495, + "loss": 2.5192, + "theoretical_loss": 3.3639909925710976, + "tokens_seen": 2542600192 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023347474747474747, + "loss": 2.6282, + "theoretical_loss": 3.3639840997094077, + "tokens_seen": 2542665728 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023345454545454545, + "loss": 2.5555, + "theoretical_loss": 3.363977207075119, + "tokens_seen": 2542731264 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023343434343434344, + "loss": 2.4653, + "theoretical_loss": 3.3639703146682187, + "tokens_seen": 2542796800 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023341414141414142, + "loss": 2.6739, + "theoretical_loss": 3.363963422488693, + "tokens_seen": 2542862336 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002333939393939394, + "loss": 2.6557, + "theoretical_loss": 3.3639565305365284, + "tokens_seen": 2542927872 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002333737373737374, + "loss": 2.6826, + "theoretical_loss": 3.363949638811712, + "tokens_seen": 2542993408 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1394002, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.704057455062866, + "objective/train/theoretical_loss": 3.36394274731423, + "objective/train/tokens_used": 901918176, + "theoretical_loss": 3.36394274731423, + "tokens_seen": 2543058944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023335353535353535, + "loss": 2.6489, + "theoretical_loss": 3.36394274731423, + "tokens_seen": 2543058944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023333333333333333, + "loss": 2.4723, + "theoretical_loss": 3.3639358560440686, + "tokens_seen": 2543124480 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023331313131313131, + "loss": 2.9125, + "theoretical_loss": 3.3639289650012154, + "tokens_seen": 2543190016 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002332929292929293, + "loss": 2.5235, + "theoretical_loss": 3.363922074185657, + "tokens_seen": 2543255552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023327272727272728, + "loss": 2.5792, + "theoretical_loss": 3.3639151835973795, + "tokens_seen": 2543321088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023325252525252524, + "loss": 2.5602, + "theoretical_loss": 3.3639082932363698, + "tokens_seen": 2543386624 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023323232323232325, + "loss": 2.4142, + "theoretical_loss": 3.363901403102614, + "tokens_seen": 2543452160 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002332121212121212, + "loss": 2.5644, + "theoretical_loss": 3.3638945131961, + "tokens_seen": 2543517696 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002331919191919192, + "loss": 2.6675, + "theoretical_loss": 3.3638876235168134, + "tokens_seen": 2543583232 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023317171717171718, + "loss": 2.635, + "theoretical_loss": 3.3638807340647414, + "tokens_seen": 2543648768 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023315151515151516, + "loss": 2.4709, + "theoretical_loss": 3.36387384483987, + "tokens_seen": 2543714304 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023313131313131314, + "loss": 2.7813, + "theoretical_loss": 3.3638669558421865, + "tokens_seen": 2543779840 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002331111111111111, + "loss": 2.5628, + "theoretical_loss": 3.3638600670716774, + "tokens_seen": 2543845376 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002330909090909091, + "loss": 2.4428, + "theoretical_loss": 3.363853178528329, + "tokens_seen": 2543910912 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023307070707070707, + "loss": 2.5687, + "theoretical_loss": 3.3638462902121287, + "tokens_seen": 2543976448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023305050505050508, + "loss": 2.3504, + "theoretical_loss": 3.3638394021230624, + "tokens_seen": 2544041984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023303030303030304, + "loss": 2.5489, + "theoretical_loss": 3.363832514261117, + "tokens_seen": 2544107520 + }, + { + "epoch": 0.54, + "learning_rate": 0.000233010101010101, + "loss": 2.6251, + "theoretical_loss": 3.3638256266262796, + "tokens_seen": 2544173056 + }, + { + "epoch": 0.54, + "learning_rate": 0.000232989898989899, + "loss": 2.6629, + "theoretical_loss": 3.3638187392185364, + "tokens_seen": 2544238592 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023296969696969696, + "loss": 2.6492, + "theoretical_loss": 3.363811852037874, + "tokens_seen": 2544304128 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023294949494949497, + "loss": 2.6118, + "theoretical_loss": 3.363804965084279, + "tokens_seen": 2544369664 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023292929292929293, + "loss": 2.566, + "theoretical_loss": 3.3637980783577386, + "tokens_seen": 2544435200 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002329090909090909, + "loss": 2.7243, + "theoretical_loss": 3.363791191858239, + "tokens_seen": 2544500736 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002328888888888889, + "loss": 2.6767, + "theoretical_loss": 3.3637843055857672, + "tokens_seen": 2544566272 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023286868686868688, + "loss": 2.538, + "theoretical_loss": 3.3637774195403094, + "tokens_seen": 2544631808 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1394146, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.181093454360962, + "objective/train/theoretical_loss": 3.3637705337218526, + "objective/train/tokens_used": 903556576, + "theoretical_loss": 3.3637705337218526, + "tokens_seen": 2544697344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023284848484848486, + "loss": 2.4151, + "theoretical_loss": 3.3637705337218526, + "tokens_seen": 2544697344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023282828282828282, + "loss": 2.7687, + "theoretical_loss": 3.3637636481303836, + "tokens_seen": 2544762880 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002328080808080808, + "loss": 2.4268, + "theoretical_loss": 3.3637567627658886, + "tokens_seen": 2544828416 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002327878787878788, + "loss": 2.7664, + "theoretical_loss": 3.3637498776283548, + "tokens_seen": 2544893952 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023276767676767677, + "loss": 2.4098, + "theoretical_loss": 3.3637429927177687, + "tokens_seen": 2544959488 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023274747474747476, + "loss": 2.5337, + "theoretical_loss": 3.363736108034117, + "tokens_seen": 2545025024 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023272727272727274, + "loss": 2.5796, + "theoretical_loss": 3.363729223577386, + "tokens_seen": 2545090560 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002327070707070707, + "loss": 2.6336, + "theoretical_loss": 3.3637223393475626, + "tokens_seen": 2545156096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002326868686868687, + "loss": 2.6943, + "theoretical_loss": 3.363715455344634, + "tokens_seen": 2545221632 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023266666666666667, + "loss": 2.609, + "theoretical_loss": 3.363708571568586, + "tokens_seen": 2545287168 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023264646464646465, + "loss": 2.392, + "theoretical_loss": 3.3637016880194057, + "tokens_seen": 2545352704 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023262626262626263, + "loss": 2.6878, + "theoretical_loss": 3.3636948046970803, + "tokens_seen": 2545418240 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002326060606060606, + "loss": 2.5737, + "theoretical_loss": 3.3636879216015956, + "tokens_seen": 2545483776 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002325858585858586, + "loss": 2.7241, + "theoretical_loss": 3.3636810387329383, + "tokens_seen": 2545549312 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023256565656565656, + "loss": 2.5932, + "theoretical_loss": 3.363674156091096, + "tokens_seen": 2545614848 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023254545454545457, + "loss": 2.6117, + "theoretical_loss": 3.3636672736760547, + "tokens_seen": 2545680384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023252525252525253, + "loss": 2.5194, + "theoretical_loss": 3.363660391487801, + "tokens_seen": 2545745920 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023250505050505048, + "loss": 2.7884, + "theoretical_loss": 3.3636535095263222, + "tokens_seen": 2545811456 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002324848484848485, + "loss": 2.5799, + "theoretical_loss": 3.3636466277916046, + "tokens_seen": 2545876992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023246464646464645, + "loss": 2.6815, + "theoretical_loss": 3.3636397462836345, + "tokens_seen": 2545942528 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023244444444444446, + "loss": 2.5922, + "theoretical_loss": 3.3636328650023994, + "tokens_seen": 2546008064 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023242424242424242, + "loss": 2.7509, + "theoretical_loss": 3.3636259839478853, + "tokens_seen": 2546073600 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002324040404040404, + "loss": 2.8669, + "theoretical_loss": 3.363619103120079, + "tokens_seen": 2546139136 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002323838383838384, + "loss": 2.3998, + "theoretical_loss": 3.3636122225189675, + "tokens_seen": 2546204672 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023236363636363637, + "loss": 2.4883, + "theoretical_loss": 3.3636053421445378, + "tokens_seen": 2546270208 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1395112, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8437981605529785, + "objective/train/theoretical_loss": 3.3635984619967756, + "objective/train/tokens_used": 905194976, + "theoretical_loss": 3.3635984619967756, + "tokens_seen": 2546335744 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023234343434343436, + "loss": 2.6607, + "theoretical_loss": 3.3635984619967756, + "tokens_seen": 2546335744 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002323232323232323, + "loss": 2.4318, + "theoretical_loss": 3.3635915820756686, + "tokens_seen": 2546401280 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023230303030303032, + "loss": 2.5522, + "theoretical_loss": 3.363584702381203, + "tokens_seen": 2546466816 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023228282828282828, + "loss": 2.8892, + "theoretical_loss": 3.3635778229133653, + "tokens_seen": 2546532352 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023226262626262626, + "loss": 2.6612, + "theoretical_loss": 3.3635709436721424, + "tokens_seen": 2546597888 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023224242424242425, + "loss": 2.6752, + "theoretical_loss": 3.3635640646575213, + "tokens_seen": 2546663424 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023222222222222223, + "loss": 2.61, + "theoretical_loss": 3.3635571858694884, + "tokens_seen": 2546728960 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023220202020202022, + "loss": 2.7056, + "theoretical_loss": 3.3635503073080306, + "tokens_seen": 2546794496 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002321818181818182, + "loss": 2.4905, + "theoretical_loss": 3.3635434289731343, + "tokens_seen": 2546860032 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023216161616161616, + "loss": 2.4825, + "theoretical_loss": 3.3635365508647865, + "tokens_seen": 2546925568 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023214141414141414, + "loss": 2.6878, + "theoretical_loss": 3.363529672982974, + "tokens_seen": 2546991104 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023212121212121212, + "loss": 2.5733, + "theoretical_loss": 3.363522795327683, + "tokens_seen": 2547056640 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002321010101010101, + "loss": 2.5229, + "theoretical_loss": 3.3635159178989005, + "tokens_seen": 2547122176 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002320808080808081, + "loss": 2.365, + "theoretical_loss": 3.3635090406966133, + "tokens_seen": 2547187712 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023206060606060605, + "loss": 2.5908, + "theoretical_loss": 3.363502163720808, + "tokens_seen": 2547253248 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023204040404040406, + "loss": 2.6327, + "theoretical_loss": 3.363495286971472, + "tokens_seen": 2547318784 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023202020202020202, + "loss": 2.5876, + "theoretical_loss": 3.3634884104485905, + "tokens_seen": 2547384320 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023200000000000003, + "loss": 2.6502, + "theoretical_loss": 3.3634815341521516, + "tokens_seen": 2547449856 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023197979797979799, + "loss": 2.5526, + "theoretical_loss": 3.3634746580821413, + "tokens_seen": 2547515392 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023195959595959594, + "loss": 2.4623, + "theoretical_loss": 3.3634677822385464, + "tokens_seen": 2547580928 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023193939393939395, + "loss": 2.589, + "theoretical_loss": 3.363460906621354, + "tokens_seen": 2547646464 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002319191919191919, + "loss": 2.5597, + "theoretical_loss": 3.3634540312305505, + "tokens_seen": 2547712000 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023189898989898992, + "loss": 2.5301, + "theoretical_loss": 3.3634471560661225, + "tokens_seen": 2547777536 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023187878787878788, + "loss": 2.6186, + "theoretical_loss": 3.3634402811280575, + "tokens_seen": 2547843072 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023185858585858586, + "loss": 2.7319, + "theoretical_loss": 3.363433406416341, + "tokens_seen": 2547908608 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1395695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3248932361602783, + "objective/train/theoretical_loss": 3.363426531930961, + "objective/train/tokens_used": 906833376, + "theoretical_loss": 3.363426531930961, + "tokens_seen": 2547974144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023183838383838385, + "loss": 2.5215, + "theoretical_loss": 3.363426531930961, + "tokens_seen": 2547974144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023181818181818183, + "loss": 2.602, + "theoretical_loss": 3.363419657671903, + "tokens_seen": 2548039680 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023179797979797981, + "loss": 2.3246, + "theoretical_loss": 3.3634127836391547, + "tokens_seen": 2548105216 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023177777777777777, + "loss": 2.7113, + "theoretical_loss": 3.3634059098327023, + "tokens_seen": 2548170752 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023175757575757575, + "loss": 2.6343, + "theoretical_loss": 3.363399036252533, + "tokens_seen": 2548236288 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023173737373737374, + "loss": 2.5076, + "theoretical_loss": 3.363392162898633, + "tokens_seen": 2548301824 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023171717171717172, + "loss": 2.7902, + "theoretical_loss": 3.363385289770989, + "tokens_seen": 2548367360 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002316969696969697, + "loss": 2.7543, + "theoretical_loss": 3.363378416869588, + "tokens_seen": 2548432896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002316767676767677, + "loss": 2.8748, + "theoretical_loss": 3.3633715441944174, + "tokens_seen": 2548498432 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023165656565656565, + "loss": 2.529, + "theoretical_loss": 3.3633646717454626, + "tokens_seen": 2548563968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023163636363636366, + "loss": 2.5737, + "theoretical_loss": 3.3633577995227113, + "tokens_seen": 2548629504 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023161616161616162, + "loss": 2.6683, + "theoretical_loss": 3.3633509275261497, + "tokens_seen": 2548695040 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002315959595959596, + "loss": 2.5447, + "theoretical_loss": 3.363344055755765, + "tokens_seen": 2548760576 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023157575757575758, + "loss": 2.5594, + "theoretical_loss": 3.363337184211544, + "tokens_seen": 2548826112 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023155555555555554, + "loss": 2.6081, + "theoretical_loss": 3.3633303128934724, + "tokens_seen": 2548891648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023153535353535355, + "loss": 2.683, + "theoretical_loss": 3.3633234418015383, + "tokens_seen": 2548957184 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002315151515151515, + "loss": 2.4854, + "theoretical_loss": 3.3633165709357273, + "tokens_seen": 2549022720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023149494949494952, + "loss": 2.5795, + "theoretical_loss": 3.3633097002960275, + "tokens_seen": 2549088256 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023147474747474748, + "loss": 2.5687, + "theoretical_loss": 3.363302829882424, + "tokens_seen": 2549153792 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023145454545454546, + "loss": 2.797, + "theoretical_loss": 3.363295959694905, + "tokens_seen": 2549219328 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023143434343434344, + "loss": 2.7385, + "theoretical_loss": 3.363289089733456, + "tokens_seen": 2549284864 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002314141414141414, + "loss": 2.8137, + "theoretical_loss": 3.363282219998065, + "tokens_seen": 2549350400 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002313939393939394, + "loss": 2.6756, + "theoretical_loss": 3.363275350488718, + "tokens_seen": 2549415936 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023137373737373737, + "loss": 2.6287, + "theoretical_loss": 3.3632684812054015, + "tokens_seen": 2549481472 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023135353535353538, + "loss": 2.5179, + "theoretical_loss": 3.363261612148103, + "tokens_seen": 2549547008 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1396968, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2947990894317627, + "objective/train/theoretical_loss": 3.3632547433168085, + "objective/train/tokens_used": 908471776, + "theoretical_loss": 3.3632547433168085, + "tokens_seen": 2549612544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023133333333333334, + "loss": 2.8115, + "theoretical_loss": 3.3632547433168085, + "tokens_seen": 2549612544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023131313131313132, + "loss": 2.6321, + "theoretical_loss": 3.3632478747115053, + "tokens_seen": 2549678080 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002312929292929293, + "loss": 2.7633, + "theoretical_loss": 3.3632410063321805, + "tokens_seen": 2549743616 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023127272727272726, + "loss": 2.4416, + "theoretical_loss": 3.3632341381788198, + "tokens_seen": 2549809152 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023125252525252527, + "loss": 2.4922, + "theoretical_loss": 3.3632272702514108, + "tokens_seen": 2549874688 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023123232323232323, + "loss": 2.555, + "theoretical_loss": 3.3632204025499397, + "tokens_seen": 2549940224 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002312121212121212, + "loss": 2.5403, + "theoretical_loss": 3.3632135350743937, + "tokens_seen": 2550005760 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002311919191919192, + "loss": 2.4247, + "theoretical_loss": 3.3632066678247594, + "tokens_seen": 2550071296 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023117171717171718, + "loss": 2.5022, + "theoretical_loss": 3.3631998008010235, + "tokens_seen": 2550136832 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023115151515151517, + "loss": 2.5859, + "theoretical_loss": 3.3631929340031728, + "tokens_seen": 2550202368 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023113131313131315, + "loss": 2.8307, + "theoretical_loss": 3.363186067431194, + "tokens_seen": 2550267904 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002311111111111111, + "loss": 2.7717, + "theoretical_loss": 3.3631792010850745, + "tokens_seen": 2550333440 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002310909090909091, + "loss": 2.6598, + "theoretical_loss": 3.3631723349648, + "tokens_seen": 2550398976 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023107070707070707, + "loss": 2.5159, + "theoretical_loss": 3.363165469070358, + "tokens_seen": 2550464512 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023105050505050506, + "loss": 2.4401, + "theoretical_loss": 3.363158603401735, + "tokens_seen": 2550530048 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023103030303030304, + "loss": 2.4653, + "theoretical_loss": 3.3631517379589178, + "tokens_seen": 2550595584 + }, + { + "epoch": 0.54, + "learning_rate": 0.000231010101010101, + "loss": 2.5412, + "theoretical_loss": 3.363144872741893, + "tokens_seen": 2550661120 + }, + { + "epoch": 0.54, + "learning_rate": 0.000230989898989899, + "loss": 2.4256, + "theoretical_loss": 3.363138007750648, + "tokens_seen": 2550726656 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023096969696969697, + "loss": 2.5071, + "theoretical_loss": 3.3631311429851687, + "tokens_seen": 2550792192 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023094949494949498, + "loss": 2.3989, + "theoretical_loss": 3.3631242784454427, + "tokens_seen": 2550857728 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023092929292929293, + "loss": 2.6109, + "theoretical_loss": 3.363117414131456, + "tokens_seen": 2550923264 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002309090909090909, + "loss": 2.6367, + "theoretical_loss": 3.3631105500431966, + "tokens_seen": 2550988800 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002308888888888889, + "loss": 2.5224, + "theoretical_loss": 3.36310368618065, + "tokens_seen": 2551054336 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023086868686868686, + "loss": 2.4318, + "theoretical_loss": 3.363096822543803, + "tokens_seen": 2551119872 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023084848484848487, + "loss": 2.468, + "theoretical_loss": 3.363089959132643, + "tokens_seen": 2551185408 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1397903, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4797961711883545, + "objective/train/theoretical_loss": 3.363083095947157, + "objective/train/tokens_used": 910110176, + "theoretical_loss": 3.363083095947157, + "tokens_seen": 2551250944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023082828282828283, + "loss": 2.6271, + "theoretical_loss": 3.363083095947157, + "tokens_seen": 2551250944 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002308080808080808, + "loss": 2.6973, + "theoretical_loss": 3.363076232987331, + "tokens_seen": 2551316480 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002307878787878788, + "loss": 2.7594, + "theoretical_loss": 3.3630693702531524, + "tokens_seen": 2551382016 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023076767676767675, + "loss": 2.4957, + "theoretical_loss": 3.3630625077446075, + "tokens_seen": 2551447552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023074747474747476, + "loss": 2.7009, + "theoretical_loss": 3.3630556454616833, + "tokens_seen": 2551513088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023072727272727272, + "loss": 2.6245, + "theoretical_loss": 3.363048783404367, + "tokens_seen": 2551578624 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002307070707070707, + "loss": 2.4169, + "theoretical_loss": 3.363041921572645, + "tokens_seen": 2551644160 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002306868686868687, + "loss": 2.4232, + "theoretical_loss": 3.3630350599665038, + "tokens_seen": 2551709696 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023066666666666667, + "loss": 2.6133, + "theoretical_loss": 3.3630281985859307, + "tokens_seen": 2551775232 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023064646464646466, + "loss": 2.545, + "theoretical_loss": 3.3630213374309124, + "tokens_seen": 2551840768 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023062626262626264, + "loss": 2.419, + "theoretical_loss": 3.3630144765014354, + "tokens_seen": 2551906304 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002306060606060606, + "loss": 2.6984, + "theoretical_loss": 3.3630076157974864, + "tokens_seen": 2551971840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023058585858585858, + "loss": 2.7016, + "theoretical_loss": 3.363000755319053, + "tokens_seen": 2552037376 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023056565656565656, + "loss": 2.4047, + "theoretical_loss": 3.362993895066121, + "tokens_seen": 2552102912 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023054545454545455, + "loss": 2.6434, + "theoretical_loss": 3.3629870350386777, + "tokens_seen": 2552168448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023052525252525253, + "loss": 2.704, + "theoretical_loss": 3.3629801752367103, + "tokens_seen": 2552233984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023050505050505052, + "loss": 2.6878, + "theoretical_loss": 3.362973315660205, + "tokens_seen": 2552299520 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002304848484848485, + "loss": 2.6135, + "theoretical_loss": 3.3629664563091484, + "tokens_seen": 2552365056 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023046464646464646, + "loss": 2.392, + "theoretical_loss": 3.362959597183528, + "tokens_seen": 2552430592 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023044444444444447, + "loss": 2.608, + "theoretical_loss": 3.36295273828333, + "tokens_seen": 2552496128 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023042424242424243, + "loss": 2.57, + "theoretical_loss": 3.3629458796085414, + "tokens_seen": 2552561664 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002304040404040404, + "loss": 2.6565, + "theoretical_loss": 3.3629390211591494, + "tokens_seen": 2552627200 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002303838383838384, + "loss": 2.4804, + "theoretical_loss": 3.3629321629351407, + "tokens_seen": 2552692736 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023036363636363635, + "loss": 2.5498, + "theoretical_loss": 3.3629253049365015, + "tokens_seen": 2552758272 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023034343434343436, + "loss": 2.6348, + "theoretical_loss": 3.362918447163219, + "tokens_seen": 2552823808 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1398368, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.575436592102051, + "objective/train/theoretical_loss": 3.36291158961528, + "objective/train/tokens_used": 911748576, + "theoretical_loss": 3.36291158961528, + "tokens_seen": 2552889344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023032323232323232, + "loss": 2.7566, + "theoretical_loss": 3.36291158961528, + "tokens_seen": 2552889344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023030303030303033, + "loss": 2.5712, + "theoretical_loss": 3.3629047322926713, + "tokens_seen": 2552954880 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023028282828282829, + "loss": 2.6174, + "theoretical_loss": 3.36289787519538, + "tokens_seen": 2553020416 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023026262626262627, + "loss": 2.3893, + "theoretical_loss": 3.3628910183233924, + "tokens_seen": 2553085952 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023024242424242425, + "loss": 2.4815, + "theoretical_loss": 3.362884161676696, + "tokens_seen": 2553151488 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002302222222222222, + "loss": 2.6386, + "theoretical_loss": 3.3628773052552763, + "tokens_seen": 2553217024 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023020202020202022, + "loss": 2.4313, + "theoretical_loss": 3.3628704490591215, + "tokens_seen": 2553282560 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023018181818181818, + "loss": 2.4207, + "theoretical_loss": 3.362863593088218, + "tokens_seen": 2553348096 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023016161616161616, + "loss": 2.7121, + "theoretical_loss": 3.362856737342552, + "tokens_seen": 2553413632 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023014141414141415, + "loss": 2.5634, + "theoretical_loss": 3.3628498818221115, + "tokens_seen": 2553479168 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023012121212121213, + "loss": 2.7912, + "theoretical_loss": 3.3628430265268823, + "tokens_seen": 2553544704 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023010101010101011, + "loss": 2.4155, + "theoretical_loss": 3.362836171456852, + "tokens_seen": 2553610240 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002300808080808081, + "loss": 2.5919, + "theoretical_loss": 3.3628293166120065, + "tokens_seen": 2553675776 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023006060606060606, + "loss": 2.6233, + "theoretical_loss": 3.3628224619923333, + "tokens_seen": 2553741312 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023004040404040404, + "loss": 2.7086, + "theoretical_loss": 3.362815607597819, + "tokens_seen": 2553806848 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023002020202020202, + "loss": 2.4947, + "theoretical_loss": 3.3628087534284505, + "tokens_seen": 2553872384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023, + "loss": 2.7466, + "theoretical_loss": 3.362801899484215, + "tokens_seen": 2553937920 + }, + { + "epoch": 0.54, + "learning_rate": 0.000229979797979798, + "loss": 2.5205, + "theoretical_loss": 3.3627950457650986, + "tokens_seen": 2554003456 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022995959595959595, + "loss": 2.3375, + "theoretical_loss": 3.3627881922710885, + "tokens_seen": 2554068992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022993939393939396, + "loss": 2.5964, + "theoretical_loss": 3.3627813390021712, + "tokens_seen": 2554134528 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022991919191919192, + "loss": 2.5728, + "theoretical_loss": 3.3627744859583344, + "tokens_seen": 2554200064 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002298989898989899, + "loss": 2.4382, + "theoretical_loss": 3.3627676331395637, + "tokens_seen": 2554265600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022987878787878788, + "loss": 2.5344, + "theoretical_loss": 3.3627607805458473, + "tokens_seen": 2554331136 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022985858585858584, + "loss": 2.3541, + "theoretical_loss": 3.3627539281771712, + "tokens_seen": 2554396672 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022983838383838385, + "loss": 2.6214, + "theoretical_loss": 3.362747076033522, + "tokens_seen": 2554462208 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1399295, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6599607467651367, + "objective/train/theoretical_loss": 3.362740224114887, + "objective/train/tokens_used": 913386976, + "theoretical_loss": 3.362740224114887, + "tokens_seen": 2554527744 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002298181818181818, + "loss": 2.4687, + "theoretical_loss": 3.362740224114887, + "tokens_seen": 2554527744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022979797979797982, + "loss": 2.5853, + "theoretical_loss": 3.362733372421253, + "tokens_seen": 2554593280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022977777777777778, + "loss": 2.8089, + "theoretical_loss": 3.3627265209526067, + "tokens_seen": 2554658816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022975757575757576, + "loss": 2.577, + "theoretical_loss": 3.3627196697089357, + "tokens_seen": 2554724352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022973737373737374, + "loss": 2.5586, + "theoretical_loss": 3.362712818690225, + "tokens_seen": 2554789888 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002297171717171717, + "loss": 2.6406, + "theoretical_loss": 3.3627059678964635, + "tokens_seen": 2554855424 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002296969696969697, + "loss": 2.7724, + "theoretical_loss": 3.362699117327637, + "tokens_seen": 2554920960 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022967676767676767, + "loss": 2.5388, + "theoretical_loss": 3.362692266983732, + "tokens_seen": 2554986496 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022965656565656568, + "loss": 2.5942, + "theoretical_loss": 3.3626854168647364, + "tokens_seen": 2555052032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022963636363636364, + "loss": 2.5285, + "theoretical_loss": 3.3626785669706365, + "tokens_seen": 2555117568 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022961616161616162, + "loss": 2.5427, + "theoretical_loss": 3.362671717301419, + "tokens_seen": 2555183104 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002295959595959596, + "loss": 2.48, + "theoretical_loss": 3.3626648678570707, + "tokens_seen": 2555248640 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002295757575757576, + "loss": 2.4674, + "theoretical_loss": 3.3626580186375787, + "tokens_seen": 2555314176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022955555555555557, + "loss": 2.6681, + "theoretical_loss": 3.36265116964293, + "tokens_seen": 2555379712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022953535353535353, + "loss": 2.5396, + "theoretical_loss": 3.362644320873111, + "tokens_seen": 2555445248 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022951515151515151, + "loss": 2.4922, + "theoretical_loss": 3.362637472328109, + "tokens_seen": 2555510784 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002294949494949495, + "loss": 2.5783, + "theoretical_loss": 3.3626306240079105, + "tokens_seen": 2555576320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022947474747474748, + "loss": 2.7018, + "theoretical_loss": 3.362623775912503, + "tokens_seen": 2555641856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022945454545454547, + "loss": 2.6705, + "theoretical_loss": 3.3626169280418723, + "tokens_seen": 2555707392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022943434343434345, + "loss": 2.6724, + "theoretical_loss": 3.362610080396006, + "tokens_seen": 2555772928 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002294141414141414, + "loss": 2.6912, + "theoretical_loss": 3.3626032329748905, + "tokens_seen": 2555838464 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022939393939393942, + "loss": 2.6626, + "theoretical_loss": 3.3625963857785135, + "tokens_seen": 2555904000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022937373737373737, + "loss": 2.4301, + "theoretical_loss": 3.362589538806861, + "tokens_seen": 2555969536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022935353535353536, + "loss": 2.4563, + "theoretical_loss": 3.36258269205992, + "tokens_seen": 2556035072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022933333333333334, + "loss": 2.588, + "theoretical_loss": 3.3625758455376777, + "tokens_seen": 2556100608 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1400625, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.826688289642334, + "objective/train/theoretical_loss": 3.362568999240121, + "objective/train/tokens_used": 915025376, + "theoretical_loss": 3.362568999240121, + "tokens_seen": 2556166144 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002293131313131313, + "loss": 2.7183, + "theoretical_loss": 3.362568999240121, + "tokens_seen": 2556166144 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002292929292929293, + "loss": 2.6652, + "theoretical_loss": 3.3625621531672367, + "tokens_seen": 2556231680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022927272727272727, + "loss": 2.4161, + "theoretical_loss": 3.362555307319011, + "tokens_seen": 2556297216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022925252525252528, + "loss": 2.526, + "theoretical_loss": 3.3625484616954315, + "tokens_seen": 2556362752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022923232323232323, + "loss": 2.714, + "theoretical_loss": 3.362541616296485, + "tokens_seen": 2556428288 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002292121212121212, + "loss": 2.6548, + "theoretical_loss": 3.3625347711221583, + "tokens_seen": 2556493824 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002291919191919192, + "loss": 2.5988, + "theoretical_loss": 3.362527926172438, + "tokens_seen": 2556559360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022917171717171716, + "loss": 2.5166, + "theoretical_loss": 3.362521081447311, + "tokens_seen": 2556624896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022915151515151517, + "loss": 2.7294, + "theoretical_loss": 3.3625142369467644, + "tokens_seen": 2556690432 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022913131313131313, + "loss": 2.7967, + "theoretical_loss": 3.3625073926707856, + "tokens_seen": 2556755968 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002291111111111111, + "loss": 2.5215, + "theoretical_loss": 3.3625005486193604, + "tokens_seen": 2556821504 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002290909090909091, + "loss": 2.4801, + "theoretical_loss": 3.3624937047924766, + "tokens_seen": 2556887040 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022907070707070708, + "loss": 2.7837, + "theoretical_loss": 3.36248686119012, + "tokens_seen": 2556952576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022905050505050506, + "loss": 2.4681, + "theoretical_loss": 3.362480017812279, + "tokens_seen": 2557018112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022903030303030302, + "loss": 2.6862, + "theoretical_loss": 3.3624731746589394, + "tokens_seen": 2557083648 + }, + { + "epoch": 0.55, + "learning_rate": 0.000229010101010101, + "loss": 2.5918, + "theoretical_loss": 3.362466331730088, + "tokens_seen": 2557149184 + }, + { + "epoch": 0.55, + "learning_rate": 0.000228989898989899, + "loss": 2.488, + "theoretical_loss": 3.362459489025712, + "tokens_seen": 2557214720 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022896969696969697, + "loss": 2.4771, + "theoretical_loss": 3.3624526465457985, + "tokens_seen": 2557280256 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022894949494949496, + "loss": 2.5262, + "theoretical_loss": 3.3624458042903336, + "tokens_seen": 2557345792 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022892929292929294, + "loss": 2.5601, + "theoretical_loss": 3.362438962259305, + "tokens_seen": 2557411328 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002289090909090909, + "loss": 2.5782, + "theoretical_loss": 3.3624321204527, + "tokens_seen": 2557476864 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002288888888888889, + "loss": 2.5581, + "theoretical_loss": 3.362425278870504, + "tokens_seen": 2557542400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022886868686868686, + "loss": 2.6424, + "theoretical_loss": 3.362418437512705, + "tokens_seen": 2557607936 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022884848484848485, + "loss": 2.6295, + "theoretical_loss": 3.3624115963792898, + "tokens_seen": 2557673472 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022882828282828283, + "loss": 2.8505, + "theoretical_loss": 3.3624047554702448, + "tokens_seen": 2557739008 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1401375, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7073214054107666, + "objective/train/theoretical_loss": 3.362397914785557, + "objective/train/tokens_used": 916663776, + "theoretical_loss": 3.362397914785557, + "tokens_seen": 2557804544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022880808080808082, + "loss": 2.7022, + "theoretical_loss": 3.362397914785557, + "tokens_seen": 2557804544 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002287878787878788, + "loss": 2.6011, + "theoretical_loss": 3.362391074325214, + "tokens_seen": 2557870080 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022876767676767676, + "loss": 2.7017, + "theoretical_loss": 3.3623842340892023, + "tokens_seen": 2557935616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022874747474747477, + "loss": 2.7039, + "theoretical_loss": 3.3623773940775084, + "tokens_seen": 2558001152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022872727272727273, + "loss": 2.5218, + "theoretical_loss": 3.362370554290119, + "tokens_seen": 2558066688 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022870707070707074, + "loss": 2.7054, + "theoretical_loss": 3.3623637147270222, + "tokens_seen": 2558132224 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002286868686868687, + "loss": 2.6012, + "theoretical_loss": 3.362356875388204, + "tokens_seen": 2558197760 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022866666666666665, + "loss": 2.5456, + "theoretical_loss": 3.362350036273651, + "tokens_seen": 2558263296 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022864646464646466, + "loss": 2.5671, + "theoretical_loss": 3.3623431973833506, + "tokens_seen": 2558328832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022862626262626262, + "loss": 2.6018, + "theoretical_loss": 3.36233635871729, + "tokens_seen": 2558394368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022860606060606063, + "loss": 2.4401, + "theoretical_loss": 3.362329520275456, + "tokens_seen": 2558459904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022858585858585859, + "loss": 2.5295, + "theoretical_loss": 3.362322682057835, + "tokens_seen": 2558525440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022856565656565657, + "loss": 2.5317, + "theoretical_loss": 3.3623158440644136, + "tokens_seen": 2558590976 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022854545454545455, + "loss": 2.6185, + "theoretical_loss": 3.36230900629518, + "tokens_seen": 2558656512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022852525252525254, + "loss": 2.5875, + "theoretical_loss": 3.3623021687501202, + "tokens_seen": 2558722048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022850505050505052, + "loss": 2.5357, + "theoretical_loss": 3.3622953314292214, + "tokens_seen": 2558787584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022848484848484848, + "loss": 2.5946, + "theoretical_loss": 3.3622884943324705, + "tokens_seen": 2558853120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022846464646464646, + "loss": 2.5611, + "theoretical_loss": 3.362281657459854, + "tokens_seen": 2558918656 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022844444444444445, + "loss": 2.4251, + "theoretical_loss": 3.3622748208113595, + "tokens_seen": 2558984192 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022842424242424243, + "loss": 2.3916, + "theoretical_loss": 3.3622679843869734, + "tokens_seen": 2559049728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022840404040404041, + "loss": 2.6423, + "theoretical_loss": 3.3622611481866826, + "tokens_seen": 2559115264 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002283838383838384, + "loss": 2.333, + "theoretical_loss": 3.3622543122104744, + "tokens_seen": 2559180800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022836363636363636, + "loss": 2.7554, + "theoretical_loss": 3.3622474764583354, + "tokens_seen": 2559246336 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022834343434343437, + "loss": 2.7019, + "theoretical_loss": 3.3622406409302528, + "tokens_seen": 2559311872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022832323232323232, + "loss": 2.7094, + "theoretical_loss": 3.3622338056262127, + "tokens_seen": 2559377408 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1402702, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7075891494750977, + "objective/train/theoretical_loss": 3.362226970546203, + "objective/train/tokens_used": 918302176, + "theoretical_loss": 3.362226970546203, + "tokens_seen": 2559442944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002283030303030303, + "loss": 2.5418, + "theoretical_loss": 3.362226970546203, + "tokens_seen": 2559442944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002282828282828283, + "loss": 2.6696, + "theoretical_loss": 3.362220135690211, + "tokens_seen": 2559508480 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022826262626262625, + "loss": 2.5328, + "theoretical_loss": 3.362213301058222, + "tokens_seen": 2559574016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022824242424242426, + "loss": 2.3914, + "theoretical_loss": 3.362206466650224, + "tokens_seen": 2559639552 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022822222222222222, + "loss": 2.5646, + "theoretical_loss": 3.362199632466204, + "tokens_seen": 2559705088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022820202020202023, + "loss": 2.6797, + "theoretical_loss": 3.3621927985061486, + "tokens_seen": 2559770624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022818181818181818, + "loss": 2.6857, + "theoretical_loss": 3.3621859647700445, + "tokens_seen": 2559836160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022816161616161614, + "loss": 2.5767, + "theoretical_loss": 3.3621791312578795, + "tokens_seen": 2559901696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022814141414141415, + "loss": 2.3725, + "theoretical_loss": 3.3621722979696393, + "tokens_seen": 2559967232 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002281212121212121, + "loss": 2.6499, + "theoretical_loss": 3.362165464905312, + "tokens_seen": 2560032768 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022810101010101012, + "loss": 2.4565, + "theoretical_loss": 3.362158632064884, + "tokens_seen": 2560098304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022808080808080808, + "loss": 2.7, + "theoretical_loss": 3.362151799448342, + "tokens_seen": 2560163840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022806060606060606, + "loss": 2.4886, + "theoretical_loss": 3.362144967055673, + "tokens_seen": 2560229376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022804040404040404, + "loss": 2.5594, + "theoretical_loss": 3.3621381348868646, + "tokens_seen": 2560294912 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022802020202020203, + "loss": 2.6457, + "theoretical_loss": 3.362131302941903, + "tokens_seen": 2560360448 + }, + { + "epoch": 0.55, + "learning_rate": 0.000228, + "loss": 2.5623, + "theoretical_loss": 3.3621244712207754, + "tokens_seen": 2560425984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022797979797979797, + "loss": 2.5425, + "theoretical_loss": 3.3621176397234684, + "tokens_seen": 2560491520 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022795959595959598, + "loss": 2.4453, + "theoretical_loss": 3.36211080844997, + "tokens_seen": 2560557056 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022793939393939394, + "loss": 2.7774, + "theoretical_loss": 3.362103977400266, + "tokens_seen": 2560622592 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022791919191919192, + "loss": 2.5743, + "theoretical_loss": 3.3620971465743437, + "tokens_seen": 2560688128 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002278989898989899, + "loss": 2.6519, + "theoretical_loss": 3.36209031597219, + "tokens_seen": 2560753664 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002278787878787879, + "loss": 2.6161, + "theoretical_loss": 3.3620834855937924, + "tokens_seen": 2560819200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022785858585858587, + "loss": 2.5052, + "theoretical_loss": 3.362076655439137, + "tokens_seen": 2560884736 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022783838383838386, + "loss": 2.7581, + "theoretical_loss": 3.362069825508211, + "tokens_seen": 2560950272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022781818181818181, + "loss": 2.4574, + "theoretical_loss": 3.3620629958010015, + "tokens_seen": 2561015808 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1403266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9517369270324707, + "objective/train/theoretical_loss": 3.3620561663174957, + "objective/train/tokens_used": 919940576, + "theoretical_loss": 3.3620561663174957, + "tokens_seen": 2561081344 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002277979797979798, + "loss": 2.7024, + "theoretical_loss": 3.3620561663174957, + "tokens_seen": 2561081344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022777777777777778, + "loss": 2.5378, + "theoretical_loss": 3.3620493370576803, + "tokens_seen": 2561146880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022775757575757577, + "loss": 2.2478, + "theoretical_loss": 3.362042508021542, + "tokens_seen": 2561212416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022773737373737375, + "loss": 2.5882, + "theoretical_loss": 3.362035679209068, + "tokens_seen": 2561277952 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002277171717171717, + "loss": 2.5611, + "theoretical_loss": 3.3620288506202454, + "tokens_seen": 2561343488 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022769696969696972, + "loss": 2.7747, + "theoretical_loss": 3.3620220222550605, + "tokens_seen": 2561409024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022767676767676767, + "loss": 2.8544, + "theoretical_loss": 3.3620151941135012, + "tokens_seen": 2561474560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022765656565656569, + "loss": 2.4052, + "theoretical_loss": 3.3620083661955538, + "tokens_seen": 2561540096 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022763636363636364, + "loss": 2.3106, + "theoretical_loss": 3.3620015385012056, + "tokens_seen": 2561605632 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002276161616161616, + "loss": 2.6862, + "theoretical_loss": 3.361994711030443, + "tokens_seen": 2561671168 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002275959595959596, + "loss": 2.5657, + "theoretical_loss": 3.3619878837832537, + "tokens_seen": 2561736704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022757575757575757, + "loss": 2.6497, + "theoretical_loss": 3.361981056759624, + "tokens_seen": 2561802240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022755555555555558, + "loss": 2.6136, + "theoretical_loss": 3.361974229959541, + "tokens_seen": 2561867776 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022753535353535354, + "loss": 2.6712, + "theoretical_loss": 3.3619674033829927, + "tokens_seen": 2561933312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022751515151515152, + "loss": 2.6558, + "theoretical_loss": 3.3619605770299645, + "tokens_seen": 2561998848 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002274949494949495, + "loss": 2.474, + "theoretical_loss": 3.3619537509004442, + "tokens_seen": 2562064384 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022747474747474746, + "loss": 2.6959, + "theoretical_loss": 3.361946924994419, + "tokens_seen": 2562129920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022745454545454547, + "loss": 2.5167, + "theoretical_loss": 3.361940099311875, + "tokens_seen": 2562195456 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022743434343434343, + "loss": 2.6456, + "theoretical_loss": 3.3619332738527996, + "tokens_seen": 2562260992 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002274141414141414, + "loss": 2.6732, + "theoretical_loss": 3.3619264486171803, + "tokens_seen": 2562326528 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002273939393939394, + "loss": 2.6386, + "theoretical_loss": 3.3619196236050035, + "tokens_seen": 2562392064 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022737373737373738, + "loss": 2.6875, + "theoretical_loss": 3.361912798816256, + "tokens_seen": 2562457600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022735353535353536, + "loss": 2.7069, + "theoretical_loss": 3.361905974250925, + "tokens_seen": 2562523136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022733333333333335, + "loss": 2.4783, + "theoretical_loss": 3.3618991499089974, + "tokens_seen": 2562588672 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002273131313131313, + "loss": 2.5364, + "theoretical_loss": 3.3618923257904605, + "tokens_seen": 2562654208 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1404500, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7659013271331787, + "objective/train/theoretical_loss": 3.3618855018953013, + "objective/train/tokens_used": 921578976, + "theoretical_loss": 3.3618855018953013, + "tokens_seen": 2562719744 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002272929292929293, + "loss": 2.5743, + "theoretical_loss": 3.3618855018953013, + "tokens_seen": 2562719744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022727272727272727, + "loss": 2.5772, + "theoretical_loss": 3.3618786782235066, + "tokens_seen": 2562785280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022725252525252526, + "loss": 2.4968, + "theoretical_loss": 3.361871854775063, + "tokens_seen": 2562850816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022723232323232324, + "loss": 2.6017, + "theoretical_loss": 3.3618650315499576, + "tokens_seen": 2562916352 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002272121212121212, + "loss": 2.5718, + "theoretical_loss": 3.3618582085481776, + "tokens_seen": 2562981888 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002271919191919192, + "loss": 2.4662, + "theoretical_loss": 3.3618513857697105, + "tokens_seen": 2563047424 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022717171717171717, + "loss": 2.5702, + "theoretical_loss": 3.361844563214542, + "tokens_seen": 2563112960 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022715151515151518, + "loss": 2.5992, + "theoretical_loss": 3.3618377408826605, + "tokens_seen": 2563178496 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022713131313131313, + "loss": 2.7946, + "theoretical_loss": 3.361830918774052, + "tokens_seen": 2563244032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022711111111111112, + "loss": 2.7117, + "theoretical_loss": 3.3618240968887037, + "tokens_seen": 2563309568 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002270909090909091, + "loss": 2.3677, + "theoretical_loss": 3.3618172752266027, + "tokens_seen": 2563375104 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022707070707070706, + "loss": 2.4435, + "theoretical_loss": 3.361810453787736, + "tokens_seen": 2563440640 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022705050505050507, + "loss": 2.5919, + "theoretical_loss": 3.361803632572091, + "tokens_seen": 2563506176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022703030303030303, + "loss": 2.6822, + "theoretical_loss": 3.3617968115796533, + "tokens_seen": 2563571712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022701010101010104, + "loss": 2.6193, + "theoretical_loss": 3.3617899908104114, + "tokens_seen": 2563637248 + }, + { + "epoch": 0.55, + "learning_rate": 0.000226989898989899, + "loss": 2.435, + "theoretical_loss": 3.3617831702643515, + "tokens_seen": 2563702784 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022696969696969698, + "loss": 2.4034, + "theoretical_loss": 3.361776349941461, + "tokens_seen": 2563768320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022694949494949496, + "loss": 2.4265, + "theoretical_loss": 3.3617695298417267, + "tokens_seen": 2563833856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022692929292929292, + "loss": 2.3804, + "theoretical_loss": 3.3617627099651353, + "tokens_seen": 2563899392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022690909090909093, + "loss": 2.721, + "theoretical_loss": 3.361755890311674, + "tokens_seen": 2563964928 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002268888888888889, + "loss": 2.5143, + "theoretical_loss": 3.3617490708813307, + "tokens_seen": 2564030464 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022686868686868687, + "loss": 2.3869, + "theoretical_loss": 3.361742251674091, + "tokens_seen": 2564096000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022684848484848485, + "loss": 2.5408, + "theoretical_loss": 3.3617354326899425, + "tokens_seen": 2564161536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022682828282828284, + "loss": 2.5471, + "theoretical_loss": 3.361728613928872, + "tokens_seen": 2564227072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022680808080808082, + "loss": 2.6331, + "theoretical_loss": 3.361721795390867, + "tokens_seen": 2564292608 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1405265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5630226135253906, + "objective/train/theoretical_loss": 3.361714977075914, + "objective/train/tokens_used": 923217376, + "theoretical_loss": 3.361714977075914, + "tokens_seen": 2564358144 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002267878787878788, + "loss": 2.7071, + "theoretical_loss": 3.361714977075914, + "tokens_seen": 2564358144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022676767676767676, + "loss": 2.5375, + "theoretical_loss": 3.3617081589840003, + "tokens_seen": 2564423680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022674747474747475, + "loss": 2.4482, + "theoretical_loss": 3.3617013411151127, + "tokens_seen": 2564489216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022672727272727273, + "loss": 2.404, + "theoretical_loss": 3.3616945234692386, + "tokens_seen": 2564554752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022670707070707072, + "loss": 2.6495, + "theoretical_loss": 3.3616877060463644, + "tokens_seen": 2564620288 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002266868686868687, + "loss": 2.5744, + "theoretical_loss": 3.361680888846477, + "tokens_seen": 2564685824 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022666666666666666, + "loss": 2.6006, + "theoretical_loss": 3.3616740718695644, + "tokens_seen": 2564751360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022664646464646467, + "loss": 2.6253, + "theoretical_loss": 3.3616672551156133, + "tokens_seen": 2564816896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022662626262626262, + "loss": 2.433, + "theoretical_loss": 3.36166043858461, + "tokens_seen": 2564882432 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002266060606060606, + "loss": 2.5285, + "theoretical_loss": 3.361653622276542, + "tokens_seen": 2564947968 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002265858585858586, + "loss": 2.5386, + "theoretical_loss": 3.361646806191396, + "tokens_seen": 2565013504 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022656565656565655, + "loss": 2.4593, + "theoretical_loss": 3.36163999032916, + "tokens_seen": 2565079040 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022654545454545456, + "loss": 2.554, + "theoretical_loss": 3.36163317468982, + "tokens_seen": 2565144576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022652525252525252, + "loss": 2.5766, + "theoretical_loss": 3.361626359273363, + "tokens_seen": 2565210112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022650505050505053, + "loss": 2.4994, + "theoretical_loss": 3.3616195440797765, + "tokens_seen": 2565275648 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022648484848484848, + "loss": 2.7954, + "theoretical_loss": 3.3616127291090474, + "tokens_seen": 2565341184 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022646464646464647, + "loss": 2.478, + "theoretical_loss": 3.3616059143611623, + "tokens_seen": 2565406720 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022644444444444445, + "loss": 2.4961, + "theoretical_loss": 3.361599099836109, + "tokens_seen": 2565472256 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002264242424242424, + "loss": 2.5967, + "theoretical_loss": 3.3615922855338742, + "tokens_seen": 2565537792 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022640404040404042, + "loss": 2.3754, + "theoretical_loss": 3.3615854714544446, + "tokens_seen": 2565603328 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022638383838383838, + "loss": 2.6623, + "theoretical_loss": 3.3615786575978075, + "tokens_seen": 2565668864 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022636363636363636, + "loss": 2.6872, + "theoretical_loss": 3.36157184396395, + "tokens_seen": 2565734400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022634343434343435, + "loss": 2.6839, + "theoretical_loss": 3.3615650305528586, + "tokens_seen": 2565799936 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022632323232323233, + "loss": 2.6944, + "theoretical_loss": 3.361558217364521, + "tokens_seen": 2565865472 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002263030303030303, + "loss": 2.5876, + "theoretical_loss": 3.361551404398924, + "tokens_seen": 2565931008 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1405984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.499677896499634, + "objective/train/theoretical_loss": 3.3615445916560547, + "objective/train/tokens_used": 924855776, + "theoretical_loss": 3.3615445916560547, + "tokens_seen": 2565996544 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002262828282828283, + "loss": 2.4251, + "theoretical_loss": 3.3615445916560547, + "tokens_seen": 2565996544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022626262626262628, + "loss": 2.2787, + "theoretical_loss": 3.3615377791359, + "tokens_seen": 2566062080 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022624242424242424, + "loss": 2.6603, + "theoretical_loss": 3.3615309668384468, + "tokens_seen": 2566127616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022622222222222222, + "loss": 2.6231, + "theoretical_loss": 3.361524154763682, + "tokens_seen": 2566193152 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002262020202020202, + "loss": 2.6168, + "theoretical_loss": 3.3615173429115934, + "tokens_seen": 2566258688 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002261818181818182, + "loss": 2.5521, + "theoretical_loss": 3.3615105312821676, + "tokens_seen": 2566324224 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022616161616161617, + "loss": 2.5061, + "theoretical_loss": 3.361503719875391, + "tokens_seen": 2566389760 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022614141414141416, + "loss": 2.7799, + "theoretical_loss": 3.3614969086912523, + "tokens_seen": 2566455296 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022612121212121211, + "loss": 2.4515, + "theoretical_loss": 3.3614900977297366, + "tokens_seen": 2566520832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022610101010101013, + "loss": 2.622, + "theoretical_loss": 3.361483286990832, + "tokens_seen": 2566586368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022608080808080808, + "loss": 2.5093, + "theoretical_loss": 3.3614764764745253, + "tokens_seen": 2566651904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022606060606060607, + "loss": 2.5789, + "theoretical_loss": 3.361469666180804, + "tokens_seen": 2566717440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022604040404040405, + "loss": 2.7051, + "theoretical_loss": 3.3614628561096542, + "tokens_seen": 2566782976 + }, + { + "epoch": 0.55, + "learning_rate": 0.000226020202020202, + "loss": 2.6851, + "theoretical_loss": 3.361456046261064, + "tokens_seen": 2566848512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022600000000000002, + "loss": 2.5935, + "theoretical_loss": 3.361449236635019, + "tokens_seen": 2566914048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022597979797979798, + "loss": 2.4982, + "theoretical_loss": 3.3614424272315078, + "tokens_seen": 2566979584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022595959595959599, + "loss": 2.5564, + "theoretical_loss": 3.3614356180505167, + "tokens_seen": 2567045120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022593939393939394, + "loss": 2.3668, + "theoretical_loss": 3.361428809092033, + "tokens_seen": 2567110656 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002259191919191919, + "loss": 2.6117, + "theoretical_loss": 3.3614220003560433, + "tokens_seen": 2567176192 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002258989898989899, + "loss": 2.5992, + "theoretical_loss": 3.3614151918425357, + "tokens_seen": 2567241728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022587878787878787, + "loss": 2.6247, + "theoretical_loss": 3.3614083835514954, + "tokens_seen": 2567307264 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022585858585858588, + "loss": 2.5183, + "theoretical_loss": 3.3614015754829114, + "tokens_seen": 2567372800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022583838383838384, + "loss": 2.562, + "theoretical_loss": 3.3613947676367695, + "tokens_seen": 2567438336 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022581818181818182, + "loss": 2.2841, + "theoretical_loss": 3.361387960013057, + "tokens_seen": 2567503872 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002257979797979798, + "loss": 2.605, + "theoretical_loss": 3.3613811526117616, + "tokens_seen": 2567569408 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1407293, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.837613105773926, + "objective/train/theoretical_loss": 3.36137434543287, + "objective/train/tokens_used": 926494176, + "theoretical_loss": 3.36137434543287, + "tokens_seen": 2567634944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002257777777777778, + "loss": 2.7259, + "theoretical_loss": 3.36137434543287, + "tokens_seen": 2567634944 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022575757575757577, + "loss": 2.508, + "theoretical_loss": 3.361367538476369, + "tokens_seen": 2567700480 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022573737373737373, + "loss": 2.3551, + "theoretical_loss": 3.3613607317422454, + "tokens_seen": 2567766016 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002257171717171717, + "loss": 2.6477, + "theoretical_loss": 3.3613539252304867, + "tokens_seen": 2567831552 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002256969696969697, + "loss": 2.3382, + "theoretical_loss": 3.3613471189410804, + "tokens_seen": 2567897088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022567676767676768, + "loss": 2.691, + "theoretical_loss": 3.3613403128740127, + "tokens_seen": 2567962624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022565656565656566, + "loss": 2.4764, + "theoretical_loss": 3.361333507029271, + "tokens_seen": 2568028160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022563636363636365, + "loss": 2.5417, + "theoretical_loss": 3.361326701406843, + "tokens_seen": 2568093696 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002256161616161616, + "loss": 2.4511, + "theoretical_loss": 3.361319896006714, + "tokens_seen": 2568159232 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022559595959595962, + "loss": 2.5368, + "theoretical_loss": 3.3613130908288733, + "tokens_seen": 2568224768 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022557575757575757, + "loss": 2.6526, + "theoretical_loss": 3.3613062858733063, + "tokens_seen": 2568290304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022555555555555556, + "loss": 2.5259, + "theoretical_loss": 3.361299481140001, + "tokens_seen": 2568355840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022553535353535354, + "loss": 2.5857, + "theoretical_loss": 3.361292676628944, + "tokens_seen": 2568421376 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002255151515151515, + "loss": 2.7697, + "theoretical_loss": 3.361285872340123, + "tokens_seen": 2568486912 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002254949494949495, + "loss": 2.5742, + "theoretical_loss": 3.361279068273524, + "tokens_seen": 2568552448 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022547474747474747, + "loss": 2.6054, + "theoretical_loss": 3.3612722644291346, + "tokens_seen": 2568617984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022545454545454548, + "loss": 2.6186, + "theoretical_loss": 3.3612654608069423, + "tokens_seen": 2568683520 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022543434343434343, + "loss": 2.3318, + "theoretical_loss": 3.3612586574069336, + "tokens_seen": 2568749056 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022541414141414144, + "loss": 2.5793, + "theoretical_loss": 3.3612518542290957, + "tokens_seen": 2568814592 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002253939393939394, + "loss": 2.4563, + "theoretical_loss": 3.361245051273416, + "tokens_seen": 2568880128 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022537373737373736, + "loss": 2.457, + "theoretical_loss": 3.361238248539881, + "tokens_seen": 2568945664 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022535353535353537, + "loss": 2.5486, + "theoretical_loss": 3.3612314460284782, + "tokens_seen": 2569011200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022533333333333333, + "loss": 2.4845, + "theoretical_loss": 3.361224643739195, + "tokens_seen": 2569076736 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022531313131313134, + "loss": 2.3753, + "theoretical_loss": 3.3612178416720178, + "tokens_seen": 2569142272 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002252929292929293, + "loss": 2.325, + "theoretical_loss": 3.3612110398269337, + "tokens_seen": 2569207808 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1408033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3271267414093018, + "objective/train/theoretical_loss": 3.36120423820393, + "objective/train/tokens_used": 928132576, + "theoretical_loss": 3.36120423820393, + "tokens_seen": 2569273344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022527272727272728, + "loss": 2.3177, + "theoretical_loss": 3.36120423820393, + "tokens_seen": 2569273344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022525252525252526, + "loss": 2.4534, + "theoretical_loss": 3.3611974368029944, + "tokens_seen": 2569338880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022523232323232325, + "loss": 2.3449, + "theoretical_loss": 3.361190635624113, + "tokens_seen": 2569404416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022521212121212123, + "loss": 2.4716, + "theoretical_loss": 3.3611838346672736, + "tokens_seen": 2569469952 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002251919191919192, + "loss": 2.4864, + "theoretical_loss": 3.3611770339324627, + "tokens_seen": 2569535488 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022517171717171717, + "loss": 2.4013, + "theoretical_loss": 3.361170233419668, + "tokens_seen": 2569601024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022515151515151515, + "loss": 2.6963, + "theoretical_loss": 3.361163433128876, + "tokens_seen": 2569666560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022513131313131314, + "loss": 2.3117, + "theoretical_loss": 3.361156633060074, + "tokens_seen": 2569732096 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022511111111111112, + "loss": 2.6837, + "theoretical_loss": 3.3611498332132497, + "tokens_seen": 2569797632 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002250909090909091, + "loss": 2.4776, + "theoretical_loss": 3.361143033588389, + "tokens_seen": 2569863168 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022507070707070706, + "loss": 2.425, + "theoretical_loss": 3.36113623418548, + "tokens_seen": 2569928704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022505050505050507, + "loss": 2.524, + "theoretical_loss": 3.361129435004509, + "tokens_seen": 2569994240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022503030303030303, + "loss": 2.6779, + "theoretical_loss": 3.361122636045464, + "tokens_seen": 2570059776 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022501010101010102, + "loss": 2.613, + "theoretical_loss": 3.3611158373083314, + "tokens_seen": 2570125312 + }, + { + "epoch": 0.55, + "learning_rate": 0.000224989898989899, + "loss": 2.6216, + "theoretical_loss": 3.3611090387930984, + "tokens_seen": 2570190848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022496969696969696, + "loss": 2.6213, + "theoretical_loss": 3.3611022404997524, + "tokens_seen": 2570256384 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022494949494949497, + "loss": 2.3933, + "theoretical_loss": 3.3610954424282804, + "tokens_seen": 2570321920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022492929292929292, + "loss": 2.6353, + "theoretical_loss": 3.361088644578669, + "tokens_seen": 2570387456 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022490909090909094, + "loss": 2.4455, + "theoretical_loss": 3.3610818469509063, + "tokens_seen": 2570452992 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002248888888888889, + "loss": 2.5442, + "theoretical_loss": 3.3610750495449784, + "tokens_seen": 2570518528 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022486868686868685, + "loss": 2.5144, + "theoretical_loss": 3.3610682523608735, + "tokens_seen": 2570584064 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022484848484848486, + "loss": 2.6568, + "theoretical_loss": 3.3610614553985774, + "tokens_seen": 2570649600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022482828282828282, + "loss": 2.8062, + "theoretical_loss": 3.361054658658078, + "tokens_seen": 2570715136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022480808080808083, + "loss": 2.6866, + "theoretical_loss": 3.3610478621393622, + "tokens_seen": 2570780672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022478787878787878, + "loss": 2.614, + "theoretical_loss": 3.3610410658424175, + "tokens_seen": 2570846208 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1409076, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9130380153656006, + "objective/train/theoretical_loss": 3.3610342697672304, + "objective/train/tokens_used": 929770976, + "theoretical_loss": 3.3610342697672304, + "tokens_seen": 2570911744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022476767676767677, + "loss": 2.477, + "theoretical_loss": 3.3610342697672304, + "tokens_seen": 2570911744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022474747474747475, + "loss": 2.6016, + "theoretical_loss": 3.361027473913788, + "tokens_seen": 2570977280 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022472727272727274, + "loss": 2.3289, + "theoretical_loss": 3.3610206782820784, + "tokens_seen": 2571042816 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022470707070707072, + "loss": 2.662, + "theoretical_loss": 3.3610138828720877, + "tokens_seen": 2571108352 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022468686868686868, + "loss": 2.4994, + "theoretical_loss": 3.361007087683803, + "tokens_seen": 2571173888 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022466666666666666, + "loss": 2.7204, + "theoretical_loss": 3.361000292717212, + "tokens_seen": 2571239424 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022464646464646465, + "loss": 2.4349, + "theoretical_loss": 3.3609934979723017, + "tokens_seen": 2571304960 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022462626262626263, + "loss": 2.5819, + "theoretical_loss": 3.360986703449059, + "tokens_seen": 2571370496 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002246060606060606, + "loss": 2.6861, + "theoretical_loss": 3.360979909147471, + "tokens_seen": 2571436032 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002245858585858586, + "loss": 2.6242, + "theoretical_loss": 3.3609731150675253, + "tokens_seen": 2571501568 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022456565656565658, + "loss": 2.4868, + "theoretical_loss": 3.3609663212092085, + "tokens_seen": 2571567104 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022454545454545457, + "loss": 2.4331, + "theoretical_loss": 3.3609595275725077, + "tokens_seen": 2571632640 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022452525252525252, + "loss": 2.5809, + "theoretical_loss": 3.36095273415741, + "tokens_seen": 2571698176 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002245050505050505, + "loss": 2.4557, + "theoretical_loss": 3.360945940963903, + "tokens_seen": 2571763712 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002244848484848485, + "loss": 2.5004, + "theoretical_loss": 3.3609391479919735, + "tokens_seen": 2571829248 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022446464646464647, + "loss": 2.6773, + "theoretical_loss": 3.3609323552416086, + "tokens_seen": 2571894784 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022444444444444446, + "loss": 2.699, + "theoretical_loss": 3.360925562712796, + "tokens_seen": 2571960320 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022442424242424241, + "loss": 2.5101, + "theoretical_loss": 3.3609187704055214, + "tokens_seen": 2572025856 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022440404040404043, + "loss": 2.4041, + "theoretical_loss": 3.3609119783197734, + "tokens_seen": 2572091392 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022438383838383838, + "loss": 2.3493, + "theoretical_loss": 3.360905186455539, + "tokens_seen": 2572156928 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002243636363636364, + "loss": 2.6747, + "theoretical_loss": 3.360898394812804, + "tokens_seen": 2572222464 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022434343434343435, + "loss": 2.6986, + "theoretical_loss": 3.360891603391557, + "tokens_seen": 2572288000 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002243232323232323, + "loss": 2.4015, + "theoretical_loss": 3.3608848121917845, + "tokens_seen": 2572353536 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022430303030303032, + "loss": 2.5643, + "theoretical_loss": 3.3608780212134737, + "tokens_seen": 2572419072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022428282828282828, + "loss": 2.6435, + "theoretical_loss": 3.3608712304566115, + "tokens_seen": 2572484608 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1409639, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9053080081939697, + "objective/train/theoretical_loss": 3.3608644399211856, + "objective/train/tokens_used": 931409376, + "theoretical_loss": 3.3608644399211856, + "tokens_seen": 2572550144 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002242626262626263, + "loss": 2.6655, + "theoretical_loss": 3.3608644399211856, + "tokens_seen": 2572550144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022424242424242424, + "loss": 2.6647, + "theoretical_loss": 3.360857649607183, + "tokens_seen": 2572615680 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022422222222222223, + "loss": 2.5757, + "theoretical_loss": 3.3608508595145903, + "tokens_seen": 2572681216 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002242020202020202, + "loss": 2.5073, + "theoretical_loss": 3.360844069643395, + "tokens_seen": 2572746752 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022418181818181817, + "loss": 2.6572, + "theoretical_loss": 3.360837279993585, + "tokens_seen": 2572812288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022416161616161618, + "loss": 2.518, + "theoretical_loss": 3.360830490565146, + "tokens_seen": 2572877824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022414141414141414, + "loss": 2.3146, + "theoretical_loss": 3.3608237013580657, + "tokens_seen": 2572943360 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022412121212121212, + "loss": 2.6971, + "theoretical_loss": 3.3608169123723317, + "tokens_seen": 2573008896 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002241010101010101, + "loss": 2.5024, + "theoretical_loss": 3.360810123607931, + "tokens_seen": 2573074432 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002240808080808081, + "loss": 2.5778, + "theoretical_loss": 3.3608033350648503, + "tokens_seen": 2573139968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022406060606060607, + "loss": 2.4016, + "theoretical_loss": 3.3607965467430767, + "tokens_seen": 2573205504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022404040404040406, + "loss": 2.6393, + "theoretical_loss": 3.360789758642598, + "tokens_seen": 2573271040 + }, + { + "epoch": 0.56, + "learning_rate": 0.000224020202020202, + "loss": 2.5162, + "theoretical_loss": 3.360782970763401, + "tokens_seen": 2573336576 + }, + { + "epoch": 0.56, + "learning_rate": 0.000224, + "loss": 2.6595, + "theoretical_loss": 3.3607761831054734, + "tokens_seen": 2573402112 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022397979797979798, + "loss": 2.3669, + "theoretical_loss": 3.360769395668801, + "tokens_seen": 2573467648 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022395959595959596, + "loss": 2.3576, + "theoretical_loss": 3.3607626084533724, + "tokens_seen": 2573533184 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022393939393939395, + "loss": 2.602, + "theoretical_loss": 3.3607558214591737, + "tokens_seen": 2573598720 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002239191919191919, + "loss": 2.7358, + "theoretical_loss": 3.360749034686193, + "tokens_seen": 2573664256 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022389898989898992, + "loss": 2.4063, + "theoretical_loss": 3.3607422481344167, + "tokens_seen": 2573729792 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022387878787878787, + "loss": 2.508, + "theoretical_loss": 3.360735461803832, + "tokens_seen": 2573795328 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022385858585858588, + "loss": 2.5933, + "theoretical_loss": 3.3607286756944266, + "tokens_seen": 2573860864 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022383838383838384, + "loss": 2.6648, + "theoretical_loss": 3.360721889806187, + "tokens_seen": 2573926400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002238181818181818, + "loss": 2.5607, + "theoretical_loss": 3.360715104139101, + "tokens_seen": 2573991936 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002237979797979798, + "loss": 2.5683, + "theoretical_loss": 3.3607083186931552, + "tokens_seen": 2574057472 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022377777777777777, + "loss": 2.5328, + "theoretical_loss": 3.3607015334683368, + "tokens_seen": 2574123008 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1410544, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5232367515563965, + "objective/train/theoretical_loss": 3.360694748464634, + "objective/train/tokens_used": 933047776, + "theoretical_loss": 3.360694748464634, + "tokens_seen": 2574188544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022375757575757578, + "loss": 2.523, + "theoretical_loss": 3.360694748464634, + "tokens_seen": 2574188544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022373737373737373, + "loss": 2.4974, + "theoretical_loss": 3.3606879636820324, + "tokens_seen": 2574254080 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022371717171717172, + "loss": 2.8157, + "theoretical_loss": 3.3606811791205202, + "tokens_seen": 2574319616 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002236969696969697, + "loss": 2.736, + "theoretical_loss": 3.360674394780084, + "tokens_seen": 2574385152 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022367676767676769, + "loss": 2.4203, + "theoretical_loss": 3.3606676106607116, + "tokens_seen": 2574450688 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022365656565656567, + "loss": 2.5239, + "theoretical_loss": 3.36066082676239, + "tokens_seen": 2574516224 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022363636363636363, + "loss": 2.2266, + "theoretical_loss": 3.3606540430851055, + "tokens_seen": 2574581760 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022361616161616164, + "loss": 2.439, + "theoretical_loss": 3.3606472596288466, + "tokens_seen": 2574647296 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002235959595959596, + "loss": 2.5288, + "theoretical_loss": 3.3606404763935993, + "tokens_seen": 2574712832 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022357575757575758, + "loss": 2.5587, + "theoretical_loss": 3.3606336933793517, + "tokens_seen": 2574778368 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022355555555555556, + "loss": 2.5504, + "theoretical_loss": 3.3606269105860904, + "tokens_seen": 2574843904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022353535353535355, + "loss": 2.57, + "theoretical_loss": 3.360620128013803, + "tokens_seen": 2574909440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022351515151515153, + "loss": 2.6005, + "theoretical_loss": 3.3606133456624763, + "tokens_seen": 2574974976 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022349494949494951, + "loss": 2.6424, + "theoretical_loss": 3.360606563532097, + "tokens_seen": 2575040512 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022347474747474747, + "loss": 2.526, + "theoretical_loss": 3.3605997816226534, + "tokens_seen": 2575106048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022345454545454546, + "loss": 2.6213, + "theoretical_loss": 3.3605929999341324, + "tokens_seen": 2575171584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022343434343434344, + "loss": 2.5482, + "theoretical_loss": 3.360586218466521, + "tokens_seen": 2575237120 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022341414141414142, + "loss": 2.6305, + "theoretical_loss": 3.3605794372198057, + "tokens_seen": 2575302656 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002233939393939394, + "loss": 2.691, + "theoretical_loss": 3.360572656193975, + "tokens_seen": 2575368192 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022337373737373736, + "loss": 2.3053, + "theoretical_loss": 3.360565875389015, + "tokens_seen": 2575433728 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022335353535353537, + "loss": 2.2463, + "theoretical_loss": 3.3605590948049135, + "tokens_seen": 2575499264 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022333333333333333, + "loss": 2.6453, + "theoretical_loss": 3.3605523144416574, + "tokens_seen": 2575564800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022331313131313132, + "loss": 2.722, + "theoretical_loss": 3.360545534299234, + "tokens_seen": 2575630336 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002232929292929293, + "loss": 2.782, + "theoretical_loss": 3.36053875437763, + "tokens_seen": 2575695872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022327272727272726, + "loss": 2.558, + "theoretical_loss": 3.3605319746768334, + "tokens_seen": 2575761408 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1411184, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.559873342514038, + "objective/train/theoretical_loss": 3.360525195196831, + "objective/train/tokens_used": 934686176, + "theoretical_loss": 3.360525195196831, + "tokens_seen": 2575826944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022325252525252527, + "loss": 2.3536, + "theoretical_loss": 3.360525195196831, + "tokens_seen": 2575826944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022323232323232322, + "loss": 2.5187, + "theoretical_loss": 3.3605184159376105, + "tokens_seen": 2575892480 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022321212121212124, + "loss": 2.7863, + "theoretical_loss": 3.360511636899158, + "tokens_seen": 2575958016 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002231919191919192, + "loss": 2.8348, + "theoretical_loss": 3.3605048580814616, + "tokens_seen": 2576023552 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022317171717171718, + "loss": 2.5115, + "theoretical_loss": 3.360498079484508, + "tokens_seen": 2576089088 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022315151515151516, + "loss": 2.61, + "theoretical_loss": 3.3604913011082846, + "tokens_seen": 2576154624 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022313131313131312, + "loss": 2.6702, + "theoretical_loss": 3.360484522952779, + "tokens_seen": 2576220160 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022311111111111113, + "loss": 2.7061, + "theoretical_loss": 3.3604777450179775, + "tokens_seen": 2576285696 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022309090909090909, + "loss": 2.5649, + "theoretical_loss": 3.3604709673038684, + "tokens_seen": 2576351232 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022307070707070707, + "loss": 2.708, + "theoretical_loss": 3.360464189810438, + "tokens_seen": 2576416768 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022305050505050505, + "loss": 2.8263, + "theoretical_loss": 3.360457412537673, + "tokens_seen": 2576482304 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022303030303030304, + "loss": 2.5526, + "theoretical_loss": 3.3604506354855626, + "tokens_seen": 2576547840 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022301010101010102, + "loss": 2.5622, + "theoretical_loss": 3.3604438586540923, + "tokens_seen": 2576613376 + }, + { + "epoch": 0.56, + "learning_rate": 0.000222989898989899, + "loss": 2.7016, + "theoretical_loss": 3.3604370820432496, + "tokens_seen": 2576678912 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022296969696969696, + "loss": 2.5369, + "theoretical_loss": 3.3604303056530225, + "tokens_seen": 2576744448 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022294949494949495, + "loss": 2.7631, + "theoretical_loss": 3.360423529483397, + "tokens_seen": 2576809984 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022292929292929293, + "loss": 2.5487, + "theoretical_loss": 3.3604167535343614, + "tokens_seen": 2576875520 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022290909090909091, + "loss": 2.5263, + "theoretical_loss": 3.3604099778059022, + "tokens_seen": 2576941056 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002228888888888889, + "loss": 2.7057, + "theoretical_loss": 3.360403202298007, + "tokens_seen": 2577006592 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022286868686868685, + "loss": 2.6283, + "theoretical_loss": 3.3603964270106625, + "tokens_seen": 2577072128 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022284848484848487, + "loss": 2.5462, + "theoretical_loss": 3.3603896519438567, + "tokens_seen": 2577137664 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022282828282828282, + "loss": 2.3673, + "theoretical_loss": 3.3603828770975763, + "tokens_seen": 2577203200 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022280808080808083, + "loss": 2.7053, + "theoretical_loss": 3.3603761024718084, + "tokens_seen": 2577268736 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002227878787878788, + "loss": 2.5939, + "theoretical_loss": 3.3603693280665405, + "tokens_seen": 2577334272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022276767676767677, + "loss": 2.5805, + "theoretical_loss": 3.3603625538817603, + "tokens_seen": 2577399808 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1412369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5855770111083984, + "objective/train/theoretical_loss": 3.3603557799174535, + "objective/train/tokens_used": 936324576, + "theoretical_loss": 3.3603557799174535, + "tokens_seen": 2577465344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022274747474747476, + "loss": 2.7335, + "theoretical_loss": 3.3603557799174535, + "tokens_seen": 2577465344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022272727272727272, + "loss": 2.5779, + "theoretical_loss": 3.360349006173609, + "tokens_seen": 2577530880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022270707070707073, + "loss": 2.6307, + "theoretical_loss": 3.360342232650213, + "tokens_seen": 2577596416 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022268686868686868, + "loss": 2.393, + "theoretical_loss": 3.360335459347253, + "tokens_seen": 2577661952 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002226666666666667, + "loss": 2.7563, + "theoretical_loss": 3.360328686264716, + "tokens_seen": 2577727488 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022264646464646465, + "loss": 2.4388, + "theoretical_loss": 3.36032191340259, + "tokens_seen": 2577793024 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002226262626262626, + "loss": 2.7172, + "theoretical_loss": 3.3603151407608616, + "tokens_seen": 2577858560 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022260606060606062, + "loss": 2.5949, + "theoretical_loss": 3.360308368339518, + "tokens_seen": 2577924096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022258585858585858, + "loss": 2.5263, + "theoretical_loss": 3.3603015961385463, + "tokens_seen": 2577989632 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002225656565656566, + "loss": 2.5724, + "theoretical_loss": 3.360294824157934, + "tokens_seen": 2578055168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022254545454545454, + "loss": 2.6198, + "theoretical_loss": 3.3602880523976686, + "tokens_seen": 2578120704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022252525252525253, + "loss": 2.4131, + "theoretical_loss": 3.3602812808577367, + "tokens_seen": 2578186240 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002225050505050505, + "loss": 2.4091, + "theoretical_loss": 3.360274509538126, + "tokens_seen": 2578251776 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002224848484848485, + "loss": 2.4155, + "theoretical_loss": 3.3602677384388233, + "tokens_seen": 2578317312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022246464646464648, + "loss": 2.5602, + "theoretical_loss": 3.3602609675598165, + "tokens_seen": 2578382848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022244444444444444, + "loss": 2.6259, + "theoretical_loss": 3.3602541969010926, + "tokens_seen": 2578448384 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022242424242424242, + "loss": 2.5302, + "theoretical_loss": 3.3602474264626383, + "tokens_seen": 2578513920 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002224040404040404, + "loss": 2.7642, + "theoretical_loss": 3.3602406562444416, + "tokens_seen": 2578579456 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002223838383838384, + "loss": 2.4968, + "theoretical_loss": 3.3602338862464887, + "tokens_seen": 2578644992 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022236363636363637, + "loss": 2.7516, + "theoretical_loss": 3.360227116468768, + "tokens_seen": 2578710528 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022234343434343436, + "loss": 2.3713, + "theoretical_loss": 3.360220346911266, + "tokens_seen": 2578776064 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002223232323232323, + "loss": 2.4961, + "theoretical_loss": 3.3602135775739708, + "tokens_seen": 2578841600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022230303030303032, + "loss": 2.5778, + "theoretical_loss": 3.3602068084568684, + "tokens_seen": 2578907136 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022228282828282828, + "loss": 2.5252, + "theoretical_loss": 3.3602000395599463, + "tokens_seen": 2578972672 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022226262626262626, + "loss": 2.6237, + "theoretical_loss": 3.360193270883193, + "tokens_seen": 2579038208 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1412775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.703599452972412, + "objective/train/theoretical_loss": 3.3601865024265942, + "objective/train/tokens_used": 937962976, + "theoretical_loss": 3.3601865024265942, + "tokens_seen": 2579103744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022224242424242425, + "loss": 2.522, + "theoretical_loss": 3.3601865024265942, + "tokens_seen": 2579103744 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002222222222222222, + "loss": 2.4549, + "theoretical_loss": 3.360179734190138, + "tokens_seen": 2579169280 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022220202020202022, + "loss": 2.5503, + "theoretical_loss": 3.3601729661738116, + "tokens_seen": 2579234816 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022218181818181817, + "loss": 2.7587, + "theoretical_loss": 3.360166198377602, + "tokens_seen": 2579300352 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022216161616161618, + "loss": 2.5149, + "theoretical_loss": 3.360159430801496, + "tokens_seen": 2579365888 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022214141414141414, + "loss": 2.4234, + "theoretical_loss": 3.360152663445482, + "tokens_seen": 2579431424 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022212121212121213, + "loss": 2.6957, + "theoretical_loss": 3.3601458963095463, + "tokens_seen": 2579496960 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002221010101010101, + "loss": 2.7463, + "theoretical_loss": 3.3601391293936764, + "tokens_seen": 2579562496 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022208080808080807, + "loss": 2.7041, + "theoretical_loss": 3.3601323626978603, + "tokens_seen": 2579628032 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022206060606060608, + "loss": 2.6664, + "theoretical_loss": 3.360125596222084, + "tokens_seen": 2579693568 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022204040404040403, + "loss": 2.3405, + "theoretical_loss": 3.3601188299663356, + "tokens_seen": 2579759104 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022202020202020202, + "loss": 2.662, + "theoretical_loss": 3.360112063930602, + "tokens_seen": 2579824640 + }, + { + "epoch": 0.56, + "learning_rate": 0.000222, + "loss": 2.5805, + "theoretical_loss": 3.3601052981148705, + "tokens_seen": 2579890176 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022197979797979799, + "loss": 2.7325, + "theoretical_loss": 3.3600985325191286, + "tokens_seen": 2579955712 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022195959595959597, + "loss": 2.5754, + "theoretical_loss": 3.360091767143363, + "tokens_seen": 2580021248 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022193939393939395, + "loss": 2.6249, + "theoretical_loss": 3.360085001987562, + "tokens_seen": 2580086784 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022191919191919194, + "loss": 2.4187, + "theoretical_loss": 3.3600782370517113, + "tokens_seen": 2580152320 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002218989898989899, + "loss": 2.671, + "theoretical_loss": 3.3600714723358, + "tokens_seen": 2580217856 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022187878787878788, + "loss": 2.4087, + "theoretical_loss": 3.360064707839814, + "tokens_seen": 2580283392 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022185858585858586, + "loss": 2.4344, + "theoretical_loss": 3.3600579435637408, + "tokens_seen": 2580348928 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022183838383838385, + "loss": 2.5499, + "theoretical_loss": 3.360051179507568, + "tokens_seen": 2580414464 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022181818181818183, + "loss": 2.5075, + "theoretical_loss": 3.3600444156712825, + "tokens_seen": 2580480000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022179797979797981, + "loss": 2.7245, + "theoretical_loss": 3.360037652054872, + "tokens_seen": 2580545536 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022177777777777777, + "loss": 2.8689, + "theoretical_loss": 3.3600308886583234, + "tokens_seen": 2580611072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022175757575757576, + "loss": 2.4853, + "theoretical_loss": 3.3600241254816243, + "tokens_seen": 2580676608 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1413675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5682265758514404, + "objective/train/theoretical_loss": 3.3600173625247622, + "objective/train/tokens_used": 939601376, + "theoretical_loss": 3.3600173625247622, + "tokens_seen": 2580742144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022173737373737374, + "loss": 2.6656, + "theoretical_loss": 3.3600173625247622, + "tokens_seen": 2580742144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022171717171717172, + "loss": 2.5645, + "theoretical_loss": 3.3600105997877234, + "tokens_seen": 2580807680 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002216969696969697, + "loss": 2.5198, + "theoretical_loss": 3.360003837270496, + "tokens_seen": 2580873216 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022167676767676766, + "loss": 2.4892, + "theoretical_loss": 3.3599970749730668, + "tokens_seen": 2580938752 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022165656565656568, + "loss": 2.2658, + "theoretical_loss": 3.359990312895423, + "tokens_seen": 2581004288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022163636363636363, + "loss": 2.5639, + "theoretical_loss": 3.359983551037553, + "tokens_seen": 2581069824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022161616161616164, + "loss": 2.7028, + "theoretical_loss": 3.3599767893994428, + "tokens_seen": 2581135360 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002215959595959596, + "loss": 2.5408, + "theoretical_loss": 3.3599700279810802, + "tokens_seen": 2581200896 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022157575757575756, + "loss": 2.5146, + "theoretical_loss": 3.359963266782452, + "tokens_seen": 2581266432 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022155555555555557, + "loss": 2.5869, + "theoretical_loss": 3.359956505803546, + "tokens_seen": 2581331968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022153535353535352, + "loss": 2.4845, + "theoretical_loss": 3.35994974504435, + "tokens_seen": 2581397504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022151515151515154, + "loss": 2.9334, + "theoretical_loss": 3.3599429845048503, + "tokens_seen": 2581463040 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002214949494949495, + "loss": 2.7303, + "theoretical_loss": 3.3599362241850343, + "tokens_seen": 2581528576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022147474747474748, + "loss": 2.6198, + "theoretical_loss": 3.35992946408489, + "tokens_seen": 2581594112 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022145454545454546, + "loss": 2.7762, + "theoretical_loss": 3.3599227042044033, + "tokens_seen": 2581659648 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022143434343434344, + "loss": 2.5838, + "theoretical_loss": 3.3599159445435633, + "tokens_seen": 2581725184 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022141414141414143, + "loss": 2.5541, + "theoretical_loss": 3.3599091851023557, + "tokens_seen": 2581790720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022139393939393939, + "loss": 2.5901, + "theoretical_loss": 3.359902425880769, + "tokens_seen": 2581856256 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022137373737373737, + "loss": 2.6997, + "theoretical_loss": 3.35989566687879, + "tokens_seen": 2581921792 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022135353535353535, + "loss": 2.5625, + "theoretical_loss": 3.3598889080964054, + "tokens_seen": 2581987328 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022133333333333334, + "loss": 2.5755, + "theoretical_loss": 3.359882149533603, + "tokens_seen": 2582052864 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022131313131313132, + "loss": 2.5887, + "theoretical_loss": 3.3598753911903705, + "tokens_seen": 2582118400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002212929292929293, + "loss": 2.531, + "theoretical_loss": 3.359868633066695, + "tokens_seen": 2582183936 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022127272727272726, + "loss": 2.6268, + "theoretical_loss": 3.3598618751625633, + "tokens_seen": 2582249472 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022125252525252527, + "loss": 2.6622, + "theoretical_loss": 3.359855117477963, + "tokens_seen": 2582315008 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1414348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.053363561630249, + "objective/train/theoretical_loss": 3.3598483600128817, + "objective/train/tokens_used": 941239776, + "theoretical_loss": 3.3598483600128817, + "tokens_seen": 2582380544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022123232323232323, + "loss": 2.4699, + "theoretical_loss": 3.3598483600128817, + "tokens_seen": 2582380544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022121212121212121, + "loss": 2.5715, + "theoretical_loss": 3.359841602767306, + "tokens_seen": 2582446080 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002211919191919192, + "loss": 2.3848, + "theoretical_loss": 3.359834845741224, + "tokens_seen": 2582511616 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022117171717171715, + "loss": 2.6419, + "theoretical_loss": 3.3598280889346226, + "tokens_seen": 2582577152 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022115151515151517, + "loss": 2.2869, + "theoretical_loss": 3.359821332347489, + "tokens_seen": 2582642688 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022113131313131312, + "loss": 2.6196, + "theoretical_loss": 3.3598145759798106, + "tokens_seen": 2582708224 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022111111111111113, + "loss": 2.592, + "theoretical_loss": 3.3598078198315746, + "tokens_seen": 2582773760 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002210909090909091, + "loss": 2.548, + "theoretical_loss": 3.3598010639027684, + "tokens_seen": 2582839296 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002210707070707071, + "loss": 2.6218, + "theoretical_loss": 3.3597943081933797, + "tokens_seen": 2582904832 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022105050505050506, + "loss": 2.701, + "theoretical_loss": 3.359787552703395, + "tokens_seen": 2582970368 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022103030303030302, + "loss": 2.5702, + "theoretical_loss": 3.3597807974328022, + "tokens_seen": 2583035904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022101010101010103, + "loss": 2.5625, + "theoretical_loss": 3.3597740423815883, + "tokens_seen": 2583101440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022098989898989898, + "loss": 2.5406, + "theoretical_loss": 3.359767287549741, + "tokens_seen": 2583166976 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220969696969697, + "loss": 2.7338, + "theoretical_loss": 3.3597605329372473, + "tokens_seen": 2583232512 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022094949494949495, + "loss": 2.3598, + "theoretical_loss": 3.3597537785440945, + "tokens_seen": 2583298048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022092929292929294, + "loss": 2.7676, + "theoretical_loss": 3.3597470243702703, + "tokens_seen": 2583363584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022090909090909092, + "loss": 2.4892, + "theoretical_loss": 3.3597402704157613, + "tokens_seen": 2583429120 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022088888888888888, + "loss": 2.5141, + "theoretical_loss": 3.3597335166805555, + "tokens_seen": 2583494656 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002208686868686869, + "loss": 2.6098, + "theoretical_loss": 3.35972676316464, + "tokens_seen": 2583560192 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022084848484848484, + "loss": 2.6547, + "theoretical_loss": 3.359720009868002, + "tokens_seen": 2583625728 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022082828282828283, + "loss": 2.5265, + "theoretical_loss": 3.3597132567906285, + "tokens_seen": 2583691264 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002208080808080808, + "loss": 2.821, + "theoretical_loss": 3.3597065039325074, + "tokens_seen": 2583756800 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002207878787878788, + "loss": 2.5841, + "theoretical_loss": 3.3596997512936264, + "tokens_seen": 2583822336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022076767676767678, + "loss": 2.7251, + "theoretical_loss": 3.3596929988739714, + "tokens_seen": 2583887872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022074747474747476, + "loss": 2.4899, + "theoretical_loss": 3.359686246673531, + "tokens_seen": 2583953408 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1415593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.620751142501831, + "objective/train/theoretical_loss": 3.3596794946922923, + "objective/train/tokens_used": 942878176, + "theoretical_loss": 3.3596794946922923, + "tokens_seen": 2584018944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022072727272727272, + "loss": 2.3497, + "theoretical_loss": 3.3596794946922923, + "tokens_seen": 2584018944 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002207070707070707, + "loss": 2.5823, + "theoretical_loss": 3.359672742930242, + "tokens_seen": 2584084480 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002206868686868687, + "loss": 2.5812, + "theoretical_loss": 3.359665991387368, + "tokens_seen": 2584150016 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022066666666666667, + "loss": 2.5143, + "theoretical_loss": 3.3596592400636576, + "tokens_seen": 2584215552 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022064646464646466, + "loss": 2.5914, + "theoretical_loss": 3.3596524889590977, + "tokens_seen": 2584281088 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002206262626262626, + "loss": 2.6124, + "theoretical_loss": 3.359645738073676, + "tokens_seen": 2584346624 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022060606060606062, + "loss": 2.4529, + "theoretical_loss": 3.3596389874073798, + "tokens_seen": 2584412160 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022058585858585858, + "loss": 2.4931, + "theoretical_loss": 3.3596322369601963, + "tokens_seen": 2584477696 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002205656565656566, + "loss": 2.5786, + "theoretical_loss": 3.359625486732113, + "tokens_seen": 2584543232 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022054545454545455, + "loss": 2.5751, + "theoretical_loss": 3.3596187367231174, + "tokens_seen": 2584608768 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002205252525252525, + "loss": 2.5049, + "theoretical_loss": 3.359611986933196, + "tokens_seen": 2584674304 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022050505050505052, + "loss": 2.5752, + "theoretical_loss": 3.359605237362337, + "tokens_seen": 2584739840 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022048484848484847, + "loss": 2.6426, + "theoretical_loss": 3.3595984880105276, + "tokens_seen": 2584805376 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022046464646464649, + "loss": 2.5223, + "theoretical_loss": 3.359591738877755, + "tokens_seen": 2584870912 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022044444444444444, + "loss": 2.5745, + "theoretical_loss": 3.3595849899640067, + "tokens_seen": 2584936448 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022042424242424243, + "loss": 2.5532, + "theoretical_loss": 3.3595782412692694, + "tokens_seen": 2585001984 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002204040404040404, + "loss": 2.5212, + "theoretical_loss": 3.359571492793531, + "tokens_seen": 2585067520 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002203838383838384, + "loss": 2.423, + "theoretical_loss": 3.359564744536779, + "tokens_seen": 2585133056 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022036363636363638, + "loss": 2.565, + "theoretical_loss": 3.3595579964990003, + "tokens_seen": 2585198592 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022034343434343433, + "loss": 2.5455, + "theoretical_loss": 3.359551248680183, + "tokens_seen": 2585264128 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022032323232323232, + "loss": 2.5565, + "theoretical_loss": 3.359544501080313, + "tokens_seen": 2585329664 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002203030303030303, + "loss": 2.6657, + "theoretical_loss": 3.359537753699379, + "tokens_seen": 2585395200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002202828282828283, + "loss": 2.7, + "theoretical_loss": 3.3595310065373676, + "tokens_seen": 2585460736 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022026262626262627, + "loss": 2.399, + "theoretical_loss": 3.3595242595942665, + "tokens_seen": 2585526272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022024242424242425, + "loss": 2.6826, + "theoretical_loss": 3.359517512870063, + "tokens_seen": 2585591808 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1416631, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3755147457122803, + "objective/train/theoretical_loss": 3.3595107663647448, + "objective/train/tokens_used": 944516576, + "theoretical_loss": 3.3595107663647448, + "tokens_seen": 2585657344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022022222222222224, + "loss": 2.4763, + "theoretical_loss": 3.3595107663647448, + "tokens_seen": 2585657344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022020202020202022, + "loss": 2.5341, + "theoretical_loss": 3.3595040200782984, + "tokens_seen": 2585722880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022018181818181818, + "loss": 2.6848, + "theoretical_loss": 3.359497274010712, + "tokens_seen": 2585788416 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022016161616161616, + "loss": 2.5841, + "theoretical_loss": 3.3594905281619725, + "tokens_seen": 2585853952 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022014141414141415, + "loss": 2.4764, + "theoretical_loss": 3.359483782532067, + "tokens_seen": 2585919488 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022012121212121213, + "loss": 2.534, + "theoretical_loss": 3.3594770371209837, + "tokens_seen": 2585985024 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022010101010101012, + "loss": 2.4264, + "theoretical_loss": 3.359470291928709, + "tokens_seen": 2586050560 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022008080808080807, + "loss": 2.7399, + "theoretical_loss": 3.359463546955231, + "tokens_seen": 2586116096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022006060606060608, + "loss": 2.3414, + "theoretical_loss": 3.3594568022005364, + "tokens_seen": 2586181632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022004040404040404, + "loss": 2.5416, + "theoretical_loss": 3.3594500576646134, + "tokens_seen": 2586247168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022002020202020202, + "loss": 2.5454, + "theoretical_loss": 3.3594433133474486, + "tokens_seen": 2586312704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022, + "loss": 2.5902, + "theoretical_loss": 3.35943656924903, + "tokens_seen": 2586378240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021997979797979796, + "loss": 2.4311, + "theoretical_loss": 3.359429825369344, + "tokens_seen": 2586443776 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021995959595959598, + "loss": 2.4764, + "theoretical_loss": 3.3594230817083788, + "tokens_seen": 2586509312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021993939393939393, + "loss": 2.6504, + "theoretical_loss": 3.3594163382661217, + "tokens_seen": 2586574848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021991919191919194, + "loss": 2.5732, + "theoretical_loss": 3.3594095950425595, + "tokens_seen": 2586640384 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002198989898989899, + "loss": 2.5213, + "theoretical_loss": 3.3594028520376806, + "tokens_seen": 2586705920 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021987878787878788, + "loss": 2.5253, + "theoretical_loss": 3.3593961092514713, + "tokens_seen": 2586771456 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021985858585858587, + "loss": 2.4615, + "theoretical_loss": 3.3593893666839194, + "tokens_seen": 2586836992 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021983838383838383, + "loss": 2.498, + "theoretical_loss": 3.3593826243350127, + "tokens_seen": 2586902528 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021981818181818184, + "loss": 2.503, + "theoretical_loss": 3.3593758822047377, + "tokens_seen": 2586968064 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002197979797979798, + "loss": 2.459, + "theoretical_loss": 3.359369140293082, + "tokens_seen": 2587033600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021977777777777778, + "loss": 2.5173, + "theoretical_loss": 3.359362398600034, + "tokens_seen": 2587099136 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021975757575757576, + "loss": 2.5156, + "theoretical_loss": 3.3593556571255796, + "tokens_seen": 2587164672 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021973737373737375, + "loss": 2.3776, + "theoretical_loss": 3.359348915869707, + "tokens_seen": 2587230208 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1417164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6016485691070557, + "objective/train/theoretical_loss": 3.3593421748324035, + "objective/train/tokens_used": 946154976, + "theoretical_loss": 3.3593421748324035, + "tokens_seen": 2587295744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021971717171717173, + "loss": 2.6303, + "theoretical_loss": 3.3593421748324035, + "tokens_seen": 2587295744 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002196969696969697, + "loss": 2.5608, + "theoretical_loss": 3.3593354340136563, + "tokens_seen": 2587361280 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021967676767676767, + "loss": 2.4379, + "theoretical_loss": 3.359328693413453, + "tokens_seen": 2587426816 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021965656565656565, + "loss": 2.3855, + "theoretical_loss": 3.3593219530317806, + "tokens_seen": 2587492352 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021963636363636364, + "loss": 2.6761, + "theoretical_loss": 3.359315212868627, + "tokens_seen": 2587557888 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021961616161616162, + "loss": 2.4719, + "theoretical_loss": 3.3593084729239795, + "tokens_seen": 2587623424 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002195959595959596, + "loss": 2.5267, + "theoretical_loss": 3.359301733197825, + "tokens_seen": 2587688960 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021957575757575756, + "loss": 2.564, + "theoretical_loss": 3.359294993690151, + "tokens_seen": 2587754496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021955555555555557, + "loss": 2.5549, + "theoretical_loss": 3.3592882544009455, + "tokens_seen": 2587820032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021953535353535353, + "loss": 2.3944, + "theoretical_loss": 3.359281515330195, + "tokens_seen": 2587885568 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021951515151515154, + "loss": 2.465, + "theoretical_loss": 3.3592747764778874, + "tokens_seen": 2587951104 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002194949494949495, + "loss": 2.6697, + "theoretical_loss": 3.3592680378440103, + "tokens_seen": 2588016640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021947474747474746, + "loss": 2.5515, + "theoretical_loss": 3.3592612994285505, + "tokens_seen": 2588082176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021945454545454547, + "loss": 2.4575, + "theoretical_loss": 3.359254561231496, + "tokens_seen": 2588147712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021943434343434342, + "loss": 2.6306, + "theoretical_loss": 3.3592478232528338, + "tokens_seen": 2588213248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021941414141414143, + "loss": 2.6545, + "theoretical_loss": 3.3592410854925516, + "tokens_seen": 2588278784 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002193939393939394, + "loss": 2.6376, + "theoretical_loss": 3.359234347950636, + "tokens_seen": 2588344320 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002193737373737374, + "loss": 2.556, + "theoretical_loss": 3.3592276106270758, + "tokens_seen": 2588409856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021935353535353536, + "loss": 2.6016, + "theoretical_loss": 3.3592208735218567, + "tokens_seen": 2588475392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021933333333333332, + "loss": 2.7429, + "theoretical_loss": 3.3592141366349675, + "tokens_seen": 2588540928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021931313131313133, + "loss": 2.4559, + "theoretical_loss": 3.3592073999663947, + "tokens_seen": 2588606464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021929292929292928, + "loss": 2.5043, + "theoretical_loss": 3.3592006635161265, + "tokens_seen": 2588672000 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002192727272727273, + "loss": 2.4034, + "theoretical_loss": 3.3591939272841493, + "tokens_seen": 2588737536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021925252525252525, + "loss": 2.4723, + "theoretical_loss": 3.359187191270452, + "tokens_seen": 2588803072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021923232323232324, + "loss": 2.7292, + "theoretical_loss": 3.35918045547502, + "tokens_seen": 2588868608 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1417967, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.8440449237823486, + "objective/train/theoretical_loss": 3.3591737198978424, + "objective/train/tokens_used": 947793376, + "theoretical_loss": 3.3591737198978424, + "tokens_seen": 2588934144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021921212121212122, + "loss": 2.4915, + "theoretical_loss": 3.3591737198978424, + "tokens_seen": 2588934144 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002191919191919192, + "loss": 2.4884, + "theoretical_loss": 3.359166984538906, + "tokens_seen": 2588999680 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002191717171717172, + "loss": 2.5033, + "theoretical_loss": 3.359160249398198, + "tokens_seen": 2589065216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021915151515151514, + "loss": 2.6362, + "theoretical_loss": 3.3591535144757056, + "tokens_seen": 2589130752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021913131313131313, + "loss": 2.4171, + "theoretical_loss": 3.359146779771417, + "tokens_seen": 2589196288 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002191111111111111, + "loss": 2.4259, + "theoretical_loss": 3.3591400452853195, + "tokens_seen": 2589261824 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002190909090909091, + "loss": 2.5817, + "theoretical_loss": 3.3591333110173998, + "tokens_seen": 2589327360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021907070707070708, + "loss": 2.7101, + "theoretical_loss": 3.3591265769676455, + "tokens_seen": 2589392896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021905050505050506, + "loss": 2.4445, + "theoretical_loss": 3.3591198431360443, + "tokens_seen": 2589458432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021903030303030302, + "loss": 2.4888, + "theoretical_loss": 3.359113109522584, + "tokens_seen": 2589523968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021901010101010103, + "loss": 2.6211, + "theoretical_loss": 3.3591063761272513, + "tokens_seen": 2589589504 + }, + { + "epoch": 0.57, + "learning_rate": 0.000218989898989899, + "loss": 2.6563, + "theoretical_loss": 3.3590996429500337, + "tokens_seen": 2589655040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021896969696969697, + "loss": 2.6382, + "theoretical_loss": 3.3590929099909186, + "tokens_seen": 2589720576 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021894949494949496, + "loss": 2.3845, + "theoretical_loss": 3.3590861772498943, + "tokens_seen": 2589786112 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021892929292929291, + "loss": 2.5932, + "theoretical_loss": 3.359079444726947, + "tokens_seen": 2589851648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021890909090909092, + "loss": 2.4212, + "theoretical_loss": 3.3590727124220647, + "tokens_seen": 2589917184 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021888888888888888, + "loss": 2.3237, + "theoretical_loss": 3.3590659803352345, + "tokens_seen": 2589982720 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002188686868686869, + "loss": 2.739, + "theoretical_loss": 3.3590592484664445, + "tokens_seen": 2590048256 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021884848484848485, + "loss": 2.6274, + "theoretical_loss": 3.3590525168156815, + "tokens_seen": 2590113792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021882828282828283, + "loss": 2.5006, + "theoretical_loss": 3.359045785382933, + "tokens_seen": 2590179328 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021880808080808082, + "loss": 2.4492, + "theoretical_loss": 3.359039054168187, + "tokens_seen": 2590244864 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021878787878787877, + "loss": 2.5897, + "theoretical_loss": 3.35903232317143, + "tokens_seen": 2590310400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021876767676767679, + "loss": 2.59, + "theoretical_loss": 3.35902559239265, + "tokens_seen": 2590375936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021874747474747474, + "loss": 2.295, + "theoretical_loss": 3.359018861831834, + "tokens_seen": 2590441472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021872727272727273, + "loss": 2.4984, + "theoretical_loss": 3.3590121314889703, + "tokens_seen": 2590507008 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1418580, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.70296573638916, + "objective/train/theoretical_loss": 3.359005401364046, + "objective/train/tokens_used": 949431776, + "theoretical_loss": 3.359005401364046, + "tokens_seen": 2590572544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002187070707070707, + "loss": 2.5571, + "theoretical_loss": 3.359005401364046, + "tokens_seen": 2590572544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002186868686868687, + "loss": 2.4321, + "theoretical_loss": 3.3589986714570474, + "tokens_seen": 2590638080 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021866666666666668, + "loss": 2.5474, + "theoretical_loss": 3.3589919417679632, + "tokens_seen": 2590703616 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021864646464646466, + "loss": 2.4746, + "theoretical_loss": 3.3589852122967807, + "tokens_seen": 2590769152 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021862626262626262, + "loss": 2.6257, + "theoretical_loss": 3.3589784830434866, + "tokens_seen": 2590834688 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002186060606060606, + "loss": 2.6667, + "theoretical_loss": 3.3589717540080692, + "tokens_seen": 2590900224 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002185858585858586, + "loss": 2.5003, + "theoretical_loss": 3.3589650251905154, + "tokens_seen": 2590965760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021856565656565657, + "loss": 2.4967, + "theoretical_loss": 3.358958296590813, + "tokens_seen": 2591031296 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021854545454545455, + "loss": 2.4113, + "theoretical_loss": 3.358951568208949, + "tokens_seen": 2591096832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021852525252525254, + "loss": 2.4337, + "theoretical_loss": 3.358944840044911, + "tokens_seen": 2591162368 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021850505050505052, + "loss": 2.486, + "theoretical_loss": 3.358938112098687, + "tokens_seen": 2591227904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021848484848484848, + "loss": 2.5468, + "theoretical_loss": 3.3589313843702637, + "tokens_seen": 2591293440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021846464646464646, + "loss": 2.5335, + "theoretical_loss": 3.358924656859629, + "tokens_seen": 2591358976 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021844444444444445, + "loss": 2.5353, + "theoretical_loss": 3.3589179295667697, + "tokens_seen": 2591424512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021842424242424243, + "loss": 2.4998, + "theoretical_loss": 3.3589112024916736, + "tokens_seen": 2591490048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021840404040404042, + "loss": 2.5761, + "theoretical_loss": 3.3589044756343287, + "tokens_seen": 2591555584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021838383838383837, + "loss": 2.5652, + "theoretical_loss": 3.3588977489947216, + "tokens_seen": 2591621120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021836363636363638, + "loss": 2.5676, + "theoretical_loss": 3.3588910225728403, + "tokens_seen": 2591686656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021834343434343434, + "loss": 2.5555, + "theoretical_loss": 3.358884296368672, + "tokens_seen": 2591752192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021832323232323235, + "loss": 2.4154, + "theoretical_loss": 3.3588775703822042, + "tokens_seen": 2591817728 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002183030303030303, + "loss": 2.6286, + "theoretical_loss": 3.358870844613424, + "tokens_seen": 2591883264 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021828282828282827, + "loss": 2.3768, + "theoretical_loss": 3.35886411906232, + "tokens_seen": 2591948800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021826262626262628, + "loss": 2.5213, + "theoretical_loss": 3.3588573937288784, + "tokens_seen": 2592014336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021824242424242423, + "loss": 2.4374, + "theoretical_loss": 3.358850668613087, + "tokens_seen": 2592079872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021822222222222224, + "loss": 2.6197, + "theoretical_loss": 3.3588439437149336, + "tokens_seen": 2592145408 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1419695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1405866146087646, + "objective/train/theoretical_loss": 3.3588372190344056, + "objective/train/tokens_used": 951070176, + "theoretical_loss": 3.3588372190344056, + "tokens_seen": 2592210944 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002182020202020202, + "loss": 2.2491, + "theoretical_loss": 3.3588372190344056, + "tokens_seen": 2592210944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021818181818181818, + "loss": 2.3255, + "theoretical_loss": 3.3588304945714897, + "tokens_seen": 2592276480 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021816161616161617, + "loss": 2.5569, + "theoretical_loss": 3.3588237703261745, + "tokens_seen": 2592342016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021814141414141415, + "loss": 2.4683, + "theoretical_loss": 3.3588170462984466, + "tokens_seen": 2592407552 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021812121212121214, + "loss": 2.5846, + "theoretical_loss": 3.358810322488294, + "tokens_seen": 2592473088 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002181010101010101, + "loss": 2.3852, + "theoretical_loss": 3.3588035988957032, + "tokens_seen": 2592538624 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021808080808080808, + "loss": 2.4947, + "theoretical_loss": 3.358796875520663, + "tokens_seen": 2592604160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021806060606060606, + "loss": 2.5856, + "theoretical_loss": 3.3587901523631603, + "tokens_seen": 2592669696 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021804040404040405, + "loss": 2.6675, + "theoretical_loss": 3.358783429423182, + "tokens_seen": 2592735232 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021802020202020203, + "loss": 2.5032, + "theoretical_loss": 3.3587767067007164, + "tokens_seen": 2592800768 + }, + { + "epoch": 0.57, + "learning_rate": 0.000218, + "loss": 2.469, + "theoretical_loss": 3.358769984195751, + "tokens_seen": 2592866304 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021797979797979797, + "loss": 2.3924, + "theoretical_loss": 3.358763261908272, + "tokens_seen": 2592931840 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021795959595959598, + "loss": 2.3487, + "theoretical_loss": 3.3587565398382684, + "tokens_seen": 2592997376 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021793939393939394, + "loss": 2.4935, + "theoretical_loss": 3.3587498179857267, + "tokens_seen": 2593062912 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021791919191919192, + "loss": 2.5652, + "theoretical_loss": 3.358743096350635, + "tokens_seen": 2593128448 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002178989898989899, + "loss": 2.572, + "theoretical_loss": 3.35873637493298, + "tokens_seen": 2593193984 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021787878787878786, + "loss": 2.1529, + "theoretical_loss": 3.3587296537327505, + "tokens_seen": 2593259520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021785858585858587, + "loss": 2.5112, + "theoretical_loss": 3.3587229327499326, + "tokens_seen": 2593325056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021783838383838383, + "loss": 2.5555, + "theoretical_loss": 3.358716211984514, + "tokens_seen": 2593390592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021781818181818184, + "loss": 2.6006, + "theoretical_loss": 3.358709491436483, + "tokens_seen": 2593456128 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002177979797979798, + "loss": 2.3527, + "theoretical_loss": 3.358702771105826, + "tokens_seen": 2593521664 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021777777777777776, + "loss": 2.506, + "theoretical_loss": 3.358696050992531, + "tokens_seen": 2593587200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021775757575757577, + "loss": 2.3082, + "theoretical_loss": 3.3586893310965857, + "tokens_seen": 2593652736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021773737373737372, + "loss": 2.488, + "theoretical_loss": 3.3586826114179775, + "tokens_seen": 2593718272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021771717171717173, + "loss": 2.4448, + "theoretical_loss": 3.3586758919566937, + "tokens_seen": 2593783808 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1420179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4729344844818115, + "objective/train/theoretical_loss": 3.358669172712722, + "objective/train/tokens_used": 952708576, + "theoretical_loss": 3.358669172712722, + "tokens_seen": 2593849344 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002176969696969697, + "loss": 2.3976, + "theoretical_loss": 3.358669172712722, + "tokens_seen": 2593849344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021767676767676768, + "loss": 2.2915, + "theoretical_loss": 3.3586624536860494, + "tokens_seen": 2593914880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021765656565656566, + "loss": 2.4951, + "theoretical_loss": 3.3586557348766637, + "tokens_seen": 2593980416 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021763636363636364, + "loss": 2.359, + "theoretical_loss": 3.358649016284552, + "tokens_seen": 2594045952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021761616161616163, + "loss": 2.4568, + "theoretical_loss": 3.358642297909703, + "tokens_seen": 2594111488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021761616161616163, + "loss": 2.2921, + "theoretical_loss": 3.358635579752103, + "tokens_seen": 2594177024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021759595959595958, + "loss": 2.6008, + "theoretical_loss": 3.3586288618117397, + "tokens_seen": 2594242560 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002175757575757576, + "loss": 2.396, + "theoretical_loss": 3.3586221440886006, + "tokens_seen": 2594308096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021755555555555555, + "loss": 2.5289, + "theoretical_loss": 3.3586154265826735, + "tokens_seen": 2594373632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021753535353535354, + "loss": 2.4621, + "theoretical_loss": 3.3586087092939456, + "tokens_seen": 2594439168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021751515151515152, + "loss": 2.4745, + "theoretical_loss": 3.358601992222405, + "tokens_seen": 2594504704 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002174949494949495, + "loss": 2.6116, + "theoretical_loss": 3.358595275368038, + "tokens_seen": 2594570240 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002174747474747475, + "loss": 2.7344, + "theoretical_loss": 3.3585885587308333, + "tokens_seen": 2594635776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021745454545454547, + "loss": 2.5481, + "theoretical_loss": 3.3585818423107776, + "tokens_seen": 2594701312 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021743434343434343, + "loss": 2.5348, + "theoretical_loss": 3.3585751261078585, + "tokens_seen": 2594766848 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002174141414141414, + "loss": 2.6566, + "theoretical_loss": 3.358568410122064, + "tokens_seen": 2594832384 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002173939393939394, + "loss": 2.619, + "theoretical_loss": 3.358561694353381, + "tokens_seen": 2594897920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021737373737373738, + "loss": 2.3552, + "theoretical_loss": 3.358554978801797, + "tokens_seen": 2594963456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021735353535353536, + "loss": 2.539, + "theoretical_loss": 3.3585482634673003, + "tokens_seen": 2595028992 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021733333333333332, + "loss": 2.437, + "theoretical_loss": 3.3585415483498777, + "tokens_seen": 2595094528 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021731313131313133, + "loss": 2.4495, + "theoretical_loss": 3.3585348334495166, + "tokens_seen": 2595160064 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002172929292929293, + "loss": 2.3601, + "theoretical_loss": 3.358528118766205, + "tokens_seen": 2595225600 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002172727272727273, + "loss": 2.6459, + "theoretical_loss": 3.35852140429993, + "tokens_seen": 2595291136 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021725252525252526, + "loss": 2.3839, + "theoretical_loss": 3.3585146900506793, + "tokens_seen": 2595356672 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021723232323232321, + "loss": 2.2672, + "theoretical_loss": 3.3585079760184406, + "tokens_seen": 2595422208 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1421066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4548287391662598, + "objective/train/theoretical_loss": 3.358501262203201, + "objective/train/tokens_used": 954346976, + "theoretical_loss": 3.358501262203201, + "tokens_seen": 2595487744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021721212121212123, + "loss": 2.596, + "theoretical_loss": 3.358501262203201, + "tokens_seen": 2595487744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021719191919191918, + "loss": 2.5514, + "theoretical_loss": 3.3584945486049484, + "tokens_seen": 2595553280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002171717171717172, + "loss": 2.3528, + "theoretical_loss": 3.35848783522367, + "tokens_seen": 2595618816 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021715151515151515, + "loss": 2.3445, + "theoretical_loss": 3.358481122059353, + "tokens_seen": 2595684352 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021713131313131313, + "loss": 2.563, + "theoretical_loss": 3.358474409111986, + "tokens_seen": 2595749888 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021711111111111112, + "loss": 2.6417, + "theoretical_loss": 3.358467696381555, + "tokens_seen": 2595815424 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002170909090909091, + "loss": 2.5867, + "theoretical_loss": 3.358460983868049, + "tokens_seen": 2595880960 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021707070707070709, + "loss": 2.4192, + "theoretical_loss": 3.3584542715714543, + "tokens_seen": 2595946496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021705050505050504, + "loss": 2.6282, + "theoretical_loss": 3.3584475594917595, + "tokens_seen": 2596012032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021703030303030303, + "loss": 2.7238, + "theoretical_loss": 3.3584408476289513, + "tokens_seen": 2596077568 + }, + { + "epoch": 0.57, + "learning_rate": 0.000217010101010101, + "loss": 2.6259, + "theoretical_loss": 3.3584341359830177, + "tokens_seen": 2596143104 + }, + { + "epoch": 0.57, + "learning_rate": 0.000216989898989899, + "loss": 2.3881, + "theoretical_loss": 3.358427424553946, + "tokens_seen": 2596208640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021696969696969698, + "loss": 2.4984, + "theoretical_loss": 3.3584207133417237, + "tokens_seen": 2596274176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021694949494949496, + "loss": 2.5496, + "theoretical_loss": 3.358414002346338, + "tokens_seen": 2596339712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021692929292929292, + "loss": 2.5617, + "theoretical_loss": 3.358407291567777, + "tokens_seen": 2596405248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021690909090909093, + "loss": 2.5783, + "theoretical_loss": 3.3584005810060282, + "tokens_seen": 2596470784 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002168888888888889, + "loss": 2.6994, + "theoretical_loss": 3.3583938706610788, + "tokens_seen": 2596536320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021686868686868687, + "loss": 2.5369, + "theoretical_loss": 3.3583871605329163, + "tokens_seen": 2596601856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021684848484848486, + "loss": 2.5248, + "theoretical_loss": 3.358380450621529, + "tokens_seen": 2596667392 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002168282828282828, + "loss": 2.7868, + "theoretical_loss": 3.358373740926903, + "tokens_seen": 2596732928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021680808080808082, + "loss": 2.6456, + "theoretical_loss": 3.358367031449027, + "tokens_seen": 2596798464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021678787878787878, + "loss": 2.5403, + "theoretical_loss": 3.358360322187888, + "tokens_seen": 2596864000 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002167676767676768, + "loss": 2.2078, + "theoretical_loss": 3.358353613143474, + "tokens_seen": 2596929536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021674747474747475, + "loss": 2.5189, + "theoretical_loss": 3.3583469043157717, + "tokens_seen": 2596995072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021672727272727273, + "loss": 2.5678, + "theoretical_loss": 3.35834019570477, + "tokens_seen": 2597060608 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1421641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6630356311798096, + "objective/train/theoretical_loss": 3.3583334873104547, + "objective/train/tokens_used": 955985376, + "theoretical_loss": 3.3583334873104547, + "tokens_seen": 2597126144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021670707070707072, + "loss": 2.5555, + "theoretical_loss": 3.3583334873104547, + "tokens_seen": 2597126144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021668686868686867, + "loss": 2.2591, + "theoretical_loss": 3.358326779132814, + "tokens_seen": 2597191680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021666666666666668, + "loss": 2.4252, + "theoretical_loss": 3.3583200711718364, + "tokens_seen": 2597257216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021664646464646464, + "loss": 2.4429, + "theoretical_loss": 3.3583133634275084, + "tokens_seen": 2597322752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021662626262626265, + "loss": 2.5285, + "theoretical_loss": 3.358306655899818, + "tokens_seen": 2597388288 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002166060606060606, + "loss": 2.6184, + "theoretical_loss": 3.3582999485887526, + "tokens_seen": 2597453824 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002165858585858586, + "loss": 2.6832, + "theoretical_loss": 3.3582932414942994, + "tokens_seen": 2597519360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021656565656565658, + "loss": 2.8435, + "theoretical_loss": 3.3582865346164463, + "tokens_seen": 2597584896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021654545454545453, + "loss": 2.4983, + "theoretical_loss": 3.358279827955181, + "tokens_seen": 2597650432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021652525252525254, + "loss": 2.4359, + "theoretical_loss": 3.35827312151049, + "tokens_seen": 2597715968 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002165050505050505, + "loss": 2.5759, + "theoretical_loss": 3.3582664152823627, + "tokens_seen": 2597781504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021648484848484849, + "loss": 2.4554, + "theoretical_loss": 3.358259709270785, + "tokens_seen": 2597847040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021646464646464647, + "loss": 2.6763, + "theoretical_loss": 3.3582530034757454, + "tokens_seen": 2597912576 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021644444444444445, + "loss": 2.5601, + "theoretical_loss": 3.358246297897231, + "tokens_seen": 2597978112 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021642424242424244, + "loss": 2.7189, + "theoretical_loss": 3.3582395925352295, + "tokens_seen": 2598043648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021640404040404042, + "loss": 2.5977, + "theoretical_loss": 3.358232887389728, + "tokens_seen": 2598109184 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021638383838383838, + "loss": 2.6685, + "theoretical_loss": 3.3582261824607147, + "tokens_seen": 2598174720 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021636363636363636, + "loss": 2.4319, + "theoretical_loss": 3.358219477748177, + "tokens_seen": 2598240256 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021634343434343435, + "loss": 2.5106, + "theoretical_loss": 3.3582127732521023, + "tokens_seen": 2598305792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021632323232323233, + "loss": 2.4512, + "theoretical_loss": 3.358206068972478, + "tokens_seen": 2598371328 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021630303030303031, + "loss": 2.4567, + "theoretical_loss": 3.358199364909292, + "tokens_seen": 2598436864 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021628282828282827, + "loss": 2.5224, + "theoretical_loss": 3.3581926610625317, + "tokens_seen": 2598502400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021626262626262628, + "loss": 2.276, + "theoretical_loss": 3.3581859574321844, + "tokens_seen": 2598567936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021624242424242424, + "loss": 2.7024, + "theoretical_loss": 3.3581792540182382, + "tokens_seen": 2598633472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021622222222222225, + "loss": 2.5148, + "theoretical_loss": 3.35817255082068, + "tokens_seen": 2598699008 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1422678, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7150049209594727, + "objective/train/theoretical_loss": 3.358165847839498, + "objective/train/tokens_used": 957623776, + "theoretical_loss": 3.358165847839498, + "tokens_seen": 2598764544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002162020202020202, + "loss": 2.4448, + "theoretical_loss": 3.358165847839498, + "tokens_seen": 2598764544 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021618181818181816, + "loss": 2.466, + "theoretical_loss": 3.3581591450746795, + "tokens_seen": 2598830080 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021616161616161617, + "loss": 2.6349, + "theoretical_loss": 3.3581524425262117, + "tokens_seen": 2598895616 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021614141414141413, + "loss": 2.4254, + "theoretical_loss": 3.358145740194083, + "tokens_seen": 2598961152 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021612121212121214, + "loss": 2.4, + "theoretical_loss": 3.3581390380782805, + "tokens_seen": 2599026688 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002161010101010101, + "loss": 2.5153, + "theoretical_loss": 3.3581323361787914, + "tokens_seen": 2599092224 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021608080808080808, + "loss": 2.4718, + "theoretical_loss": 3.3581256344956034, + "tokens_seen": 2599157760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021606060606060607, + "loss": 2.5701, + "theoretical_loss": 3.3581189330287047, + "tokens_seen": 2599223296 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021604040404040402, + "loss": 2.4253, + "theoretical_loss": 3.3581122317780823, + "tokens_seen": 2599288832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021602020202020204, + "loss": 2.491, + "theoretical_loss": 3.3581055307437238, + "tokens_seen": 2599354368 + }, + { + "epoch": 0.57, + "learning_rate": 0.000216, + "loss": 2.6047, + "theoretical_loss": 3.3580988299256167, + "tokens_seen": 2599419904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021597979797979798, + "loss": 2.3913, + "theoretical_loss": 3.358092129323749, + "tokens_seen": 2599485440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021595959595959596, + "loss": 2.5345, + "theoretical_loss": 3.358085428938108, + "tokens_seen": 2599550976 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021593939393939394, + "loss": 2.5214, + "theoretical_loss": 3.358078728768681, + "tokens_seen": 2599616512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021591919191919193, + "loss": 2.6337, + "theoretical_loss": 3.358072028815456, + "tokens_seen": 2599682048 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002158989898989899, + "loss": 2.574, + "theoretical_loss": 3.358065329078421, + "tokens_seen": 2599747584 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002158787878787879, + "loss": 2.6739, + "theoretical_loss": 3.3580586295575623, + "tokens_seen": 2599813120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021585858585858585, + "loss": 2.5347, + "theoretical_loss": 3.358051930252868, + "tokens_seen": 2599878656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021583838383838384, + "loss": 2.5135, + "theoretical_loss": 3.3580452311643265, + "tokens_seen": 2599944192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021581818181818182, + "loss": 2.0731, + "theoretical_loss": 3.358038532291924, + "tokens_seen": 2600009728 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002157979797979798, + "loss": 2.5628, + "theoretical_loss": 3.3580318336356494, + "tokens_seen": 2600075264 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002157777777777778, + "loss": 2.6217, + "theoretical_loss": 3.358025135195489, + "tokens_seen": 2600140800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021575757575757577, + "loss": 2.5824, + "theoretical_loss": 3.358018436971432, + "tokens_seen": 2600206336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021573737373737373, + "loss": 2.4306, + "theoretical_loss": 3.3580117389634645, + "tokens_seen": 2600271872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021571717171717174, + "loss": 2.4561, + "theoretical_loss": 3.3580050411715745, + "tokens_seen": 2600337408 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1423150, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.528973340988159, + "objective/train/theoretical_loss": 3.35799834359575, + "objective/train/tokens_used": 959262176, + "theoretical_loss": 3.35799834359575, + "tokens_seen": 2600402944 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002156969696969697, + "loss": 2.4335, + "theoretical_loss": 3.35799834359575, + "tokens_seen": 2600402944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021567676767676768, + "loss": 2.4746, + "theoretical_loss": 3.3579916462359782, + "tokens_seen": 2600468480 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021565656565656567, + "loss": 2.3377, + "theoretical_loss": 3.3579849490922467, + "tokens_seen": 2600534016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021563636363636362, + "loss": 2.3646, + "theoretical_loss": 3.357978252164543, + "tokens_seen": 2600599552 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021561616161616163, + "loss": 2.4497, + "theoretical_loss": 3.3579715554528553, + "tokens_seen": 2600665088 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002155959595959596, + "loss": 2.7239, + "theoretical_loss": 3.3579648589571707, + "tokens_seen": 2600730624 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002155757575757576, + "loss": 2.3426, + "theoretical_loss": 3.3579581626774764, + "tokens_seen": 2600796160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021555555555555556, + "loss": 2.4124, + "theoretical_loss": 3.357951466613761, + "tokens_seen": 2600861696 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021553535353535354, + "loss": 2.6073, + "theoretical_loss": 3.357944770766011, + "tokens_seen": 2600927232 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021551515151515153, + "loss": 2.3444, + "theoretical_loss": 3.3579380751342147, + "tokens_seen": 2600992768 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021549494949494948, + "loss": 2.4205, + "theoretical_loss": 3.3579313797183596, + "tokens_seen": 2601058304 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002154747474747475, + "loss": 2.6766, + "theoretical_loss": 3.3579246845184327, + "tokens_seen": 2601123840 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021545454545454545, + "loss": 2.4486, + "theoretical_loss": 3.357917989534423, + "tokens_seen": 2601189376 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021543434343434343, + "loss": 2.4147, + "theoretical_loss": 3.3579112947663163, + "tokens_seen": 2601254912 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021541414141414142, + "loss": 2.6322, + "theoretical_loss": 3.3579046002141015, + "tokens_seen": 2601320448 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002153939393939394, + "loss": 2.5092, + "theoretical_loss": 3.357897905877766, + "tokens_seen": 2601385984 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021537373737373739, + "loss": 2.3349, + "theoretical_loss": 3.3578912117572965, + "tokens_seen": 2601451520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021535353535353537, + "loss": 2.51, + "theoretical_loss": 3.357884517852682, + "tokens_seen": 2601517056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021533333333333333, + "loss": 2.3547, + "theoretical_loss": 3.3578778241639093, + "tokens_seen": 2601582592 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002153131313131313, + "loss": 2.62, + "theoretical_loss": 3.357871130690966, + "tokens_seen": 2601648128 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002152929292929293, + "loss": 2.6658, + "theoretical_loss": 3.3578644374338396, + "tokens_seen": 2601713664 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021527272727272728, + "loss": 2.6041, + "theoretical_loss": 3.357857744392518, + "tokens_seen": 2601779200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021525252525252526, + "loss": 2.3339, + "theoretical_loss": 3.357851051566989, + "tokens_seen": 2601844736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021523232323232322, + "loss": 2.6773, + "theoretical_loss": 3.3578443589572395, + "tokens_seen": 2601910272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021521212121212123, + "loss": 2.6162, + "theoretical_loss": 3.357837666563258, + "tokens_seen": 2601975808 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1424283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8036410808563232, + "objective/train/theoretical_loss": 3.357830974385031, + "objective/train/tokens_used": 960900576, + "theoretical_loss": 3.357830974385031, + "tokens_seen": 2602041344 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002151919191919192, + "loss": 2.629, + "theoretical_loss": 3.357830974385031, + "tokens_seen": 2602041344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021517171717171717, + "loss": 2.6666, + "theoretical_loss": 3.357824282422547, + "tokens_seen": 2602106880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021515151515151516, + "loss": 2.5084, + "theoretical_loss": 3.3578175906757934, + "tokens_seen": 2602172416 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002151313131313131, + "loss": 2.473, + "theoretical_loss": 3.357810899144758, + "tokens_seen": 2602237952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021511111111111112, + "loss": 2.5214, + "theoretical_loss": 3.357804207829428, + "tokens_seen": 2602303488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021509090909090908, + "loss": 2.6761, + "theoretical_loss": 3.357797516729791, + "tokens_seen": 2602369024 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002150707070707071, + "loss": 2.4958, + "theoretical_loss": 3.3577908258458353, + "tokens_seen": 2602434560 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021505050505050505, + "loss": 2.5877, + "theoretical_loss": 3.357784135177548, + "tokens_seen": 2602500096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021503030303030306, + "loss": 2.4545, + "theoretical_loss": 3.357777444724916, + "tokens_seen": 2602565632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021501010101010102, + "loss": 2.562, + "theoretical_loss": 3.3577707544879285, + "tokens_seen": 2602631168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021498989898989897, + "loss": 2.4617, + "theoretical_loss": 3.3577640644665716, + "tokens_seen": 2602696704 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021496969696969698, + "loss": 2.3442, + "theoretical_loss": 3.3577573746608342, + "tokens_seen": 2602762240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021494949494949494, + "loss": 2.4284, + "theoretical_loss": 3.357750685070703, + "tokens_seen": 2602827776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021492929292929295, + "loss": 2.4988, + "theoretical_loss": 3.357743995696166, + "tokens_seen": 2602893312 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002149090909090909, + "loss": 2.3649, + "theoretical_loss": 3.357737306537211, + "tokens_seen": 2602958848 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002148888888888889, + "loss": 2.4496, + "theoretical_loss": 3.357730617593825, + "tokens_seen": 2603024384 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021486868686868688, + "loss": 2.6176, + "theoretical_loss": 3.3577239288659966, + "tokens_seen": 2603089920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021484848484848486, + "loss": 2.4863, + "theoretical_loss": 3.357717240353712, + "tokens_seen": 2603155456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021482828282828284, + "loss": 2.4509, + "theoretical_loss": 3.3577105520569606, + "tokens_seen": 2603220992 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002148080808080808, + "loss": 2.531, + "theoretical_loss": 3.3577038639757286, + "tokens_seen": 2603286528 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021478787878787879, + "loss": 2.4638, + "theoretical_loss": 3.3576971761100043, + "tokens_seen": 2603352064 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021476767676767677, + "loss": 2.4935, + "theoretical_loss": 3.357690488459775, + "tokens_seen": 2603417600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021474747474747475, + "loss": 2.6634, + "theoretical_loss": 3.3576838010250287, + "tokens_seen": 2603483136 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021472727272727274, + "loss": 2.5092, + "theoretical_loss": 3.3576771138057526, + "tokens_seen": 2603548672 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021470707070707072, + "loss": 2.5816, + "theoretical_loss": 3.357670426801935, + "tokens_seen": 2603614208 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1425394, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.416898250579834, + "objective/train/theoretical_loss": 3.357663740013563, + "objective/train/tokens_used": 962538976, + "theoretical_loss": 3.357663740013563, + "tokens_seen": 2603679744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021468686868686868, + "loss": 2.6167, + "theoretical_loss": 3.357663740013563, + "tokens_seen": 2603679744 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002146666666666667, + "loss": 2.3749, + "theoretical_loss": 3.357657053440624, + "tokens_seen": 2603745280 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021464646464646465, + "loss": 2.5893, + "theoretical_loss": 3.3576503670831057, + "tokens_seen": 2603810816 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021462626262626263, + "loss": 2.3739, + "theoretical_loss": 3.357643680940997, + "tokens_seen": 2603876352 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021460606060606061, + "loss": 2.5018, + "theoretical_loss": 3.3576369950142837, + "tokens_seen": 2603941888 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021458585858585857, + "loss": 2.452, + "theoretical_loss": 3.3576303093029547, + "tokens_seen": 2604007424 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021456565656565658, + "loss": 2.6996, + "theoretical_loss": 3.3576236238069974, + "tokens_seen": 2604072960 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021454545454545454, + "loss": 2.5707, + "theoretical_loss": 3.357616938526399, + "tokens_seen": 2604138496 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021452525252525255, + "loss": 2.549, + "theoretical_loss": 3.3576102534611474, + "tokens_seen": 2604204032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002145050505050505, + "loss": 2.7563, + "theoretical_loss": 3.35760356861123, + "tokens_seen": 2604269568 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021448484848484846, + "loss": 2.6428, + "theoretical_loss": 3.357596883976635, + "tokens_seen": 2604335104 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021446464646464647, + "loss": 2.5993, + "theoretical_loss": 3.3575901995573503, + "tokens_seen": 2604400640 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021444444444444443, + "loss": 2.5414, + "theoretical_loss": 3.357583515353362, + "tokens_seen": 2604466176 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021442424242424244, + "loss": 2.5438, + "theoretical_loss": 3.3575768313646597, + "tokens_seen": 2604531712 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002144040404040404, + "loss": 2.4132, + "theoretical_loss": 3.3575701475912294, + "tokens_seen": 2604597248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021438383838383838, + "loss": 2.5396, + "theoretical_loss": 3.35756346403306, + "tokens_seen": 2604662784 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021436363636363637, + "loss": 2.2879, + "theoretical_loss": 3.357556780690138, + "tokens_seen": 2604728320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021434343434343435, + "loss": 2.7272, + "theoretical_loss": 3.357550097562452, + "tokens_seen": 2604793856 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021432323232323234, + "loss": 2.5865, + "theoretical_loss": 3.357543414649989, + "tokens_seen": 2604859392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002143030303030303, + "loss": 2.5205, + "theoretical_loss": 3.3575367319527376, + "tokens_seen": 2604924928 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021428282828282828, + "loss": 2.757, + "theoretical_loss": 3.3575300494706846, + "tokens_seen": 2604990464 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021426262626262626, + "loss": 2.8165, + "theoretical_loss": 3.3575233672038176, + "tokens_seen": 2605056000 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021424242424242424, + "loss": 2.3575, + "theoretical_loss": 3.3575166851521248, + "tokens_seen": 2605121536 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021422222222222223, + "loss": 2.4721, + "theoretical_loss": 3.3575100033155936, + "tokens_seen": 2605187072 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002142020202020202, + "loss": 2.4847, + "theoretical_loss": 3.357503321694211, + "tokens_seen": 2605252608 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1425905, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4751393795013428, + "objective/train/theoretical_loss": 3.357496640287966, + "objective/train/tokens_used": 964177376, + "theoretical_loss": 3.357496640287966, + "tokens_seen": 2605318144 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002141818181818182, + "loss": 2.2784, + "theoretical_loss": 3.357496640287966, + "tokens_seen": 2605318144 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021416161616161618, + "loss": 2.5948, + "theoretical_loss": 3.3574899590968457, + "tokens_seen": 2605383680 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021414141414141414, + "loss": 2.5617, + "theoretical_loss": 3.3574832781208372, + "tokens_seen": 2605449216 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021412121212121212, + "loss": 2.6004, + "theoretical_loss": 3.3574765973599288, + "tokens_seen": 2605514752 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002141010101010101, + "loss": 2.5505, + "theoretical_loss": 3.357469916814108, + "tokens_seen": 2605580288 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002140808080808081, + "loss": 2.7487, + "theoretical_loss": 3.3574632364833623, + "tokens_seen": 2605645824 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021406060606060607, + "loss": 2.55, + "theoretical_loss": 3.3574565563676795, + "tokens_seen": 2605711360 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021404040404040403, + "loss": 2.5279, + "theoretical_loss": 3.3574498764670473, + "tokens_seen": 2605776896 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021402020202020204, + "loss": 2.6366, + "theoretical_loss": 3.3574431967814533, + "tokens_seen": 2605842432 + }, + { + "epoch": 0.58, + "learning_rate": 0.000214, + "loss": 2.702, + "theoretical_loss": 3.3574365173108855, + "tokens_seen": 2605907968 + }, + { + "epoch": 0.58, + "learning_rate": 0.000213979797979798, + "loss": 2.6411, + "theoretical_loss": 3.3574298380553307, + "tokens_seen": 2605973504 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021395959595959597, + "loss": 2.3604, + "theoretical_loss": 3.3574231590147776, + "tokens_seen": 2606039040 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021393939393939392, + "loss": 2.3543, + "theoretical_loss": 3.3574164801892135, + "tokens_seen": 2606104576 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021391919191919193, + "loss": 2.6456, + "theoretical_loss": 3.3574098015786253, + "tokens_seen": 2606170112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002138989898989899, + "loss": 2.6066, + "theoretical_loss": 3.357403123183002, + "tokens_seen": 2606235648 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002138787878787879, + "loss": 2.6323, + "theoretical_loss": 3.3573964450023306, + "tokens_seen": 2606301184 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021385858585858586, + "loss": 2.3246, + "theoretical_loss": 3.3573897670365986, + "tokens_seen": 2606366720 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021383838383838384, + "loss": 2.4493, + "theoretical_loss": 3.3573830892857943, + "tokens_seen": 2606432256 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021381818181818183, + "loss": 2.4728, + "theoretical_loss": 3.3573764117499043, + "tokens_seen": 2606497792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002137979797979798, + "loss": 2.6322, + "theoretical_loss": 3.3573697344289175, + "tokens_seen": 2606563328 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002137777777777778, + "loss": 2.572, + "theoretical_loss": 3.3573630573228206, + "tokens_seen": 2606628864 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021375757575757575, + "loss": 2.8425, + "theoretical_loss": 3.357356380431602, + "tokens_seen": 2606694400 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021373737373737373, + "loss": 2.7532, + "theoretical_loss": 3.3573497037552493, + "tokens_seen": 2606759936 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021371717171717172, + "loss": 2.5353, + "theoretical_loss": 3.3573430272937497, + "tokens_seen": 2606825472 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002136969696969697, + "loss": 2.4463, + "theoretical_loss": 3.357336351047091, + "tokens_seen": 2606891008 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1426905, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.029381275177002, + "objective/train/theoretical_loss": 3.3573296750152615, + "objective/train/tokens_used": 965815776, + "theoretical_loss": 3.3573296750152615, + "tokens_seen": 2606956544 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002136767676767677, + "loss": 2.7504, + "theoretical_loss": 3.3573296750152615, + "tokens_seen": 2606956544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021365656565656567, + "loss": 2.5439, + "theoretical_loss": 3.3573229991982485, + "tokens_seen": 2607022080 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021363636363636363, + "loss": 2.4821, + "theoretical_loss": 3.3573163235960393, + "tokens_seen": 2607087616 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021361616161616164, + "loss": 2.5, + "theoretical_loss": 3.3573096482086218, + "tokens_seen": 2607153152 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002135959595959596, + "loss": 2.5273, + "theoretical_loss": 3.357302973035984, + "tokens_seen": 2607218688 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021357575757575758, + "loss": 2.4311, + "theoretical_loss": 3.357296298078113, + "tokens_seen": 2607284224 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021355555555555556, + "loss": 2.5382, + "theoretical_loss": 3.3572896233349976, + "tokens_seen": 2607349760 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021353535353535352, + "loss": 2.4258, + "theoretical_loss": 3.3572829488066245, + "tokens_seen": 2607415296 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021351515151515153, + "loss": 2.5728, + "theoretical_loss": 3.357276274492982, + "tokens_seen": 2607480832 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002134949494949495, + "loss": 2.8131, + "theoretical_loss": 3.3572696003940568, + "tokens_seen": 2607546368 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002134747474747475, + "loss": 2.6259, + "theoretical_loss": 3.3572629265098377, + "tokens_seen": 2607611904 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021345454545454546, + "loss": 2.501, + "theoretical_loss": 3.357256252840312, + "tokens_seen": 2607677440 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002134343434343434, + "loss": 2.5461, + "theoretical_loss": 3.357249579385467, + "tokens_seen": 2607742976 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021341414141414142, + "loss": 2.4399, + "theoretical_loss": 3.357242906145291, + "tokens_seen": 2607808512 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021339393939393938, + "loss": 2.4862, + "theoretical_loss": 3.3572362331197714, + "tokens_seen": 2607874048 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002133737373737374, + "loss": 2.461, + "theoretical_loss": 3.357229560308896, + "tokens_seen": 2607939584 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021335353535353535, + "loss": 2.5046, + "theoretical_loss": 3.3572228877126524, + "tokens_seen": 2608005120 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021333333333333336, + "loss": 2.4246, + "theoretical_loss": 3.3572162153310288, + "tokens_seen": 2608070656 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021331313131313132, + "loss": 2.5559, + "theoretical_loss": 3.3572095431640117, + "tokens_seen": 2608136192 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002132929292929293, + "loss": 2.557, + "theoretical_loss": 3.35720287121159, + "tokens_seen": 2608201728 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021327272727272728, + "loss": 2.4461, + "theoretical_loss": 3.3571961994737514, + "tokens_seen": 2608267264 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021325252525252524, + "loss": 2.5888, + "theoretical_loss": 3.3571895279504824, + "tokens_seen": 2608332800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021323232323232325, + "loss": 2.5643, + "theoretical_loss": 3.357182856641772, + "tokens_seen": 2608398336 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002132121212121212, + "loss": 2.6168, + "theoretical_loss": 3.3571761855476074, + "tokens_seen": 2608463872 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002131919191919192, + "loss": 2.6193, + "theoretical_loss": 3.357169514667976, + "tokens_seen": 2608529408 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1427573, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6366658210754395, + "objective/train/theoretical_loss": 3.3571628440028665, + "objective/train/tokens_used": 967454176, + "theoretical_loss": 3.3571628440028665, + "tokens_seen": 2608594944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021317171717171718, + "loss": 2.6232, + "theoretical_loss": 3.3571628440028665, + "tokens_seen": 2608594944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021315151515151516, + "loss": 2.371, + "theoretical_loss": 3.3571561735522653, + "tokens_seen": 2608660480 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021313131313131315, + "loss": 2.5256, + "theoretical_loss": 3.357149503316161, + "tokens_seen": 2608726016 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021311111111111113, + "loss": 2.317, + "theoretical_loss": 3.357142833294541, + "tokens_seen": 2608791552 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021309090909090909, + "loss": 2.3651, + "theoretical_loss": 3.357136163487393, + "tokens_seen": 2608857088 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021307070707070707, + "loss": 2.5298, + "theoretical_loss": 3.357129493894705, + "tokens_seen": 2608922624 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021305050505050505, + "loss": 2.3274, + "theoretical_loss": 3.3571228245164644, + "tokens_seen": 2608988160 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021303030303030304, + "loss": 2.3741, + "theoretical_loss": 3.357116155352659, + "tokens_seen": 2609053696 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021301010101010102, + "loss": 2.9016, + "theoretical_loss": 3.357109486403277, + "tokens_seen": 2609119232 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021298989898989898, + "loss": 2.4458, + "theoretical_loss": 3.3571028176683053, + "tokens_seen": 2609184768 + }, + { + "epoch": 0.58, + "learning_rate": 0.000212969696969697, + "loss": 2.5049, + "theoretical_loss": 3.357096149147732, + "tokens_seen": 2609250304 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021294949494949495, + "loss": 2.5534, + "theoretical_loss": 3.3570894808415446, + "tokens_seen": 2609315840 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021292929292929296, + "loss": 2.4275, + "theoretical_loss": 3.3570828127497316, + "tokens_seen": 2609381376 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021290909090909091, + "loss": 2.6579, + "theoretical_loss": 3.3570761448722797, + "tokens_seen": 2609446912 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021288888888888887, + "loss": 2.5843, + "theoretical_loss": 3.3570694772091776, + "tokens_seen": 2609512448 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021286868686868688, + "loss": 2.6099, + "theoretical_loss": 3.357062809760412, + "tokens_seen": 2609577984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021284848484848484, + "loss": 2.6261, + "theoretical_loss": 3.3570561425259715, + "tokens_seen": 2609643520 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021282828282828285, + "loss": 2.3038, + "theoretical_loss": 3.357049475505843, + "tokens_seen": 2609709056 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002128080808080808, + "loss": 2.35, + "theoretical_loss": 3.3570428087000153, + "tokens_seen": 2609774592 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002127878787878788, + "loss": 2.4055, + "theoretical_loss": 3.357036142108475, + "tokens_seen": 2609840128 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021276767676767678, + "loss": 2.6026, + "theoretical_loss": 3.3570294757312107, + "tokens_seen": 2609905664 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021274747474747473, + "loss": 2.447, + "theoretical_loss": 3.35702280956821, + "tokens_seen": 2609971200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021272727272727274, + "loss": 2.6633, + "theoretical_loss": 3.35701614361946, + "tokens_seen": 2610036736 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002127070707070707, + "loss": 2.5256, + "theoretical_loss": 3.357009477884949, + "tokens_seen": 2610102272 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021268686868686868, + "loss": 2.6561, + "theoretical_loss": 3.357002812364665, + "tokens_seen": 2610167808 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1428899, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.595594882965088, + "objective/train/theoretical_loss": 3.3569961470585947, + "objective/train/tokens_used": 969092576, + "theoretical_loss": 3.3569961470585947, + "tokens_seen": 2610233344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021266666666666667, + "loss": 2.6376, + "theoretical_loss": 3.3569961470585947, + "tokens_seen": 2610233344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021264646464646465, + "loss": 2.6591, + "theoretical_loss": 3.356989481966727, + "tokens_seen": 2610298880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021262626262626264, + "loss": 2.6441, + "theoretical_loss": 3.3569828170890488, + "tokens_seen": 2610364416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021260606060606062, + "loss": 2.4531, + "theoretical_loss": 3.356976152425548, + "tokens_seen": 2610429952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021258585858585858, + "loss": 2.4429, + "theoretical_loss": 3.356969487976213, + "tokens_seen": 2610495488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021256565656565656, + "loss": 2.5106, + "theoretical_loss": 3.3569628237410307, + "tokens_seen": 2610561024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021254545454545454, + "loss": 2.6624, + "theoretical_loss": 3.3569561597199895, + "tokens_seen": 2610626560 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021252525252525253, + "loss": 2.4727, + "theoretical_loss": 3.3569494959130766, + "tokens_seen": 2610692096 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002125050505050505, + "loss": 2.4208, + "theoretical_loss": 3.35694283232028, + "tokens_seen": 2610757632 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002124848484848485, + "loss": 2.5561, + "theoretical_loss": 3.356936168941587, + "tokens_seen": 2610823168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021246464646464648, + "loss": 2.6174, + "theoretical_loss": 3.3569295057769866, + "tokens_seen": 2610888704 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021244444444444444, + "loss": 2.5465, + "theoretical_loss": 3.356922842826465, + "tokens_seen": 2610954240 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021242424242424245, + "loss": 2.4753, + "theoretical_loss": 3.3569161800900114, + "tokens_seen": 2611019776 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002124040404040404, + "loss": 2.567, + "theoretical_loss": 3.3569095175676122, + "tokens_seen": 2611085312 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002123838383838384, + "loss": 2.537, + "theoretical_loss": 3.3569028552592557, + "tokens_seen": 2611150848 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021236363636363637, + "loss": 2.4309, + "theoretical_loss": 3.35689619316493, + "tokens_seen": 2611216384 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021234343434343433, + "loss": 2.4974, + "theoretical_loss": 3.3568895312846223, + "tokens_seen": 2611281920 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021232323232323234, + "loss": 2.569, + "theoretical_loss": 3.356882869618321, + "tokens_seen": 2611347456 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002123030303030303, + "loss": 2.5171, + "theoretical_loss": 3.3568762081660135, + "tokens_seen": 2611412992 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002122828282828283, + "loss": 2.514, + "theoretical_loss": 3.3568695469276872, + "tokens_seen": 2611478528 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021226262626262627, + "loss": 2.4508, + "theoretical_loss": 3.3568628859033303, + "tokens_seen": 2611544064 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021224242424242425, + "loss": 2.5712, + "theoretical_loss": 3.3568562250929306, + "tokens_seen": 2611609600 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021222222222222223, + "loss": 2.4007, + "theoretical_loss": 3.3568495644964753, + "tokens_seen": 2611675136 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002122020202020202, + "loss": 2.4889, + "theoretical_loss": 3.356842904113953, + "tokens_seen": 2611740672 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002121818181818182, + "loss": 2.5711, + "theoretical_loss": 3.3568362439453505, + "tokens_seen": 2611806208 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1430125, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6226844787597656, + "objective/train/theoretical_loss": 3.3568295839906566, + "objective/train/tokens_used": 970730976, + "theoretical_loss": 3.3568295839906566, + "tokens_seen": 2611871744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021216161616161616, + "loss": 2.613, + "theoretical_loss": 3.3568295839906566, + "tokens_seen": 2611871744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021214141414141414, + "loss": 2.5875, + "theoretical_loss": 3.356822924249858, + "tokens_seen": 2611937280 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021212121212121213, + "loss": 2.5992, + "theoretical_loss": 3.356816264722944, + "tokens_seen": 2612002816 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002121010101010101, + "loss": 2.5432, + "theoretical_loss": 3.3568096054099006, + "tokens_seen": 2612068352 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002120808080808081, + "loss": 2.5754, + "theoretical_loss": 3.3568029463107165, + "tokens_seen": 2612133888 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021206060606060608, + "loss": 2.5232, + "theoretical_loss": 3.356796287425379, + "tokens_seen": 2612199424 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021204040404040404, + "loss": 2.3733, + "theoretical_loss": 3.3567896287538765, + "tokens_seen": 2612264960 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021202020202020202, + "loss": 2.4386, + "theoretical_loss": 3.3567829702961967, + "tokens_seen": 2612330496 + }, + { + "epoch": 0.58, + "learning_rate": 0.000212, + "loss": 2.6223, + "theoretical_loss": 3.3567763120523266, + "tokens_seen": 2612396032 + }, + { + "epoch": 0.58, + "learning_rate": 0.000211979797979798, + "loss": 2.5512, + "theoretical_loss": 3.356769654022255, + "tokens_seen": 2612461568 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021195959595959597, + "loss": 2.428, + "theoretical_loss": 3.356762996205969, + "tokens_seen": 2612527104 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021193939393939393, + "loss": 2.3907, + "theoretical_loss": 3.3567563386034562, + "tokens_seen": 2612592640 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021191919191919194, + "loss": 2.6224, + "theoretical_loss": 3.356749681214705, + "tokens_seen": 2612658176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002118989898989899, + "loss": 2.4833, + "theoretical_loss": 3.356743024039703, + "tokens_seen": 2612723712 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021187878787878788, + "loss": 2.5694, + "theoretical_loss": 3.3567363670784376, + "tokens_seen": 2612789248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021185858585858586, + "loss": 2.4991, + "theoretical_loss": 3.356729710330897, + "tokens_seen": 2612854784 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021183838383838382, + "loss": 2.4517, + "theoretical_loss": 3.3567230537970683, + "tokens_seen": 2612920320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021181818181818183, + "loss": 2.589, + "theoretical_loss": 3.3567163974769403, + "tokens_seen": 2612985856 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002117979797979798, + "loss": 2.6619, + "theoretical_loss": 3.3567097413705, + "tokens_seen": 2613051392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002117777777777778, + "loss": 2.4908, + "theoretical_loss": 3.356703085477736, + "tokens_seen": 2613116928 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021175757575757576, + "loss": 2.5084, + "theoretical_loss": 3.3566964297986353, + "tokens_seen": 2613182464 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021173737373737374, + "loss": 2.4844, + "theoretical_loss": 3.3566897743331854, + "tokens_seen": 2613248000 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021171717171717172, + "loss": 2.6649, + "theoretical_loss": 3.356683119081375, + "tokens_seen": 2613313536 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021169696969696968, + "loss": 2.8031, + "theoretical_loss": 3.3566764640431916, + "tokens_seen": 2613379072 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002116767676767677, + "loss": 2.6248, + "theoretical_loss": 3.3566698092186225, + "tokens_seen": 2613444608 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1430667, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.676116943359375, + "objective/train/theoretical_loss": 3.356663154607656, + "objective/train/tokens_used": 972369376, + "theoretical_loss": 3.356663154607656, + "tokens_seen": 2613510144 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021165656565656565, + "loss": 2.7086, + "theoretical_loss": 3.356663154607656, + "tokens_seen": 2613510144 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021163636363636366, + "loss": 2.6487, + "theoretical_loss": 3.35665650021028, + "tokens_seen": 2613575680 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021161616161616162, + "loss": 2.4514, + "theoretical_loss": 3.3566498460264818, + "tokens_seen": 2613641216 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002115959595959596, + "loss": 2.3919, + "theoretical_loss": 3.356643192056249, + "tokens_seen": 2613706752 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021157575757575758, + "loss": 2.4953, + "theoretical_loss": 3.3566365382995706, + "tokens_seen": 2613772288 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021155555555555557, + "loss": 2.708, + "theoretical_loss": 3.3566298847564333, + "tokens_seen": 2613837824 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021153535353535355, + "loss": 2.3042, + "theoretical_loss": 3.356623231426825, + "tokens_seen": 2613903360 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002115151515151515, + "loss": 2.3695, + "theoretical_loss": 3.3566165783107342, + "tokens_seen": 2613968896 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002114949494949495, + "loss": 2.6424, + "theoretical_loss": 3.3566099254081476, + "tokens_seen": 2614034432 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021147474747474748, + "loss": 2.5789, + "theoretical_loss": 3.3566032727190542, + "tokens_seen": 2614099968 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021145454545454546, + "loss": 2.4884, + "theoretical_loss": 3.3565966202434407, + "tokens_seen": 2614165504 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021143434343434345, + "loss": 2.4687, + "theoretical_loss": 3.3565899679812956, + "tokens_seen": 2614231040 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021141414141414143, + "loss": 2.5493, + "theoretical_loss": 3.356583315932606, + "tokens_seen": 2614296576 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021139393939393939, + "loss": 2.4455, + "theoretical_loss": 3.3565766640973607, + "tokens_seen": 2614362112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002113737373737374, + "loss": 2.4711, + "theoretical_loss": 3.356570012475547, + "tokens_seen": 2614427648 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021135353535353535, + "loss": 2.7244, + "theoretical_loss": 3.3565633610671526, + "tokens_seen": 2614493184 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021133333333333334, + "loss": 2.4952, + "theoretical_loss": 3.356556709872165, + "tokens_seen": 2614558720 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021131313131313132, + "loss": 2.5953, + "theoretical_loss": 3.356550058890573, + "tokens_seen": 2614624256 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021129292929292928, + "loss": 2.4256, + "theoretical_loss": 3.3565434081223633, + "tokens_seen": 2614689792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002112727272727273, + "loss": 2.4189, + "theoretical_loss": 3.3565367575675245, + "tokens_seen": 2614755328 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021125252525252525, + "loss": 2.6432, + "theoretical_loss": 3.356530107226044, + "tokens_seen": 2614820864 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021123232323232326, + "loss": 2.6864, + "theoretical_loss": 3.3565234570979094, + "tokens_seen": 2614886400 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021121212121212121, + "loss": 2.8216, + "theoretical_loss": 3.3565168071831093, + "tokens_seen": 2614951936 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021119191919191917, + "loss": 2.5928, + "theoretical_loss": 3.356510157481631, + "tokens_seen": 2615017472 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021117171717171718, + "loss": 2.4133, + "theoretical_loss": 3.356503507993462, + "tokens_seen": 2615083008 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1431756, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4766592979431152, + "objective/train/theoretical_loss": 3.3564968587185904, + "objective/train/tokens_used": 974007776, + "theoretical_loss": 3.3564968587185904, + "tokens_seen": 2615148544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021115151515151514, + "loss": 2.6954, + "theoretical_loss": 3.3564968587185904, + "tokens_seen": 2615148544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021113131313131315, + "loss": 2.4157, + "theoretical_loss": 3.3564902096570046, + "tokens_seen": 2615214080 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002111111111111111, + "loss": 2.578, + "theoretical_loss": 3.356483560808692, + "tokens_seen": 2615279616 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002110909090909091, + "loss": 2.5581, + "theoretical_loss": 3.3564769121736395, + "tokens_seen": 2615345152 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021107070707070708, + "loss": 2.4454, + "theoretical_loss": 3.356470263751836, + "tokens_seen": 2615410688 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021105050505050506, + "loss": 2.5662, + "theoretical_loss": 3.3564636155432694, + "tokens_seen": 2615476224 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021103030303030304, + "loss": 2.5524, + "theoretical_loss": 3.3564569675479268, + "tokens_seen": 2615541760 + }, + { + "epoch": 0.58, + "learning_rate": 0.000211010101010101, + "loss": 2.5799, + "theoretical_loss": 3.356450319765796, + "tokens_seen": 2615607296 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021098989898989898, + "loss": 2.2762, + "theoretical_loss": 3.356443672196866, + "tokens_seen": 2615672832 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021096969696969697, + "loss": 2.574, + "theoretical_loss": 3.3564370248411235, + "tokens_seen": 2615738368 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021094949494949495, + "loss": 2.8441, + "theoretical_loss": 3.3564303776985565, + "tokens_seen": 2615803904 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021092929292929294, + "loss": 2.722, + "theoretical_loss": 3.356423730769153, + "tokens_seen": 2615869440 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021090909090909092, + "loss": 2.5251, + "theoretical_loss": 3.356417084052901, + "tokens_seen": 2615934976 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021088888888888888, + "loss": 2.3438, + "theoretical_loss": 3.356410437549788, + "tokens_seen": 2616000512 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002108686868686869, + "loss": 2.7276, + "theoretical_loss": 3.3564037912598015, + "tokens_seen": 2616066048 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021084848484848484, + "loss": 2.5053, + "theoretical_loss": 3.35639714518293, + "tokens_seen": 2616131584 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021082828282828283, + "loss": 2.4028, + "theoretical_loss": 3.3563904993191613, + "tokens_seen": 2616197120 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002108080808080808, + "loss": 2.4793, + "theoretical_loss": 3.3563838536684827, + "tokens_seen": 2616262656 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021078787878787877, + "loss": 2.3259, + "theoretical_loss": 3.356377208230883, + "tokens_seen": 2616328192 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021076767676767678, + "loss": 2.4687, + "theoretical_loss": 3.3563705630063487, + "tokens_seen": 2616393728 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021074747474747474, + "loss": 2.4625, + "theoretical_loss": 3.3563639179948686, + "tokens_seen": 2616459264 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021072727272727275, + "loss": 2.6138, + "theoretical_loss": 3.35635727319643, + "tokens_seen": 2616524800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002107070707070707, + "loss": 2.7129, + "theoretical_loss": 3.356350628611021, + "tokens_seen": 2616590336 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021068686868686872, + "loss": 2.5521, + "theoretical_loss": 3.3563439842386296, + "tokens_seen": 2616655872 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021066666666666667, + "loss": 2.5632, + "theoretical_loss": 3.3563373400792433, + "tokens_seen": 2616721408 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1432270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.063832998275757, + "objective/train/theoretical_loss": 3.3563306961328503, + "objective/train/tokens_used": 975646176, + "theoretical_loss": 3.3563306961328503, + "tokens_seen": 2616786944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021064646464646463, + "loss": 2.4609, + "theoretical_loss": 3.3563306961328503, + "tokens_seen": 2616786944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021062626262626264, + "loss": 2.4333, + "theoretical_loss": 3.356324052399438, + "tokens_seen": 2616852480 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002106060606060606, + "loss": 2.2694, + "theoretical_loss": 3.3563174088789944, + "tokens_seen": 2616918016 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002105858585858586, + "loss": 2.5277, + "theoretical_loss": 3.3563107655715076, + "tokens_seen": 2616983552 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021056565656565657, + "loss": 2.6887, + "theoretical_loss": 3.356304122476965, + "tokens_seen": 2617049088 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021054545454545455, + "loss": 2.5384, + "theoretical_loss": 3.356297479595355, + "tokens_seen": 2617114624 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021052525252525253, + "loss": 2.7132, + "theoretical_loss": 3.356290836926665, + "tokens_seen": 2617180160 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021050505050505052, + "loss": 2.7574, + "theoretical_loss": 3.356284194470883, + "tokens_seen": 2617245696 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002104848484848485, + "loss": 2.5398, + "theoretical_loss": 3.3562775522279966, + "tokens_seen": 2617311232 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021046464646464646, + "loss": 2.5217, + "theoretical_loss": 3.356270910197994, + "tokens_seen": 2617376768 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021044444444444444, + "loss": 2.57, + "theoretical_loss": 3.356264268380863, + "tokens_seen": 2617442304 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021042424242424243, + "loss": 2.659, + "theoretical_loss": 3.3562576267765913, + "tokens_seen": 2617507840 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002104040404040404, + "loss": 2.8339, + "theoretical_loss": 3.3562509853851665, + "tokens_seen": 2617573376 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002103838383838384, + "loss": 2.6079, + "theoretical_loss": 3.3562443442065772, + "tokens_seen": 2617638912 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021036363636363638, + "loss": 2.5655, + "theoretical_loss": 3.3562377032408106, + "tokens_seen": 2617704448 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021034343434343434, + "loss": 2.3997, + "theoretical_loss": 3.3562310624878546, + "tokens_seen": 2617769984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021032323232323232, + "loss": 2.7091, + "theoretical_loss": 3.3562244219476973, + "tokens_seen": 2617835520 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002103030303030303, + "loss": 2.6427, + "theoretical_loss": 3.3562177816203267, + "tokens_seen": 2617901056 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002102828282828283, + "loss": 2.6266, + "theoretical_loss": 3.3562111415057303, + "tokens_seen": 2617966592 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021026262626262627, + "loss": 2.6516, + "theoretical_loss": 3.356204501603896, + "tokens_seen": 2618032128 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021024242424242423, + "loss": 2.6616, + "theoretical_loss": 3.356197861914812, + "tokens_seen": 2618097664 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021022222222222224, + "loss": 2.3793, + "theoretical_loss": 3.356191222438466, + "tokens_seen": 2618163200 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002102020202020202, + "loss": 2.6618, + "theoretical_loss": 3.356184583174845, + "tokens_seen": 2618228736 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002101818181818182, + "loss": 2.5448, + "theoretical_loss": 3.356177944123938, + "tokens_seen": 2618294272 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021016161616161616, + "loss": 2.5587, + "theoretical_loss": 3.3561713052857325, + "tokens_seen": 2618359808 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1433381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.570761203765869, + "objective/train/theoretical_loss": 3.356164666660216, + "objective/train/tokens_used": 977284576, + "theoretical_loss": 3.356164666660216, + "tokens_seen": 2618425344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021014141414141412, + "loss": 2.7156, + "theoretical_loss": 3.356164666660216, + "tokens_seen": 2618425344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021012121212121213, + "loss": 2.3835, + "theoretical_loss": 3.356158028247377, + "tokens_seen": 2618490880 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002101010101010101, + "loss": 2.4358, + "theoretical_loss": 3.356151390047203, + "tokens_seen": 2618556416 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002100808080808081, + "loss": 2.434, + "theoretical_loss": 3.356144752059682, + "tokens_seen": 2618621952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021006060606060606, + "loss": 2.7625, + "theoretical_loss": 3.3561381142848017, + "tokens_seen": 2618687488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021004040404040404, + "loss": 2.5602, + "theoretical_loss": 3.3561314767225503, + "tokens_seen": 2618753024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021002020202020202, + "loss": 2.3882, + "theoretical_loss": 3.356124839372915, + "tokens_seen": 2618818560 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021, + "loss": 2.4798, + "theoretical_loss": 3.3561182022358844, + "tokens_seen": 2618884096 + }, + { + "epoch": 0.58, + "learning_rate": 0.000209979797979798, + "loss": 2.632, + "theoretical_loss": 3.356111565311446, + "tokens_seen": 2618949632 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020995959595959595, + "loss": 2.485, + "theoretical_loss": 3.3561049285995876, + "tokens_seen": 2619015168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020993939393939393, + "loss": 2.6499, + "theoretical_loss": 3.3560982921002975, + "tokens_seen": 2619080704 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020991919191919192, + "loss": 2.6056, + "theoretical_loss": 3.356091655813563, + "tokens_seen": 2619146240 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002098989898989899, + "loss": 2.6201, + "theoretical_loss": 3.3560850197393726, + "tokens_seen": 2619211776 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020987878787878789, + "loss": 2.5483, + "theoretical_loss": 3.3560783838777137, + "tokens_seen": 2619277312 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020985858585858587, + "loss": 2.4559, + "theoretical_loss": 3.356071748228574, + "tokens_seen": 2619342848 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020983838383838385, + "loss": 2.7005, + "theoretical_loss": 3.3560651127919416, + "tokens_seen": 2619408384 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020981818181818184, + "loss": 2.6068, + "theoretical_loss": 3.356058477567805, + "tokens_seen": 2619473920 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002097979797979798, + "loss": 2.7769, + "theoretical_loss": 3.356051842556151, + "tokens_seen": 2619539456 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020977777777777778, + "loss": 2.5373, + "theoretical_loss": 3.3560452077569685, + "tokens_seen": 2619604992 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020975757575757576, + "loss": 2.3276, + "theoretical_loss": 3.3560385731702445, + "tokens_seen": 2619670528 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020973737373737375, + "loss": 2.5158, + "theoretical_loss": 3.3560319387959674, + "tokens_seen": 2619736064 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020971717171717173, + "loss": 2.4936, + "theoretical_loss": 3.356025304634125, + "tokens_seen": 2619801600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002096969696969697, + "loss": 2.5438, + "theoretical_loss": 3.3560186706847053, + "tokens_seen": 2619867136 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002096767676767677, + "loss": 2.4555, + "theoretical_loss": 3.356012036947696, + "tokens_seen": 2619932672 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020965656565656565, + "loss": 2.6299, + "theoretical_loss": 3.356005403423085, + "tokens_seen": 2619998208 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1434670, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.737292766571045, + "objective/train/theoretical_loss": 3.3559987701108596, + "objective/train/tokens_used": 978922976, + "theoretical_loss": 3.3559987701108596, + "tokens_seen": 2620063744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020963636363636367, + "loss": 2.5781, + "theoretical_loss": 3.3559987701108596, + "tokens_seen": 2620063744 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020961616161616162, + "loss": 2.6748, + "theoretical_loss": 3.355992137011009, + "tokens_seen": 2620129280 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020959595959595958, + "loss": 2.5884, + "theoretical_loss": 3.3559855041235203, + "tokens_seen": 2620194816 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002095757575757576, + "loss": 2.6832, + "theoretical_loss": 3.355978871448381, + "tokens_seen": 2620260352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020955555555555555, + "loss": 2.5558, + "theoretical_loss": 3.3559722389855797, + "tokens_seen": 2620325888 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020953535353535356, + "loss": 2.5155, + "theoretical_loss": 3.3559656067351042, + "tokens_seen": 2620391424 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020951515151515152, + "loss": 2.6279, + "theoretical_loss": 3.355958974696942, + "tokens_seen": 2620456960 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002094949494949495, + "loss": 2.7061, + "theoretical_loss": 3.3559523428710816, + "tokens_seen": 2620522496 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020947474747474748, + "loss": 2.3612, + "theoretical_loss": 3.3559457112575104, + "tokens_seen": 2620588032 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020945454545454544, + "loss": 2.4493, + "theoretical_loss": 3.355939079856216, + "tokens_seen": 2620653568 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020943434343434345, + "loss": 2.3312, + "theoretical_loss": 3.355932448667187, + "tokens_seen": 2620719104 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002094141414141414, + "loss": 2.349, + "theoretical_loss": 3.3559258176904114, + "tokens_seen": 2620784640 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002093939393939394, + "loss": 2.3716, + "theoretical_loss": 3.3559191869258767, + "tokens_seen": 2620850176 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020937373737373738, + "loss": 2.4964, + "theoretical_loss": 3.3559125563735703, + "tokens_seen": 2620915712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020935353535353536, + "loss": 2.4622, + "theoretical_loss": 3.355905926033481, + "tokens_seen": 2620981248 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020933333333333334, + "loss": 2.3406, + "theoretical_loss": 3.355899295905596, + "tokens_seen": 2621046784 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020931313131313133, + "loss": 2.4408, + "theoretical_loss": 3.355892665989904, + "tokens_seen": 2621112320 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020929292929292928, + "loss": 2.4602, + "theoretical_loss": 3.355886036286392, + "tokens_seen": 2621177856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020927272727272727, + "loss": 2.591, + "theoretical_loss": 3.3558794067950486, + "tokens_seen": 2621243392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020925252525252525, + "loss": 2.6486, + "theoretical_loss": 3.355872777515861, + "tokens_seen": 2621308928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020923232323232324, + "loss": 2.5668, + "theoretical_loss": 3.3558661484488184, + "tokens_seen": 2621374464 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020921212121212122, + "loss": 2.7602, + "theoretical_loss": 3.355859519593907, + "tokens_seen": 2621440000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020919191919191918, + "loss": 2.8709, + "theoretical_loss": 3.355852890951116, + "tokens_seen": 2621505536 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002091717171717172, + "loss": 2.4494, + "theoretical_loss": 3.3558462625204326, + "tokens_seen": 2621571072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020915151515151515, + "loss": 2.6668, + "theoretical_loss": 3.3558396343018453, + "tokens_seen": 2621636608 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1435251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4519340991973877, + "objective/train/theoretical_loss": 3.3558330062953416, + "objective/train/tokens_used": 980561376, + "theoretical_loss": 3.3558330062953416, + "tokens_seen": 2621702144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020913131313131316, + "loss": 2.3416, + "theoretical_loss": 3.3558330062953416, + "tokens_seen": 2621702144 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002091111111111111, + "loss": 2.6353, + "theoretical_loss": 3.3558263785009093, + "tokens_seen": 2621767680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020909090909090907, + "loss": 2.4991, + "theoretical_loss": 3.3558197509185366, + "tokens_seen": 2621833216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020907070707070708, + "loss": 2.2038, + "theoretical_loss": 3.3558131235482116, + "tokens_seen": 2621898752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020905050505050504, + "loss": 2.316, + "theoretical_loss": 3.3558064963899215, + "tokens_seen": 2621964288 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020903030303030305, + "loss": 2.3229, + "theoretical_loss": 3.355799869443655, + "tokens_seen": 2622029824 + }, + { + "epoch": 0.59, + "learning_rate": 0.000209010101010101, + "loss": 2.483, + "theoretical_loss": 3.3557932427093995, + "tokens_seen": 2622095360 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020898989898989902, + "loss": 2.517, + "theoretical_loss": 3.3557866161871432, + "tokens_seen": 2622160896 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020896969696969697, + "loss": 2.4839, + "theoretical_loss": 3.3557799898768743, + "tokens_seen": 2622226432 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020894949494949496, + "loss": 2.5892, + "theoretical_loss": 3.35577336377858, + "tokens_seen": 2622291968 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020892929292929294, + "loss": 2.3282, + "theoretical_loss": 3.3557667378922487, + "tokens_seen": 2622357504 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002089090909090909, + "loss": 2.7857, + "theoretical_loss": 3.355760112217868, + "tokens_seen": 2622423040 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002088888888888889, + "loss": 2.6895, + "theoretical_loss": 3.355753486755426, + "tokens_seen": 2622488576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020886868686868687, + "loss": 2.4773, + "theoretical_loss": 3.3557468615049104, + "tokens_seen": 2622554112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020884848484848485, + "loss": 2.8306, + "theoretical_loss": 3.3557402364663096, + "tokens_seen": 2622619648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020882828282828283, + "loss": 2.3545, + "theoretical_loss": 3.3557336116396117, + "tokens_seen": 2622685184 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020880808080808082, + "loss": 2.7187, + "theoretical_loss": 3.355726987024804, + "tokens_seen": 2622750720 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002087878787878788, + "loss": 2.6083, + "theoretical_loss": 3.355720362621874, + "tokens_seen": 2622816256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020876767676767679, + "loss": 2.5964, + "theoretical_loss": 3.355713738430811, + "tokens_seen": 2622881792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020874747474747474, + "loss": 2.5436, + "theoretical_loss": 3.3557071144516017, + "tokens_seen": 2622947328 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020872727272727273, + "loss": 2.53, + "theoretical_loss": 3.355700490684235, + "tokens_seen": 2623012864 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002087070707070707, + "loss": 2.5755, + "theoretical_loss": 3.355693867128698, + "tokens_seen": 2623078400 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002086868686868687, + "loss": 2.6648, + "theoretical_loss": 3.3556872437849794, + "tokens_seen": 2623143936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020866666666666668, + "loss": 2.7903, + "theoretical_loss": 3.3556806206530667, + "tokens_seen": 2623209472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020864646464646464, + "loss": 2.6338, + "theoretical_loss": 3.3556739977329477, + "tokens_seen": 2623275008 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1436460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.362396478652954, + "objective/train/theoretical_loss": 3.3556673750246104, + "objective/train/tokens_used": 982199776, + "theoretical_loss": 3.3556673750246104, + "tokens_seen": 2623340544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020862626262626265, + "loss": 2.5939, + "theoretical_loss": 3.3556673750246104, + "tokens_seen": 2623340544 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002086060606060606, + "loss": 2.58, + "theoretical_loss": 3.355660752528043, + "tokens_seen": 2623406080 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002085858585858586, + "loss": 2.3428, + "theoretical_loss": 3.3556541302432334, + "tokens_seen": 2623471616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020856565656565657, + "loss": 2.4109, + "theoretical_loss": 3.3556475081701693, + "tokens_seen": 2623537152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020854545454545453, + "loss": 2.5077, + "theoretical_loss": 3.3556408863088385, + "tokens_seen": 2623602688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020852525252525254, + "loss": 2.4767, + "theoretical_loss": 3.3556342646592294, + "tokens_seen": 2623668224 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002085050505050505, + "loss": 2.4522, + "theoretical_loss": 3.3556276432213297, + "tokens_seen": 2623733760 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002084848484848485, + "loss": 2.4326, + "theoretical_loss": 3.3556210219951277, + "tokens_seen": 2623799296 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020846464646464646, + "loss": 2.7576, + "theoretical_loss": 3.3556144009806106, + "tokens_seen": 2623864832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020844444444444445, + "loss": 2.586, + "theoretical_loss": 3.355607780177767, + "tokens_seen": 2623930368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020842424242424243, + "loss": 2.4097, + "theoretical_loss": 3.355601159586585, + "tokens_seen": 2623995904 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002084040404040404, + "loss": 2.5562, + "theoretical_loss": 3.3555945392070514, + "tokens_seen": 2624061440 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002083838383838384, + "loss": 2.556, + "theoretical_loss": 3.3555879190391553, + "tokens_seen": 2624126976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020836363636363636, + "loss": 2.4892, + "theoretical_loss": 3.3555812990828846, + "tokens_seen": 2624192512 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020834343434343434, + "loss": 2.3191, + "theoretical_loss": 3.3555746793382264, + "tokens_seen": 2624258048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020832323232323233, + "loss": 2.7068, + "theoretical_loss": 3.355568059805169, + "tokens_seen": 2624323584 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002083030303030303, + "loss": 2.5889, + "theoretical_loss": 3.3555614404837013, + "tokens_seen": 2624389120 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002082828282828283, + "loss": 2.3777, + "theoretical_loss": 3.35555482137381, + "tokens_seen": 2624454656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020826262626262628, + "loss": 2.545, + "theoretical_loss": 3.3555482024754832, + "tokens_seen": 2624520192 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020824242424242423, + "loss": 2.2774, + "theoretical_loss": 3.35554158378871, + "tokens_seen": 2624585728 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020822222222222222, + "loss": 2.4767, + "theoretical_loss": 3.355534965313477, + "tokens_seen": 2624651264 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002082020202020202, + "loss": 2.5686, + "theoretical_loss": 3.355528347049773, + "tokens_seen": 2624716800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020818181818181819, + "loss": 2.518, + "theoretical_loss": 3.3555217289975854, + "tokens_seen": 2624782336 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020816161616161617, + "loss": 2.6216, + "theoretical_loss": 3.3555151111569024, + "tokens_seen": 2624847872 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020814141414141415, + "loss": 2.3936, + "theoretical_loss": 3.3555084935277124, + "tokens_seen": 2624913408 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1436953, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.385594606399536, + "objective/train/theoretical_loss": 3.3555018761100026, + "objective/train/tokens_used": 983838176, + "theoretical_loss": 3.3555018761100026, + "tokens_seen": 2624978944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020812121212121214, + "loss": 2.589, + "theoretical_loss": 3.3555018761100026, + "tokens_seen": 2624978944 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002081010101010101, + "loss": 2.4644, + "theoretical_loss": 3.355495258903761, + "tokens_seen": 2625044480 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002080808080808081, + "loss": 2.5098, + "theoretical_loss": 3.3554886419089764, + "tokens_seen": 2625110016 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020806060606060606, + "loss": 2.4511, + "theoretical_loss": 3.355482025125636, + "tokens_seen": 2625175552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020804040404040405, + "loss": 2.625, + "theoretical_loss": 3.355475408553728, + "tokens_seen": 2625241088 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020802020202020203, + "loss": 2.8077, + "theoretical_loss": 3.35546879219324, + "tokens_seen": 2625306624 + }, + { + "epoch": 0.59, + "learning_rate": 0.000208, + "loss": 2.6063, + "theoretical_loss": 3.3554621760441607, + "tokens_seen": 2625372160 + }, + { + "epoch": 0.59, + "learning_rate": 0.000207979797979798, + "loss": 2.4482, + "theoretical_loss": 3.3554555601064777, + "tokens_seen": 2625437696 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020795959595959596, + "loss": 2.4554, + "theoretical_loss": 3.3554489443801794, + "tokens_seen": 2625503232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020793939393939397, + "loss": 2.3918, + "theoretical_loss": 3.3554423288652524, + "tokens_seen": 2625568768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020791919191919192, + "loss": 2.6476, + "theoretical_loss": 3.355435713561686, + "tokens_seen": 2625634304 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020789898989898988, + "loss": 2.596, + "theoretical_loss": 3.355429098469468, + "tokens_seen": 2625699840 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002078787878787879, + "loss": 2.6903, + "theoretical_loss": 3.3554224835885864, + "tokens_seen": 2625765376 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020785858585858585, + "loss": 2.5218, + "theoretical_loss": 3.3554158689190285, + "tokens_seen": 2625830912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020783838383838386, + "loss": 2.5325, + "theoretical_loss": 3.3554092544607825, + "tokens_seen": 2625896448 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020781818181818182, + "loss": 2.4557, + "theoretical_loss": 3.355402640213837, + "tokens_seen": 2625961984 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002077979797979798, + "loss": 2.6001, + "theoretical_loss": 3.3553960261781794, + "tokens_seen": 2626027520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020777777777777778, + "loss": 2.6986, + "theoretical_loss": 3.355389412353798, + "tokens_seen": 2626093056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020775757575757577, + "loss": 2.5085, + "theoretical_loss": 3.35538279874068, + "tokens_seen": 2626158592 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020773737373737375, + "loss": 2.6162, + "theoretical_loss": 3.355376185338814, + "tokens_seen": 2626224128 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002077171717171717, + "loss": 2.1777, + "theoretical_loss": 3.355369572148189, + "tokens_seen": 2626289664 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002076969696969697, + "loss": 2.364, + "theoretical_loss": 3.355362959168791, + "tokens_seen": 2626355200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020767676767676768, + "loss": 2.6279, + "theoretical_loss": 3.355356346400609, + "tokens_seen": 2626420736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020765656565656566, + "loss": 2.3316, + "theoretical_loss": 3.3553497338436316, + "tokens_seen": 2626486272 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020763636363636364, + "loss": 2.7823, + "theoretical_loss": 3.3553431214978455, + "tokens_seen": 2626551808 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1438355, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3611056804656982, + "objective/train/theoretical_loss": 3.3553365093632395, + "objective/train/tokens_used": 985476576, + "theoretical_loss": 3.3553365093632395, + "tokens_seen": 2626617344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020761616161616163, + "loss": 2.3885, + "theoretical_loss": 3.3553365093632395, + "tokens_seen": 2626617344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020759595959595959, + "loss": 2.7687, + "theoretical_loss": 3.355329897439801, + "tokens_seen": 2626682880 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002075757575757576, + "loss": 2.5705, + "theoretical_loss": 3.355323285727519, + "tokens_seen": 2626748416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020755555555555555, + "loss": 2.6344, + "theoretical_loss": 3.3553166742263802, + "tokens_seen": 2626813952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020753535353535354, + "loss": 2.3722, + "theoretical_loss": 3.3553100629363737, + "tokens_seen": 2626879488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020751515151515152, + "loss": 2.4149, + "theoretical_loss": 3.3553034518574867, + "tokens_seen": 2626945024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020749494949494948, + "loss": 2.6055, + "theoretical_loss": 3.3552968409897077, + "tokens_seen": 2627010560 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002074747474747475, + "loss": 2.5863, + "theoretical_loss": 3.3552902303330248, + "tokens_seen": 2627076096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020745454545454545, + "loss": 2.4718, + "theoretical_loss": 3.355283619887425, + "tokens_seen": 2627141632 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020743434343434346, + "loss": 2.5338, + "theoretical_loss": 3.3552770096528977, + "tokens_seen": 2627207168 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020741414141414141, + "loss": 2.3503, + "theoretical_loss": 3.35527039962943, + "tokens_seen": 2627272704 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002073939393939394, + "loss": 2.5626, + "theoretical_loss": 3.35526378981701, + "tokens_seen": 2627338240 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020737373737373738, + "loss": 2.3726, + "theoretical_loss": 3.3552571802156255, + "tokens_seen": 2627403776 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020735353535353534, + "loss": 2.309, + "theoretical_loss": 3.355250570825265, + "tokens_seen": 2627469312 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020733333333333335, + "loss": 2.6262, + "theoretical_loss": 3.3552439616459164, + "tokens_seen": 2627534848 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002073131313131313, + "loss": 2.6787, + "theoretical_loss": 3.355237352677568, + "tokens_seen": 2627600384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020729292929292932, + "loss": 2.753, + "theoretical_loss": 3.355230743920207, + "tokens_seen": 2627665920 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020727272727272727, + "loss": 2.3945, + "theoretical_loss": 3.3552241353738212, + "tokens_seen": 2627731456 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020725252525252526, + "loss": 2.3233, + "theoretical_loss": 3.3552175270383997, + "tokens_seen": 2627796992 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020723232323232324, + "loss": 2.515, + "theoretical_loss": 3.3552109189139303, + "tokens_seen": 2627862528 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020721212121212123, + "loss": 2.4881, + "theoretical_loss": 3.3552043110004, + "tokens_seen": 2627928064 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002071919191919192, + "loss": 2.7409, + "theoretical_loss": 3.355197703297798, + "tokens_seen": 2627993600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020717171717171717, + "loss": 2.2419, + "theoretical_loss": 3.355191095806112, + "tokens_seen": 2628059136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020715151515151515, + "loss": 2.8155, + "theoretical_loss": 3.35518448852533, + "tokens_seen": 2628124672 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020713131313131313, + "loss": 2.4631, + "theoretical_loss": 3.355177881455439, + "tokens_seen": 2628190208 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1438882, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7245101928710938, + "objective/train/theoretical_loss": 3.3551712745964286, + "objective/train/tokens_used": 987114976, + "theoretical_loss": 3.3551712745964286, + "tokens_seen": 2628255744 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020711111111111112, + "loss": 2.488, + "theoretical_loss": 3.3551712745964286, + "tokens_seen": 2628255744 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002070909090909091, + "loss": 2.647, + "theoretical_loss": 3.3551646679482854, + "tokens_seen": 2628321280 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002070707070707071, + "loss": 2.5554, + "theoretical_loss": 3.3551580615109984, + "tokens_seen": 2628386816 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020705050505050504, + "loss": 2.4623, + "theoretical_loss": 3.355151455284555, + "tokens_seen": 2628452352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020703030303030303, + "loss": 2.3514, + "theoretical_loss": 3.355144849268944, + "tokens_seen": 2628517888 + }, + { + "epoch": 0.59, + "learning_rate": 0.000207010101010101, + "loss": 2.5696, + "theoretical_loss": 3.355138243464153, + "tokens_seen": 2628583424 + }, + { + "epoch": 0.59, + "learning_rate": 0.000206989898989899, + "loss": 2.628, + "theoretical_loss": 3.3551316378701697, + "tokens_seen": 2628648960 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020696969696969698, + "loss": 2.6639, + "theoretical_loss": 3.355125032486982, + "tokens_seen": 2628714496 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020694949494949494, + "loss": 2.4404, + "theoretical_loss": 3.3551184273145784, + "tokens_seen": 2628780032 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020692929292929295, + "loss": 2.5456, + "theoretical_loss": 3.355111822352947, + "tokens_seen": 2628845568 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002069090909090909, + "loss": 2.6819, + "theoretical_loss": 3.355105217602076, + "tokens_seen": 2628911104 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020688888888888892, + "loss": 2.5105, + "theoretical_loss": 3.3550986130619522, + "tokens_seen": 2628976640 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020686868686868687, + "loss": 2.5711, + "theoretical_loss": 3.355092008732565, + "tokens_seen": 2629042176 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020684848484848483, + "loss": 2.5188, + "theoretical_loss": 3.3550854046139014, + "tokens_seen": 2629107712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020682828282828284, + "loss": 2.526, + "theoretical_loss": 3.35507880070595, + "tokens_seen": 2629173248 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002068080808080808, + "loss": 2.4181, + "theoretical_loss": 3.355072197008699, + "tokens_seen": 2629238784 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002067878787878788, + "loss": 2.6339, + "theoretical_loss": 3.355065593522136, + "tokens_seen": 2629304320 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020676767676767676, + "loss": 2.4246, + "theoretical_loss": 3.3550589902462495, + "tokens_seen": 2629369856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020674747474747475, + "loss": 2.4179, + "theoretical_loss": 3.355052387181027, + "tokens_seen": 2629435392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020672727272727273, + "loss": 2.5745, + "theoretical_loss": 3.3550457843264563, + "tokens_seen": 2629500928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020670707070707072, + "loss": 2.4752, + "theoretical_loss": 3.3550391816825265, + "tokens_seen": 2629566464 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002066868686868687, + "loss": 2.7124, + "theoretical_loss": 3.355032579249224, + "tokens_seen": 2629632000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020666666666666666, + "loss": 2.5756, + "theoretical_loss": 3.3550259770265387, + "tokens_seen": 2629697536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020664646464646464, + "loss": 2.6481, + "theoretical_loss": 3.3550193750144577, + "tokens_seen": 2629763072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020662626262626263, + "loss": 2.5412, + "theoretical_loss": 3.355012773212969, + "tokens_seen": 2629828608 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1440194, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7252695560455322, + "objective/train/theoretical_loss": 3.3550061716220605, + "objective/train/tokens_used": 988753376, + "theoretical_loss": 3.3550061716220605, + "tokens_seen": 2629894144 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002066060606060606, + "loss": 2.6699, + "theoretical_loss": 3.3550061716220605, + "tokens_seen": 2629894144 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002065858585858586, + "loss": 2.498, + "theoretical_loss": 3.354999570241721, + "tokens_seen": 2629959680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020656565656565658, + "loss": 2.5242, + "theoretical_loss": 3.3549929690719376, + "tokens_seen": 2630025216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020654545454545453, + "loss": 2.8215, + "theoretical_loss": 3.3549863681126983, + "tokens_seen": 2630090752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020652525252525255, + "loss": 2.5494, + "theoretical_loss": 3.354979767363992, + "tokens_seen": 2630156288 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002065050505050505, + "loss": 2.6365, + "theoretical_loss": 3.3549731668258063, + "tokens_seen": 2630221824 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020648484848484849, + "loss": 2.5794, + "theoretical_loss": 3.3549665664981294, + "tokens_seen": 2630287360 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020646464646464647, + "loss": 2.4213, + "theoretical_loss": 3.354959966380949, + "tokens_seen": 2630352896 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020644444444444445, + "loss": 2.5761, + "theoretical_loss": 3.3549533664742532, + "tokens_seen": 2630418432 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020642424242424244, + "loss": 2.3287, + "theoretical_loss": 3.3549467667780304, + "tokens_seen": 2630483968 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002064040404040404, + "loss": 2.6289, + "theoretical_loss": 3.3549401672922685, + "tokens_seen": 2630549504 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002063838383838384, + "loss": 2.6217, + "theoretical_loss": 3.354933568016955, + "tokens_seen": 2630615040 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020636363636363636, + "loss": 2.4354, + "theoretical_loss": 3.3549269689520793, + "tokens_seen": 2630680576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020634343434343437, + "loss": 2.6467, + "theoretical_loss": 3.354920370097628, + "tokens_seen": 2630746112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020632323232323233, + "loss": 2.4941, + "theoretical_loss": 3.3549137714535897, + "tokens_seen": 2630811648 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002063030303030303, + "loss": 2.3973, + "theoretical_loss": 3.354907173019952, + "tokens_seen": 2630877184 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002062828282828283, + "loss": 2.4841, + "theoretical_loss": 3.3549005747967042, + "tokens_seen": 2630942720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020626262626262626, + "loss": 2.5298, + "theoretical_loss": 3.354893976783833, + "tokens_seen": 2631008256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020624242424242427, + "loss": 2.7488, + "theoretical_loss": 3.3548873789813274, + "tokens_seen": 2631073792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020622222222222222, + "loss": 2.6124, + "theoretical_loss": 3.3548807813891752, + "tokens_seen": 2631139328 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002062020202020202, + "loss": 2.6101, + "theoretical_loss": 3.3548741840073637, + "tokens_seen": 2631204864 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002061818181818182, + "loss": 2.5685, + "theoretical_loss": 3.354867586835882, + "tokens_seen": 2631270400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020616161616161615, + "loss": 2.4826, + "theoretical_loss": 3.3548609898747177, + "tokens_seen": 2631335936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020614141414141416, + "loss": 2.4373, + "theoretical_loss": 3.3548543931238592, + "tokens_seen": 2631401472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020612121212121212, + "loss": 2.1639, + "theoretical_loss": 3.354847796583294, + "tokens_seen": 2631467008 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1441640, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7747178077697754, + "objective/train/theoretical_loss": 3.35484120025301, + "objective/train/tokens_used": 990391776, + "theoretical_loss": 3.35484120025301, + "tokens_seen": 2631532544 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002061010101010101, + "loss": 2.7235, + "theoretical_loss": 3.35484120025301, + "tokens_seen": 2631532544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020608080808080808, + "loss": 2.5871, + "theoretical_loss": 3.354834604132996, + "tokens_seen": 2631598080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020606060606060607, + "loss": 2.478, + "theoretical_loss": 3.35482800822324, + "tokens_seen": 2631663616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020604040404040405, + "loss": 2.6366, + "theoretical_loss": 3.3548214125237292, + "tokens_seen": 2631729152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020602020202020204, + "loss": 2.4885, + "theoretical_loss": 3.3548148170344523, + "tokens_seen": 2631794688 + }, + { + "epoch": 0.59, + "learning_rate": 0.000206, + "loss": 2.5617, + "theoretical_loss": 3.354808221755398, + "tokens_seen": 2631860224 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020597979797979798, + "loss": 2.513, + "theoretical_loss": 3.354801626686553, + "tokens_seen": 2631925760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020595959595959596, + "loss": 2.7004, + "theoretical_loss": 3.3547950318279063, + "tokens_seen": 2631991296 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020593939393939394, + "loss": 2.6097, + "theoretical_loss": 3.3547884371794456, + "tokens_seen": 2632056832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020591919191919193, + "loss": 2.7247, + "theoretical_loss": 3.3547818427411595, + "tokens_seen": 2632122368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020589898989898989, + "loss": 2.4718, + "theoretical_loss": 3.354775248513035, + "tokens_seen": 2632187904 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002058787878787879, + "loss": 2.5828, + "theoretical_loss": 3.3547686544950612, + "tokens_seen": 2632253440 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020585858585858585, + "loss": 2.6441, + "theoretical_loss": 3.354762060687226, + "tokens_seen": 2632318976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020583838383838386, + "loss": 2.4276, + "theoretical_loss": 3.3547554670895168, + "tokens_seen": 2632384512 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020581818181818182, + "loss": 2.6191, + "theoretical_loss": 3.354748873701922, + "tokens_seen": 2632450048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020579797979797978, + "loss": 2.544, + "theoretical_loss": 3.3547422805244307, + "tokens_seen": 2632515584 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002057777777777778, + "loss": 2.6791, + "theoretical_loss": 3.3547356875570293, + "tokens_seen": 2632581120 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020575757575757575, + "loss": 2.3915, + "theoretical_loss": 3.354729094799707, + "tokens_seen": 2632646656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020573737373737376, + "loss": 2.4431, + "theoretical_loss": 3.3547225022524514, + "tokens_seen": 2632712192 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020571717171717171, + "loss": 2.7013, + "theoretical_loss": 3.354715909915251, + "tokens_seen": 2632777728 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002056969696969697, + "loss": 2.5844, + "theoretical_loss": 3.3547093177880933, + "tokens_seen": 2632843264 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020567676767676768, + "loss": 2.5503, + "theoretical_loss": 3.3547027258709665, + "tokens_seen": 2632908800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020565656565656567, + "loss": 2.6289, + "theoretical_loss": 3.3546961341638593, + "tokens_seen": 2632974336 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020563636363636365, + "loss": 2.614, + "theoretical_loss": 3.354689542666759, + "tokens_seen": 2633039872 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002056161616161616, + "loss": 2.4605, + "theoretical_loss": 3.354682951379654, + "tokens_seen": 2633105408 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1442322, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3413121700286865, + "objective/train/theoretical_loss": 3.3546763603025327, + "objective/train/tokens_used": 992030176, + "theoretical_loss": 3.3546763603025327, + "tokens_seen": 2633170944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020559595959595962, + "loss": 2.5204, + "theoretical_loss": 3.3546763603025327, + "tokens_seen": 2633170944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020557575757575757, + "loss": 2.6001, + "theoretical_loss": 3.354669769435383, + "tokens_seen": 2633236480 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020555555555555556, + "loss": 2.5761, + "theoretical_loss": 3.354663178778192, + "tokens_seen": 2633302016 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020553535353535354, + "loss": 2.4491, + "theoretical_loss": 3.3546565883309496, + "tokens_seen": 2633367552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020551515151515153, + "loss": 2.5183, + "theoretical_loss": 3.3546499980936426, + "tokens_seen": 2633433088 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002054949494949495, + "loss": 2.6907, + "theoretical_loss": 3.3546434080662593, + "tokens_seen": 2633498624 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002054747474747475, + "loss": 2.305, + "theoretical_loss": 3.354636818248788, + "tokens_seen": 2633564160 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020545454545454545, + "loss": 2.3771, + "theoretical_loss": 3.354630228641217, + "tokens_seen": 2633629696 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020543434343434344, + "loss": 2.5314, + "theoretical_loss": 3.354623639243534, + "tokens_seen": 2633695232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020541414141414142, + "loss": 2.5414, + "theoretical_loss": 3.354617050055727, + "tokens_seen": 2633760768 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053939393939394, + "loss": 2.5888, + "theoretical_loss": 3.354610461077784, + "tokens_seen": 2633826304 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053737373737374, + "loss": 2.4271, + "theoretical_loss": 3.354603872309694, + "tokens_seen": 2633891840 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020535353535353534, + "loss": 2.5103, + "theoretical_loss": 3.354597283751444, + "tokens_seen": 2633957376 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020533333333333336, + "loss": 2.37, + "theoretical_loss": 3.354590695403023, + "tokens_seen": 2634022912 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053131313131313, + "loss": 2.6337, + "theoretical_loss": 3.3545841072644187, + "tokens_seen": 2634088448 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002052929292929293, + "loss": 2.4478, + "theoretical_loss": 3.354577519335619, + "tokens_seen": 2634153984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020527272727272728, + "loss": 2.4778, + "theoretical_loss": 3.3545709316166117, + "tokens_seen": 2634219520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020525252525252524, + "loss": 2.4299, + "theoretical_loss": 3.354564344107386, + "tokens_seen": 2634285056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020523232323232325, + "loss": 2.3935, + "theoretical_loss": 3.354557756807929, + "tokens_seen": 2634350592 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002052121212121212, + "loss": 2.6921, + "theoretical_loss": 3.3545511697182295, + "tokens_seen": 2634416128 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020519191919191922, + "loss": 2.6472, + "theoretical_loss": 3.354544582838275, + "tokens_seen": 2634481664 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020517171717171717, + "loss": 2.6854, + "theoretical_loss": 3.354537996168054, + "tokens_seen": 2634547200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020515151515151516, + "loss": 2.5316, + "theoretical_loss": 3.3545314097075547, + "tokens_seen": 2634612736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020513131313131314, + "loss": 2.6331, + "theoretical_loss": 3.3545248234567646, + "tokens_seen": 2634678272 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002051111111111111, + "loss": 2.6417, + "theoretical_loss": 3.354518237415672, + "tokens_seen": 2634743808 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1443625, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.782468795776367, + "objective/train/theoretical_loss": 3.3545116515842657, + "objective/train/tokens_used": 993668576, + "theoretical_loss": 3.3545116515842657, + "tokens_seen": 2634809344 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002050909090909091, + "loss": 2.6177, + "theoretical_loss": 3.3545116515842657, + "tokens_seen": 2634809344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020507070707070707, + "loss": 2.5574, + "theoretical_loss": 3.3545050659625333, + "tokens_seen": 2634874880 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020505050505050505, + "loss": 2.5084, + "theoretical_loss": 3.354498480550463, + "tokens_seen": 2634940416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020503030303030303, + "loss": 2.6944, + "theoretical_loss": 3.3544918953480423, + "tokens_seen": 2635005952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020501010101010102, + "loss": 2.3932, + "theoretical_loss": 3.35448531035526, + "tokens_seen": 2635071488 + }, + { + "epoch": 0.59, + "learning_rate": 0.000204989898989899, + "loss": 2.4214, + "theoretical_loss": 3.3544787255721045, + "tokens_seen": 2635137024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020496969696969699, + "loss": 2.8083, + "theoretical_loss": 3.354472140998563, + "tokens_seen": 2635202560 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020494949494949494, + "loss": 2.483, + "theoretical_loss": 3.3544655566346244, + "tokens_seen": 2635268096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020492929292929293, + "loss": 2.6708, + "theoretical_loss": 3.3544589724802765, + "tokens_seen": 2635333632 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002049090909090909, + "loss": 2.3091, + "theoretical_loss": 3.354452388535507, + "tokens_seen": 2635399168 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002048888888888889, + "loss": 2.3091, + "theoretical_loss": 3.3544458048003047, + "tokens_seen": 2635464704 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020486868686868688, + "loss": 2.4276, + "theoretical_loss": 3.3544392212746574, + "tokens_seen": 2635530240 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020484848484848483, + "loss": 2.4445, + "theoretical_loss": 3.3544326379585536, + "tokens_seen": 2635595776 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020482828282828285, + "loss": 2.5868, + "theoretical_loss": 3.354426054851981, + "tokens_seen": 2635661312 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002048080808080808, + "loss": 2.4394, + "theoretical_loss": 3.3544194719549276, + "tokens_seen": 2635726848 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020478787878787881, + "loss": 2.6389, + "theoretical_loss": 3.354412889267382, + "tokens_seen": 2635792384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020476767676767677, + "loss": 2.3571, + "theoretical_loss": 3.3544063067893317, + "tokens_seen": 2635857920 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020474747474747473, + "loss": 2.5287, + "theoretical_loss": 3.354399724520765, + "tokens_seen": 2635923456 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020472727272727274, + "loss": 2.4407, + "theoretical_loss": 3.354393142461671, + "tokens_seen": 2635988992 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002047070707070707, + "loss": 2.4813, + "theoretical_loss": 3.3543865606120367, + "tokens_seen": 2636054528 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002046868686868687, + "loss": 2.5608, + "theoretical_loss": 3.3543799789718505, + "tokens_seen": 2636120064 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020466666666666666, + "loss": 2.4228, + "theoretical_loss": 3.3543733975411003, + "tokens_seen": 2636185600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020464646464646467, + "loss": 2.9393, + "theoretical_loss": 3.3543668163197746, + "tokens_seen": 2636251136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020462626262626263, + "loss": 2.6287, + "theoretical_loss": 3.354360235307862, + "tokens_seen": 2636316672 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002046060606060606, + "loss": 2.523, + "theoretical_loss": 3.3543536545053496, + "tokens_seen": 2636382208 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1444226, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1333506107330322, + "objective/train/theoretical_loss": 3.3543470739122263, + "objective/train/tokens_used": 995306976, + "theoretical_loss": 3.3543470739122263, + "tokens_seen": 2636447744 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002045858585858586, + "loss": 2.4369, + "theoretical_loss": 3.3543470739122263, + "tokens_seen": 2636447744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020456565656565656, + "loss": 2.4366, + "theoretical_loss": 3.3543404935284795, + "tokens_seen": 2636513280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020454545454545457, + "loss": 2.4689, + "theoretical_loss": 3.354333913354098, + "tokens_seen": 2636578816 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020452525252525252, + "loss": 2.7063, + "theoretical_loss": 3.3543273333890697, + "tokens_seen": 2636644352 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002045050505050505, + "loss": 2.513, + "theoretical_loss": 3.354320753633383, + "tokens_seen": 2636709888 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002044848484848485, + "loss": 2.618, + "theoretical_loss": 3.3543141740870257, + "tokens_seen": 2636775424 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020446464646464648, + "loss": 2.6782, + "theoretical_loss": 3.354307594749986, + "tokens_seen": 2636840960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020444444444444446, + "loss": 2.4553, + "theoretical_loss": 3.354301015622252, + "tokens_seen": 2636906496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020442424242424242, + "loss": 2.5374, + "theoretical_loss": 3.3542944367038117, + "tokens_seen": 2636972032 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002044040404040404, + "loss": 2.535, + "theoretical_loss": 3.354287857994654, + "tokens_seen": 2637037568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020438383838383838, + "loss": 2.4309, + "theoretical_loss": 3.354281279494766, + "tokens_seen": 2637103104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020436363636363637, + "loss": 2.8039, + "theoretical_loss": 3.3542747012041363, + "tokens_seen": 2637168640 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020434343434343435, + "loss": 2.672, + "theoretical_loss": 3.354268123122753, + "tokens_seen": 2637234176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020432323232323234, + "loss": 2.3302, + "theoretical_loss": 3.354261545250605, + "tokens_seen": 2637299712 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002043030303030303, + "loss": 2.4494, + "theoretical_loss": 3.3542549675876794, + "tokens_seen": 2637365248 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002042828282828283, + "loss": 2.3474, + "theoretical_loss": 3.3542483901339644, + "tokens_seen": 2637430784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020426262626262626, + "loss": 2.6203, + "theoretical_loss": 3.3542418128894487, + "tokens_seen": 2637496320 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020424242424242425, + "loss": 2.7205, + "theoretical_loss": 3.3542352358541203, + "tokens_seen": 2637561856 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020422222222222223, + "loss": 2.3857, + "theoretical_loss": 3.3542286590279673, + "tokens_seen": 2637627392 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020420202020202019, + "loss": 2.6463, + "theoretical_loss": 3.3542220824109776, + "tokens_seen": 2637692928 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002041818181818182, + "loss": 2.3769, + "theoretical_loss": 3.3542155060031393, + "tokens_seen": 2637758464 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020416161616161615, + "loss": 2.4234, + "theoretical_loss": 3.3542089298044413, + "tokens_seen": 2637824000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020414141414141416, + "loss": 2.714, + "theoretical_loss": 3.354202353814871, + "tokens_seen": 2637889536 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020412121212121212, + "loss": 2.5929, + "theoretical_loss": 3.354195778034417, + "tokens_seen": 2637955072 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002041010101010101, + "loss": 2.4963, + "theoretical_loss": 3.354189202463067, + "tokens_seen": 2638020608 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1445429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.842092275619507, + "objective/train/theoretical_loss": 3.3541826271008097, + "objective/train/tokens_used": 996945376, + "theoretical_loss": 3.3541826271008097, + "tokens_seen": 2638086144 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002040808080808081, + "loss": 2.5018, + "theoretical_loss": 3.3541826271008097, + "tokens_seen": 2638086144 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020406060606060605, + "loss": 2.4667, + "theoretical_loss": 3.3541760519476327, + "tokens_seen": 2638151680 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020404040404040406, + "loss": 2.4265, + "theoretical_loss": 3.354169477003525, + "tokens_seen": 2638217216 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020402020202020201, + "loss": 2.6254, + "theoretical_loss": 3.354162902268474, + "tokens_seen": 2638282752 + }, + { + "epoch": 0.6, + "learning_rate": 0.000204, + "loss": 2.5333, + "theoretical_loss": 3.3541563277424675, + "tokens_seen": 2638348288 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020397979797979798, + "loss": 2.5255, + "theoretical_loss": 3.354149753425495, + "tokens_seen": 2638413824 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020395959595959597, + "loss": 2.4482, + "theoretical_loss": 3.354143179317543, + "tokens_seen": 2638479360 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020393939393939395, + "loss": 2.7569, + "theoretical_loss": 3.354136605418601, + "tokens_seen": 2638544896 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020391919191919193, + "loss": 2.4789, + "theoretical_loss": 3.354130031728657, + "tokens_seen": 2638610432 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002038989898989899, + "loss": 2.509, + "theoretical_loss": 3.354123458247699, + "tokens_seen": 2638675968 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020387878787878788, + "loss": 2.3724, + "theoretical_loss": 3.3541168849757144, + "tokens_seen": 2638741504 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020385858585858586, + "loss": 2.5621, + "theoretical_loss": 3.3541103119126925, + "tokens_seen": 2638807040 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020383838383838384, + "loss": 2.4991, + "theoretical_loss": 3.354103739058621, + "tokens_seen": 2638872576 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020381818181818183, + "loss": 2.5027, + "theoretical_loss": 3.3540971664134873, + "tokens_seen": 2638938112 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002037979797979798, + "loss": 2.3328, + "theoretical_loss": 3.354090593977281, + "tokens_seen": 2639003648 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002037777777777778, + "loss": 2.5546, + "theoretical_loss": 3.3540840217499897, + "tokens_seen": 2639069184 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020375757575757575, + "loss": 2.4876, + "theoretical_loss": 3.3540774497316006, + "tokens_seen": 2639134720 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020373737373737374, + "loss": 2.4025, + "theoretical_loss": 3.3540708779221036, + "tokens_seen": 2639200256 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020371717171717172, + "loss": 2.6314, + "theoretical_loss": 3.3540643063214857, + "tokens_seen": 2639265792 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002036969696969697, + "loss": 2.4343, + "theoretical_loss": 3.3540577349297354, + "tokens_seen": 2639331328 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002036767676767677, + "loss": 2.4641, + "theoretical_loss": 3.3540511637468406, + "tokens_seen": 2639396864 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020365656565656564, + "loss": 2.6703, + "theoretical_loss": 3.35404459277279, + "tokens_seen": 2639462400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020363636363636366, + "loss": 2.6479, + "theoretical_loss": 3.354038022007572, + "tokens_seen": 2639527936 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002036161616161616, + "loss": 2.549, + "theoretical_loss": 3.3540314514511733, + "tokens_seen": 2639593472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020359595959595962, + "loss": 2.5515, + "theoretical_loss": 3.3540248811035838, + "tokens_seen": 2639659008 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1446170, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.584564685821533, + "objective/train/theoretical_loss": 3.3540183109647908, + "objective/train/tokens_used": 998583776, + "theoretical_loss": 3.3540183109647908, + "tokens_seen": 2639724544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020357575757575758, + "loss": 2.6946, + "theoretical_loss": 3.3540183109647908, + "tokens_seen": 2639724544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020355555555555554, + "loss": 2.47, + "theoretical_loss": 3.3540117410347823, + "tokens_seen": 2639790080 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020353535353535355, + "loss": 2.4332, + "theoretical_loss": 3.3540051713135473, + "tokens_seen": 2639855616 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002035151515151515, + "loss": 2.4539, + "theoretical_loss": 3.353998601801073, + "tokens_seen": 2639921152 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020349494949494952, + "loss": 2.4892, + "theoretical_loss": 3.3539920324973482, + "tokens_seen": 2639986688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020347474747474747, + "loss": 2.4403, + "theoretical_loss": 3.3539854634023607, + "tokens_seen": 2640052224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020345454545454546, + "loss": 2.5336, + "theoretical_loss": 3.3539788945160995, + "tokens_seen": 2640117760 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020343434343434344, + "loss": 2.4967, + "theoretical_loss": 3.3539723258385523, + "tokens_seen": 2640183296 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020341414141414142, + "loss": 2.4621, + "theoretical_loss": 3.3539657573697066, + "tokens_seen": 2640248832 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002033939393939394, + "loss": 2.4746, + "theoretical_loss": 3.3539591891095517, + "tokens_seen": 2640314368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020337373737373737, + "loss": 2.7718, + "theoretical_loss": 3.3539526210580752, + "tokens_seen": 2640379904 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020335353535353535, + "loss": 2.5249, + "theoretical_loss": 3.353946053215265, + "tokens_seen": 2640445440 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020333333333333333, + "loss": 2.5948, + "theoretical_loss": 3.3539394855811104, + "tokens_seen": 2640510976 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020331313131313132, + "loss": 2.5381, + "theoretical_loss": 3.3539329181555986, + "tokens_seen": 2640576512 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002032929292929293, + "loss": 2.6449, + "theoretical_loss": 3.3539263509387176, + "tokens_seen": 2640642048 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020327272727272729, + "loss": 2.3962, + "theoretical_loss": 3.353919783930457, + "tokens_seen": 2640707584 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020325252525252524, + "loss": 2.3859, + "theoretical_loss": 3.3539132171308035, + "tokens_seen": 2640773120 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020323232323232325, + "loss": 2.5057, + "theoretical_loss": 3.3539066505397455, + "tokens_seen": 2640838656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002032121212121212, + "loss": 2.6876, + "theoretical_loss": 3.353900084157272, + "tokens_seen": 2640904192 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002031919191919192, + "loss": 2.666, + "theoretical_loss": 3.3538935179833707, + "tokens_seen": 2640969728 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020317171717171718, + "loss": 2.4959, + "theoretical_loss": 3.35388695201803, + "tokens_seen": 2641035264 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020315151515151514, + "loss": 2.6454, + "theoretical_loss": 3.3538803862612374, + "tokens_seen": 2641100800 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020313131313131315, + "loss": 2.3571, + "theoretical_loss": 3.353873820712982, + "tokens_seen": 2641166336 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002031111111111111, + "loss": 2.341, + "theoretical_loss": 3.3538672553732516, + "tokens_seen": 2641231872 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020309090909090911, + "loss": 2.3788, + "theoretical_loss": 3.3538606902420347, + "tokens_seen": 2641297408 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1447240, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6527767181396484, + "objective/train/theoretical_loss": 3.3538541253193195, + "objective/train/tokens_used": 1000222176, + "theoretical_loss": 3.3538541253193195, + "tokens_seen": 2641362944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020307070707070707, + "loss": 2.5712, + "theoretical_loss": 3.3538541253193195, + "tokens_seen": 2641362944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020305050505050503, + "loss": 2.5421, + "theoretical_loss": 3.353847560605093, + "tokens_seen": 2641428480 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020303030303030304, + "loss": 2.2935, + "theoretical_loss": 3.353840996099345, + "tokens_seen": 2641494016 + }, + { + "epoch": 0.6, + "learning_rate": 0.000203010101010101, + "loss": 2.3186, + "theoretical_loss": 3.3538344318020634, + "tokens_seen": 2641559552 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202989898989899, + "loss": 2.6663, + "theoretical_loss": 3.3538278677132354, + "tokens_seen": 2641625088 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020296969696969696, + "loss": 2.5128, + "theoretical_loss": 3.3538213038328504, + "tokens_seen": 2641690624 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020294949494949497, + "loss": 2.4634, + "theoretical_loss": 3.353814740160896, + "tokens_seen": 2641756160 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020292929292929293, + "loss": 2.5404, + "theoretical_loss": 3.3538081766973606, + "tokens_seen": 2641821696 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020290909090909092, + "loss": 2.4918, + "theoretical_loss": 3.353801613442232, + "tokens_seen": 2641887232 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002028888888888889, + "loss": 2.4896, + "theoretical_loss": 3.353795050395499, + "tokens_seen": 2641952768 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020286868686868686, + "loss": 2.4449, + "theoretical_loss": 3.3537884875571495, + "tokens_seen": 2642018304 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020284848484848487, + "loss": 2.6791, + "theoretical_loss": 3.353781924927172, + "tokens_seen": 2642083840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020282828282828282, + "loss": 2.4065, + "theoretical_loss": 3.353775362505554, + "tokens_seen": 2642149376 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002028080808080808, + "loss": 2.4809, + "theoretical_loss": 3.3537688002922845, + "tokens_seen": 2642214912 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002027878787878788, + "loss": 2.7207, + "theoretical_loss": 3.3537622382873513, + "tokens_seen": 2642280448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020276767676767678, + "loss": 2.6247, + "theoretical_loss": 3.353755676490743, + "tokens_seen": 2642345984 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020274747474747476, + "loss": 2.8295, + "theoretical_loss": 3.353749114902447, + "tokens_seen": 2642411520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020272727272727274, + "loss": 2.3515, + "theoretical_loss": 3.353742553522453, + "tokens_seen": 2642477056 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002027070707070707, + "loss": 2.5542, + "theoretical_loss": 3.3537359923507477, + "tokens_seen": 2642542592 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020268686868686868, + "loss": 2.363, + "theoretical_loss": 3.35372943138732, + "tokens_seen": 2642608128 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020266666666666667, + "loss": 2.4245, + "theoretical_loss": 3.3537228706321582, + "tokens_seen": 2642673664 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020264646464646465, + "loss": 2.4782, + "theoretical_loss": 3.3537163100852503, + "tokens_seen": 2642739200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020262626262626264, + "loss": 2.7439, + "theoretical_loss": 3.353709749746584, + "tokens_seen": 2642804736 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002026060606060606, + "loss": 2.4381, + "theoretical_loss": 3.353703189616149, + "tokens_seen": 2642870272 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002025858585858586, + "loss": 2.2415, + "theoretical_loss": 3.3536966296939323, + "tokens_seen": 2642935808 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1447769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.581217050552368, + "objective/train/theoretical_loss": 3.3536900699799226, + "objective/train/tokens_used": 1001860576, + "theoretical_loss": 3.3536900699799226, + "tokens_seen": 2643001344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020256565656565656, + "loss": 2.4888, + "theoretical_loss": 3.3536900699799226, + "tokens_seen": 2643001344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020254545454545457, + "loss": 2.316, + "theoretical_loss": 3.353683510474108, + "tokens_seen": 2643066880 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020252525252525253, + "loss": 2.481, + "theoretical_loss": 3.353676951176477, + "tokens_seen": 2643132416 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020250505050505049, + "loss": 2.2993, + "theoretical_loss": 3.353670392087017, + "tokens_seen": 2643197952 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002024848484848485, + "loss": 2.4659, + "theoretical_loss": 3.353663833205717, + "tokens_seen": 2643263488 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020246464646464645, + "loss": 2.41, + "theoretical_loss": 3.3536572745325652, + "tokens_seen": 2643329024 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020244444444444447, + "loss": 2.4892, + "theoretical_loss": 3.3536507160675493, + "tokens_seen": 2643394560 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020242424242424242, + "loss": 2.5475, + "theoretical_loss": 3.3536441578106584, + "tokens_seen": 2643460096 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002024040404040404, + "loss": 2.46, + "theoretical_loss": 3.35363759976188, + "tokens_seen": 2643525632 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002023838383838384, + "loss": 2.4332, + "theoretical_loss": 3.3536310419212025, + "tokens_seen": 2643591168 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020236363636363637, + "loss": 2.4998, + "theoretical_loss": 3.3536244842886145, + "tokens_seen": 2643656704 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020234343434343436, + "loss": 2.3399, + "theoretical_loss": 3.3536179268641035, + "tokens_seen": 2643722240 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020232323232323231, + "loss": 2.5412, + "theoretical_loss": 3.3536113696476586, + "tokens_seen": 2643787776 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002023030303030303, + "loss": 2.4482, + "theoretical_loss": 3.3536048126392677, + "tokens_seen": 2643853312 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020228282828282828, + "loss": 2.4939, + "theoretical_loss": 3.353598255838919, + "tokens_seen": 2643918848 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020226262626262627, + "loss": 2.5698, + "theoretical_loss": 3.3535916992466, + "tokens_seen": 2643984384 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020224242424242425, + "loss": 2.3163, + "theoretical_loss": 3.3535851428623005, + "tokens_seen": 2644049920 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020222222222222223, + "loss": 2.5823, + "theoretical_loss": 3.3535785866860075, + "tokens_seen": 2644115456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002022020202020202, + "loss": 2.3124, + "theoretical_loss": 3.3535720307177095, + "tokens_seen": 2644180992 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020218181818181818, + "loss": 2.4176, + "theoretical_loss": 3.3535654749573953, + "tokens_seen": 2644246528 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020216161616161616, + "loss": 2.6747, + "theoretical_loss": 3.3535589194050526, + "tokens_seen": 2644312064 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020214141414141414, + "loss": 2.7327, + "theoretical_loss": 3.3535523640606697, + "tokens_seen": 2644377600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020212121212121213, + "loss": 2.4384, + "theoretical_loss": 3.353545808924235, + "tokens_seen": 2644443136 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002021010101010101, + "loss": 2.5577, + "theoretical_loss": 3.353539253995737, + "tokens_seen": 2644508672 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002020808080808081, + "loss": 2.6404, + "theoretical_loss": 3.353532699275163, + "tokens_seen": 2644574208 + }, + { + "debugging/Self-BLEU-5": 0.3806857716898066, + "debugging/distinct-1-grams": 0.7114641316153453, + "debugging/distinct-2-grams": 0.8748620261829383, + "debugging/entropy-1-grams": 5.322027392293219, + "debugging/entropy-2-grams": 5.946451670482928, + "debugging/length": 544.0, + "debugging/num_segments": 6, + "epoch": 0.6, + "objective/train/docs_used": 1449122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.866844892501831, + "objective/train/theoretical_loss": 3.3535261447625024, + "objective/train/tokens_used": 1003498976, + "theoretical_loss": 3.3535261447625024, + "tokens_seen": 2644639744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020206060606060605, + "loss": 2.4781, + "theoretical_loss": 3.3535261447625024, + "tokens_seen": 2644639744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020204040404040406, + "loss": 2.5342, + "theoretical_loss": 3.353519590457743, + "tokens_seen": 2644705280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020202020202020202, + "loss": 2.3997, + "theoretical_loss": 3.353513036360873, + "tokens_seen": 2644770816 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202, + "loss": 2.5918, + "theoretical_loss": 3.3535064824718805, + "tokens_seen": 2644836352 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201979797979798, + "loss": 2.2975, + "theoretical_loss": 3.353499928790754, + "tokens_seen": 2644901888 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020195959595959594, + "loss": 2.4527, + "theoretical_loss": 3.3534933753174814, + "tokens_seen": 2644967424 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020193939393939396, + "loss": 2.4123, + "theoretical_loss": 3.3534868220520515, + "tokens_seen": 2645032960 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002019191919191919, + "loss": 2.5549, + "theoretical_loss": 3.353480268994452, + "tokens_seen": 2645098496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020189898989898992, + "loss": 2.281, + "theoretical_loss": 3.3534737161446717, + "tokens_seen": 2645164032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020187878787878788, + "loss": 2.526, + "theoretical_loss": 3.3534671635026987, + "tokens_seen": 2645229568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020185858585858586, + "loss": 2.4262, + "theoretical_loss": 3.3534606110685212, + "tokens_seen": 2645295104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020183838383838385, + "loss": 2.4313, + "theoretical_loss": 3.353454058842127, + "tokens_seen": 2645360640 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002018181818181818, + "loss": 2.4439, + "theoretical_loss": 3.353447506823505, + "tokens_seen": 2645426176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020179797979797982, + "loss": 2.4488, + "theoretical_loss": 3.3534409550126436, + "tokens_seen": 2645491712 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020177777777777777, + "loss": 2.5502, + "theoretical_loss": 3.3534344034095303, + "tokens_seen": 2645557248 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020175757575757576, + "loss": 2.3812, + "theoretical_loss": 3.3534278520141543, + "tokens_seen": 2645622784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020173737373737374, + "loss": 2.2579, + "theoretical_loss": 3.353421300826503, + "tokens_seen": 2645688320 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020171717171717173, + "loss": 2.4524, + "theoretical_loss": 3.353414749846565, + "tokens_seen": 2645753856 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002016969696969697, + "loss": 2.3762, + "theoretical_loss": 3.3534081990743285, + "tokens_seen": 2645819392 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002016767676767677, + "loss": 2.4941, + "theoretical_loss": 3.353401648509782, + "tokens_seen": 2645884928 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020165656565656565, + "loss": 2.431, + "theoretical_loss": 3.353395098152914, + "tokens_seen": 2645950464 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020163636363636363, + "loss": 2.5094, + "theoretical_loss": 3.353388548003712, + "tokens_seen": 2646016000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020161616161616162, + "loss": 2.4778, + "theoretical_loss": 3.353381998062165, + "tokens_seen": 2646081536 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002015959595959596, + "loss": 2.3115, + "theoretical_loss": 3.3533754483282605, + "tokens_seen": 2646147072 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020157575757575759, + "loss": 2.5795, + "theoretical_loss": 3.3533688988019876, + "tokens_seen": 2646212608 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1449979, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7366762161254883, + "objective/train/theoretical_loss": 3.3533623494833344, + "objective/train/tokens_used": 1005137376, + "theoretical_loss": 3.3533623494833344, + "tokens_seen": 2646278144 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020155555555555554, + "loss": 2.4919, + "theoretical_loss": 3.3533623494833344, + "tokens_seen": 2646278144 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020153535353535355, + "loss": 2.771, + "theoretical_loss": 3.3533558003722885, + "tokens_seen": 2646343680 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002015151515151515, + "loss": 2.3588, + "theoretical_loss": 3.353349251468839, + "tokens_seen": 2646409216 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020149494949494952, + "loss": 2.5566, + "theoretical_loss": 3.3533427027729736, + "tokens_seen": 2646474752 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020147474747474748, + "loss": 2.5956, + "theoretical_loss": 3.353336154284681, + "tokens_seen": 2646540288 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020145454545454544, + "loss": 2.6503, + "theoretical_loss": 3.3533296060039492, + "tokens_seen": 2646605824 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020143434343434345, + "loss": 2.3714, + "theoretical_loss": 3.353323057930767, + "tokens_seen": 2646671360 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002014141414141414, + "loss": 2.564, + "theoretical_loss": 3.3533165100651217, + "tokens_seen": 2646736896 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020139393939393941, + "loss": 2.5456, + "theoretical_loss": 3.3533099624070024, + "tokens_seen": 2646802432 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020137373737373737, + "loss": 2.5628, + "theoretical_loss": 3.353303414956397, + "tokens_seen": 2646867968 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020135353535353536, + "loss": 2.5234, + "theoretical_loss": 3.3532968677132944, + "tokens_seen": 2646933504 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020133333333333334, + "loss": 2.5044, + "theoretical_loss": 3.353290320677682, + "tokens_seen": 2646999040 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002013131313131313, + "loss": 2.4042, + "theoretical_loss": 3.3532837738495487, + "tokens_seen": 2647064576 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002012929292929293, + "loss": 2.3805, + "theoretical_loss": 3.353277227228882, + "tokens_seen": 2647130112 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020127272727272726, + "loss": 2.5289, + "theoretical_loss": 3.3532706808156716, + "tokens_seen": 2647195648 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020125252525252528, + "loss": 2.4921, + "theoretical_loss": 3.353264134609905, + "tokens_seen": 2647261184 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020123232323232323, + "loss": 2.5631, + "theoretical_loss": 3.3532575886115694, + "tokens_seen": 2647326720 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020121212121212122, + "loss": 2.3107, + "theoretical_loss": 3.3532510428206552, + "tokens_seen": 2647392256 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002011919191919192, + "loss": 2.5545, + "theoretical_loss": 3.3532444972371493, + "tokens_seen": 2647457792 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020117171717171718, + "loss": 2.5281, + "theoretical_loss": 3.3532379518610402, + "tokens_seen": 2647523328 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020115151515151517, + "loss": 2.5125, + "theoretical_loss": 3.3532314066923163, + "tokens_seen": 2647588864 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020113131313131312, + "loss": 2.5857, + "theoretical_loss": 3.353224861730966, + "tokens_seen": 2647654400 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002011111111111111, + "loss": 2.4776, + "theoretical_loss": 3.3532183169769776, + "tokens_seen": 2647719936 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002010909090909091, + "loss": 2.5633, + "theoretical_loss": 3.3532117724303396, + "tokens_seen": 2647785472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020107070707070708, + "loss": 2.3298, + "theoretical_loss": 3.35320522809104, + "tokens_seen": 2647851008 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1450923, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.731349468231201, + "objective/train/theoretical_loss": 3.3531986839590666, + "objective/train/tokens_used": 1006775776, + "theoretical_loss": 3.3531986839590666, + "tokens_seen": 2647916544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020105050505050506, + "loss": 2.7316, + "theoretical_loss": 3.3531986839590666, + "tokens_seen": 2647916544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020103030303030304, + "loss": 2.5103, + "theoretical_loss": 3.3531921400344085, + "tokens_seen": 2647982080 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201010101010101, + "loss": 2.463, + "theoretical_loss": 3.353185596317054, + "tokens_seen": 2648047616 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200989898989899, + "loss": 2.3869, + "theoretical_loss": 3.3531790528069907, + "tokens_seen": 2648113152 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020096969696969697, + "loss": 2.6476, + "theoretical_loss": 3.3531725095042075, + "tokens_seen": 2648178688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020094949494949495, + "loss": 2.43, + "theoretical_loss": 3.3531659664086924, + "tokens_seen": 2648244224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020092929292929294, + "loss": 2.515, + "theoretical_loss": 3.353159423520434, + "tokens_seen": 2648309760 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002009090909090909, + "loss": 2.581, + "theoretical_loss": 3.3531528808394206, + "tokens_seen": 2648375296 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002008888888888889, + "loss": 2.5799, + "theoretical_loss": 3.35314633836564, + "tokens_seen": 2648440832 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020086868686868686, + "loss": 2.3902, + "theoretical_loss": 3.353139796099081, + "tokens_seen": 2648506368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020084848484848487, + "loss": 2.3964, + "theoretical_loss": 3.353133254039732, + "tokens_seen": 2648571904 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020082828282828283, + "loss": 2.6709, + "theoretical_loss": 3.3531267121875805, + "tokens_seen": 2648637440 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020080808080808081, + "loss": 2.4107, + "theoretical_loss": 3.353120170542616, + "tokens_seen": 2648702976 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002007878787878788, + "loss": 2.4521, + "theoretical_loss": 3.353113629104826, + "tokens_seen": 2648768512 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020076767676767675, + "loss": 2.4593, + "theoretical_loss": 3.353107087874199, + "tokens_seen": 2648834048 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020074747474747477, + "loss": 2.4129, + "theoretical_loss": 3.3531005468507233, + "tokens_seen": 2648899584 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020072727272727272, + "loss": 2.6882, + "theoretical_loss": 3.3530940060343872, + "tokens_seen": 2648965120 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002007070707070707, + "loss": 2.489, + "theoretical_loss": 3.353087465425179, + "tokens_seen": 2649030656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002006868686868687, + "loss": 2.5607, + "theoretical_loss": 3.3530809250230877, + "tokens_seen": 2649096192 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020066666666666667, + "loss": 2.5956, + "theoretical_loss": 3.3530743848281004, + "tokens_seen": 2649161728 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020064646464646466, + "loss": 2.36, + "theoretical_loss": 3.353067844840206, + "tokens_seen": 2649227264 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020062626262626264, + "loss": 2.4052, + "theoretical_loss": 3.353061305059393, + "tokens_seen": 2649292800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002006060606060606, + "loss": 2.5173, + "theoretical_loss": 3.3530547654856497, + "tokens_seen": 2649358336 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020058585858585858, + "loss": 2.4487, + "theoretical_loss": 3.3530482261189642, + "tokens_seen": 2649423872 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020056565656565657, + "loss": 2.3324, + "theoretical_loss": 3.353041686959325, + "tokens_seen": 2649489408 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1451404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.839794635772705, + "objective/train/theoretical_loss": 3.35303514800672, + "objective/train/tokens_used": 1008414176, + "theoretical_loss": 3.35303514800672, + "tokens_seen": 2649554944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020054545454545455, + "loss": 2.6139, + "theoretical_loss": 3.35303514800672, + "tokens_seen": 2649554944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020052525252525254, + "loss": 2.7389, + "theoretical_loss": 3.353028609261138, + "tokens_seen": 2649620480 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002005050505050505, + "loss": 2.3895, + "theoretical_loss": 3.3530220707225675, + "tokens_seen": 2649686016 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002004848484848485, + "loss": 2.6618, + "theoretical_loss": 3.353015532390996, + "tokens_seen": 2649751552 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020046464646464646, + "loss": 2.3531, + "theoretical_loss": 3.3530089942664123, + "tokens_seen": 2649817088 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020044444444444444, + "loss": 2.4143, + "theoretical_loss": 3.353002456348805, + "tokens_seen": 2649882624 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020042424242424243, + "loss": 2.5058, + "theoretical_loss": 3.3529959186381624, + "tokens_seen": 2649948160 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002004040404040404, + "loss": 2.3157, + "theoretical_loss": 3.3529893811344724, + "tokens_seen": 2650013696 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002003838383838384, + "loss": 2.4094, + "theoretical_loss": 3.3529828438377236, + "tokens_seen": 2650079232 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020036363636363635, + "loss": 2.5233, + "theoretical_loss": 3.3529763067479044, + "tokens_seen": 2650144768 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020034343434343436, + "loss": 2.4015, + "theoretical_loss": 3.3529697698650027, + "tokens_seen": 2650210304 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020032323232323232, + "loss": 2.6235, + "theoretical_loss": 3.352963233189007, + "tokens_seen": 2650275840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020030303030303033, + "loss": 2.6242, + "theoretical_loss": 3.352956696719906, + "tokens_seen": 2650341376 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002002828282828283, + "loss": 2.6108, + "theoretical_loss": 3.352950160457688, + "tokens_seen": 2650406912 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020026262626262625, + "loss": 2.3372, + "theoretical_loss": 3.352943624402341, + "tokens_seen": 2650472448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020024242424242426, + "loss": 2.5075, + "theoretical_loss": 3.352937088553854, + "tokens_seen": 2650537984 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002002222222222222, + "loss": 2.5586, + "theoretical_loss": 3.352930552912214, + "tokens_seen": 2650603520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020020202020202022, + "loss": 2.6873, + "theoretical_loss": 3.3529240174774104, + "tokens_seen": 2650669056 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020018181818181818, + "loss": 2.5061, + "theoretical_loss": 3.3529174822494316, + "tokens_seen": 2650734592 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020016161616161616, + "loss": 2.4104, + "theoretical_loss": 3.352910947228265, + "tokens_seen": 2650800128 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020014141414141415, + "loss": 2.5027, + "theoretical_loss": 3.3529044124139005, + "tokens_seen": 2650865664 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020012121212121213, + "loss": 2.5356, + "theoretical_loss": 3.3528978778063254, + "tokens_seen": 2650931200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020010101010101012, + "loss": 2.5219, + "theoretical_loss": 3.3528913434055276, + "tokens_seen": 2650996736 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020008080808080807, + "loss": 2.6471, + "theoretical_loss": 3.3528848092114965, + "tokens_seen": 2651062272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020006060606060606, + "loss": 2.5698, + "theoretical_loss": 3.35287827522422, + "tokens_seen": 2651127808 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1452746, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.292234420776367, + "objective/train/theoretical_loss": 3.352871741443686, + "objective/train/tokens_used": 1010052576, + "theoretical_loss": 3.352871741443686, + "tokens_seen": 2651193344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020004040404040404, + "loss": 2.6127, + "theoretical_loss": 3.352871741443686, + "tokens_seen": 2651193344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020002020202020203, + "loss": 2.4828, + "theoretical_loss": 3.3528652078698835, + "tokens_seen": 2651258880 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002, + "loss": 2.548, + "theoretical_loss": 3.3528586745028006, + "tokens_seen": 2651324416 + }, + { + "epoch": 0.6, + "learning_rate": 0.000199979797979798, + "loss": 2.6384, + "theoretical_loss": 3.3528521413424257, + "tokens_seen": 2651389952 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019995959595959595, + "loss": 2.478, + "theoretical_loss": 3.3528456083887472, + "tokens_seen": 2651455488 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019993939393939396, + "loss": 2.6538, + "theoretical_loss": 3.352839075641753, + "tokens_seen": 2651521024 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019991919191919192, + "loss": 2.611, + "theoretical_loss": 3.3528325431014325, + "tokens_seen": 2651586560 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001998989898989899, + "loss": 2.6428, + "theoretical_loss": 3.3528260107677728, + "tokens_seen": 2651652096 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019987878787878789, + "loss": 2.6609, + "theoretical_loss": 3.352819478640763, + "tokens_seen": 2651717632 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019985858585858584, + "loss": 2.4714, + "theoretical_loss": 3.3528129467203915, + "tokens_seen": 2651783168 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019983838383838385, + "loss": 2.4131, + "theoretical_loss": 3.352806415006646, + "tokens_seen": 2651848704 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001998181818181818, + "loss": 2.4969, + "theoretical_loss": 3.352799883499516, + "tokens_seen": 2651914240 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019979797979797982, + "loss": 2.5064, + "theoretical_loss": 3.3527933521989883, + "tokens_seen": 2651979776 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019977777777777778, + "loss": 2.5161, + "theoretical_loss": 3.3527868211050524, + "tokens_seen": 2652045312 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019975757575757574, + "loss": 2.7671, + "theoretical_loss": 3.352780290217697, + "tokens_seen": 2652110848 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019973737373737375, + "loss": 2.4429, + "theoretical_loss": 3.352773759536909, + "tokens_seen": 2652176384 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001997171717171717, + "loss": 2.4732, + "theoretical_loss": 3.352767229062678, + "tokens_seen": 2652241920 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019969696969696971, + "loss": 2.5058, + "theoretical_loss": 3.352760698794992, + "tokens_seen": 2652307456 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019967676767676767, + "loss": 2.4427, + "theoretical_loss": 3.3527541687338394, + "tokens_seen": 2652372992 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019965656565656566, + "loss": 2.6361, + "theoretical_loss": 3.352747638879208, + "tokens_seen": 2652438528 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019963636363636364, + "loss": 2.528, + "theoretical_loss": 3.3527411092310873, + "tokens_seen": 2652504064 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019961616161616162, + "loss": 2.118, + "theoretical_loss": 3.3527345797894648, + "tokens_seen": 2652569600 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001995959595959596, + "loss": 2.6372, + "theoretical_loss": 3.352728050554329, + "tokens_seen": 2652635136 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019957575757575756, + "loss": 2.3212, + "theoretical_loss": 3.3527215215256687, + "tokens_seen": 2652700672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019955555555555558, + "loss": 2.6494, + "theoretical_loss": 3.352714992703471, + "tokens_seen": 2652766208 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1453835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6858787536621094, + "objective/train/theoretical_loss": 3.3527084640877263, + "objective/train/tokens_used": 1011690976, + "theoretical_loss": 3.3527084640877263, + "tokens_seen": 2652831744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019953535353535353, + "loss": 2.5642, + "theoretical_loss": 3.3527084640877263, + "tokens_seen": 2652831744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019951515151515152, + "loss": 2.5744, + "theoretical_loss": 3.3527019356784216, + "tokens_seen": 2652897280 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001994949494949495, + "loss": 2.3795, + "theoretical_loss": 3.352695407475545, + "tokens_seen": 2652962816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019947474747474748, + "loss": 2.3516, + "theoretical_loss": 3.352688879479086, + "tokens_seen": 2653028352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019945454545454547, + "loss": 2.5893, + "theoretical_loss": 3.3526823516890323, + "tokens_seen": 2653093888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019943434343434345, + "loss": 2.5813, + "theoretical_loss": 3.352675824105372, + "tokens_seen": 2653159424 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001994141414141414, + "loss": 2.463, + "theoretical_loss": 3.352669296728094, + "tokens_seen": 2653224960 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001993939393939394, + "loss": 2.5026, + "theoretical_loss": 3.352662769557187, + "tokens_seen": 2653290496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019937373737373738, + "loss": 2.4246, + "theoretical_loss": 3.3526562425926385, + "tokens_seen": 2653356032 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019935353535353536, + "loss": 2.5525, + "theoretical_loss": 3.3526497158344375, + "tokens_seen": 2653421568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019933333333333334, + "loss": 2.6686, + "theoretical_loss": 3.352643189282572, + "tokens_seen": 2653487104 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001993131313131313, + "loss": 2.5692, + "theoretical_loss": 3.3526366629370306, + "tokens_seen": 2653552640 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001992929292929293, + "loss": 2.4908, + "theoretical_loss": 3.3526301367978015, + "tokens_seen": 2653618176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019927272727272727, + "loss": 2.3672, + "theoretical_loss": 3.3526236108648733, + "tokens_seen": 2653683712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019925252525252528, + "loss": 2.5324, + "theoretical_loss": 3.352617085138234, + "tokens_seen": 2653749248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019923232323232324, + "loss": 2.6159, + "theoretical_loss": 3.352610559617873, + "tokens_seen": 2653814784 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001992121212121212, + "loss": 2.4106, + "theoretical_loss": 3.3526040343037775, + "tokens_seen": 2653880320 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001991919191919192, + "loss": 2.5256, + "theoretical_loss": 3.3525975091959364, + "tokens_seen": 2653945856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019917171717171716, + "loss": 2.4398, + "theoretical_loss": 3.352590984294338, + "tokens_seen": 2654011392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019915151515151517, + "loss": 2.4575, + "theoretical_loss": 3.3525844595989707, + "tokens_seen": 2654076928 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019913131313131313, + "loss": 2.4998, + "theoretical_loss": 3.352577935109823, + "tokens_seen": 2654142464 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019911111111111111, + "loss": 2.4917, + "theoretical_loss": 3.3525714108268834, + "tokens_seen": 2654208000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001990909090909091, + "loss": 2.5708, + "theoretical_loss": 3.35256488675014, + "tokens_seen": 2654273536 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019907070707070708, + "loss": 2.2758, + "theoretical_loss": 3.3525583628795808, + "tokens_seen": 2654339072 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019905050505050507, + "loss": 2.4219, + "theoretical_loss": 3.352551839215195, + "tokens_seen": 2654404608 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1454467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1566362380981445, + "objective/train/theoretical_loss": 3.352545315756971, + "objective/train/tokens_used": 1013329376, + "theoretical_loss": 3.352545315756971, + "tokens_seen": 2654470144 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019903030303030302, + "loss": 2.7777, + "theoretical_loss": 3.352545315756971, + "tokens_seen": 2654470144 + }, + { + "epoch": 0.61, + "learning_rate": 0.000199010101010101, + "loss": 2.5015, + "theoretical_loss": 3.352538792504897, + "tokens_seen": 2654535680 + }, + { + "epoch": 0.61, + "learning_rate": 0.000198989898989899, + "loss": 2.6502, + "theoretical_loss": 3.3525322694589605, + "tokens_seen": 2654601216 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019896969696969697, + "loss": 2.7015, + "theoretical_loss": 3.3525257466191514, + "tokens_seen": 2654666752 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019894949494949496, + "loss": 2.5124, + "theoretical_loss": 3.352519223985457, + "tokens_seen": 2654732288 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019892929292929294, + "loss": 2.4434, + "theoretical_loss": 3.352512701557866, + "tokens_seen": 2654797824 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001989090909090909, + "loss": 2.4979, + "theoretical_loss": 3.352506179336367, + "tokens_seen": 2654863360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019888888888888888, + "loss": 2.5917, + "theoretical_loss": 3.3524996573209482, + "tokens_seen": 2654928896 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019886868686868687, + "loss": 2.2612, + "theoretical_loss": 3.3524931355115983, + "tokens_seen": 2654994432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019884848484848485, + "loss": 2.5707, + "theoretical_loss": 3.3524866139083054, + "tokens_seen": 2655059968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019882828282828284, + "loss": 2.5653, + "theoretical_loss": 3.3524800925110574, + "tokens_seen": 2655125504 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001988080808080808, + "loss": 2.6521, + "theoretical_loss": 3.352473571319844, + "tokens_seen": 2655191040 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001987878787878788, + "loss": 2.6293, + "theoretical_loss": 3.3524670503346528, + "tokens_seen": 2655256576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019876767676767676, + "loss": 2.4631, + "theoretical_loss": 3.352460529555472, + "tokens_seen": 2655322112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019874747474747477, + "loss": 2.5296, + "theoretical_loss": 3.3524540089822903, + "tokens_seen": 2655387648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019872727272727273, + "loss": 2.5384, + "theoretical_loss": 3.3524474886150966, + "tokens_seen": 2655453184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019870707070707068, + "loss": 2.2901, + "theoretical_loss": 3.352440968453878, + "tokens_seen": 2655518720 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001986868686868687, + "loss": 2.6048, + "theoretical_loss": 3.3524344484986246, + "tokens_seen": 2655584256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019866666666666665, + "loss": 2.5828, + "theoretical_loss": 3.3524279287493233, + "tokens_seen": 2655649792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019864646464646466, + "loss": 2.7395, + "theoretical_loss": 3.3524214092059634, + "tokens_seen": 2655715328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019862626262626262, + "loss": 2.5691, + "theoretical_loss": 3.352414889868533, + "tokens_seen": 2655780864 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019860606060606063, + "loss": 2.4172, + "theoretical_loss": 3.3524083707370207, + "tokens_seen": 2655846400 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001985858585858586, + "loss": 2.402, + "theoretical_loss": 3.3524018518114147, + "tokens_seen": 2655911936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019856565656565657, + "loss": 2.534, + "theoretical_loss": 3.3523953330917036, + "tokens_seen": 2655977472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019854545454545456, + "loss": 2.5576, + "theoretical_loss": 3.3523888145778757, + "tokens_seen": 2656043008 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1455118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3296263217926025, + "objective/train/theoretical_loss": 3.3523822962699192, + "objective/train/tokens_used": 1014967776, + "theoretical_loss": 3.3523822962699192, + "tokens_seen": 2656108544 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001985252525252525, + "loss": 2.443, + "theoretical_loss": 3.3523822962699192, + "tokens_seen": 2656108544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019850505050505052, + "loss": 2.3405, + "theoretical_loss": 3.352375778167823, + "tokens_seen": 2656174080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019848484848484848, + "loss": 2.551, + "theoretical_loss": 3.3523692602715753, + "tokens_seen": 2656239616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019846464646464647, + "loss": 2.6063, + "theoretical_loss": 3.3523627425811644, + "tokens_seen": 2656305152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019844444444444445, + "loss": 2.5995, + "theoretical_loss": 3.3523562250965786, + "tokens_seen": 2656370688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019842424242424243, + "loss": 2.3433, + "theoretical_loss": 3.352349707817807, + "tokens_seen": 2656436224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019840404040404042, + "loss": 2.5972, + "theoretical_loss": 3.352343190744837, + "tokens_seen": 2656501760 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001983838383838384, + "loss": 2.6779, + "theoretical_loss": 3.3523366738776583, + "tokens_seen": 2656567296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019836363636363636, + "loss": 2.5159, + "theoretical_loss": 3.352330157216258, + "tokens_seen": 2656632832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019834343434343434, + "loss": 2.4608, + "theoretical_loss": 3.3523236407606256, + "tokens_seen": 2656698368 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019832323232323233, + "loss": 2.4275, + "theoretical_loss": 3.352317124510749, + "tokens_seen": 2656763904 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019832323232323233, + "loss": 2.3229, + "theoretical_loss": 3.3523106084666168, + "tokens_seen": 2656829440 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001983030303030303, + "loss": 2.61, + "theoretical_loss": 3.352304092628217, + "tokens_seen": 2656894976 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001982828282828283, + "loss": 2.4089, + "theoretical_loss": 3.3522975769955385, + "tokens_seen": 2656960512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019826262626262625, + "loss": 2.5355, + "theoretical_loss": 3.3522910615685695, + "tokens_seen": 2657026048 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019824242424242426, + "loss": 2.5925, + "theoretical_loss": 3.3522845463472986, + "tokens_seen": 2657091584 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019822222222222222, + "loss": 2.5389, + "theoretical_loss": 3.352278031331714, + "tokens_seen": 2657157120 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019820202020202023, + "loss": 2.5292, + "theoretical_loss": 3.3522715165218044, + "tokens_seen": 2657222656 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001981818181818182, + "loss": 2.4939, + "theoretical_loss": 3.352265001917558, + "tokens_seen": 2657288192 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019816161616161614, + "loss": 2.3584, + "theoretical_loss": 3.352258487518964, + "tokens_seen": 2657353728 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019814141414141415, + "loss": 2.3721, + "theoretical_loss": 3.3522519733260094, + "tokens_seen": 2657419264 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001981212121212121, + "loss": 2.6653, + "theoretical_loss": 3.352245459338684, + "tokens_seen": 2657484800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019810101010101012, + "loss": 2.5444, + "theoretical_loss": 3.352238945556975, + "tokens_seen": 2657550336 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019808080808080808, + "loss": 2.6636, + "theoretical_loss": 3.3522324319808723, + "tokens_seen": 2657615872 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019806060606060606, + "loss": 2.4531, + "theoretical_loss": 3.352225918610363, + "tokens_seen": 2657681408 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1456223, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8828985691070557, + "objective/train/theoretical_loss": 3.352219405445436, + "objective/train/tokens_used": 1016606176, + "theoretical_loss": 3.352219405445436, + "tokens_seen": 2657746944 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019804040404040405, + "loss": 2.5353, + "theoretical_loss": 3.352219405445436, + "tokens_seen": 2657746944 + }, + { + "epoch": 0.61, + "learning_rate": 0.000198020202020202, + "loss": 2.4716, + "theoretical_loss": 3.3522128924860803, + "tokens_seen": 2657812480 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019800000000000002, + "loss": 2.5462, + "theoretical_loss": 3.3522063797322836, + "tokens_seen": 2657878016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019797979797979797, + "loss": 2.4209, + "theoretical_loss": 3.3521998671840345, + "tokens_seen": 2657943552 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019795959595959596, + "loss": 2.3932, + "theoretical_loss": 3.3521933548413223, + "tokens_seen": 2658009088 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019793939393939394, + "loss": 2.3762, + "theoretical_loss": 3.3521868427041337, + "tokens_seen": 2658074624 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019791919191919192, + "loss": 2.4539, + "theoretical_loss": 3.352180330772459, + "tokens_seen": 2658140160 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001978989898989899, + "loss": 2.4222, + "theoretical_loss": 3.3521738190462855, + "tokens_seen": 2658205696 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001978787878787879, + "loss": 2.3635, + "theoretical_loss": 3.3521673075256015, + "tokens_seen": 2658271232 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019785858585858585, + "loss": 2.4932, + "theoretical_loss": 3.3521607962103963, + "tokens_seen": 2658336768 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019783838383838383, + "loss": 2.4453, + "theoretical_loss": 3.3521542851006583, + "tokens_seen": 2658402304 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019781818181818182, + "loss": 2.4196, + "theoretical_loss": 3.352147774196375, + "tokens_seen": 2658467840 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001977979797979798, + "loss": 2.4653, + "theoretical_loss": 3.3521412634975354, + "tokens_seen": 2658533376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019777777777777778, + "loss": 2.4768, + "theoretical_loss": 3.3521347530041288, + "tokens_seen": 2658598912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019775757575757577, + "loss": 2.5614, + "theoretical_loss": 3.3521282427161423, + "tokens_seen": 2658664448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019773737373737375, + "loss": 2.6416, + "theoretical_loss": 3.352121732633565, + "tokens_seen": 2658729984 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001977171717171717, + "loss": 2.4865, + "theoretical_loss": 3.3521152227563853, + "tokens_seen": 2658795520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019769696969696972, + "loss": 2.7369, + "theoretical_loss": 3.3521087130845912, + "tokens_seen": 2658861056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019767676767676768, + "loss": 2.3634, + "theoretical_loss": 3.352102203618172, + "tokens_seen": 2658926592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019765656565656566, + "loss": 2.5013, + "theoretical_loss": 3.352095694357116, + "tokens_seen": 2658992128 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019763636363636365, + "loss": 2.5832, + "theoretical_loss": 3.352089185301411, + "tokens_seen": 2659057664 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001976161616161616, + "loss": 2.484, + "theoretical_loss": 3.352082676451046, + "tokens_seen": 2659123200 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001975959595959596, + "loss": 2.682, + "theoretical_loss": 3.3520761678060094, + "tokens_seen": 2659188736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019757575757575757, + "loss": 2.6926, + "theoretical_loss": 3.3520696593662898, + "tokens_seen": 2659254272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019755555555555558, + "loss": 2.4182, + "theoretical_loss": 3.352063151131875, + "tokens_seen": 2659319808 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1457493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5922701358795166, + "objective/train/theoretical_loss": 3.352056643102754, + "objective/train/tokens_used": 1018244576, + "theoretical_loss": 3.352056643102754, + "tokens_seen": 2659385344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019753535353535354, + "loss": 2.4809, + "theoretical_loss": 3.352056643102754, + "tokens_seen": 2659385344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019751515151515152, + "loss": 2.9616, + "theoretical_loss": 3.3520501352789154, + "tokens_seen": 2659450880 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001974949494949495, + "loss": 2.2786, + "theoretical_loss": 3.3520436276603474, + "tokens_seen": 2659516416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019747474747474746, + "loss": 2.3407, + "theoretical_loss": 3.3520371202470383, + "tokens_seen": 2659581952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019745454545454547, + "loss": 2.4585, + "theoretical_loss": 3.352030613038977, + "tokens_seen": 2659647488 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019743434343434343, + "loss": 2.7334, + "theoretical_loss": 3.3520241060361515, + "tokens_seen": 2659713024 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019741414141414141, + "loss": 2.5413, + "theoretical_loss": 3.3520175992385512, + "tokens_seen": 2659778560 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001973939393939394, + "loss": 2.5969, + "theoretical_loss": 3.3520110926461633, + "tokens_seen": 2659844096 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019737373737373738, + "loss": 2.4151, + "theoretical_loss": 3.352004586258977, + "tokens_seen": 2659909632 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019735353535353537, + "loss": 2.6478, + "theoretical_loss": 3.3519980800769806, + "tokens_seen": 2659975168 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019733333333333335, + "loss": 2.2895, + "theoretical_loss": 3.351991574100163, + "tokens_seen": 2660040704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001973131313131313, + "loss": 2.489, + "theoretical_loss": 3.3519850683285117, + "tokens_seen": 2660106240 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001972929292929293, + "loss": 2.3768, + "theoretical_loss": 3.351978562762016, + "tokens_seen": 2660171776 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019727272727272728, + "loss": 2.634, + "theoretical_loss": 3.351972057400664, + "tokens_seen": 2660237312 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019725252525252526, + "loss": 2.5941, + "theoretical_loss": 3.3519655522444447, + "tokens_seen": 2660302848 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019723232323232324, + "loss": 2.5072, + "theoretical_loss": 3.351959047293346, + "tokens_seen": 2660368384 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001972121212121212, + "loss": 2.6271, + "theoretical_loss": 3.3519525425473566, + "tokens_seen": 2660433920 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001971919191919192, + "loss": 2.5462, + "theoretical_loss": 3.3519460380064654, + "tokens_seen": 2660499456 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019717171717171717, + "loss": 2.4667, + "theoretical_loss": 3.35193953367066, + "tokens_seen": 2660564992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019715151515151515, + "loss": 2.4661, + "theoretical_loss": 3.351933029539929, + "tokens_seen": 2660630528 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019713131313131314, + "loss": 2.4824, + "theoretical_loss": 3.3519265256142616, + "tokens_seen": 2660696064 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001971111111111111, + "loss": 2.3649, + "theoretical_loss": 3.351920021893646, + "tokens_seen": 2660761600 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001970909090909091, + "loss": 2.5306, + "theoretical_loss": 3.35191351837807, + "tokens_seen": 2660827136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019707070707070706, + "loss": 2.5332, + "theoretical_loss": 3.351907015067523, + "tokens_seen": 2660892672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019705050505050507, + "loss": 2.6748, + "theoretical_loss": 3.3519005119619933, + "tokens_seen": 2660958208 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1458037, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.740746259689331, + "objective/train/theoretical_loss": 3.3518940090614695, + "objective/train/tokens_used": 1019882976, + "theoretical_loss": 3.3518940090614695, + "tokens_seen": 2661023744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019703030303030303, + "loss": 2.5832, + "theoretical_loss": 3.3518940090614695, + "tokens_seen": 2661023744 + }, + { + "epoch": 0.61, + "learning_rate": 0.000197010101010101, + "loss": 2.5883, + "theoretical_loss": 3.3518875063659395, + "tokens_seen": 2661089280 + }, + { + "epoch": 0.61, + "learning_rate": 0.000196989898989899, + "loss": 2.4978, + "theoretical_loss": 3.351881003875392, + "tokens_seen": 2661154816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019696969696969695, + "loss": 2.367, + "theoretical_loss": 3.3518745015898155, + "tokens_seen": 2661220352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019694949494949496, + "loss": 2.5075, + "theoretical_loss": 3.351867999509199, + "tokens_seen": 2661285888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019692929292929292, + "loss": 2.5259, + "theoretical_loss": 3.3518614976335304, + "tokens_seen": 2661351424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019690909090909093, + "loss": 2.3953, + "theoretical_loss": 3.3518549959627983, + "tokens_seen": 2661416960 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001968888888888889, + "loss": 2.3894, + "theoretical_loss": 3.3518484944969913, + "tokens_seen": 2661482496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019686868686868687, + "loss": 2.6628, + "theoretical_loss": 3.351841993236098, + "tokens_seen": 2661548032 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019684848484848486, + "loss": 2.2204, + "theoretical_loss": 3.351835492180107, + "tokens_seen": 2661613568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019682828282828284, + "loss": 2.6109, + "theoretical_loss": 3.351828991329006, + "tokens_seen": 2661679104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019680808080808082, + "loss": 2.5322, + "theoretical_loss": 3.351822490682784, + "tokens_seen": 2661744640 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019678787878787878, + "loss": 2.3903, + "theoretical_loss": 3.3518159902414304, + "tokens_seen": 2661810176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019676767676767677, + "loss": 2.4966, + "theoretical_loss": 3.351809490004932, + "tokens_seen": 2661875712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019674747474747475, + "loss": 2.5301, + "theoretical_loss": 3.3518029899732786, + "tokens_seen": 2661941248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019672727272727273, + "loss": 2.4648, + "theoretical_loss": 3.3517964901464583, + "tokens_seen": 2662006784 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019670707070707072, + "loss": 2.4157, + "theoretical_loss": 3.351789990524459, + "tokens_seen": 2662072320 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001966868686868687, + "loss": 2.5671, + "theoretical_loss": 3.3517834911072706, + "tokens_seen": 2662137856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019666666666666666, + "loss": 2.5077, + "theoretical_loss": 3.35177699189488, + "tokens_seen": 2662203392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019664646464646467, + "loss": 2.489, + "theoretical_loss": 3.351770492887277, + "tokens_seen": 2662268928 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019662626262626263, + "loss": 2.545, + "theoretical_loss": 3.3517639940844495, + "tokens_seen": 2662334464 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001966060606060606, + "loss": 2.3762, + "theoretical_loss": 3.3517574954863862, + "tokens_seen": 2662400000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001965858585858586, + "loss": 2.4357, + "theoretical_loss": 3.3517509970930752, + "tokens_seen": 2662465536 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019656565656565655, + "loss": 2.6609, + "theoretical_loss": 3.3517444989045053, + "tokens_seen": 2662531072 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019654545454545456, + "loss": 2.5247, + "theoretical_loss": 3.3517380009206654, + "tokens_seen": 2662596608 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1459076, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5953822135925293, + "objective/train/theoretical_loss": 3.3517315031415436, + "objective/train/tokens_used": 1021521376, + "theoretical_loss": 3.3517315031415436, + "tokens_seen": 2662662144 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019652525252525252, + "loss": 2.7188, + "theoretical_loss": 3.3517315031415436, + "tokens_seen": 2662662144 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019650505050505053, + "loss": 2.4411, + "theoretical_loss": 3.351725005567128, + "tokens_seen": 2662727680 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001964848484848485, + "loss": 2.5941, + "theoretical_loss": 3.3517185081974077, + "tokens_seen": 2662793216 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019646464646464644, + "loss": 2.6263, + "theoretical_loss": 3.3517120110323715, + "tokens_seen": 2662858752 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019644444444444445, + "loss": 2.3823, + "theoretical_loss": 3.351705514072007, + "tokens_seen": 2662924288 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001964242424242424, + "loss": 2.7029, + "theoretical_loss": 3.3516990173163035, + "tokens_seen": 2662989824 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019640404040404042, + "loss": 2.4713, + "theoretical_loss": 3.351692520765249, + "tokens_seen": 2663055360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019638383838383838, + "loss": 2.793, + "theoretical_loss": 3.3516860244188322, + "tokens_seen": 2663120896 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019636363636363636, + "loss": 2.6391, + "theoretical_loss": 3.351679528277042, + "tokens_seen": 2663186432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019634343434343435, + "loss": 2.5277, + "theoretical_loss": 3.3516730323398662, + "tokens_seen": 2663251968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019632323232323233, + "loss": 2.4362, + "theoretical_loss": 3.351666536607294, + "tokens_seen": 2663317504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019630303030303032, + "loss": 2.5871, + "theoretical_loss": 3.351660041079313, + "tokens_seen": 2663383040 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019628282828282827, + "loss": 2.7775, + "theoretical_loss": 3.351653545755913, + "tokens_seen": 2663448576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019626262626262626, + "loss": 2.5223, + "theoretical_loss": 3.351647050637082, + "tokens_seen": 2663514112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019624242424242424, + "loss": 2.4433, + "theoretical_loss": 3.351640555722808, + "tokens_seen": 2663579648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019622222222222222, + "loss": 2.5425, + "theoretical_loss": 3.3516340610130797, + "tokens_seen": 2663645184 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001962020202020202, + "loss": 2.5632, + "theoretical_loss": 3.3516275665078865, + "tokens_seen": 2663710720 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001961818181818182, + "loss": 2.5539, + "theoretical_loss": 3.3516210722072155, + "tokens_seen": 2663776256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019616161616161615, + "loss": 2.6064, + "theoretical_loss": 3.3516145781110565, + "tokens_seen": 2663841792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019614141414141416, + "loss": 2.4596, + "theoretical_loss": 3.3516080842193974, + "tokens_seen": 2663907328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019612121212121212, + "loss": 2.3964, + "theoretical_loss": 3.3516015905322267, + "tokens_seen": 2663972864 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001961010101010101, + "loss": 2.4594, + "theoretical_loss": 3.3515950970495334, + "tokens_seen": 2664038400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019608080808080808, + "loss": 2.4477, + "theoretical_loss": 3.3515886037713054, + "tokens_seen": 2664103936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019606060606060607, + "loss": 2.3682, + "theoretical_loss": 3.3515821106975316, + "tokens_seen": 2664169472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019604040404040405, + "loss": 2.4924, + "theoretical_loss": 3.3515756178282006, + "tokens_seen": 2664235008 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1459609, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.219696283340454, + "objective/train/theoretical_loss": 3.3515691251633006, + "objective/train/tokens_used": 1023159776, + "theoretical_loss": 3.3515691251633006, + "tokens_seen": 2664300544 + }, + { + "epoch": 0.61, + "learning_rate": 0.000196020202020202, + "loss": 2.5064, + "theoretical_loss": 3.3515691251633006, + "tokens_seen": 2664300544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019600000000000002, + "loss": 2.3594, + "theoretical_loss": 3.3515626327028203, + "tokens_seen": 2664366080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019597979797979798, + "loss": 2.5238, + "theoretical_loss": 3.3515561404467484, + "tokens_seen": 2664431616 + }, + { + "epoch": 0.61, + "learning_rate": 0.000195959595959596, + "loss": 2.6275, + "theoretical_loss": 3.3515496483950735, + "tokens_seen": 2664497152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019593939393939395, + "loss": 2.5507, + "theoretical_loss": 3.351543156547784, + "tokens_seen": 2664562688 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001959191919191919, + "loss": 2.5008, + "theoretical_loss": 3.351536664904868, + "tokens_seen": 2664628224 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001958989898989899, + "loss": 2.4596, + "theoretical_loss": 3.3515301734663145, + "tokens_seen": 2664693760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019587878787878787, + "loss": 2.2915, + "theoretical_loss": 3.351523682232112, + "tokens_seen": 2664759296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019585858585858588, + "loss": 2.637, + "theoretical_loss": 3.3515171912022494, + "tokens_seen": 2664824832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019583838383838384, + "loss": 2.477, + "theoretical_loss": 3.3515107003767146, + "tokens_seen": 2664890368 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019581818181818182, + "loss": 2.3855, + "theoretical_loss": 3.3515042097554963, + "tokens_seen": 2664955904 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001957979797979798, + "loss": 2.6494, + "theoretical_loss": 3.351497719338583, + "tokens_seen": 2665021440 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001957777777777778, + "loss": 2.5398, + "theoretical_loss": 3.3514912291259638, + "tokens_seen": 2665086976 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019575757575757577, + "loss": 2.4313, + "theoretical_loss": 3.351484739117627, + "tokens_seen": 2665152512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019573737373737373, + "loss": 2.6649, + "theoretical_loss": 3.35147824931356, + "tokens_seen": 2665218048 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019571717171717171, + "loss": 2.6761, + "theoretical_loss": 3.3514717597137533, + "tokens_seen": 2665283584 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001956969696969697, + "loss": 2.4756, + "theoretical_loss": 3.351465270318194, + "tokens_seen": 2665349120 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019567676767676768, + "loss": 2.4651, + "theoretical_loss": 3.3514587811268712, + "tokens_seen": 2665414656 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019565656565656567, + "loss": 2.6329, + "theoretical_loss": 3.351452292139774, + "tokens_seen": 2665480192 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019563636363636365, + "loss": 2.6946, + "theoretical_loss": 3.3514458033568895, + "tokens_seen": 2665545728 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001956161616161616, + "loss": 2.5515, + "theoretical_loss": 3.351439314778207, + "tokens_seen": 2665611264 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001955959595959596, + "loss": 2.4736, + "theoretical_loss": 3.3514328264037156, + "tokens_seen": 2665676800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019557575757575758, + "loss": 2.5012, + "theoretical_loss": 3.3514263382334035, + "tokens_seen": 2665742336 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019555555555555556, + "loss": 2.4619, + "theoretical_loss": 3.3514198502672587, + "tokens_seen": 2665807872 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019553535353535354, + "loss": 2.5445, + "theoretical_loss": 3.3514133625052707, + "tokens_seen": 2665873408 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1460723, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5888512134552, + "objective/train/theoretical_loss": 3.3514068749474273, + "objective/train/tokens_used": 1024798176, + "theoretical_loss": 3.3514068749474273, + "tokens_seen": 2665938944 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001955151515151515, + "loss": 2.5897, + "theoretical_loss": 3.3514068749474273, + "tokens_seen": 2665938944 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001954949494949495, + "loss": 2.4827, + "theoretical_loss": 3.351400387593717, + "tokens_seen": 2666004480 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019547474747474747, + "loss": 2.589, + "theoretical_loss": 3.3513939004441293, + "tokens_seen": 2666070016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019545454545454548, + "loss": 2.6115, + "theoretical_loss": 3.351387413498652, + "tokens_seen": 2666135552 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019543434343434344, + "loss": 2.2911, + "theoretical_loss": 3.3513809267572734, + "tokens_seen": 2666201088 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001954141414141414, + "loss": 2.6373, + "theoretical_loss": 3.351374440219983, + "tokens_seen": 2666266624 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001953939393939394, + "loss": 2.247, + "theoretical_loss": 3.3513679538867684, + "tokens_seen": 2666332160 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019537373737373736, + "loss": 2.5155, + "theoretical_loss": 3.3513614677576187, + "tokens_seen": 2666397696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019535353535353537, + "loss": 2.7287, + "theoretical_loss": 3.3513549818325226, + "tokens_seen": 2666463232 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019533333333333333, + "loss": 2.4993, + "theoretical_loss": 3.351348496111468, + "tokens_seen": 2666528768 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001953131313131313, + "loss": 2.4415, + "theoretical_loss": 3.3513420105944443, + "tokens_seen": 2666594304 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001952929292929293, + "loss": 2.3402, + "theoretical_loss": 3.3513355252814394, + "tokens_seen": 2666659840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019527272727272728, + "loss": 2.3422, + "theoretical_loss": 3.351329040172442, + "tokens_seen": 2666725376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019525252525252526, + "loss": 2.5642, + "theoretical_loss": 3.351322555267441, + "tokens_seen": 2666790912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019523232323232322, + "loss": 2.5479, + "theoretical_loss": 3.3513160705664244, + "tokens_seen": 2666856448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019521212121212123, + "loss": 2.7494, + "theoretical_loss": 3.3513095860693816, + "tokens_seen": 2666921984 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001951919191919192, + "loss": 2.617, + "theoretical_loss": 3.3513031017763004, + "tokens_seen": 2666987520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019517171717171717, + "loss": 2.5739, + "theoretical_loss": 3.35129661768717, + "tokens_seen": 2667053056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019515151515151516, + "loss": 2.5076, + "theoretical_loss": 3.3512901338019785, + "tokens_seen": 2667118592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019513131313131314, + "loss": 2.3524, + "theoretical_loss": 3.3512836501207146, + "tokens_seen": 2667184128 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019511111111111113, + "loss": 2.5395, + "theoretical_loss": 3.3512771666433667, + "tokens_seen": 2667249664 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001950909090909091, + "loss": 2.6239, + "theoretical_loss": 3.3512706833699237, + "tokens_seen": 2667315200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019507070707070707, + "loss": 2.5807, + "theoretical_loss": 3.351264200300374, + "tokens_seen": 2667380736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019505050505050505, + "loss": 2.5741, + "theoretical_loss": 3.3512577174347067, + "tokens_seen": 2667446272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019503030303030303, + "loss": 2.5306, + "theoretical_loss": 3.3512512347729095, + "tokens_seen": 2667511808 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1462120, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3585121631622314, + "objective/train/theoretical_loss": 3.3512447523149715, + "objective/train/tokens_used": 1026436576, + "theoretical_loss": 3.3512447523149715, + "tokens_seen": 2667577344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019501010101010102, + "loss": 2.4041, + "theoretical_loss": 3.3512447523149715, + "tokens_seen": 2667577344 + }, + { + "epoch": 0.61, + "learning_rate": 0.000194989898989899, + "loss": 2.4452, + "theoretical_loss": 3.351238270060881, + "tokens_seen": 2667642880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019496969696969696, + "loss": 2.3971, + "theoretical_loss": 3.351231788010627, + "tokens_seen": 2667708416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019494949494949497, + "loss": 2.435, + "theoretical_loss": 3.351225306164198, + "tokens_seen": 2667773952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019492929292929293, + "loss": 2.4446, + "theoretical_loss": 3.351218824521582, + "tokens_seen": 2667839488 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019490909090909094, + "loss": 2.2768, + "theoretical_loss": 3.351212343082768, + "tokens_seen": 2667905024 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001948888888888889, + "loss": 2.5078, + "theoretical_loss": 3.351205861847745, + "tokens_seen": 2667970560 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019486868686868685, + "loss": 2.6464, + "theoretical_loss": 3.351199380816501, + "tokens_seen": 2668036096 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019484848484848486, + "loss": 2.5922, + "theoretical_loss": 3.3511928999890244, + "tokens_seen": 2668101632 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019482828282828282, + "loss": 2.6065, + "theoretical_loss": 3.3511864193653045, + "tokens_seen": 2668167168 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019480808080808083, + "loss": 2.6774, + "theoretical_loss": 3.3511799389453296, + "tokens_seen": 2668232704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001947878787878788, + "loss": 2.4449, + "theoretical_loss": 3.351173458729088, + "tokens_seen": 2668298240 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019476767676767677, + "loss": 2.4654, + "theoretical_loss": 3.3511669787165688, + "tokens_seen": 2668363776 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019474747474747476, + "loss": 2.4668, + "theoretical_loss": 3.35116049890776, + "tokens_seen": 2668429312 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001947272727272727, + "loss": 2.522, + "theoretical_loss": 3.3511540193026508, + "tokens_seen": 2668494848 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019470707070707072, + "loss": 2.722, + "theoretical_loss": 3.3511475399012296, + "tokens_seen": 2668560384 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019468686868686868, + "loss": 2.5914, + "theoretical_loss": 3.351141060703484, + "tokens_seen": 2668625920 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019466666666666666, + "loss": 2.5427, + "theoretical_loss": 3.3511345817094047, + "tokens_seen": 2668691456 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019464646464646465, + "loss": 2.6406, + "theoretical_loss": 3.3511281029189783, + "tokens_seen": 2668756992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019462626262626263, + "loss": 2.2987, + "theoretical_loss": 3.351121624332195, + "tokens_seen": 2668822528 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019460606060606062, + "loss": 2.4408, + "theoretical_loss": 3.351115145949042, + "tokens_seen": 2668888064 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001945858585858586, + "loss": 2.3816, + "theoretical_loss": 3.351108667769508, + "tokens_seen": 2668953600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019456565656565656, + "loss": 2.7126, + "theoretical_loss": 3.351102189793583, + "tokens_seen": 2669019136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019454545454545454, + "loss": 2.4215, + "theoretical_loss": 3.3510957120212543, + "tokens_seen": 2669084672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019452525252525252, + "loss": 2.4052, + "theoretical_loss": 3.3510892344525107, + "tokens_seen": 2669150208 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1463069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5914080142974854, + "objective/train/theoretical_loss": 3.3510827570873416, + "objective/train/tokens_used": 1028074976, + "theoretical_loss": 3.3510827570873416, + "tokens_seen": 2669215744 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001945050505050505, + "loss": 2.5217, + "theoretical_loss": 3.3510827570873416, + "tokens_seen": 2669215744 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001944848484848485, + "loss": 2.4921, + "theoretical_loss": 3.3510762799257345, + "tokens_seen": 2669281280 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019446464646464645, + "loss": 2.5745, + "theoretical_loss": 3.351069802967679, + "tokens_seen": 2669346816 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019444444444444446, + "loss": 2.4156, + "theoretical_loss": 3.3510633262131626, + "tokens_seen": 2669412352 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019442424242424242, + "loss": 2.4641, + "theoretical_loss": 3.3510568496621747, + "tokens_seen": 2669477888 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019440404040404043, + "loss": 2.5401, + "theoretical_loss": 3.351050373314704, + "tokens_seen": 2669543424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019438383838383839, + "loss": 2.6074, + "theoretical_loss": 3.3510438971707384, + "tokens_seen": 2669608960 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019436363636363637, + "loss": 2.6119, + "theoretical_loss": 3.3510374212302674, + "tokens_seen": 2669674496 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019434343434343435, + "loss": 2.4556, + "theoretical_loss": 3.351030945493279, + "tokens_seen": 2669740032 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001943232323232323, + "loss": 2.3706, + "theoretical_loss": 3.3510244699597616, + "tokens_seen": 2669805568 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019430303030303032, + "loss": 2.5111, + "theoretical_loss": 3.3510179946297045, + "tokens_seen": 2669871104 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019428282828282828, + "loss": 2.6926, + "theoretical_loss": 3.3510115195030963, + "tokens_seen": 2669936640 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001942626262626263, + "loss": 2.6848, + "theoretical_loss": 3.351005044579925, + "tokens_seen": 2670002176 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019424242424242425, + "loss": 2.3939, + "theoretical_loss": 3.350998569860179, + "tokens_seen": 2670067712 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019422222222222223, + "loss": 2.5511, + "theoretical_loss": 3.350992095343848, + "tokens_seen": 2670133248 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019420202020202021, + "loss": 2.7412, + "theoretical_loss": 3.3509856210309206, + "tokens_seen": 2670198784 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019418181818181817, + "loss": 2.6661, + "theoretical_loss": 3.350979146921384, + "tokens_seen": 2670264320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019416161616161618, + "loss": 2.4929, + "theoretical_loss": 3.350972673015228, + "tokens_seen": 2670329856 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019414141414141414, + "loss": 2.6713, + "theoretical_loss": 3.350966199312441, + "tokens_seen": 2670395392 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019412121212121212, + "loss": 2.4785, + "theoretical_loss": 3.3509597258130115, + "tokens_seen": 2670460928 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001941010101010101, + "loss": 2.4691, + "theoretical_loss": 3.350953252516928, + "tokens_seen": 2670526464 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001940808080808081, + "loss": 2.5176, + "theoretical_loss": 3.3509467794241794, + "tokens_seen": 2670592000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019406060606060607, + "loss": 2.5314, + "theoretical_loss": 3.350940306534754, + "tokens_seen": 2670657536 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019404040404040406, + "loss": 2.5522, + "theoretical_loss": 3.3509338338486407, + "tokens_seen": 2670723072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019402020202020202, + "loss": 2.7239, + "theoretical_loss": 3.3509273613658284, + "tokens_seen": 2670788608 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1463760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.706230878829956, + "objective/train/theoretical_loss": 3.350920889086305, + "objective/train/tokens_used": 1029713376, + "theoretical_loss": 3.350920889086305, + "tokens_seen": 2670854144 + }, + { + "epoch": 0.62, + "learning_rate": 0.000194, + "loss": 2.4463, + "theoretical_loss": 3.350920889086305, + "tokens_seen": 2670854144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019397979797979798, + "loss": 2.4683, + "theoretical_loss": 3.3509144170100598, + "tokens_seen": 2670919680 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019395959595959597, + "loss": 2.5659, + "theoretical_loss": 3.350907945137081, + "tokens_seen": 2670985216 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019393939393939395, + "loss": 2.3787, + "theoretical_loss": 3.3509014734673572, + "tokens_seen": 2671050752 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001939191919191919, + "loss": 2.6089, + "theoretical_loss": 3.350895002000877, + "tokens_seen": 2671116288 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019389898989898992, + "loss": 2.6349, + "theoretical_loss": 3.3508885307376297, + "tokens_seen": 2671181824 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019387878787878788, + "loss": 2.6378, + "theoretical_loss": 3.350882059677603, + "tokens_seen": 2671247360 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019385858585858586, + "loss": 2.818, + "theoretical_loss": 3.3508755888207866, + "tokens_seen": 2671312896 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019383838383838384, + "loss": 2.3639, + "theoretical_loss": 3.350869118167168, + "tokens_seen": 2671378432 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001938181818181818, + "loss": 2.439, + "theoretical_loss": 3.350862647716737, + "tokens_seen": 2671443968 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001937979797979798, + "loss": 2.5524, + "theoretical_loss": 3.3508561774694807, + "tokens_seen": 2671509504 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019377777777777777, + "loss": 2.5102, + "theoretical_loss": 3.350849707425389, + "tokens_seen": 2671575040 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019375757575757578, + "loss": 2.4817, + "theoretical_loss": 3.35084323758445, + "tokens_seen": 2671640576 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019373737373737374, + "loss": 2.4164, + "theoretical_loss": 3.350836767946653, + "tokens_seen": 2671706112 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019371717171717172, + "loss": 2.5292, + "theoretical_loss": 3.350830298511986, + "tokens_seen": 2671771648 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001936969696969697, + "loss": 2.586, + "theoretical_loss": 3.3508238292804378, + "tokens_seen": 2671837184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019367676767676766, + "loss": 2.5946, + "theoretical_loss": 3.3508173602519964, + "tokens_seen": 2671902720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019365656565656567, + "loss": 2.548, + "theoretical_loss": 3.3508108914266517, + "tokens_seen": 2671968256 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019363636363636363, + "loss": 2.5511, + "theoretical_loss": 3.3508044228043916, + "tokens_seen": 2672033792 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001936161616161616, + "loss": 2.3673, + "theoretical_loss": 3.3507979543852047, + "tokens_seen": 2672099328 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001935959595959596, + "loss": 2.5926, + "theoretical_loss": 3.35079148616908, + "tokens_seen": 2672164864 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019357575757575758, + "loss": 2.5628, + "theoretical_loss": 3.3507850181560057, + "tokens_seen": 2672230400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019355555555555557, + "loss": 2.42, + "theoretical_loss": 3.3507785503459706, + "tokens_seen": 2672295936 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019353535353535355, + "loss": 2.5504, + "theoretical_loss": 3.3507720827389638, + "tokens_seen": 2672361472 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019351515151515153, + "loss": 2.4483, + "theoretical_loss": 3.350765615334973, + "tokens_seen": 2672427008 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1465035, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.667522668838501, + "objective/train/theoretical_loss": 3.3507591481339882, + "objective/train/tokens_used": 1031351776, + "theoretical_loss": 3.3507591481339882, + "tokens_seen": 2672492544 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001934949494949495, + "loss": 2.664, + "theoretical_loss": 3.3507591481339882, + "tokens_seen": 2672492544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019347474747474747, + "loss": 2.6409, + "theoretical_loss": 3.350752681135997, + "tokens_seen": 2672558080 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019345454545454546, + "loss": 2.5572, + "theoretical_loss": 3.350746214340988, + "tokens_seen": 2672623616 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019343434343434344, + "loss": 2.588, + "theoretical_loss": 3.3507397477489507, + "tokens_seen": 2672689152 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019341414141414143, + "loss": 2.624, + "theoretical_loss": 3.3507332813598727, + "tokens_seen": 2672754688 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001933939393939394, + "loss": 2.4797, + "theoretical_loss": 3.3507268151737435, + "tokens_seen": 2672820224 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019337373737373737, + "loss": 2.4654, + "theoretical_loss": 3.350720349190551, + "tokens_seen": 2672885760 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019335353535353538, + "loss": 2.444, + "theoretical_loss": 3.350713883410285, + "tokens_seen": 2672951296 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019333333333333333, + "loss": 2.5188, + "theoretical_loss": 3.350707417832933, + "tokens_seen": 2673016832 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019331313131313132, + "loss": 2.4828, + "theoretical_loss": 3.3507009524584843, + "tokens_seen": 2673082368 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001932929292929293, + "loss": 2.5014, + "theoretical_loss": 3.350694487286927, + "tokens_seen": 2673147904 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019327272727272726, + "loss": 2.496, + "theoretical_loss": 3.3506880223182507, + "tokens_seen": 2673213440 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019325252525252527, + "loss": 2.5516, + "theoretical_loss": 3.350681557552443, + "tokens_seen": 2673278976 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019323232323232323, + "loss": 2.6604, + "theoretical_loss": 3.3506750929894933, + "tokens_seen": 2673344512 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019321212121212124, + "loss": 2.5113, + "theoretical_loss": 3.35066862862939, + "tokens_seen": 2673410048 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001931919191919192, + "loss": 2.6697, + "theoretical_loss": 3.350662164472122, + "tokens_seen": 2673475584 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019317171717171715, + "loss": 2.6501, + "theoretical_loss": 3.3506557005176774, + "tokens_seen": 2673541120 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019315151515151516, + "loss": 2.6429, + "theoretical_loss": 3.350649236766045, + "tokens_seen": 2673606656 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019313131313131312, + "loss": 2.4218, + "theoretical_loss": 3.3506427732172144, + "tokens_seen": 2673672192 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019311111111111113, + "loss": 2.5571, + "theoretical_loss": 3.350636309871173, + "tokens_seen": 2673737728 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001930909090909091, + "loss": 2.4029, + "theoretical_loss": 3.35062984672791, + "tokens_seen": 2673803264 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019307070707070707, + "loss": 2.4958, + "theoretical_loss": 3.350623383787414, + "tokens_seen": 2673868800 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019305050505050506, + "loss": 2.4506, + "theoretical_loss": 3.350616921049674, + "tokens_seen": 2673934336 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019303030303030304, + "loss": 2.5517, + "theoretical_loss": 3.3506104585146783, + "tokens_seen": 2673999872 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019301010101010102, + "loss": 2.4225, + "theoretical_loss": 3.3506039961824157, + "tokens_seen": 2674065408 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1465709, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.590970993041992, + "objective/train/theoretical_loss": 3.3505975340528744, + "objective/train/tokens_used": 1032990176, + "theoretical_loss": 3.3505975340528744, + "tokens_seen": 2674130944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019298989898989898, + "loss": 2.751, + "theoretical_loss": 3.3505975340528744, + "tokens_seen": 2674130944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019296969696969696, + "loss": 2.5505, + "theoretical_loss": 3.3505910721260443, + "tokens_seen": 2674196480 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019294949494949495, + "loss": 2.3657, + "theoretical_loss": 3.3505846104019126, + "tokens_seen": 2674262016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019292929292929293, + "loss": 2.385, + "theoretical_loss": 3.3505781488804693, + "tokens_seen": 2674327552 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019290909090909092, + "loss": 2.4174, + "theoretical_loss": 3.350571687561702, + "tokens_seen": 2674393088 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928888888888889, + "loss": 2.5721, + "theoretical_loss": 3.3505652264456, + "tokens_seen": 2674458624 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019286868686868686, + "loss": 2.1511, + "theoretical_loss": 3.350558765532152, + "tokens_seen": 2674524160 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019284848484848487, + "loss": 2.5657, + "theoretical_loss": 3.350552304821346, + "tokens_seen": 2674589696 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019282828282828283, + "loss": 2.4621, + "theoretical_loss": 3.3505458443131717, + "tokens_seen": 2674655232 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928080808080808, + "loss": 2.3927, + "theoretical_loss": 3.3505393840076167, + "tokens_seen": 2674720768 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001927878787878788, + "loss": 2.4952, + "theoretical_loss": 3.3505329239046704, + "tokens_seen": 2674786304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019276767676767675, + "loss": 2.6709, + "theoretical_loss": 3.3505264640043215, + "tokens_seen": 2674851840 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019274747474747476, + "loss": 2.3387, + "theoretical_loss": 3.350520004306558, + "tokens_seen": 2674917376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019272727272727272, + "loss": 2.5509, + "theoretical_loss": 3.3505135448113696, + "tokens_seen": 2674982912 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019270707070707073, + "loss": 2.417, + "theoretical_loss": 3.3505070855187444, + "tokens_seen": 2675048448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019268686868686869, + "loss": 2.5144, + "theoretical_loss": 3.350500626428671, + "tokens_seen": 2675113984 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001926666666666667, + "loss": 2.6062, + "theoretical_loss": 3.350494167541138, + "tokens_seen": 2675179520 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019264646464646465, + "loss": 2.5596, + "theoretical_loss": 3.350487708856135, + "tokens_seen": 2675245056 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001926262626262626, + "loss": 2.5749, + "theoretical_loss": 3.350481250373649, + "tokens_seen": 2675310592 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019260606060606062, + "loss": 2.4904, + "theoretical_loss": 3.3504747920936704, + "tokens_seen": 2675376128 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019258585858585858, + "loss": 2.5758, + "theoretical_loss": 3.350468334016187, + "tokens_seen": 2675441664 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001925656565656566, + "loss": 2.5619, + "theoretical_loss": 3.3504618761411877, + "tokens_seen": 2675507200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019254545454545455, + "loss": 2.349, + "theoretical_loss": 3.350455418468661, + "tokens_seen": 2675572736 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019252525252525253, + "loss": 2.7623, + "theoretical_loss": 3.350448960998596, + "tokens_seen": 2675638272 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019250505050505051, + "loss": 2.7085, + "theoretical_loss": 3.350442503730981, + "tokens_seen": 2675703808 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1466786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8028337955474854, + "objective/train/theoretical_loss": 3.3504360466658047, + "objective/train/tokens_used": 1034628576, + "theoretical_loss": 3.3504360466658047, + "tokens_seen": 2675769344 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001924848484848485, + "loss": 2.744, + "theoretical_loss": 3.3504360466658047, + "tokens_seen": 2675769344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019246464646464648, + "loss": 2.5138, + "theoretical_loss": 3.3504295898030563, + "tokens_seen": 2675834880 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019244444444444444, + "loss": 2.4419, + "theoretical_loss": 3.3504231331427237, + "tokens_seen": 2675900416 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019242424242424242, + "loss": 2.4719, + "theoretical_loss": 3.3504166766847963, + "tokens_seen": 2675965952 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001924040404040404, + "loss": 2.5871, + "theoretical_loss": 3.3504102204292625, + "tokens_seen": 2676031488 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001923838383838384, + "loss": 2.6861, + "theoretical_loss": 3.350403764376111, + "tokens_seen": 2676097024 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019236363636363637, + "loss": 2.6325, + "theoretical_loss": 3.3503973085253307, + "tokens_seen": 2676162560 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019234343434343436, + "loss": 2.2442, + "theoretical_loss": 3.35039085287691, + "tokens_seen": 2676228096 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019232323232323232, + "loss": 2.5936, + "theoretical_loss": 3.3503843974308376, + "tokens_seen": 2676293632 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001923030303030303, + "loss": 2.6557, + "theoretical_loss": 3.3503779421871025, + "tokens_seen": 2676359168 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019228282828282828, + "loss": 2.6018, + "theoretical_loss": 3.350371487145693, + "tokens_seen": 2676424704 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019226262626262627, + "loss": 2.375, + "theoretical_loss": 3.3503650323065983, + "tokens_seen": 2676490240 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019224242424242425, + "loss": 2.5115, + "theoretical_loss": 3.3503585776698066, + "tokens_seen": 2676555776 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001922222222222222, + "loss": 2.5157, + "theoretical_loss": 3.3503521232353073, + "tokens_seen": 2676621312 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019220202020202022, + "loss": 2.5028, + "theoretical_loss": 3.3503456690030884, + "tokens_seen": 2676686848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019218181818181818, + "loss": 2.5938, + "theoretical_loss": 3.350339214973139, + "tokens_seen": 2676752384 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001921616161616162, + "loss": 2.6377, + "theoretical_loss": 3.350332761145447, + "tokens_seen": 2676817920 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019214141414141414, + "loss": 2.4994, + "theoretical_loss": 3.3503263075200023, + "tokens_seen": 2676883456 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001921212121212121, + "loss": 2.5887, + "theoretical_loss": 3.350319854096793, + "tokens_seen": 2676948992 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001921010101010101, + "loss": 2.4202, + "theoretical_loss": 3.350313400875808, + "tokens_seen": 2677014528 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019208080808080807, + "loss": 2.5599, + "theoretical_loss": 3.350306947857036, + "tokens_seen": 2677080064 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019206060606060608, + "loss": 2.4609, + "theoretical_loss": 3.350300495040466, + "tokens_seen": 2677145600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019204040404040404, + "loss": 2.5329, + "theoretical_loss": 3.3502940424260856, + "tokens_seen": 2677211136 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019202020202020202, + "loss": 2.7932, + "theoretical_loss": 3.3502875900138847, + "tokens_seen": 2677276672 + }, + { + "epoch": 0.62, + "learning_rate": 0.000192, + "loss": 2.4324, + "theoretical_loss": 3.3502811378038513, + "tokens_seen": 2677342208 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1468239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.355358600616455, + "objective/train/theoretical_loss": 3.3502746857959744, + "objective/train/tokens_used": 1036266976, + "theoretical_loss": 3.3502746857959744, + "tokens_seen": 2677407744 + }, + { + "epoch": 0.62, + "learning_rate": 0.000191979797979798, + "loss": 2.5288, + "theoretical_loss": 3.3502746857959744, + "tokens_seen": 2677407744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019195959595959597, + "loss": 2.5757, + "theoretical_loss": 3.350268233990243, + "tokens_seen": 2677473280 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019193939393939393, + "loss": 2.4711, + "theoretical_loss": 3.3502617823866454, + "tokens_seen": 2677538816 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019191919191919191, + "loss": 2.5556, + "theoretical_loss": 3.3502553309851706, + "tokens_seen": 2677604352 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001918989898989899, + "loss": 2.4789, + "theoretical_loss": 3.350248879785807, + "tokens_seen": 2677669888 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001918989898989899, + "loss": 2.6691, + "theoretical_loss": 3.3502424287885435, + "tokens_seen": 2677735424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019187878787878788, + "loss": 2.4576, + "theoretical_loss": 3.3502359779933686, + "tokens_seen": 2677800960 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019185858585858587, + "loss": 2.4932, + "theoretical_loss": 3.3502295274002716, + "tokens_seen": 2677866496 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019183838383838385, + "loss": 2.5311, + "theoretical_loss": 3.350223077009241, + "tokens_seen": 2677932032 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001918181818181818, + "loss": 2.5367, + "theoretical_loss": 3.350216626820265, + "tokens_seen": 2677997568 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019179797979797982, + "loss": 2.5406, + "theoretical_loss": 3.350210176833333, + "tokens_seen": 2678063104 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019177777777777777, + "loss": 2.3385, + "theoretical_loss": 3.3502037270484335, + "tokens_seen": 2678128640 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019175757575757576, + "loss": 2.3386, + "theoretical_loss": 3.350197277465555, + "tokens_seen": 2678194176 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019173737373737374, + "loss": 2.5596, + "theoretical_loss": 3.350190828084686, + "tokens_seen": 2678259712 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019171717171717173, + "loss": 2.6733, + "theoretical_loss": 3.350184378905816, + "tokens_seen": 2678325248 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001916969696969697, + "loss": 2.5132, + "theoretical_loss": 3.350177929928934, + "tokens_seen": 2678390784 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019167676767676767, + "loss": 2.3979, + "theoretical_loss": 3.3501714811540273, + "tokens_seen": 2678456320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019165656565656568, + "loss": 2.5014, + "theoretical_loss": 3.3501650325810854, + "tokens_seen": 2678521856 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019163636363636363, + "loss": 2.563, + "theoretical_loss": 3.3501585842100976, + "tokens_seen": 2678587392 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019161616161616165, + "loss": 2.423, + "theoretical_loss": 3.3501521360410518, + "tokens_seen": 2678652928 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001915959595959596, + "loss": 2.1714, + "theoretical_loss": 3.350145688073937, + "tokens_seen": 2678718464 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019157575757575756, + "loss": 2.6184, + "theoretical_loss": 3.350139240308742, + "tokens_seen": 2678784000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019155555555555557, + "loss": 2.4138, + "theoretical_loss": 3.3501327927454554, + "tokens_seen": 2678849536 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019153535353535353, + "loss": 2.5737, + "theoretical_loss": 3.350126345384066, + "tokens_seen": 2678915072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019151515151515154, + "loss": 2.5624, + "theoretical_loss": 3.350119898224563, + "tokens_seen": 2678980608 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1468865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.819284677505493, + "objective/train/theoretical_loss": 3.3501134512669344, + "objective/train/tokens_used": 1037905376, + "theoretical_loss": 3.3501134512669344, + "tokens_seen": 2679046144 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001914949494949495, + "loss": 2.4153, + "theoretical_loss": 3.3501134512669344, + "tokens_seen": 2679046144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019147474747474748, + "loss": 2.5791, + "theoretical_loss": 3.3501070045111696, + "tokens_seen": 2679111680 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019145454545454546, + "loss": 2.4898, + "theoretical_loss": 3.3501005579572567, + "tokens_seen": 2679177216 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019143434343434342, + "loss": 2.4189, + "theoretical_loss": 3.3500941116051846, + "tokens_seen": 2679242752 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019141414141414143, + "loss": 2.4052, + "theoretical_loss": 3.3500876654549425, + "tokens_seen": 2679308288 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001913939393939394, + "loss": 2.4495, + "theoretical_loss": 3.350081219506519, + "tokens_seen": 2679373824 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019137373737373737, + "loss": 2.6772, + "theoretical_loss": 3.350074773759902, + "tokens_seen": 2679439360 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019135353535353536, + "loss": 2.5668, + "theoretical_loss": 3.3500683282150816, + "tokens_seen": 2679504896 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019133333333333334, + "loss": 2.4562, + "theoretical_loss": 3.3500618828720454, + "tokens_seen": 2679570432 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019131313131313132, + "loss": 2.3516, + "theoretical_loss": 3.350055437730783, + "tokens_seen": 2679635968 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001912929292929293, + "loss": 2.6521, + "theoretical_loss": 3.3500489927912827, + "tokens_seen": 2679701504 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019127272727272726, + "loss": 2.5249, + "theoretical_loss": 3.350042548053533, + "tokens_seen": 2679767040 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019125252525252525, + "loss": 2.5869, + "theoretical_loss": 3.3500361035175232, + "tokens_seen": 2679832576 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019123232323232323, + "loss": 2.6017, + "theoretical_loss": 3.350029659183242, + "tokens_seen": 2679898112 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019121212121212122, + "loss": 2.4304, + "theoretical_loss": 3.350023215050678, + "tokens_seen": 2679963648 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001911919191919192, + "loss": 2.3659, + "theoretical_loss": 3.3500167711198197, + "tokens_seen": 2680029184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019117171717171716, + "loss": 2.3833, + "theoretical_loss": 3.350010327390656, + "tokens_seen": 2680094720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019115151515151517, + "loss": 2.641, + "theoretical_loss": 3.3500038838631756, + "tokens_seen": 2680160256 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019113131313131313, + "loss": 2.5394, + "theoretical_loss": 3.3499974405373676, + "tokens_seen": 2680225792 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019111111111111114, + "loss": 2.5248, + "theoretical_loss": 3.3499909974132205, + "tokens_seen": 2680291328 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001910909090909091, + "loss": 2.469, + "theoretical_loss": 3.3499845544907236, + "tokens_seen": 2680356864 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019107070707070705, + "loss": 2.4934, + "theoretical_loss": 3.3499781117698646, + "tokens_seen": 2680422400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019105050505050506, + "loss": 2.4445, + "theoretical_loss": 3.349971669250633, + "tokens_seen": 2680487936 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019103030303030302, + "loss": 2.2473, + "theoretical_loss": 3.3499652269330173, + "tokens_seen": 2680553472 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019101010101010103, + "loss": 2.2994, + "theoretical_loss": 3.3499587848170065, + "tokens_seen": 2680619008 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1469157, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4436075687408447, + "objective/train/theoretical_loss": 3.3499523429025895, + "objective/train/tokens_used": 1039543776, + "theoretical_loss": 3.3499523429025895, + "tokens_seen": 2680684544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019098989898989899, + "loss": 2.3605, + "theoretical_loss": 3.3499523429025895, + "tokens_seen": 2680684544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019096969696969697, + "loss": 2.5353, + "theoretical_loss": 3.3499459011897543, + "tokens_seen": 2680750080 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019094949494949495, + "loss": 2.5845, + "theoretical_loss": 3.3499394596784904, + "tokens_seen": 2680815616 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019092929292929294, + "loss": 2.3071, + "theoretical_loss": 3.349933018368786, + "tokens_seen": 2680881152 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019090909090909092, + "loss": 2.2449, + "theoretical_loss": 3.3499265772606304, + "tokens_seen": 2680946688 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019088888888888888, + "loss": 2.3663, + "theoretical_loss": 3.349920136354012, + "tokens_seen": 2681012224 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001908686868686869, + "loss": 2.5076, + "theoretical_loss": 3.3499136956489197, + "tokens_seen": 2681077760 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019084848484848485, + "loss": 2.4093, + "theoretical_loss": 3.3499072551453426, + "tokens_seen": 2681143296 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019082828282828283, + "loss": 2.3722, + "theoretical_loss": 3.349900814843269, + "tokens_seen": 2681208832 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019080808080808081, + "loss": 2.6375, + "theoretical_loss": 3.3498943747426875, + "tokens_seen": 2681274368 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001907878787878788, + "loss": 2.4641, + "theoretical_loss": 3.3498879348435873, + "tokens_seen": 2681339904 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019076767676767678, + "loss": 2.4708, + "theoretical_loss": 3.3498814951459575, + "tokens_seen": 2681405440 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019074747474747474, + "loss": 2.0143, + "theoretical_loss": 3.349875055649786, + "tokens_seen": 2681470976 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019072727272727272, + "loss": 2.558, + "theoretical_loss": 3.349868616355062, + "tokens_seen": 2681536512 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001907070707070707, + "loss": 2.3703, + "theoretical_loss": 3.3498621772617745, + "tokens_seen": 2681602048 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001906868686868687, + "loss": 2.5092, + "theoretical_loss": 3.349855738369912, + "tokens_seen": 2681667584 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019066666666666668, + "loss": 2.6118, + "theoretical_loss": 3.3498492996794633, + "tokens_seen": 2681733120 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019064646464646466, + "loss": 2.4143, + "theoretical_loss": 3.3498428611904174, + "tokens_seen": 2681798656 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019062626262626262, + "loss": 2.3872, + "theoretical_loss": 3.3498364229027624, + "tokens_seen": 2681864192 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019060606060606063, + "loss": 2.4397, + "theoretical_loss": 3.349829984816488, + "tokens_seen": 2681929728 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019058585858585858, + "loss": 2.3165, + "theoretical_loss": 3.3498235469315825, + "tokens_seen": 2681995264 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019056565656565657, + "loss": 2.3202, + "theoretical_loss": 3.3498171092480344, + "tokens_seen": 2682060800 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019054545454545455, + "loss": 2.4997, + "theoretical_loss": 3.3498106717658334, + "tokens_seen": 2682126336 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001905252525252525, + "loss": 2.6068, + "theoretical_loss": 3.3498042344849672, + "tokens_seen": 2682191872 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019050505050505052, + "loss": 2.5874, + "theoretical_loss": 3.3497977974054254, + "tokens_seen": 2682257408 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1470338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2492127418518066, + "objective/train/theoretical_loss": 3.3497913605271963, + "objective/train/tokens_used": 1041182176, + "theoretical_loss": 3.3497913605271963, + "tokens_seen": 2682322944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019048484848484848, + "loss": 2.3631, + "theoretical_loss": 3.3497913605271963, + "tokens_seen": 2682322944 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001904646464646465, + "loss": 2.4778, + "theoretical_loss": 3.349784923850269, + "tokens_seen": 2682388480 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019044444444444444, + "loss": 2.3509, + "theoretical_loss": 3.349778487374632, + "tokens_seen": 2682454016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019042424242424243, + "loss": 2.7854, + "theoretical_loss": 3.349772051100274, + "tokens_seen": 2682519552 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001904040404040404, + "loss": 2.2866, + "theoretical_loss": 3.3497656150271844, + "tokens_seen": 2682585088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019038383838383837, + "loss": 2.4393, + "theoretical_loss": 3.3497591791553516, + "tokens_seen": 2682650624 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019036363636363638, + "loss": 2.537, + "theoretical_loss": 3.349752743484764, + "tokens_seen": 2682716160 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019034343434343434, + "loss": 2.4212, + "theoretical_loss": 3.349746308015411, + "tokens_seen": 2682781696 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019032323232323232, + "loss": 2.5074, + "theoretical_loss": 3.349739872747281, + "tokens_seen": 2682847232 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001903030303030303, + "loss": 2.4737, + "theoretical_loss": 3.3497334376803636, + "tokens_seen": 2682912768 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001902828282828283, + "loss": 2.4547, + "theoretical_loss": 3.3497270028146464, + "tokens_seen": 2682978304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019026262626262627, + "loss": 2.3392, + "theoretical_loss": 3.349720568150119, + "tokens_seen": 2683043840 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019024242424242426, + "loss": 2.4746, + "theoretical_loss": 3.3497141336867697, + "tokens_seen": 2683109376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019022222222222221, + "loss": 2.3469, + "theoretical_loss": 3.3497076994245876, + "tokens_seen": 2683174912 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001902020202020202, + "loss": 2.4776, + "theoretical_loss": 3.349701265363562, + "tokens_seen": 2683240448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019018181818181818, + "loss": 2.4887, + "theoretical_loss": 3.3496948315036805, + "tokens_seen": 2683305984 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019016161616161617, + "loss": 2.2377, + "theoretical_loss": 3.3496883978449326, + "tokens_seen": 2683371520 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019014141414141415, + "loss": 2.481, + "theoretical_loss": 3.3496819643873073, + "tokens_seen": 2683437056 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001901212121212121, + "loss": 2.2663, + "theoretical_loss": 3.349675531130793, + "tokens_seen": 2683502592 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019010101010101012, + "loss": 2.3078, + "theoretical_loss": 3.3496690980753785, + "tokens_seen": 2683568128 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019008080808080807, + "loss": 2.5163, + "theoretical_loss": 3.349662665221053, + "tokens_seen": 2683633664 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019006060606060609, + "loss": 2.581, + "theoretical_loss": 3.349656232567805, + "tokens_seen": 2683699200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019004040404040404, + "loss": 2.6033, + "theoretical_loss": 3.3496498001156234, + "tokens_seen": 2683764736 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019002020202020203, + "loss": 2.2805, + "theoretical_loss": 3.349643367864497, + "tokens_seen": 2683830272 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019, + "loss": 2.7595, + "theoretical_loss": 3.3496369358144142, + "tokens_seen": 2683895808 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1470971, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.071140766143799, + "objective/train/theoretical_loss": 3.349630503965365, + "objective/train/tokens_used": 1042820576, + "theoretical_loss": 3.349630503965365, + "tokens_seen": 2683961344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018997979797979797, + "loss": 2.3349, + "theoretical_loss": 3.349630503965365, + "tokens_seen": 2683961344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018995959595959598, + "loss": 2.6993, + "theoretical_loss": 3.3496240723173365, + "tokens_seen": 2684026880 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018993939393939394, + "loss": 2.52, + "theoretical_loss": 3.349617640870319, + "tokens_seen": 2684092416 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018991919191919195, + "loss": 2.5645, + "theoretical_loss": 3.3496112096243005, + "tokens_seen": 2684157952 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001898989898989899, + "loss": 2.5377, + "theoretical_loss": 3.3496047785792697, + "tokens_seen": 2684223488 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018987878787878786, + "loss": 2.2663, + "theoretical_loss": 3.349598347735216, + "tokens_seen": 2684289024 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018985858585858587, + "loss": 2.5405, + "theoretical_loss": 3.3495919170921282, + "tokens_seen": 2684354560 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018983838383838383, + "loss": 2.6888, + "theoretical_loss": 3.3495854866499943, + "tokens_seen": 2684420096 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018981818181818184, + "loss": 2.3649, + "theoretical_loss": 3.349579056408804, + "tokens_seen": 2684485632 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001897979797979798, + "loss": 2.7438, + "theoretical_loss": 3.349572626368546, + "tokens_seen": 2684551168 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018977777777777778, + "loss": 2.6526, + "theoretical_loss": 3.3495661965292087, + "tokens_seen": 2684616704 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018975757575757576, + "loss": 2.5452, + "theoretical_loss": 3.349559766890781, + "tokens_seen": 2684682240 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018973737373737375, + "loss": 2.6702, + "theoretical_loss": 3.3495533374532522, + "tokens_seen": 2684747776 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018971717171717173, + "loss": 2.5469, + "theoretical_loss": 3.3495469082166105, + "tokens_seen": 2684813312 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001896969696969697, + "loss": 2.2317, + "theoretical_loss": 3.3495404791808445, + "tokens_seen": 2684878848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018967676767676767, + "loss": 2.495, + "theoretical_loss": 3.349534050345944, + "tokens_seen": 2684944384 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018965656565656566, + "loss": 2.2157, + "theoretical_loss": 3.3495276217118977, + "tokens_seen": 2685009920 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018963636363636364, + "loss": 2.5796, + "theoretical_loss": 3.3495211932786937, + "tokens_seen": 2685075456 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018961616161616162, + "loss": 2.23, + "theoretical_loss": 3.3495147650463206, + "tokens_seen": 2685140992 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001895959595959596, + "loss": 2.6779, + "theoretical_loss": 3.3495083370147682, + "tokens_seen": 2685206528 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018957575757575757, + "loss": 2.6145, + "theoretical_loss": 3.3495019091840255, + "tokens_seen": 2685272064 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018955555555555558, + "loss": 2.5596, + "theoretical_loss": 3.3494954815540803, + "tokens_seen": 2685337600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018953535353535353, + "loss": 2.4005, + "theoretical_loss": 3.3494890541249216, + "tokens_seen": 2685403136 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018951515151515152, + "loss": 2.4784, + "theoretical_loss": 3.3494826268965388, + "tokens_seen": 2685468672 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001894949494949495, + "loss": 2.5334, + "theoretical_loss": 3.34947619986892, + "tokens_seen": 2685534208 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1472231, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9826325178146362, + "objective/train/theoretical_loss": 3.349469773042055, + "objective/train/tokens_used": 1044458976, + "theoretical_loss": 3.349469773042055, + "tokens_seen": 2685599744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018947474747474746, + "loss": 2.487, + "theoretical_loss": 3.349469773042055, + "tokens_seen": 2685599744 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018945454545454547, + "loss": 2.3575, + "theoretical_loss": 3.3494633464159316, + "tokens_seen": 2685665280 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018943434343434343, + "loss": 2.1996, + "theoretical_loss": 3.3494569199905397, + "tokens_seen": 2685730816 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018941414141414144, + "loss": 2.4829, + "theoretical_loss": 3.3494504937658673, + "tokens_seen": 2685796352 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001893939393939394, + "loss": 2.3797, + "theoretical_loss": 3.3494440677419033, + "tokens_seen": 2685861888 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018937373737373738, + "loss": 2.5123, + "theoretical_loss": 3.349437641918637, + "tokens_seen": 2685927424 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018935353535353536, + "loss": 2.5254, + "theoretical_loss": 3.3494312162960567, + "tokens_seen": 2685992960 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018933333333333332, + "loss": 2.4118, + "theoretical_loss": 3.3494247908741515, + "tokens_seen": 2686058496 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018931313131313133, + "loss": 2.2464, + "theoretical_loss": 3.34941836565291, + "tokens_seen": 2686124032 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001892929292929293, + "loss": 2.5086, + "theoretical_loss": 3.3494119406323217, + "tokens_seen": 2686189568 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018927272727272727, + "loss": 2.4341, + "theoretical_loss": 3.349405515812375, + "tokens_seen": 2686255104 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018925252525252525, + "loss": 2.4747, + "theoretical_loss": 3.3493990911930585, + "tokens_seen": 2686320640 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018923232323232324, + "loss": 2.3618, + "theoretical_loss": 3.3493926667743614, + "tokens_seen": 2686386176 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018921212121212122, + "loss": 2.2016, + "theoretical_loss": 3.349386242556273, + "tokens_seen": 2686451712 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891919191919192, + "loss": 2.4484, + "theoretical_loss": 3.3493798185387806, + "tokens_seen": 2686517248 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891717171717172, + "loss": 2.5259, + "theoretical_loss": 3.3493733947218742, + "tokens_seen": 2686582784 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018915151515151515, + "loss": 2.4334, + "theoretical_loss": 3.3493669711055425, + "tokens_seen": 2686648320 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018913131313131313, + "loss": 2.3139, + "theoretical_loss": 3.3493605476897748, + "tokens_seen": 2686713856 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018911111111111112, + "loss": 2.4416, + "theoretical_loss": 3.349354124474559, + "tokens_seen": 2686779392 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001890909090909091, + "loss": 2.4614, + "theoretical_loss": 3.3493477014598843, + "tokens_seen": 2686844928 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018907070707070708, + "loss": 2.4307, + "theoretical_loss": 3.34934127864574, + "tokens_seen": 2686910464 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018905050505050507, + "loss": 2.441, + "theoretical_loss": 3.3493348560321143, + "tokens_seen": 2686976000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018903030303030302, + "loss": 2.6179, + "theoretical_loss": 3.3493284336189966, + "tokens_seen": 2687041536 + }, + { + "epoch": 0.63, + "learning_rate": 0.000189010101010101, + "loss": 2.2023, + "theoretical_loss": 3.3493220114063753, + "tokens_seen": 2687107072 + }, + { + "epoch": 0.63, + "learning_rate": 0.000188989898989899, + "loss": 2.4574, + "theoretical_loss": 3.3493155893942395, + "tokens_seen": 2687172608 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1473498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5242228507995605, + "objective/train/theoretical_loss": 3.349309167582578, + "objective/train/tokens_used": 1046097376, + "theoretical_loss": 3.349309167582578, + "tokens_seen": 2687238144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018896969696969698, + "loss": 2.4017, + "theoretical_loss": 3.349309167582578, + "tokens_seen": 2687238144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018894949494949496, + "loss": 2.4346, + "theoretical_loss": 3.349302745971379, + "tokens_seen": 2687303680 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018892929292929292, + "loss": 2.5537, + "theoretical_loss": 3.349296324560633, + "tokens_seen": 2687369216 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018890909090909093, + "loss": 2.4903, + "theoretical_loss": 3.349289903350327, + "tokens_seen": 2687434752 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018888888888888888, + "loss": 2.3062, + "theoretical_loss": 3.3492834823404514, + "tokens_seen": 2687500288 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001888686868686869, + "loss": 2.3404, + "theoretical_loss": 3.349277061530994, + "tokens_seen": 2687565824 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018884848484848485, + "loss": 2.7609, + "theoretical_loss": 3.349270640921944, + "tokens_seen": 2687631360 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001888282828282828, + "loss": 2.4021, + "theoretical_loss": 3.3492642205132905, + "tokens_seen": 2687696896 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018880808080808082, + "loss": 2.3577, + "theoretical_loss": 3.349257800305022, + "tokens_seen": 2687762432 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018878787878787878, + "loss": 2.4386, + "theoretical_loss": 3.3492513802971278, + "tokens_seen": 2687827968 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001887676767676768, + "loss": 2.2587, + "theoretical_loss": 3.349244960489596, + "tokens_seen": 2687893504 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018874747474747475, + "loss": 2.2947, + "theoretical_loss": 3.349238540882416, + "tokens_seen": 2687959040 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018872727272727273, + "loss": 2.2574, + "theoretical_loss": 3.349232121475577, + "tokens_seen": 2688024576 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001887070707070707, + "loss": 2.2962, + "theoretical_loss": 3.3492257022690666, + "tokens_seen": 2688090112 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001886868686868687, + "loss": 2.3811, + "theoretical_loss": 3.349219283262875, + "tokens_seen": 2688155648 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018866666666666668, + "loss": 2.4024, + "theoretical_loss": 3.349212864456991, + "tokens_seen": 2688221184 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018864646464646464, + "loss": 2.4508, + "theoretical_loss": 3.3492064458514026, + "tokens_seen": 2688286720 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018862626262626262, + "loss": 2.6113, + "theoretical_loss": 3.349200027446099, + "tokens_seen": 2688352256 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001886060606060606, + "loss": 2.6725, + "theoretical_loss": 3.3491936092410692, + "tokens_seen": 2688417792 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001885858585858586, + "loss": 2.5437, + "theoretical_loss": 3.349187191236302, + "tokens_seen": 2688483328 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018856565656565657, + "loss": 2.4646, + "theoretical_loss": 3.3491807734317867, + "tokens_seen": 2688548864 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018854545454545456, + "loss": 2.3727, + "theoretical_loss": 3.3491743558275116, + "tokens_seen": 2688614400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018852525252525251, + "loss": 2.3472, + "theoretical_loss": 3.3491679384234656, + "tokens_seen": 2688679936 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018850505050505053, + "loss": 2.1493, + "theoretical_loss": 3.349161521219638, + "tokens_seen": 2688745472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018848484848484848, + "loss": 2.3411, + "theoretical_loss": 3.3491551042160173, + "tokens_seen": 2688811008 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1474112, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3311548233032227, + "objective/train/theoretical_loss": 3.349148687412592, + "objective/train/tokens_used": 1047735776, + "theoretical_loss": 3.349148687412592, + "tokens_seen": 2688876544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018846464646464647, + "loss": 2.2077, + "theoretical_loss": 3.349148687412592, + "tokens_seen": 2688876544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018844444444444445, + "loss": 2.4615, + "theoretical_loss": 3.3491422708093523, + "tokens_seen": 2688942080 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001884242424242424, + "loss": 2.4166, + "theoretical_loss": 3.3491358544062857, + "tokens_seen": 2689007616 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018840404040404042, + "loss": 2.7887, + "theoretical_loss": 3.3491294382033816, + "tokens_seen": 2689073152 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018838383838383838, + "loss": 2.355, + "theoretical_loss": 3.3491230222006294, + "tokens_seen": 2689138688 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018836363636363639, + "loss": 2.5055, + "theoretical_loss": 3.349116606398017, + "tokens_seen": 2689204224 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018834343434343434, + "loss": 2.3707, + "theoretical_loss": 3.3491101907955336, + "tokens_seen": 2689269760 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018832323232323235, + "loss": 2.389, + "theoretical_loss": 3.3491037753931687, + "tokens_seen": 2689335296 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001883030303030303, + "loss": 2.4807, + "theoretical_loss": 3.3490973601909104, + "tokens_seen": 2689400832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018828282828282827, + "loss": 2.3522, + "theoretical_loss": 3.349090945188748, + "tokens_seen": 2689466368 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018826262626262628, + "loss": 2.5167, + "theoretical_loss": 3.34908453038667, + "tokens_seen": 2689531904 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018824242424242424, + "loss": 2.6034, + "theoretical_loss": 3.349078115784666, + "tokens_seen": 2689597440 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018822222222222225, + "loss": 2.2696, + "theoretical_loss": 3.3490717013827243, + "tokens_seen": 2689662976 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001882020202020202, + "loss": 2.5018, + "theoretical_loss": 3.3490652871808337, + "tokens_seen": 2689728512 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001881818181818182, + "loss": 2.406, + "theoretical_loss": 3.3490588731789837, + "tokens_seen": 2689794048 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018816161616161617, + "loss": 2.464, + "theoretical_loss": 3.3490524593771624, + "tokens_seen": 2689859584 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018814141414141413, + "loss": 2.5187, + "theoretical_loss": 3.349046045775359, + "tokens_seen": 2689925120 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018812121212121214, + "loss": 2.5216, + "theoretical_loss": 3.3490396323735627, + "tokens_seen": 2689990656 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001881010101010101, + "loss": 2.3615, + "theoretical_loss": 3.3490332191717624, + "tokens_seen": 2690056192 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018808080808080808, + "loss": 2.584, + "theoretical_loss": 3.3490268061699466, + "tokens_seen": 2690121728 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018806060606060606, + "loss": 2.2031, + "theoretical_loss": 3.349020393368104, + "tokens_seen": 2690187264 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018804040404040405, + "loss": 2.4321, + "theoretical_loss": 3.349013980766224, + "tokens_seen": 2690252800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018802020202020203, + "loss": 2.3794, + "theoretical_loss": 3.349007568364296, + "tokens_seen": 2690318336 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018800000000000002, + "loss": 2.3306, + "theoretical_loss": 3.3490011561623074, + "tokens_seen": 2690383872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018797979797979797, + "loss": 2.6744, + "theoretical_loss": 3.3489947441602483, + "tokens_seen": 2690449408 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1475320, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5901687145233154, + "objective/train/theoretical_loss": 3.348988332358107, + "objective/train/tokens_used": 1049374176, + "theoretical_loss": 3.348988332358107, + "tokens_seen": 2690514944 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018795959595959596, + "loss": 2.7233, + "theoretical_loss": 3.348988332358107, + "tokens_seen": 2690514944 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018793939393939394, + "loss": 2.6352, + "theoretical_loss": 3.348981920755873, + "tokens_seen": 2690580480 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018791919191919192, + "loss": 2.5087, + "theoretical_loss": 3.3489755093535347, + "tokens_seen": 2690646016 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001878989898989899, + "loss": 2.523, + "theoretical_loss": 3.348969098151081, + "tokens_seen": 2690711552 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018787878787878787, + "loss": 2.3825, + "theoretical_loss": 3.348962687148501, + "tokens_seen": 2690777088 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018785858585858588, + "loss": 2.539, + "theoretical_loss": 3.3489562763457834, + "tokens_seen": 2690842624 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018783838383838383, + "loss": 2.151, + "theoretical_loss": 3.348949865742917, + "tokens_seen": 2690908160 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018781818181818184, + "loss": 2.4829, + "theoretical_loss": 3.3489434553398914, + "tokens_seen": 2690973696 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001877979797979798, + "loss": 2.412, + "theoretical_loss": 3.3489370451366947, + "tokens_seen": 2691039232 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018777777777777776, + "loss": 2.6671, + "theoretical_loss": 3.3489306351333163, + "tokens_seen": 2691104768 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018775757575757577, + "loss": 2.5635, + "theoretical_loss": 3.3489242253297453, + "tokens_seen": 2691170304 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018773737373737373, + "loss": 2.5179, + "theoretical_loss": 3.3489178157259696, + "tokens_seen": 2691235840 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018771717171717174, + "loss": 2.4773, + "theoretical_loss": 3.348911406321979, + "tokens_seen": 2691301376 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001876969696969697, + "loss": 2.5822, + "theoretical_loss": 3.3489049971177622, + "tokens_seen": 2691366912 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018767676767676768, + "loss": 2.3443, + "theoretical_loss": 3.348898588113308, + "tokens_seen": 2691432448 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018765656565656566, + "loss": 2.486, + "theoretical_loss": 3.3488921793086055, + "tokens_seen": 2691497984 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018763636363636365, + "loss": 2.7644, + "theoretical_loss": 3.3488857707036432, + "tokens_seen": 2691563520 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018761616161616163, + "loss": 2.3513, + "theoretical_loss": 3.348879362298411, + "tokens_seen": 2691629056 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001875959595959596, + "loss": 2.57, + "theoretical_loss": 3.3488729540928963, + "tokens_seen": 2691694592 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018757575757575757, + "loss": 2.4854, + "theoretical_loss": 3.3488665460870894, + "tokens_seen": 2691760128 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018755555555555555, + "loss": 2.5739, + "theoretical_loss": 3.3488601382809784, + "tokens_seen": 2691825664 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018753535353535354, + "loss": 2.4218, + "theoretical_loss": 3.348853730674552, + "tokens_seen": 2691891200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018751515151515152, + "loss": 2.2809, + "theoretical_loss": 3.3488473232678, + "tokens_seen": 2691956736 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001874949494949495, + "loss": 2.6068, + "theoretical_loss": 3.3488409160607113, + "tokens_seen": 2692022272 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001874747474747475, + "loss": 2.512, + "theoretical_loss": 3.348834509053274, + "tokens_seen": 2692087808 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1476480, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.195870876312256, + "objective/train/theoretical_loss": 3.3488281022454776, + "objective/train/tokens_used": 1051012576, + "theoretical_loss": 3.3488281022454776, + "tokens_seen": 2692153344 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018745454545454545, + "loss": 2.2735, + "theoretical_loss": 3.3488281022454776, + "tokens_seen": 2692153344 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018743434343434343, + "loss": 2.5262, + "theoretical_loss": 3.3488216956373105, + "tokens_seen": 2692218880 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018741414141414142, + "loss": 2.4886, + "theoretical_loss": 3.3488152892287624, + "tokens_seen": 2692284416 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001873939393939394, + "loss": 2.4564, + "theoretical_loss": 3.3488088830198213, + "tokens_seen": 2692349952 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018737373737373738, + "loss": 2.3529, + "theoretical_loss": 3.348802477010477, + "tokens_seen": 2692415488 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018735353535353537, + "loss": 2.3423, + "theoretical_loss": 3.348796071200718, + "tokens_seen": 2692481024 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018733333333333332, + "loss": 2.48, + "theoretical_loss": 3.348789665590533, + "tokens_seen": 2692546560 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018731313131313134, + "loss": 2.4425, + "theoretical_loss": 3.3487832601799115, + "tokens_seen": 2692612096 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001872929292929293, + "loss": 2.3937, + "theoretical_loss": 3.3487768549688415, + "tokens_seen": 2692677632 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018727272727272728, + "loss": 2.6744, + "theoretical_loss": 3.348770449957313, + "tokens_seen": 2692743168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018725252525252526, + "loss": 2.4658, + "theoretical_loss": 3.3487640451453147, + "tokens_seen": 2692808704 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018723232323232322, + "loss": 2.5764, + "theoretical_loss": 3.3487576405328348, + "tokens_seen": 2692874240 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018721212121212123, + "loss": 2.52, + "theoretical_loss": 3.3487512361198633, + "tokens_seen": 2692939776 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018719191919191918, + "loss": 2.5945, + "theoretical_loss": 3.348744831906388, + "tokens_seen": 2693005312 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001871717171717172, + "loss": 2.6165, + "theoretical_loss": 3.348738427892399, + "tokens_seen": 2693070848 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018715151515151515, + "loss": 2.4594, + "theoretical_loss": 3.3487320240778837, + "tokens_seen": 2693136384 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018713131313131314, + "loss": 2.3541, + "theoretical_loss": 3.3487256204628326, + "tokens_seen": 2693201920 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018711111111111112, + "loss": 2.4568, + "theoretical_loss": 3.3487192170472335, + "tokens_seen": 2693267456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018709090909090908, + "loss": 2.5012, + "theoretical_loss": 3.348712813831076, + "tokens_seen": 2693332992 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001870707070707071, + "loss": 2.4288, + "theoretical_loss": 3.3487064108143496, + "tokens_seen": 2693398528 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018705050505050505, + "loss": 2.7131, + "theoretical_loss": 3.3487000079970417, + "tokens_seen": 2693464064 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018703030303030303, + "loss": 2.1204, + "theoretical_loss": 3.3486936053791423, + "tokens_seen": 2693529600 + }, + { + "epoch": 0.63, + "learning_rate": 0.000187010101010101, + "loss": 2.4358, + "theoretical_loss": 3.34868720296064, + "tokens_seen": 2693595136 + }, + { + "epoch": 0.63, + "learning_rate": 0.000186989898989899, + "loss": 2.1905, + "theoretical_loss": 3.3486808007415236, + "tokens_seen": 2693660672 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018696969696969698, + "loss": 2.3522, + "theoretical_loss": 3.348674398721782, + "tokens_seen": 2693726208 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1476942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.778862237930298, + "objective/train/theoretical_loss": 3.348667996901405, + "objective/train/tokens_used": 1052650976, + "theoretical_loss": 3.348667996901405, + "tokens_seen": 2693791744 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018694949494949497, + "loss": 2.6326, + "theoretical_loss": 3.348667996901405, + "tokens_seen": 2693791744 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018692929292929292, + "loss": 2.2662, + "theoretical_loss": 3.348661595280381, + "tokens_seen": 2693857280 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001869090909090909, + "loss": 2.5923, + "theoretical_loss": 3.3486551938586984, + "tokens_seen": 2693922816 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001868888888888889, + "loss": 2.2519, + "theoretical_loss": 3.3486487926363466, + "tokens_seen": 2693988352 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018686868686868687, + "loss": 2.4467, + "theoretical_loss": 3.348642391613315, + "tokens_seen": 2694053888 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018684848484848486, + "loss": 2.4839, + "theoretical_loss": 3.3486359907895915, + "tokens_seen": 2694119424 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018682828282828281, + "loss": 2.2833, + "theoretical_loss": 3.348629590165166, + "tokens_seen": 2694184960 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018680808080808083, + "loss": 2.4663, + "theoretical_loss": 3.348623189740027, + "tokens_seen": 2694250496 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018678787878787878, + "loss": 2.5509, + "theoretical_loss": 3.3486167895141636, + "tokens_seen": 2694316032 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001867676767676768, + "loss": 2.5238, + "theoretical_loss": 3.3486103894875647, + "tokens_seen": 2694381568 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018674747474747475, + "loss": 2.4002, + "theoretical_loss": 3.3486039896602193, + "tokens_seen": 2694447104 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001867272727272727, + "loss": 2.2114, + "theoretical_loss": 3.3485975900321163, + "tokens_seen": 2694512640 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018670707070707072, + "loss": 2.559, + "theoretical_loss": 3.348591190603244, + "tokens_seen": 2694578176 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018668686868686868, + "loss": 2.4803, + "theoretical_loss": 3.348584791373593, + "tokens_seen": 2694643712 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001866666666666667, + "loss": 2.5313, + "theoretical_loss": 3.3485783923431507, + "tokens_seen": 2694709248 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018664646464646464, + "loss": 2.5669, + "theoretical_loss": 3.3485719935119067, + "tokens_seen": 2694774784 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018662626262626265, + "loss": 2.7007, + "theoretical_loss": 3.3485655948798496, + "tokens_seen": 2694840320 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001866060606060606, + "loss": 2.5186, + "theoretical_loss": 3.3485591964469688, + "tokens_seen": 2694905856 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018658585858585857, + "loss": 2.4547, + "theoretical_loss": 3.348552798213253, + "tokens_seen": 2694971392 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018656565656565658, + "loss": 2.587, + "theoretical_loss": 3.3485464001786918, + "tokens_seen": 2695036928 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018654545454545454, + "loss": 2.353, + "theoretical_loss": 3.348540002343273, + "tokens_seen": 2695102464 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018652525252525255, + "loss": 2.1833, + "theoretical_loss": 3.348533604706986, + "tokens_seen": 2695168000 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001865050505050505, + "loss": 2.434, + "theoretical_loss": 3.3485272072698202, + "tokens_seen": 2695233536 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001864848484848485, + "loss": 2.5977, + "theoretical_loss": 3.348520810031764, + "tokens_seen": 2695299072 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018646464646464647, + "loss": 2.2726, + "theoretical_loss": 3.348514412992807, + "tokens_seen": 2695364608 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1478003, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5612661838531494, + "objective/train/theoretical_loss": 3.3485080161529375, + "objective/train/tokens_used": 1054289376, + "theoretical_loss": 3.3485080161529375, + "tokens_seen": 2695430144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018644444444444446, + "loss": 2.4453, + "theoretical_loss": 3.3485080161529375, + "tokens_seen": 2695430144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018642424242424244, + "loss": 2.5392, + "theoretical_loss": 3.348501619512145, + "tokens_seen": 2695495680 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001864040404040404, + "loss": 2.5594, + "theoretical_loss": 3.348495223070418, + "tokens_seen": 2695561216 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018638383838383838, + "loss": 2.3624, + "theoretical_loss": 3.3484888268277455, + "tokens_seen": 2695626752 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018636363636363636, + "loss": 2.6489, + "theoretical_loss": 3.348482430784117, + "tokens_seen": 2695692288 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018634343434343435, + "loss": 2.4926, + "theoretical_loss": 3.3484760349395213, + "tokens_seen": 2695757824 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018632323232323233, + "loss": 2.5222, + "theoretical_loss": 3.3484696392939464, + "tokens_seen": 2695823360 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018630303030303032, + "loss": 2.3682, + "theoretical_loss": 3.348463243847383, + "tokens_seen": 2695888896 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018628282828282827, + "loss": 2.5248, + "theoretical_loss": 3.3484568485998185, + "tokens_seen": 2695954432 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018626262626262628, + "loss": 2.5558, + "theoretical_loss": 3.3484504535512425, + "tokens_seen": 2696019968 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018624242424242424, + "loss": 2.4428, + "theoretical_loss": 3.348444058701644, + "tokens_seen": 2696085504 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018622222222222223, + "loss": 2.3469, + "theoretical_loss": 3.348437664051012, + "tokens_seen": 2696151040 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001862020202020202, + "loss": 2.4062, + "theoretical_loss": 3.3484312695993355, + "tokens_seen": 2696216576 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018618181818181817, + "loss": 2.6158, + "theoretical_loss": 3.3484248753466033, + "tokens_seen": 2696282112 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018616161616161618, + "loss": 2.444, + "theoretical_loss": 3.3484184812928044, + "tokens_seen": 2696347648 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018614141414141413, + "loss": 2.5906, + "theoretical_loss": 3.348412087437928, + "tokens_seen": 2696413184 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018612121212121214, + "loss": 2.6042, + "theoretical_loss": 3.348405693781963, + "tokens_seen": 2696478720 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001861010101010101, + "loss": 2.6892, + "theoretical_loss": 3.348399300324898, + "tokens_seen": 2696544256 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018608080808080809, + "loss": 2.6522, + "theoretical_loss": 3.348392907066722, + "tokens_seen": 2696609792 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018606060606060607, + "loss": 2.6185, + "theoretical_loss": 3.3483865140074247, + "tokens_seen": 2696675328 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018604040404040403, + "loss": 2.5195, + "theoretical_loss": 3.348380121146995, + "tokens_seen": 2696740864 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018602020202020204, + "loss": 2.336, + "theoretical_loss": 3.348373728485421, + "tokens_seen": 2696806400 + }, + { + "epoch": 0.63, + "learning_rate": 0.000186, + "loss": 2.5492, + "theoretical_loss": 3.348367336022692, + "tokens_seen": 2696871936 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018597979797979798, + "loss": 2.3328, + "theoretical_loss": 3.3483609437587973, + "tokens_seen": 2696937472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018595959595959596, + "loss": 2.4845, + "theoretical_loss": 3.348354551693726, + "tokens_seen": 2697003008 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1478515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5991339683532715, + "objective/train/theoretical_loss": 3.348348159827467, + "objective/train/tokens_used": 1055927776, + "theoretical_loss": 3.348348159827467, + "tokens_seen": 2697068544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018593939393939395, + "loss": 2.5127, + "theoretical_loss": 3.348348159827467, + "tokens_seen": 2697068544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018591919191919193, + "loss": 2.3531, + "theoretical_loss": 3.3483417681600085, + "tokens_seen": 2697134080 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018589898989898991, + "loss": 2.4944, + "theoretical_loss": 3.348335376691341, + "tokens_seen": 2697199616 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018587878787878787, + "loss": 2.5394, + "theoretical_loss": 3.3483289854214515, + "tokens_seen": 2697265152 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018585858585858586, + "loss": 2.6762, + "theoretical_loss": 3.3483225943503307, + "tokens_seen": 2697330688 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018583838383838384, + "loss": 2.3825, + "theoretical_loss": 3.348316203477967, + "tokens_seen": 2697396224 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018581818181818182, + "loss": 2.4218, + "theoretical_loss": 3.3483098128043496, + "tokens_seen": 2697461760 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001857979797979798, + "loss": 2.176, + "theoretical_loss": 3.3483034223294665, + "tokens_seen": 2697527296 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018577777777777776, + "loss": 2.2991, + "theoretical_loss": 3.348297032053308, + "tokens_seen": 2697592832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018575757575757577, + "loss": 2.4322, + "theoretical_loss": 3.3482906419758627, + "tokens_seen": 2697658368 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018573737373737373, + "loss": 2.5228, + "theoretical_loss": 3.3482842520971188, + "tokens_seen": 2697723904 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018571717171717172, + "loss": 2.5293, + "theoretical_loss": 3.3482778624170666, + "tokens_seen": 2697789440 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001856969696969697, + "loss": 2.4668, + "theoretical_loss": 3.348271472935694, + "tokens_seen": 2697854976 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018567676767676768, + "loss": 2.6747, + "theoretical_loss": 3.348265083652991, + "tokens_seen": 2697920512 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018565656565656567, + "loss": 2.6877, + "theoretical_loss": 3.3482586945689454, + "tokens_seen": 2697986048 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018563636363636362, + "loss": 2.2513, + "theoretical_loss": 3.348252305683547, + "tokens_seen": 2698051584 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018561616161616164, + "loss": 2.3112, + "theoretical_loss": 3.3482459169967846, + "tokens_seen": 2698117120 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001855959595959596, + "loss": 2.4948, + "theoretical_loss": 3.348239528508647, + "tokens_seen": 2698182656 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001855757575757576, + "loss": 2.4935, + "theoretical_loss": 3.348233140219124, + "tokens_seen": 2698248192 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018555555555555556, + "loss": 2.6531, + "theoretical_loss": 3.3482267521282036, + "tokens_seen": 2698313728 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018553535353535352, + "loss": 2.4196, + "theoretical_loss": 3.348220364235875, + "tokens_seen": 2698379264 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018551515151515153, + "loss": 2.419, + "theoretical_loss": 3.348213976542128, + "tokens_seen": 2698444800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018549494949494949, + "loss": 2.4405, + "theoretical_loss": 3.3482075890469507, + "tokens_seen": 2698510336 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001854747474747475, + "loss": 2.3262, + "theoretical_loss": 3.3482012017503324, + "tokens_seen": 2698575872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018545454545454545, + "loss": 2.4757, + "theoretical_loss": 3.3481948146522624, + "tokens_seen": 2698641408 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1479744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.583965301513672, + "objective/train/theoretical_loss": 3.348188427752729, + "objective/train/tokens_used": 1057566176, + "theoretical_loss": 3.348188427752729, + "tokens_seen": 2698706944 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018543434343434344, + "loss": 2.4659, + "theoretical_loss": 3.348188427752729, + "tokens_seen": 2698706944 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018541414141414142, + "loss": 2.4803, + "theoretical_loss": 3.348182041051722, + "tokens_seen": 2698772480 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001853939393939394, + "loss": 2.6625, + "theoretical_loss": 3.34817565454923, + "tokens_seen": 2698838016 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001853737373737374, + "loss": 2.5764, + "theoretical_loss": 3.348169268245242, + "tokens_seen": 2698903552 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018535353535353535, + "loss": 2.2431, + "theoretical_loss": 3.3481628821397473, + "tokens_seen": 2698969088 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018533333333333333, + "loss": 2.3912, + "theoretical_loss": 3.3481564962327344, + "tokens_seen": 2699034624 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018531313131313131, + "loss": 2.3538, + "theoretical_loss": 3.348150110524193, + "tokens_seen": 2699100160 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001852929292929293, + "loss": 2.4493, + "theoretical_loss": 3.3481437250141113, + "tokens_seen": 2699165696 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018527272727272728, + "loss": 2.4769, + "theoretical_loss": 3.348137339702479, + "tokens_seen": 2699231232 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018525252525252527, + "loss": 2.416, + "theoretical_loss": 3.3481309545892843, + "tokens_seen": 2699296768 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018523232323232322, + "loss": 2.3809, + "theoretical_loss": 3.348124569674517, + "tokens_seen": 2699362304 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018521212121212123, + "loss": 2.3255, + "theoretical_loss": 3.3481181849581665, + "tokens_seen": 2699427840 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001851919191919192, + "loss": 2.4066, + "theoretical_loss": 3.3481118004402206, + "tokens_seen": 2699493376 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018517171717171717, + "loss": 2.6292, + "theoretical_loss": 3.348105416120669, + "tokens_seen": 2699558912 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018515151515151516, + "loss": 2.4581, + "theoretical_loss": 3.3480990319995008, + "tokens_seen": 2699624448 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018513131313131312, + "loss": 2.5486, + "theoretical_loss": 3.3480926480767046, + "tokens_seen": 2699689984 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018511111111111113, + "loss": 2.3949, + "theoretical_loss": 3.34808626435227, + "tokens_seen": 2699755520 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018509090909090908, + "loss": 2.4495, + "theoretical_loss": 3.3480798808261856, + "tokens_seen": 2699821056 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001850707070707071, + "loss": 2.582, + "theoretical_loss": 3.34807349749844, + "tokens_seen": 2699886592 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018505050505050505, + "loss": 2.3803, + "theoretical_loss": 3.3480671143690235, + "tokens_seen": 2699952128 + }, + { + "epoch": 0.63, + "learning_rate": 0.000185030303030303, + "loss": 2.5114, + "theoretical_loss": 3.348060731437924, + "tokens_seen": 2700017664 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018501010101010102, + "loss": 2.2333, + "theoretical_loss": 3.348054348705131, + "tokens_seen": 2700083200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018498989898989898, + "loss": 2.4672, + "theoretical_loss": 3.3480479661706335, + "tokens_seen": 2700148736 + }, + { + "epoch": 0.63, + "learning_rate": 0.000184969696969697, + "loss": 2.2601, + "theoretical_loss": 3.34804158383442, + "tokens_seen": 2700214272 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018494949494949494, + "loss": 2.4793, + "theoretical_loss": 3.3480352016964803, + "tokens_seen": 2700279808 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1480347, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4358396530151367, + "objective/train/theoretical_loss": 3.348028819756803, + "objective/train/tokens_used": 1059204576, + "theoretical_loss": 3.348028819756803, + "tokens_seen": 2700345344 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018492929292929293, + "loss": 2.3853, + "theoretical_loss": 3.348028819756803, + "tokens_seen": 2700345344 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001849090909090909, + "loss": 2.4037, + "theoretical_loss": 3.348022438015377, + "tokens_seen": 2700410880 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001848888888888889, + "loss": 2.5219, + "theoretical_loss": 3.348016056472192, + "tokens_seen": 2700476416 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018486868686868688, + "loss": 2.5485, + "theoretical_loss": 3.3480096751272366, + "tokens_seen": 2700541952 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018484848484848484, + "loss": 2.4248, + "theoretical_loss": 3.3480032939805, + "tokens_seen": 2700607488 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018482828282828285, + "loss": 2.5916, + "theoretical_loss": 3.3479969130319702, + "tokens_seen": 2700673024 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001848080808080808, + "loss": 2.4734, + "theoretical_loss": 3.3479905322816377, + "tokens_seen": 2700738560 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001847878787878788, + "loss": 2.3459, + "theoretical_loss": 3.3479841517294906, + "tokens_seen": 2700804096 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018476767676767677, + "loss": 2.7078, + "theoretical_loss": 3.3479777713755188, + "tokens_seen": 2700869632 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018474747474747476, + "loss": 2.6226, + "theoretical_loss": 3.34797139121971, + "tokens_seen": 2700935168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018472727272727274, + "loss": 2.4492, + "theoretical_loss": 3.3479650112620547, + "tokens_seen": 2701000704 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018470707070707072, + "loss": 2.4536, + "theoretical_loss": 3.347958631502541, + "tokens_seen": 2701066240 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018468686868686868, + "loss": 2.3387, + "theoretical_loss": 3.3479522519411584, + "tokens_seen": 2701131776 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018466666666666666, + "loss": 2.4472, + "theoretical_loss": 3.3479458725778954, + "tokens_seen": 2701197312 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018464646464646465, + "loss": 2.355, + "theoretical_loss": 3.3479394934127416, + "tokens_seen": 2701262848 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018462626262626263, + "loss": 2.372, + "theoretical_loss": 3.347933114445686, + "tokens_seen": 2701328384 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018460606060606062, + "loss": 2.4098, + "theoretical_loss": 3.347926735676717, + "tokens_seen": 2701393920 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018458585858585857, + "loss": 2.5509, + "theoretical_loss": 3.3479203571058247, + "tokens_seen": 2701459456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018456565656565658, + "loss": 2.3439, + "theoretical_loss": 3.347913978732997, + "tokens_seen": 2701524992 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018454545454545454, + "loss": 2.5472, + "theoretical_loss": 3.347907600558224, + "tokens_seen": 2701590528 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018452525252525255, + "loss": 2.7265, + "theoretical_loss": 3.347901222581494, + "tokens_seen": 2701656064 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001845050505050505, + "loss": 2.4103, + "theoretical_loss": 3.3478948448027963, + "tokens_seen": 2701721600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018448484848484847, + "loss": 2.5985, + "theoretical_loss": 3.34788846722212, + "tokens_seen": 2701787136 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018446464646464648, + "loss": 2.4802, + "theoretical_loss": 3.3478820898394543, + "tokens_seen": 2701852672 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018444444444444443, + "loss": 2.5821, + "theoretical_loss": 3.347875712654788, + "tokens_seen": 2701918208 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1481517, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.663572311401367, + "objective/train/theoretical_loss": 3.3478693356681095, + "objective/train/tokens_used": 1060842976, + "theoretical_loss": 3.3478693356681095, + "tokens_seen": 2701983744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018442424242424245, + "loss": 2.449, + "theoretical_loss": 3.3478693356681095, + "tokens_seen": 2701983744 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001844040404040404, + "loss": 2.44, + "theoretical_loss": 3.3478629588794093, + "tokens_seen": 2702049280 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018438383838383839, + "loss": 2.698, + "theoretical_loss": 3.347856582288675, + "tokens_seen": 2702114816 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018436363636363637, + "loss": 2.6436, + "theoretical_loss": 3.347850205895897, + "tokens_seen": 2702180352 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018434343434343435, + "loss": 2.6549, + "theoretical_loss": 3.3478438297010635, + "tokens_seen": 2702245888 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018432323232323234, + "loss": 2.4352, + "theoretical_loss": 3.347837453704164, + "tokens_seen": 2702311424 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001843030303030303, + "loss": 2.4105, + "theoretical_loss": 3.347831077905187, + "tokens_seen": 2702376960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018428282828282828, + "loss": 2.585, + "theoretical_loss": 3.3478247023041217, + "tokens_seen": 2702442496 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018426262626262626, + "loss": 2.2564, + "theoretical_loss": 3.347818326900958, + "tokens_seen": 2702508032 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018424242424242425, + "loss": 2.5492, + "theoretical_loss": 3.3478119516956837, + "tokens_seen": 2702573568 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018422222222222223, + "loss": 2.6858, + "theoretical_loss": 3.3478055766882884, + "tokens_seen": 2702639104 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018420202020202021, + "loss": 2.6041, + "theoretical_loss": 3.3477992018787615, + "tokens_seen": 2702704640 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018418181818181817, + "loss": 2.6253, + "theoretical_loss": 3.347792827267092, + "tokens_seen": 2702770176 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018416161616161616, + "loss": 2.306, + "theoretical_loss": 3.3477864528532684, + "tokens_seen": 2702835712 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018414141414141414, + "loss": 2.5949, + "theoretical_loss": 3.34778007863728, + "tokens_seen": 2702901248 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018412121212121212, + "loss": 2.4144, + "theoretical_loss": 3.347773704619116, + "tokens_seen": 2702966784 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001841010101010101, + "loss": 2.384, + "theoretical_loss": 3.3477673307987654, + "tokens_seen": 2703032320 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018408080808080806, + "loss": 2.5751, + "theoretical_loss": 3.3477609571762175, + "tokens_seen": 2703097856 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018406060606060608, + "loss": 2.4568, + "theoretical_loss": 3.347754583751461, + "tokens_seen": 2703163392 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018404040404040403, + "loss": 2.5424, + "theoretical_loss": 3.347748210524485, + "tokens_seen": 2703228928 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018402020202020204, + "loss": 2.4473, + "theoretical_loss": 3.347741837495279, + "tokens_seen": 2703294464 + }, + { + "epoch": 0.64, + "learning_rate": 0.000184, + "loss": 2.427, + "theoretical_loss": 3.3477354646638315, + "tokens_seen": 2703360000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018397979797979798, + "loss": 2.49, + "theoretical_loss": 3.347729092030132, + "tokens_seen": 2703425536 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018395959595959597, + "loss": 2.4909, + "theoretical_loss": 3.3477227195941692, + "tokens_seen": 2703491072 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018393939393939392, + "loss": 2.5905, + "theoretical_loss": 3.3477163473559326, + "tokens_seen": 2703556608 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1482062, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9572315216064453, + "objective/train/theoretical_loss": 3.3477099753154107, + "objective/train/tokens_used": 1062481376, + "theoretical_loss": 3.3477099753154107, + "tokens_seen": 2703622144 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018391919191919194, + "loss": 2.3128, + "theoretical_loss": 3.3477099753154107, + "tokens_seen": 2703622144 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001838989898989899, + "loss": 2.4784, + "theoretical_loss": 3.347703603472593, + "tokens_seen": 2703687680 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001838787878787879, + "loss": 2.3243, + "theoretical_loss": 3.3476972318274685, + "tokens_seen": 2703753216 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018385858585858586, + "loss": 2.6366, + "theoretical_loss": 3.3476908603800264, + "tokens_seen": 2703818752 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018383838383838384, + "loss": 2.473, + "theoretical_loss": 3.3476844891302555, + "tokens_seen": 2703884288 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018381818181818183, + "loss": 2.7443, + "theoretical_loss": 3.347678118078145, + "tokens_seen": 2703949824 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018379797979797979, + "loss": 2.5302, + "theoretical_loss": 3.3476717472236843, + "tokens_seen": 2704015360 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001837777777777778, + "loss": 2.465, + "theoretical_loss": 3.347665376566862, + "tokens_seen": 2704080896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018375757575757575, + "loss": 2.4601, + "theoretical_loss": 3.3476590061076674, + "tokens_seen": 2704146432 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018373737373737374, + "loss": 2.5635, + "theoretical_loss": 3.347652635846089, + "tokens_seen": 2704211968 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018371717171717172, + "loss": 2.5015, + "theoretical_loss": 3.3476462657821173, + "tokens_seen": 2704277504 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001836969696969697, + "loss": 2.5868, + "theoretical_loss": 3.34763989591574, + "tokens_seen": 2704343040 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001836767676767677, + "loss": 2.515, + "theoretical_loss": 3.3476335262469465, + "tokens_seen": 2704408576 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018365656565656567, + "loss": 2.533, + "theoretical_loss": 3.3476271567757263, + "tokens_seen": 2704474112 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018363636363636363, + "loss": 2.4978, + "theoretical_loss": 3.347620787502068, + "tokens_seen": 2704539648 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018361616161616161, + "loss": 2.5581, + "theoretical_loss": 3.347614418425961, + "tokens_seen": 2704605184 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001835959595959596, + "loss": 2.4623, + "theoretical_loss": 3.3476080495473943, + "tokens_seen": 2704670720 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018357575757575758, + "loss": 2.7358, + "theoretical_loss": 3.347601680866357, + "tokens_seen": 2704736256 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018355555555555557, + "loss": 2.4622, + "theoretical_loss": 3.347595312382839, + "tokens_seen": 2704801792 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018353535353535352, + "loss": 2.2301, + "theoretical_loss": 3.3475889440968274, + "tokens_seen": 2704867328 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018351515151515153, + "loss": 2.3475, + "theoretical_loss": 3.347582576008313, + "tokens_seen": 2704932864 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001834949494949495, + "loss": 2.8454, + "theoretical_loss": 3.3475762081172844, + "tokens_seen": 2704998400 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001834747474747475, + "loss": 2.4604, + "theoretical_loss": 3.34756984042373, + "tokens_seen": 2705063936 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018345454545454546, + "loss": 2.5308, + "theoretical_loss": 3.3475634729276402, + "tokens_seen": 2705129472 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018343434343434342, + "loss": 2.5021, + "theoretical_loss": 3.3475571056290034, + "tokens_seen": 2705195008 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1483223, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.250884771347046, + "objective/train/theoretical_loss": 3.3475507385278087, + "objective/train/tokens_used": 1064119776, + "theoretical_loss": 3.3475507385278087, + "tokens_seen": 2705260544 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018341414141414143, + "loss": 2.4728, + "theoretical_loss": 3.3475507385278087, + "tokens_seen": 2705260544 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018339393939393938, + "loss": 2.4835, + "theoretical_loss": 3.347544371624045, + "tokens_seen": 2705326080 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001833737373737374, + "loss": 2.3555, + "theoretical_loss": 3.347538004917702, + "tokens_seen": 2705391616 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018335353535353535, + "loss": 2.4698, + "theoretical_loss": 3.3475316384087677, + "tokens_seen": 2705457152 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018333333333333334, + "loss": 2.5391, + "theoretical_loss": 3.3475252720972324, + "tokens_seen": 2705522688 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018331313131313132, + "loss": 2.5753, + "theoretical_loss": 3.3475189059830845, + "tokens_seen": 2705588224 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018329292929292928, + "loss": 2.4569, + "theoretical_loss": 3.3475125400663135, + "tokens_seen": 2705653760 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001832727272727273, + "loss": 2.7072, + "theoretical_loss": 3.3475061743469086, + "tokens_seen": 2705719296 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018325252525252524, + "loss": 2.3648, + "theoretical_loss": 3.3474998088248578, + "tokens_seen": 2705784832 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018323232323232323, + "loss": 2.618, + "theoretical_loss": 3.3474934435001513, + "tokens_seen": 2705850368 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001832121212121212, + "loss": 2.5072, + "theoretical_loss": 3.347487078372778, + "tokens_seen": 2705915904 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001831919191919192, + "loss": 2.5421, + "theoretical_loss": 3.347480713442727, + "tokens_seen": 2705981440 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018317171717171718, + "loss": 2.4983, + "theoretical_loss": 3.3474743487099876, + "tokens_seen": 2706046976 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018315151515151516, + "loss": 2.1668, + "theoretical_loss": 3.347467984174548, + "tokens_seen": 2706112512 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018313131313131315, + "loss": 2.5562, + "theoretical_loss": 3.3474616198363982, + "tokens_seen": 2706178048 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001831111111111111, + "loss": 2.4365, + "theoretical_loss": 3.347455255695527, + "tokens_seen": 2706243584 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001830909090909091, + "loss": 2.493, + "theoretical_loss": 3.347448891751924, + "tokens_seen": 2706309120 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018307070707070707, + "loss": 2.4579, + "theoretical_loss": 3.347442528005577, + "tokens_seen": 2706374656 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018305050505050506, + "loss": 2.5042, + "theoretical_loss": 3.3474361644564765, + "tokens_seen": 2706440192 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018303030303030304, + "loss": 2.6841, + "theoretical_loss": 3.347429801104611, + "tokens_seen": 2706505728 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018301010101010102, + "loss": 2.4324, + "theoretical_loss": 3.3474234379499697, + "tokens_seen": 2706571264 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018298989898989898, + "loss": 2.2687, + "theoretical_loss": 3.3474170749925416, + "tokens_seen": 2706636800 + }, + { + "epoch": 0.64, + "learning_rate": 0.000182969696969697, + "loss": 2.6401, + "theoretical_loss": 3.3474107122323162, + "tokens_seen": 2706702336 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018294949494949495, + "loss": 2.4576, + "theoretical_loss": 3.347404349669282, + "tokens_seen": 2706767872 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018292929292929293, + "loss": 2.4525, + "theoretical_loss": 3.3473979873034283, + "tokens_seen": 2706833408 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1483803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.663747787475586, + "objective/train/theoretical_loss": 3.347391625134745, + "objective/train/tokens_used": 1065758176, + "theoretical_loss": 3.347391625134745, + "tokens_seen": 2706898944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018290909090909092, + "loss": 2.5869, + "theoretical_loss": 3.347391625134745, + "tokens_seen": 2706898944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018288888888888887, + "loss": 2.1871, + "theoretical_loss": 3.3473852631632197, + "tokens_seen": 2706964480 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018286868686868689, + "loss": 2.4987, + "theoretical_loss": 3.3473789013888426, + "tokens_seen": 2707030016 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018284848484848484, + "loss": 2.4858, + "theoretical_loss": 3.347372539811603, + "tokens_seen": 2707095552 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018282828282828285, + "loss": 2.565, + "theoretical_loss": 3.3473661784314896, + "tokens_seen": 2707161088 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001828080808080808, + "loss": 2.574, + "theoretical_loss": 3.347359817248491, + "tokens_seen": 2707226624 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001827878787878788, + "loss": 2.551, + "theoretical_loss": 3.3473534562625975, + "tokens_seen": 2707292160 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018276767676767678, + "loss": 2.68, + "theoretical_loss": 3.347347095473797, + "tokens_seen": 2707357696 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018274747474747473, + "loss": 2.4394, + "theoretical_loss": 3.3473407348820796, + "tokens_seen": 2707423232 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018272727272727275, + "loss": 2.3885, + "theoretical_loss": 3.347334374487434, + "tokens_seen": 2707488768 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001827070707070707, + "loss": 2.2589, + "theoretical_loss": 3.3473280142898494, + "tokens_seen": 2707554304 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001826868686868687, + "loss": 2.605, + "theoretical_loss": 3.3473216542893147, + "tokens_seen": 2707619840 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018266666666666667, + "loss": 2.5402, + "theoretical_loss": 3.347315294485819, + "tokens_seen": 2707685376 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018264646464646465, + "loss": 2.6106, + "theoretical_loss": 3.347308934879352, + "tokens_seen": 2707750912 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018262626262626264, + "loss": 2.4888, + "theoretical_loss": 3.347302575469902, + "tokens_seen": 2707816448 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018260606060606062, + "loss": 2.6001, + "theoretical_loss": 3.347296216257459, + "tokens_seen": 2707881984 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018258585858585858, + "loss": 2.3081, + "theoretical_loss": 3.3472898572420116, + "tokens_seen": 2707947520 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018256565656565656, + "loss": 2.629, + "theoretical_loss": 3.347283498423549, + "tokens_seen": 2708013056 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018254545454545455, + "loss": 2.5801, + "theoretical_loss": 3.3472771398020607, + "tokens_seen": 2708078592 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018252525252525253, + "loss": 2.4218, + "theoretical_loss": 3.347270781377535, + "tokens_seen": 2708144128 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018250505050505052, + "loss": 2.4671, + "theoretical_loss": 3.347264423149962, + "tokens_seen": 2708209664 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018248484848484847, + "loss": 2.2512, + "theoretical_loss": 3.34725806511933, + "tokens_seen": 2708275200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018246464646464648, + "loss": 2.5189, + "theoretical_loss": 3.3472517072856287, + "tokens_seen": 2708340736 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018244444444444444, + "loss": 2.6719, + "theoretical_loss": 3.347245349648847, + "tokens_seen": 2708406272 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018242424242424242, + "loss": 2.7243, + "theoretical_loss": 3.347238992208974, + "tokens_seen": 2708471808 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1485001, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9092819690704346, + "objective/train/theoretical_loss": 3.347232634965999, + "objective/train/tokens_used": 1067396576, + "theoretical_loss": 3.347232634965999, + "tokens_seen": 2708537344 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001824040404040404, + "loss": 2.3197, + "theoretical_loss": 3.347232634965999, + "tokens_seen": 2708537344 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018238383838383836, + "loss": 2.4197, + "theoretical_loss": 3.347226277919911, + "tokens_seen": 2708602880 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018236363636363638, + "loss": 2.365, + "theoretical_loss": 3.347219921070699, + "tokens_seen": 2708668416 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018234343434343433, + "loss": 2.4228, + "theoretical_loss": 3.347213564418353, + "tokens_seen": 2708733952 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018232323232323234, + "loss": 2.4779, + "theoretical_loss": 3.347207207962861, + "tokens_seen": 2708799488 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001823030303030303, + "loss": 2.4212, + "theoretical_loss": 3.3472008517042124, + "tokens_seen": 2708865024 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001822828282828283, + "loss": 2.345, + "theoretical_loss": 3.3471944956423973, + "tokens_seen": 2708930560 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018226262626262627, + "loss": 2.3913, + "theoretical_loss": 3.3471881397774035, + "tokens_seen": 2708996096 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018224242424242423, + "loss": 2.4625, + "theoretical_loss": 3.3471817841092206, + "tokens_seen": 2709061632 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018222222222222224, + "loss": 2.5894, + "theoretical_loss": 3.347175428637838, + "tokens_seen": 2709127168 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001822020202020202, + "loss": 2.294, + "theoretical_loss": 3.3471690733632453, + "tokens_seen": 2709192704 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001821818181818182, + "loss": 2.4062, + "theoretical_loss": 3.34716271828543, + "tokens_seen": 2709258240 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018216161616161616, + "loss": 2.3877, + "theoretical_loss": 3.3471563634043835, + "tokens_seen": 2709323776 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018214141414141415, + "loss": 2.5089, + "theoretical_loss": 3.347150008720093, + "tokens_seen": 2709389312 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018212121212121213, + "loss": 2.3448, + "theoretical_loss": 3.3471436542325486, + "tokens_seen": 2709454848 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001821010101010101, + "loss": 2.5059, + "theoretical_loss": 3.3471372999417395, + "tokens_seen": 2709520384 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001820808080808081, + "loss": 2.5152, + "theoretical_loss": 3.3471309458476544, + "tokens_seen": 2709585920 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018206060606060605, + "loss": 2.2663, + "theoretical_loss": 3.347124591950283, + "tokens_seen": 2709651456 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018204040404040404, + "loss": 2.564, + "theoretical_loss": 3.347118238249614, + "tokens_seen": 2709716992 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018202020202020202, + "loss": 2.7046, + "theoretical_loss": 3.3471118847456363, + "tokens_seen": 2709782528 + }, + { + "epoch": 0.64, + "learning_rate": 0.000182, + "loss": 2.7923, + "theoretical_loss": 3.3471055314383396, + "tokens_seen": 2709848064 + }, + { + "epoch": 0.64, + "learning_rate": 0.000181979797979798, + "loss": 2.4299, + "theoretical_loss": 3.347099178327713, + "tokens_seen": 2709913600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018195959595959597, + "loss": 2.4373, + "theoretical_loss": 3.3470928254137453, + "tokens_seen": 2709979136 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018193939393939393, + "loss": 2.4307, + "theoretical_loss": 3.3470864726964265, + "tokens_seen": 2710044672 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018191919191919194, + "loss": 2.5111, + "theoretical_loss": 3.3470801201757445, + "tokens_seen": 2710110208 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1485587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5603830814361572, + "objective/train/theoretical_loss": 3.3470737678516898, + "objective/train/tokens_used": 1069034976, + "theoretical_loss": 3.3470737678516898, + "tokens_seen": 2710175744 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001818989898989899, + "loss": 2.4378, + "theoretical_loss": 3.3470737678516898, + "tokens_seen": 2710175744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018187878787878788, + "loss": 2.63, + "theoretical_loss": 3.34706741572425, + "tokens_seen": 2710241280 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018185858585858587, + "loss": 2.476, + "theoretical_loss": 3.347061063793416, + "tokens_seen": 2710306816 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018183838383838382, + "loss": 2.4691, + "theoretical_loss": 3.3470547120591756, + "tokens_seen": 2710372352 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018181818181818183, + "loss": 2.4109, + "theoretical_loss": 3.3470483605215184, + "tokens_seen": 2710437888 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001817979797979798, + "loss": 2.3391, + "theoretical_loss": 3.3470420091804343, + "tokens_seen": 2710503424 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001817777777777778, + "loss": 2.4988, + "theoretical_loss": 3.347035658035911, + "tokens_seen": 2710568960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018175757575757576, + "loss": 2.249, + "theoretical_loss": 3.3470293070879387, + "tokens_seen": 2710634496 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018173737373737372, + "loss": 2.5243, + "theoretical_loss": 3.3470229563365064, + "tokens_seen": 2710700032 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018171717171717173, + "loss": 2.6344, + "theoretical_loss": 3.347016605781603, + "tokens_seen": 2710765568 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018169696969696968, + "loss": 2.609, + "theoretical_loss": 3.347010255423218, + "tokens_seen": 2710831104 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001816767676767677, + "loss": 2.1508, + "theoretical_loss": 3.3470039052613405, + "tokens_seen": 2710896640 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018165656565656565, + "loss": 2.4535, + "theoretical_loss": 3.3469975552959594, + "tokens_seen": 2710962176 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018163636363636364, + "loss": 2.5204, + "theoretical_loss": 3.3469912055270647, + "tokens_seen": 2711027712 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018161616161616162, + "loss": 2.4313, + "theoretical_loss": 3.346984855954644, + "tokens_seen": 2711093248 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001815959595959596, + "loss": 2.4751, + "theoretical_loss": 3.3469785065786883, + "tokens_seen": 2711158784 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001815757575757576, + "loss": 2.3699, + "theoretical_loss": 3.3469721573991853, + "tokens_seen": 2711224320 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018155555555555554, + "loss": 2.552, + "theoretical_loss": 3.3469658084161247, + "tokens_seen": 2711289856 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018153535353535353, + "loss": 2.4496, + "theoretical_loss": 3.346959459629496, + "tokens_seen": 2711355392 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001815151515151515, + "loss": 2.353, + "theoretical_loss": 3.346953111039288, + "tokens_seen": 2711420928 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001814949494949495, + "loss": 2.4166, + "theoretical_loss": 3.34694676264549, + "tokens_seen": 2711486464 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018147474747474748, + "loss": 2.4535, + "theoretical_loss": 3.346940414448091, + "tokens_seen": 2711552000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018145454545454546, + "loss": 2.381, + "theoretical_loss": 3.3469340664470804, + "tokens_seen": 2711617536 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018143434343434345, + "loss": 2.3516, + "theoretical_loss": 3.3469277186424473, + "tokens_seen": 2711683072 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018141414141414143, + "loss": 2.6999, + "theoretical_loss": 3.3469213710341807, + "tokens_seen": 2711748608 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1487028, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.833167552947998, + "objective/train/theoretical_loss": 3.3469150236222704, + "objective/train/tokens_used": 1070673376, + "theoretical_loss": 3.3469150236222704, + "tokens_seen": 2711814144 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001813939393939394, + "loss": 2.5415, + "theoretical_loss": 3.3469150236222704, + "tokens_seen": 2711814144 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018137373737373737, + "loss": 2.4226, + "theoretical_loss": 3.346908676406705, + "tokens_seen": 2711879680 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018135353535353536, + "loss": 2.5692, + "theoretical_loss": 3.3469023293874733, + "tokens_seen": 2711945216 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018133333333333334, + "loss": 2.3508, + "theoretical_loss": 3.3468959825645657, + "tokens_seen": 2712010752 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018131313131313132, + "loss": 2.4882, + "theoretical_loss": 3.3468896359379703, + "tokens_seen": 2712076288 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018129292929292928, + "loss": 2.4915, + "theoretical_loss": 3.3468832895076766, + "tokens_seen": 2712141824 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001812727272727273, + "loss": 2.7021, + "theoretical_loss": 3.3468769432736742, + "tokens_seen": 2712207360 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018125252525252525, + "loss": 2.2858, + "theoretical_loss": 3.346870597235952, + "tokens_seen": 2712272896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018123232323232326, + "loss": 2.5339, + "theoretical_loss": 3.346864251394499, + "tokens_seen": 2712338432 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018121212121212122, + "loss": 2.5822, + "theoretical_loss": 3.3468579057493044, + "tokens_seen": 2712403968 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018119191919191917, + "loss": 2.5105, + "theoretical_loss": 3.3468515603003572, + "tokens_seen": 2712469504 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018117171717171719, + "loss": 2.4078, + "theoretical_loss": 3.3468452150476478, + "tokens_seen": 2712535040 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018115151515151514, + "loss": 2.2959, + "theoretical_loss": 3.3468388699911635, + "tokens_seen": 2712600576 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018113131313131315, + "loss": 2.3479, + "theoretical_loss": 3.3468325251308952, + "tokens_seen": 2712666112 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001811111111111111, + "loss": 2.4701, + "theoretical_loss": 3.3468261804668313, + "tokens_seen": 2712731648 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001810909090909091, + "loss": 2.3747, + "theoretical_loss": 3.3468198359989607, + "tokens_seen": 2712797184 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018107070707070708, + "loss": 2.5224, + "theoretical_loss": 3.346813491727273, + "tokens_seen": 2712862720 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018105050505050506, + "loss": 2.525, + "theoretical_loss": 3.3468071476517576, + "tokens_seen": 2712928256 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018103030303030305, + "loss": 2.2141, + "theoretical_loss": 3.3468008037724033, + "tokens_seen": 2712993792 + }, + { + "epoch": 0.64, + "learning_rate": 0.000181010101010101, + "loss": 2.4438, + "theoretical_loss": 3.3467944600891997, + "tokens_seen": 2713059328 + }, + { + "epoch": 0.64, + "learning_rate": 0.000180989898989899, + "loss": 2.4823, + "theoretical_loss": 3.3467881166021356, + "tokens_seen": 2713124864 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018096969696969697, + "loss": 2.6494, + "theoretical_loss": 3.3467817733112, + "tokens_seen": 2713190400 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018094949494949495, + "loss": 2.4183, + "theoretical_loss": 3.346775430216383, + "tokens_seen": 2713255936 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018092929292929294, + "loss": 2.6091, + "theoretical_loss": 3.346769087317673, + "tokens_seen": 2713321472 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018090909090909092, + "loss": 2.4667, + "theoretical_loss": 3.3467627446150594, + "tokens_seen": 2713387008 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1487700, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3146984577178955, + "objective/train/theoretical_loss": 3.3467564021085314, + "objective/train/tokens_used": 1072311776, + "theoretical_loss": 3.3467564021085314, + "tokens_seen": 2713452544 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018088888888888888, + "loss": 2.2269, + "theoretical_loss": 3.3467564021085314, + "tokens_seen": 2713452544 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018086868686868686, + "loss": 2.5734, + "theoretical_loss": 3.3467500597980786, + "tokens_seen": 2713518080 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018084848484848485, + "loss": 2.3583, + "theoretical_loss": 3.3467437176836894, + "tokens_seen": 2713583616 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018082828282828283, + "loss": 2.3448, + "theoretical_loss": 3.3467373757653536, + "tokens_seen": 2713649152 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018080808080808082, + "loss": 2.7295, + "theoretical_loss": 3.34673103404306, + "tokens_seen": 2713714688 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018078787878787877, + "loss": 2.4642, + "theoretical_loss": 3.3467246925167986, + "tokens_seen": 2713780224 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018076767676767678, + "loss": 2.3671, + "theoretical_loss": 3.3467183511865577, + "tokens_seen": 2713845760 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018074747474747474, + "loss": 2.5793, + "theoretical_loss": 3.346712010052327, + "tokens_seen": 2713911296 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018072727272727275, + "loss": 2.3826, + "theoretical_loss": 3.3467056691140957, + "tokens_seen": 2713976832 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001807070707070707, + "loss": 2.5504, + "theoretical_loss": 3.3466993283718525, + "tokens_seen": 2714042368 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018068686868686867, + "loss": 2.4045, + "theoretical_loss": 3.3466929878255876, + "tokens_seen": 2714107904 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018066666666666668, + "loss": 2.4039, + "theoretical_loss": 3.346686647475289, + "tokens_seen": 2714173440 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018064646464646463, + "loss": 2.124, + "theoretical_loss": 3.3466803073209466, + "tokens_seen": 2714238976 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018062626262626264, + "loss": 2.5203, + "theoretical_loss": 3.34667396736255, + "tokens_seen": 2714304512 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001806060606060606, + "loss": 2.3105, + "theoretical_loss": 3.3466676276000875, + "tokens_seen": 2714370048 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001805858585858586, + "loss": 2.46, + "theoretical_loss": 3.346661288033549, + "tokens_seen": 2714435584 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018056565656565657, + "loss": 2.4667, + "theoretical_loss": 3.3466549486629233, + "tokens_seen": 2714501120 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018054545454545455, + "loss": 2.2639, + "theoretical_loss": 3.3466486094882, + "tokens_seen": 2714566656 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018052525252525254, + "loss": 2.6771, + "theoretical_loss": 3.346642270509368, + "tokens_seen": 2714632192 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001805050505050505, + "loss": 2.5091, + "theoretical_loss": 3.346635931726417, + "tokens_seen": 2714697728 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001804848484848485, + "loss": 2.476, + "theoretical_loss": 3.346629593139335, + "tokens_seen": 2714763264 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018046464646464646, + "loss": 2.6063, + "theoretical_loss": 3.346623254748113, + "tokens_seen": 2714828800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018044444444444445, + "loss": 2.4468, + "theoretical_loss": 3.3466169165527386, + "tokens_seen": 2714894336 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018042424242424243, + "loss": 2.3416, + "theoretical_loss": 3.3466105785532023, + "tokens_seen": 2714959872 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001804040404040404, + "loss": 2.6436, + "theoretical_loss": 3.3466042407494925, + "tokens_seen": 2715025408 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1488584, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8602285385131836, + "objective/train/theoretical_loss": 3.3465979031415984, + "objective/train/tokens_used": 1073950176, + "theoretical_loss": 3.3465979031415984, + "tokens_seen": 2715090944 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001803838383838384, + "loss": 2.4297, + "theoretical_loss": 3.3465979031415984, + "tokens_seen": 2715090944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018036363636363638, + "loss": 2.3063, + "theoretical_loss": 3.3465915657295096, + "tokens_seen": 2715156480 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018034343434343434, + "loss": 2.7623, + "theoretical_loss": 3.346585228513215, + "tokens_seen": 2715222016 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018032323232323232, + "loss": 2.4433, + "theoretical_loss": 3.3465788914927046, + "tokens_seen": 2715287552 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001803030303030303, + "loss": 2.5431, + "theoretical_loss": 3.3465725546679668, + "tokens_seen": 2715353088 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001802828282828283, + "loss": 2.5037, + "theoretical_loss": 3.346566218038991, + "tokens_seen": 2715418624 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018026262626262627, + "loss": 2.7025, + "theoretical_loss": 3.3465598816057667, + "tokens_seen": 2715484160 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018024242424242423, + "loss": 2.7195, + "theoretical_loss": 3.3465535453682826, + "tokens_seen": 2715549696 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018022222222222224, + "loss": 2.5807, + "theoretical_loss": 3.3465472093265287, + "tokens_seen": 2715615232 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001802020202020202, + "loss": 2.432, + "theoretical_loss": 3.3465408734804933, + "tokens_seen": 2715680768 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001801818181818182, + "loss": 2.3513, + "theoretical_loss": 3.3465345378301663, + "tokens_seen": 2715746304 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018016161616161617, + "loss": 2.5359, + "theoretical_loss": 3.346528202375537, + "tokens_seen": 2715811840 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018014141414141412, + "loss": 2.6096, + "theoretical_loss": 3.346521867116594, + "tokens_seen": 2715877376 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018012121212121213, + "loss": 2.604, + "theoretical_loss": 3.3465155320533273, + "tokens_seen": 2715942912 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001801010101010101, + "loss": 2.6083, + "theoretical_loss": 3.3465091971857257, + "tokens_seen": 2716008448 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001800808080808081, + "loss": 2.1912, + "theoretical_loss": 3.3465028625137787, + "tokens_seen": 2716073984 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018006060606060606, + "loss": 2.4818, + "theoretical_loss": 3.3464965280374748, + "tokens_seen": 2716139520 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018004040404040404, + "loss": 2.2881, + "theoretical_loss": 3.346490193756804, + "tokens_seen": 2716205056 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018002020202020203, + "loss": 2.465, + "theoretical_loss": 3.3464838596717557, + "tokens_seen": 2716270592 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017999999999999998, + "loss": 2.6133, + "theoretical_loss": 3.3464775257823183, + "tokens_seen": 2716336128 + }, + { + "epoch": 0.64, + "learning_rate": 0.000179979797979798, + "loss": 2.3731, + "theoretical_loss": 3.3464711920884813, + "tokens_seen": 2716401664 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017995959595959595, + "loss": 2.4465, + "theoretical_loss": 3.3464648585902346, + "tokens_seen": 2716467200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017993939393939394, + "loss": 2.565, + "theoretical_loss": 3.346458525287567, + "tokens_seen": 2716532736 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017991919191919192, + "loss": 2.552, + "theoretical_loss": 3.3464521921804673, + "tokens_seen": 2716598272 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001798989898989899, + "loss": 2.5929, + "theoretical_loss": 3.3464458592689255, + "tokens_seen": 2716663808 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1489419, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.378549337387085, + "objective/train/theoretical_loss": 3.3464395265529303, + "objective/train/tokens_used": 1075588576, + "theoretical_loss": 3.3464395265529303, + "tokens_seen": 2716729344 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001798787878787879, + "loss": 2.3665, + "theoretical_loss": 3.3464395265529303, + "tokens_seen": 2716729344 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017985858585858587, + "loss": 2.5537, + "theoretical_loss": 3.346433194032471, + "tokens_seen": 2716794880 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017983838383838383, + "loss": 2.4699, + "theoretical_loss": 3.3464268617075374, + "tokens_seen": 2716860416 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001798181818181818, + "loss": 2.5484, + "theoretical_loss": 3.3464205295781184, + "tokens_seen": 2716925952 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001797979797979798, + "loss": 2.5882, + "theoretical_loss": 3.346414197644203, + "tokens_seen": 2716991488 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017977777777777778, + "loss": 2.4017, + "theoretical_loss": 3.3464078659057805, + "tokens_seen": 2717057024 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017975757575757576, + "loss": 2.5802, + "theoretical_loss": 3.3464015343628404, + "tokens_seen": 2717122560 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017973737373737375, + "loss": 2.4962, + "theoretical_loss": 3.346395203015372, + "tokens_seen": 2717188096 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017971717171717173, + "loss": 2.3226, + "theoretical_loss": 3.346388871863364, + "tokens_seen": 2717253632 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001796969696969697, + "loss": 2.4841, + "theoretical_loss": 3.346382540906806, + "tokens_seen": 2717319168 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001796767676767677, + "loss": 2.3176, + "theoretical_loss": 3.3463762101456878, + "tokens_seen": 2717384704 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017965656565656566, + "loss": 2.6631, + "theoretical_loss": 3.346369879579998, + "tokens_seen": 2717450240 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017963636363636364, + "loss": 2.4856, + "theoretical_loss": 3.346363549209726, + "tokens_seen": 2717515776 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017961616161616163, + "loss": 2.5622, + "theoretical_loss": 3.3463572190348607, + "tokens_seen": 2717581312 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017959595959595958, + "loss": 2.4627, + "theoretical_loss": 3.346350889055392, + "tokens_seen": 2717646848 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001795757575757576, + "loss": 2.2519, + "theoretical_loss": 3.3463445592713086, + "tokens_seen": 2717712384 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017955555555555555, + "loss": 2.6045, + "theoretical_loss": 3.3463382296826003, + "tokens_seen": 2717777920 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017953535353535356, + "loss": 2.4538, + "theoretical_loss": 3.3463319002892558, + "tokens_seen": 2717843456 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017951515151515152, + "loss": 2.4552, + "theoretical_loss": 3.346325571091265, + "tokens_seen": 2717908992 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001794949494949495, + "loss": 2.4055, + "theoretical_loss": 3.346319242088617, + "tokens_seen": 2717974528 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017947474747474749, + "loss": 2.5917, + "theoretical_loss": 3.3463129132813005, + "tokens_seen": 2718040064 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017945454545454544, + "loss": 2.6042, + "theoretical_loss": 3.3463065846693048, + "tokens_seen": 2718105600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017943434343434345, + "loss": 2.4016, + "theoretical_loss": 3.34630025625262, + "tokens_seen": 2718171136 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001794141414141414, + "loss": 2.4873, + "theoretical_loss": 3.3462939280312347, + "tokens_seen": 2718236672 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001793939393939394, + "loss": 2.6103, + "theoretical_loss": 3.3462876000051383, + "tokens_seen": 2718302208 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1489769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7963850498199463, + "objective/train/theoretical_loss": 3.34628127217432, + "objective/train/tokens_used": 1077226976, + "theoretical_loss": 3.34628127217432, + "tokens_seen": 2718367744 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017937373737373738, + "loss": 2.6305, + "theoretical_loss": 3.34628127217432, + "tokens_seen": 2718367744 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017935353535353536, + "loss": 2.2062, + "theoretical_loss": 3.346274944538769, + "tokens_seen": 2718433280 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017933333333333335, + "loss": 2.4493, + "theoretical_loss": 3.3462686170984752, + "tokens_seen": 2718498816 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001793131313131313, + "loss": 2.7743, + "theoretical_loss": 3.346262289853427, + "tokens_seen": 2718564352 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001792929292929293, + "loss": 2.5875, + "theoretical_loss": 3.3462559628036144, + "tokens_seen": 2718629888 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017927272727272727, + "loss": 2.4126, + "theoretical_loss": 3.346249635949026, + "tokens_seen": 2718695424 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017925252525252526, + "loss": 2.6688, + "theoretical_loss": 3.3462433092896515, + "tokens_seen": 2718760960 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017923232323232324, + "loss": 2.4169, + "theoretical_loss": 3.34623698282548, + "tokens_seen": 2718826496 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017921212121212122, + "loss": 2.4834, + "theoretical_loss": 3.346230656556501, + "tokens_seen": 2718892032 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017919191919191918, + "loss": 2.5111, + "theoretical_loss": 3.3462243304827033, + "tokens_seen": 2718957568 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001791717171717172, + "loss": 2.4619, + "theoretical_loss": 3.3462180046040766, + "tokens_seen": 2719023104 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017915151515151515, + "loss": 2.564, + "theoretical_loss": 3.34621167892061, + "tokens_seen": 2719088640 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017913131313131313, + "loss": 2.3703, + "theoretical_loss": 3.346205353432293, + "tokens_seen": 2719154176 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017911111111111112, + "loss": 2.5059, + "theoretical_loss": 3.346199028139115, + "tokens_seen": 2719219712 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017909090909090907, + "loss": 2.624, + "theoretical_loss": 3.3461927030410648, + "tokens_seen": 2719285248 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017907070707070708, + "loss": 2.6148, + "theoretical_loss": 3.346186378138132, + "tokens_seen": 2719350784 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017905050505050504, + "loss": 2.3291, + "theoretical_loss": 3.346180053430305, + "tokens_seen": 2719416320 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017903030303030305, + "loss": 2.4597, + "theoretical_loss": 3.3461737289175746, + "tokens_seen": 2719481856 + }, + { + "epoch": 0.65, + "learning_rate": 0.000179010101010101, + "loss": 2.3228, + "theoretical_loss": 3.3461674045999286, + "tokens_seen": 2719547392 + }, + { + "epoch": 0.65, + "learning_rate": 0.000178989898989899, + "loss": 2.1143, + "theoretical_loss": 3.3461610804773576, + "tokens_seen": 2719612928 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017896969696969698, + "loss": 2.4988, + "theoretical_loss": 3.34615475654985, + "tokens_seen": 2719678464 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017894949494949493, + "loss": 2.6052, + "theoretical_loss": 3.3461484328173956, + "tokens_seen": 2719744000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017892929292929294, + "loss": 2.3632, + "theoretical_loss": 3.346142109279983, + "tokens_seen": 2719809536 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001789090909090909, + "loss": 2.3987, + "theoretical_loss": 3.3461357859376024, + "tokens_seen": 2719875072 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017888888888888889, + "loss": 2.2826, + "theoretical_loss": 3.346129462790242, + "tokens_seen": 2719940608 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1491149, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4027974605560303, + "objective/train/theoretical_loss": 3.346123139837892, + "objective/train/tokens_used": 1078865376, + "theoretical_loss": 3.346123139837892, + "tokens_seen": 2720006144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017886868686868687, + "loss": 2.3313, + "theoretical_loss": 3.346123139837892, + "tokens_seen": 2720006144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017884848484848485, + "loss": 2.2946, + "theoretical_loss": 3.3461168170805418, + "tokens_seen": 2720071680 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017882828282828284, + "loss": 2.6664, + "theoretical_loss": 3.3461104945181797, + "tokens_seen": 2720137216 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017880808080808082, + "loss": 2.554, + "theoretical_loss": 3.346104172150796, + "tokens_seen": 2720202752 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001787878787878788, + "loss": 2.5731, + "theoretical_loss": 3.346097849978379, + "tokens_seen": 2720268288 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017876767676767676, + "loss": 2.5387, + "theoretical_loss": 3.346091528000919, + "tokens_seen": 2720333824 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017874747474747475, + "loss": 2.622, + "theoretical_loss": 3.3460852062184046, + "tokens_seen": 2720399360 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017872727272727273, + "loss": 2.3865, + "theoretical_loss": 3.3460788846308254, + "tokens_seen": 2720464896 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017870707070707071, + "loss": 2.6337, + "theoretical_loss": 3.3460725632381703, + "tokens_seen": 2720530432 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001786868686868687, + "loss": 2.4348, + "theoretical_loss": 3.3460662420404295, + "tokens_seen": 2720595968 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017866666666666668, + "loss": 2.2586, + "theoretical_loss": 3.3460599210375914, + "tokens_seen": 2720661504 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017864646464646464, + "loss": 2.5056, + "theoretical_loss": 3.3460536002296455, + "tokens_seen": 2720727040 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017862626262626265, + "loss": 2.3681, + "theoretical_loss": 3.3460472796165814, + "tokens_seen": 2720792576 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001786060606060606, + "loss": 2.4087, + "theoretical_loss": 3.346040959198388, + "tokens_seen": 2720858112 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001785858585858586, + "loss": 2.7711, + "theoretical_loss": 3.346034638975055, + "tokens_seen": 2720923648 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017856565656565657, + "loss": 2.673, + "theoretical_loss": 3.3460283189465714, + "tokens_seen": 2720989184 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017854545454545453, + "loss": 2.4926, + "theoretical_loss": 3.3460219991129265, + "tokens_seen": 2721054720 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017852525252525254, + "loss": 2.7564, + "theoretical_loss": 3.34601567947411, + "tokens_seen": 2721120256 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001785050505050505, + "loss": 2.3423, + "theoretical_loss": 3.34600936003011, + "tokens_seen": 2721185792 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001784848484848485, + "loss": 2.4657, + "theoretical_loss": 3.346003040780918, + "tokens_seen": 2721251328 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017846464646464647, + "loss": 2.4228, + "theoretical_loss": 3.345996721726521, + "tokens_seen": 2721316864 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017844444444444442, + "loss": 2.4939, + "theoretical_loss": 3.3459904028669096, + "tokens_seen": 2721382400 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017842424242424244, + "loss": 2.5651, + "theoretical_loss": 3.345984084202073, + "tokens_seen": 2721447936 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001784040404040404, + "loss": 2.4951, + "theoretical_loss": 3.345977765732, + "tokens_seen": 2721513472 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001783838383838384, + "loss": 2.448, + "theoretical_loss": 3.3459714474566806, + "tokens_seen": 2721579008 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1491715, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6255178451538086, + "objective/train/theoretical_loss": 3.3459651293761032, + "objective/train/tokens_used": 1080503776, + "theoretical_loss": 3.3459651293761032, + "tokens_seen": 2721644544 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017836363636363636, + "loss": 2.6509, + "theoretical_loss": 3.3459651293761032, + "tokens_seen": 2721644544 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017834343434343434, + "loss": 2.277, + "theoretical_loss": 3.345958811490258, + "tokens_seen": 2721710080 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017832323232323233, + "loss": 2.5304, + "theoretical_loss": 3.345952493799134, + "tokens_seen": 2721775616 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001783030303030303, + "loss": 2.4756, + "theoretical_loss": 3.3459461763027205, + "tokens_seen": 2721841152 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001782828282828283, + "loss": 2.5428, + "theoretical_loss": 3.3459398590010068, + "tokens_seen": 2721906688 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017826262626262625, + "loss": 2.3946, + "theoretical_loss": 3.3459335418939817, + "tokens_seen": 2721972224 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017824242424242424, + "loss": 2.4894, + "theoretical_loss": 3.3459272249816356, + "tokens_seen": 2722037760 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017822222222222222, + "loss": 2.4735, + "theoretical_loss": 3.345920908263957, + "tokens_seen": 2722103296 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001782020202020202, + "loss": 2.3455, + "theoretical_loss": 3.345914591740935, + "tokens_seen": 2722168832 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001781818181818182, + "loss": 2.5208, + "theoretical_loss": 3.34590827541256, + "tokens_seen": 2722234368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017816161616161617, + "loss": 2.2801, + "theoretical_loss": 3.3459019592788204, + "tokens_seen": 2722299904 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017814141414141413, + "loss": 2.522, + "theoretical_loss": 3.3458956433397056, + "tokens_seen": 2722365440 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017812121212121214, + "loss": 2.457, + "theoretical_loss": 3.3458893275952053, + "tokens_seen": 2722430976 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001781010101010101, + "loss": 2.3007, + "theoretical_loss": 3.3458830120453085, + "tokens_seen": 2722496512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017808080808080808, + "loss": 2.5132, + "theoretical_loss": 3.3458766966900044, + "tokens_seen": 2722562048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017806060606060607, + "loss": 2.6731, + "theoretical_loss": 3.345870381529283, + "tokens_seen": 2722627584 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017804040404040402, + "loss": 2.4723, + "theoretical_loss": 3.3458640665631325, + "tokens_seen": 2722693120 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017802020202020203, + "loss": 2.6628, + "theoretical_loss": 3.3458577517915438, + "tokens_seen": 2722758656 + }, + { + "epoch": 0.65, + "learning_rate": 0.000178, + "loss": 2.4993, + "theoretical_loss": 3.3458514372145047, + "tokens_seen": 2722824192 + }, + { + "epoch": 0.65, + "learning_rate": 0.000177979797979798, + "loss": 2.3658, + "theoretical_loss": 3.345845122832005, + "tokens_seen": 2722889728 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017795959595959596, + "loss": 2.5685, + "theoretical_loss": 3.3458388086440345, + "tokens_seen": 2722955264 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017793939393939397, + "loss": 2.3321, + "theoretical_loss": 3.3458324946505824, + "tokens_seen": 2723020800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017791919191919193, + "loss": 2.6086, + "theoretical_loss": 3.3458261808516374, + "tokens_seen": 2723086336 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017789898989898988, + "loss": 2.3901, + "theoretical_loss": 3.3458198672471893, + "tokens_seen": 2723151872 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001778787878787879, + "loss": 2.4532, + "theoretical_loss": 3.3458135538372273, + "tokens_seen": 2723217408 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1492888, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6339731216430664, + "objective/train/theoretical_loss": 3.345807240621741, + "objective/train/tokens_used": 1082142176, + "theoretical_loss": 3.345807240621741, + "tokens_seen": 2723282944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017785858585858585, + "loss": 2.4715, + "theoretical_loss": 3.345807240621741, + "tokens_seen": 2723282944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017783838383838386, + "loss": 2.3342, + "theoretical_loss": 3.3458009276007195, + "tokens_seen": 2723348480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017781818181818182, + "loss": 2.4575, + "theoretical_loss": 3.345794614774152, + "tokens_seen": 2723414016 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001777979797979798, + "loss": 2.5119, + "theoretical_loss": 3.3457883021420285, + "tokens_seen": 2723479552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017777777777777779, + "loss": 2.4197, + "theoretical_loss": 3.3457819897043373, + "tokens_seen": 2723545088 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017775757575757577, + "loss": 2.4893, + "theoretical_loss": 3.3457756774610683, + "tokens_seen": 2723610624 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017773737373737375, + "loss": 2.543, + "theoretical_loss": 3.345769365412211, + "tokens_seen": 2723676160 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001777171717171717, + "loss": 2.3916, + "theoretical_loss": 3.3457630535577545, + "tokens_seen": 2723741696 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001776969696969697, + "loss": 2.3492, + "theoretical_loss": 3.345756741897688, + "tokens_seen": 2723807232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017767676767676768, + "loss": 2.5064, + "theoretical_loss": 3.345750430432001, + "tokens_seen": 2723872768 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017765656565656566, + "loss": 2.447, + "theoretical_loss": 3.345744119160683, + "tokens_seen": 2723938304 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017763636363636365, + "loss": 2.498, + "theoretical_loss": 3.345737808083723, + "tokens_seen": 2724003840 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017761616161616163, + "loss": 2.0675, + "theoretical_loss": 3.3457314972011103, + "tokens_seen": 2724069376 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001775959595959596, + "loss": 2.2933, + "theoretical_loss": 3.345725186512835, + "tokens_seen": 2724134912 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017757575757575757, + "loss": 2.4245, + "theoretical_loss": 3.3457188760188856, + "tokens_seen": 2724200448 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017755555555555556, + "loss": 2.4136, + "theoretical_loss": 3.3457125657192512, + "tokens_seen": 2724265984 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017753535353535354, + "loss": 2.3768, + "theoretical_loss": 3.3457062556139223, + "tokens_seen": 2724331520 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017751515151515152, + "loss": 2.3923, + "theoretical_loss": 3.3456999457028878, + "tokens_seen": 2724397056 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017749494949494948, + "loss": 2.5208, + "theoretical_loss": 3.3456936359861364, + "tokens_seen": 2724462592 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001774747474747475, + "loss": 2.4495, + "theoretical_loss": 3.345687326463658, + "tokens_seen": 2724528128 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017745454545454545, + "loss": 2.4585, + "theoretical_loss": 3.345681017135442, + "tokens_seen": 2724593664 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017743434343434346, + "loss": 2.5762, + "theoretical_loss": 3.3456747080014773, + "tokens_seen": 2724659200 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017741414141414142, + "loss": 2.4656, + "theoretical_loss": 3.3456683990617537, + "tokens_seen": 2724724736 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017739393939393937, + "loss": 2.2935, + "theoretical_loss": 3.3456620903162606, + "tokens_seen": 2724790272 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017737373737373738, + "loss": 2.4084, + "theoretical_loss": 3.345655781764987, + "tokens_seen": 2724855808 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1493539, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.459871768951416, + "objective/train/theoretical_loss": 3.3456494734079225, + "objective/train/tokens_used": 1083780576, + "theoretical_loss": 3.3456494734079225, + "tokens_seen": 2724921344 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017735353535353534, + "loss": 2.5583, + "theoretical_loss": 3.3456494734079225, + "tokens_seen": 2724921344 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017733333333333335, + "loss": 2.4765, + "theoretical_loss": 3.345643165245056, + "tokens_seen": 2724986880 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001773131313131313, + "loss": 2.3926, + "theoretical_loss": 3.3456368572763773, + "tokens_seen": 2725052416 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001772929292929293, + "loss": 2.2519, + "theoretical_loss": 3.345630549501876, + "tokens_seen": 2725117952 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017727272727272728, + "loss": 2.6669, + "theoretical_loss": 3.3456242419215405, + "tokens_seen": 2725183488 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017725252525252526, + "loss": 2.6278, + "theoretical_loss": 3.3456179345353614, + "tokens_seen": 2725249024 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017723232323232324, + "loss": 2.4478, + "theoretical_loss": 3.345611627343327, + "tokens_seen": 2725314560 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001772121212121212, + "loss": 2.5039, + "theoretical_loss": 3.345605320345427, + "tokens_seen": 2725380096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017719191919191919, + "loss": 2.6057, + "theoretical_loss": 3.345599013541651, + "tokens_seen": 2725445632 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017717171717171717, + "loss": 2.3337, + "theoretical_loss": 3.345592706931988, + "tokens_seen": 2725511168 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017715151515151515, + "loss": 2.4373, + "theoretical_loss": 3.345586400516428, + "tokens_seen": 2725576704 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017713131313131314, + "loss": 2.3604, + "theoretical_loss": 3.3455800942949594, + "tokens_seen": 2725642240 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017711111111111112, + "loss": 2.4601, + "theoretical_loss": 3.345573788267572, + "tokens_seen": 2725707776 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001770909090909091, + "loss": 2.578, + "theoretical_loss": 3.3455674824342556, + "tokens_seen": 2725773312 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001770707070707071, + "loss": 2.4867, + "theoretical_loss": 3.345561176794999, + "tokens_seen": 2725838848 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017705050505050505, + "loss": 2.3889, + "theoretical_loss": 3.3455548713497913, + "tokens_seen": 2725904384 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017703030303030303, + "loss": 2.4814, + "theoretical_loss": 3.345548566098623, + "tokens_seen": 2725969920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017701010101010101, + "loss": 2.3653, + "theoretical_loss": 3.3455422610414827, + "tokens_seen": 2726035456 + }, + { + "epoch": 0.65, + "learning_rate": 0.000176989898989899, + "loss": 2.3246, + "theoretical_loss": 3.3455359561783595, + "tokens_seen": 2726100992 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017696969696969698, + "loss": 2.3211, + "theoretical_loss": 3.345529651509243, + "tokens_seen": 2726166528 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017694949494949494, + "loss": 2.3399, + "theoretical_loss": 3.3455233470341232, + "tokens_seen": 2726232064 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017692929292929295, + "loss": 2.3903, + "theoretical_loss": 3.3455170427529883, + "tokens_seen": 2726297600 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001769090909090909, + "loss": 2.454, + "theoretical_loss": 3.345510738665829, + "tokens_seen": 2726363136 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017688888888888892, + "loss": 2.7146, + "theoretical_loss": 3.3455044347726335, + "tokens_seen": 2726428672 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017686868686868687, + "loss": 2.4177, + "theoretical_loss": 3.345498131073392, + "tokens_seen": 2726494208 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1494687, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.248422622680664, + "objective/train/theoretical_loss": 3.3454918275680927, + "objective/train/tokens_used": 1085418976, + "theoretical_loss": 3.3454918275680927, + "tokens_seen": 2726559744 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017684848484848483, + "loss": 2.6784, + "theoretical_loss": 3.3454918275680927, + "tokens_seen": 2726559744 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017682828282828284, + "loss": 2.5723, + "theoretical_loss": 3.3454855242567265, + "tokens_seen": 2726625280 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001768080808080808, + "loss": 2.6373, + "theoretical_loss": 3.3454792211392816, + "tokens_seen": 2726690816 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001767878787878788, + "loss": 2.255, + "theoretical_loss": 3.3454729182157483, + "tokens_seen": 2726756352 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017676767676767677, + "loss": 2.6882, + "theoretical_loss": 3.3454666154861155, + "tokens_seen": 2726821888 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017674747474747475, + "loss": 2.5089, + "theoretical_loss": 3.345460312950372, + "tokens_seen": 2726887424 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017672727272727274, + "loss": 2.4428, + "theoretical_loss": 3.345454010608508, + "tokens_seen": 2726952960 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001767070707070707, + "loss": 2.1321, + "theoretical_loss": 3.345447708460513, + "tokens_seen": 2727018496 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001766868686868687, + "loss": 2.2741, + "theoretical_loss": 3.3454414065063753, + "tokens_seen": 2727084032 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017666666666666666, + "loss": 2.5295, + "theoretical_loss": 3.3454351047460857, + "tokens_seen": 2727149568 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017664646464646464, + "loss": 2.3459, + "theoretical_loss": 3.3454288031796326, + "tokens_seen": 2727215104 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017662626262626263, + "loss": 2.5008, + "theoretical_loss": 3.3454225018070054, + "tokens_seen": 2727280640 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001766060606060606, + "loss": 2.5305, + "theoretical_loss": 3.3454162006281942, + "tokens_seen": 2727346176 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001765858585858586, + "loss": 2.5268, + "theoretical_loss": 3.3454098996431876, + "tokens_seen": 2727411712 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017656565656565658, + "loss": 2.5351, + "theoretical_loss": 3.3454035988519752, + "tokens_seen": 2727477248 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017654545454545454, + "loss": 2.5015, + "theoretical_loss": 3.3453972982545466, + "tokens_seen": 2727542784 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017652525252525252, + "loss": 2.3463, + "theoretical_loss": 3.345390997850891, + "tokens_seen": 2727608320 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001765050505050505, + "loss": 2.4823, + "theoretical_loss": 3.345384697640998, + "tokens_seen": 2727673856 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001764848484848485, + "loss": 2.5309, + "theoretical_loss": 3.3453783976248563, + "tokens_seen": 2727739392 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017646464646464647, + "loss": 2.7278, + "theoretical_loss": 3.345372097802456, + "tokens_seen": 2727804928 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017644444444444443, + "loss": 2.3179, + "theoretical_loss": 3.3453657981737868, + "tokens_seen": 2727870464 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017642424242424244, + "loss": 2.6319, + "theoretical_loss": 3.3453594987388375, + "tokens_seen": 2727936000 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001764040404040404, + "loss": 2.4107, + "theoretical_loss": 3.3453531994975974, + "tokens_seen": 2728001536 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001763838383838384, + "loss": 2.4977, + "theoretical_loss": 3.345346900450056, + "tokens_seen": 2728067072 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017636363636363637, + "loss": 2.484, + "theoretical_loss": 3.345340601596203, + "tokens_seen": 2728132608 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1495196, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6293399333953857, + "objective/train/theoretical_loss": 3.3453343029360267, + "objective/train/tokens_used": 1087057376, + "theoretical_loss": 3.3453343029360267, + "tokens_seen": 2728198144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017634343434343432, + "loss": 2.6006, + "theoretical_loss": 3.3453343029360267, + "tokens_seen": 2728198144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017632323232323233, + "loss": 2.2541, + "theoretical_loss": 3.3453280044695184, + "tokens_seen": 2728263680 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001763030303030303, + "loss": 2.6532, + "theoretical_loss": 3.345321706196666, + "tokens_seen": 2728329216 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001762828282828283, + "loss": 2.6359, + "theoretical_loss": 3.345315408117459, + "tokens_seen": 2728394752 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017626262626262626, + "loss": 2.3939, + "theoretical_loss": 3.345309110231888, + "tokens_seen": 2728460288 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017624242424242427, + "loss": 2.6302, + "theoretical_loss": 3.3453028125399404, + "tokens_seen": 2728525824 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017622222222222223, + "loss": 2.3937, + "theoretical_loss": 3.345296515041608, + "tokens_seen": 2728591360 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001762020202020202, + "loss": 2.4791, + "theoretical_loss": 3.3452902177368777, + "tokens_seen": 2728656896 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001761818181818182, + "loss": 2.292, + "theoretical_loss": 3.3452839206257408, + "tokens_seen": 2728722432 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017616161616161615, + "loss": 2.368, + "theoretical_loss": 3.345277623708186, + "tokens_seen": 2728787968 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017614141414141416, + "loss": 2.3361, + "theoretical_loss": 3.3452713269842023, + "tokens_seen": 2728853504 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017612121212121212, + "loss": 2.5059, + "theoretical_loss": 3.34526503045378, + "tokens_seen": 2728919040 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001761010101010101, + "loss": 2.3078, + "theoretical_loss": 3.3452587341169076, + "tokens_seen": 2728984576 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001760808080808081, + "loss": 2.3487, + "theoretical_loss": 3.345252437973575, + "tokens_seen": 2729050112 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017606060606060607, + "loss": 2.3791, + "theoretical_loss": 3.345246142023772, + "tokens_seen": 2729115648 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017604040404040405, + "loss": 2.5723, + "theoretical_loss": 3.3452398462674866, + "tokens_seen": 2729181184 + }, + { + "epoch": 0.65, + "learning_rate": 0.000176020202020202, + "loss": 2.5314, + "theoretical_loss": 3.3452335507047097, + "tokens_seen": 2729246720 + }, + { + "epoch": 0.65, + "learning_rate": 0.000176, + "loss": 2.5865, + "theoretical_loss": 3.34522725533543, + "tokens_seen": 2729312256 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017597979797979798, + "loss": 2.513, + "theoretical_loss": 3.3452209601596374, + "tokens_seen": 2729377792 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017595959595959596, + "loss": 2.5898, + "theoretical_loss": 3.3452146651773207, + "tokens_seen": 2729443328 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017593939393939395, + "loss": 2.2181, + "theoretical_loss": 3.3452083703884696, + "tokens_seen": 2729508864 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017591919191919193, + "loss": 2.5111, + "theoretical_loss": 3.345202075793073, + "tokens_seen": 2729574400 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001758989898989899, + "loss": 2.294, + "theoretical_loss": 3.3451957813911215, + "tokens_seen": 2729639936 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001758787878787879, + "loss": 2.513, + "theoretical_loss": 3.3451894871826036, + "tokens_seen": 2729705472 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017585858585858586, + "loss": 2.5602, + "theoretical_loss": 3.3451831931675087, + "tokens_seen": 2729771008 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1496443, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.345172882080078, + "objective/train/theoretical_loss": 3.3451768993458266, + "objective/train/tokens_used": 1088695776, + "theoretical_loss": 3.3451768993458266, + "tokens_seen": 2729836544 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017583838383838384, + "loss": 2.5411, + "theoretical_loss": 3.3451768993458266, + "tokens_seen": 2729836544 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017581818181818182, + "loss": 2.6169, + "theoretical_loss": 3.3451706057175463, + "tokens_seen": 2729902080 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017579797979797978, + "loss": 2.4187, + "theoretical_loss": 3.3451643122826575, + "tokens_seen": 2729967616 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001757777777777778, + "loss": 2.5986, + "theoretical_loss": 3.3451580190411496, + "tokens_seen": 2730033152 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017575757575757575, + "loss": 2.6854, + "theoretical_loss": 3.345151725993012, + "tokens_seen": 2730098688 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017573737373737376, + "loss": 2.6153, + "theoretical_loss": 3.345145433138234, + "tokens_seen": 2730164224 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017571717171717172, + "loss": 2.5143, + "theoretical_loss": 3.345139140476805, + "tokens_seen": 2730229760 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001756969696969697, + "loss": 2.7027, + "theoretical_loss": 3.345132848008715, + "tokens_seen": 2730295296 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017567676767676768, + "loss": 2.2668, + "theoretical_loss": 3.3451265557339527, + "tokens_seen": 2730360832 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017565656565656564, + "loss": 2.501, + "theoretical_loss": 3.3451202636525075, + "tokens_seen": 2730426368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017563636363636365, + "loss": 2.5209, + "theoretical_loss": 3.3451139717643694, + "tokens_seen": 2730491904 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001756161616161616, + "loss": 2.5928, + "theoretical_loss": 3.3451076800695274, + "tokens_seen": 2730557440 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001755959595959596, + "loss": 2.276, + "theoretical_loss": 3.3451013885679712, + "tokens_seen": 2730622976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017557575757575758, + "loss": 2.2878, + "theoretical_loss": 3.34509509725969, + "tokens_seen": 2730688512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017555555555555556, + "loss": 2.6027, + "theoretical_loss": 3.345088806144673, + "tokens_seen": 2730754048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017553535353535355, + "loss": 2.7153, + "theoretical_loss": 3.34508251522291, + "tokens_seen": 2730819584 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017551515151515153, + "loss": 2.4384, + "theoretical_loss": 3.3450762244943903, + "tokens_seen": 2730885120 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017549494949494949, + "loss": 2.5186, + "theoretical_loss": 3.3450699339591035, + "tokens_seen": 2730950656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017547474747474747, + "loss": 2.3682, + "theoretical_loss": 3.3450636436170385, + "tokens_seen": 2731016192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017545454545454545, + "loss": 2.5585, + "theoretical_loss": 3.3450573534681856, + "tokens_seen": 2731081728 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017543434343434344, + "loss": 2.3031, + "theoretical_loss": 3.3450510635125337, + "tokens_seen": 2731147264 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017541414141414142, + "loss": 2.48, + "theoretical_loss": 3.3450447737500717, + "tokens_seen": 2731212800 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001753939393939394, + "loss": 2.3915, + "theoretical_loss": 3.3450384841807903, + "tokens_seen": 2731278336 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001753737373737374, + "loss": 2.6075, + "theoretical_loss": 3.345032194804678, + "tokens_seen": 2731343872 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017535353535353535, + "loss": 2.3865, + "theoretical_loss": 3.3450259056217244, + "tokens_seen": 2731409408 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1497834, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.771178960800171, + "objective/train/theoretical_loss": 3.345019616631919, + "objective/train/tokens_used": 1090334176, + "theoretical_loss": 3.345019616631919, + "tokens_seen": 2731474944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017533333333333336, + "loss": 2.5906, + "theoretical_loss": 3.345019616631919, + "tokens_seen": 2731474944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017531313131313131, + "loss": 2.5288, + "theoretical_loss": 3.345013327835251, + "tokens_seen": 2731540480 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001752929292929293, + "loss": 2.4116, + "theoretical_loss": 3.3450070392317106, + "tokens_seen": 2731606016 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017527272727272728, + "loss": 2.5544, + "theoretical_loss": 3.3450007508212862, + "tokens_seen": 2731671552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017525252525252524, + "loss": 2.3793, + "theoretical_loss": 3.3449944626039683, + "tokens_seen": 2731737088 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017523232323232325, + "loss": 2.6695, + "theoretical_loss": 3.344988174579745, + "tokens_seen": 2731802624 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001752121212121212, + "loss": 2.6443, + "theoretical_loss": 3.344981886748607, + "tokens_seen": 2731868160 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017519191919191922, + "loss": 2.3688, + "theoretical_loss": 3.344975599110543, + "tokens_seen": 2731933696 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017517171717171718, + "loss": 2.4677, + "theoretical_loss": 3.344969311665543, + "tokens_seen": 2731999232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017515151515151513, + "loss": 2.7443, + "theoretical_loss": 3.3449630244135964, + "tokens_seen": 2732064768 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017513131313131314, + "loss": 2.4406, + "theoretical_loss": 3.344956737354692, + "tokens_seen": 2732130304 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001751111111111111, + "loss": 2.6151, + "theoretical_loss": 3.3449504504888194, + "tokens_seen": 2732195840 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001750909090909091, + "loss": 2.334, + "theoretical_loss": 3.3449441638159687, + "tokens_seen": 2732261376 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017507070707070707, + "loss": 2.4341, + "theoretical_loss": 3.344937877336129, + "tokens_seen": 2732326912 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017505050505050505, + "loss": 2.556, + "theoretical_loss": 3.344931591049289, + "tokens_seen": 2732392448 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017503030303030304, + "loss": 2.5427, + "theoretical_loss": 3.344925304955439, + "tokens_seen": 2732457984 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017501010101010102, + "loss": 2.5749, + "theoretical_loss": 3.3449190190545686, + "tokens_seen": 2732523520 + }, + { + "epoch": 0.65, + "learning_rate": 0.000174989898989899, + "loss": 2.4344, + "theoretical_loss": 3.344912733346667, + "tokens_seen": 2732589056 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017496969696969696, + "loss": 2.5658, + "theoretical_loss": 3.3449064478317228, + "tokens_seen": 2732654592 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017494949494949494, + "loss": 2.4196, + "theoretical_loss": 3.3449001625097265, + "tokens_seen": 2732720128 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017492929292929293, + "loss": 2.3603, + "theoretical_loss": 3.3448938773806676, + "tokens_seen": 2732785664 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001749090909090909, + "loss": 2.5436, + "theoretical_loss": 3.344887592444535, + "tokens_seen": 2732851200 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001748888888888889, + "loss": 2.449, + "theoretical_loss": 3.344881307701318, + "tokens_seen": 2732916736 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017486868686868688, + "loss": 2.6443, + "theoretical_loss": 3.344875023151007, + "tokens_seen": 2732982272 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017484848484848484, + "loss": 2.5382, + "theoretical_loss": 3.3448687387935903, + "tokens_seen": 2733047808 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1498342, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.313222885131836, + "objective/train/theoretical_loss": 3.344862454629058, + "objective/train/tokens_used": 1091972576, + "theoretical_loss": 3.344862454629058, + "tokens_seen": 2733113344 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017482828282828285, + "loss": 2.5151, + "theoretical_loss": 3.344862454629058, + "tokens_seen": 2733113344 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001748080808080808, + "loss": 2.582, + "theoretical_loss": 3.3448561706574, + "tokens_seen": 2733178880 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001747878787878788, + "loss": 2.5593, + "theoretical_loss": 3.3448498868786047, + "tokens_seen": 2733244416 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017476767676767677, + "loss": 2.5863, + "theoretical_loss": 3.344843603292662, + "tokens_seen": 2733309952 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017474747474747473, + "loss": 2.4391, + "theoretical_loss": 3.3448373198995616, + "tokens_seen": 2733375488 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017472727272727274, + "loss": 2.5547, + "theoretical_loss": 3.3448310366992926, + "tokens_seen": 2733441024 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001747070707070707, + "loss": 2.3641, + "theoretical_loss": 3.344824753691845, + "tokens_seen": 2733506560 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001746868686868687, + "loss": 2.5537, + "theoretical_loss": 3.3448184708772075, + "tokens_seen": 2733572096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017466666666666667, + "loss": 2.4433, + "theoretical_loss": 3.3448121882553705, + "tokens_seen": 2733637632 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017464646464646465, + "loss": 2.3546, + "theoretical_loss": 3.3448059058263224, + "tokens_seen": 2733703168 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017462626262626263, + "loss": 2.3936, + "theoretical_loss": 3.3447996235900535, + "tokens_seen": 2733768704 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001746060606060606, + "loss": 2.5517, + "theoretical_loss": 3.344793341546553, + "tokens_seen": 2733834240 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001745858585858586, + "loss": 2.4965, + "theoretical_loss": 3.3447870596958102, + "tokens_seen": 2733899776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017456565656565656, + "loss": 2.7289, + "theoretical_loss": 3.3447807780378147, + "tokens_seen": 2733965312 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017454545454545457, + "loss": 2.343, + "theoretical_loss": 3.3447744965725557, + "tokens_seen": 2734030848 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017452525252525253, + "loss": 2.4557, + "theoretical_loss": 3.3447682153000233, + "tokens_seen": 2734096384 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001745050505050505, + "loss": 2.2667, + "theoretical_loss": 3.344761934220206, + "tokens_seen": 2734161920 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001744848484848485, + "loss": 2.4983, + "theoretical_loss": 3.3447556533330944, + "tokens_seen": 2734227456 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017446464646464648, + "loss": 2.581, + "theoretical_loss": 3.3447493726386774, + "tokens_seen": 2734292992 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017444444444444446, + "loss": 2.6461, + "theoretical_loss": 3.3447430921369445, + "tokens_seen": 2734358528 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017442424242424242, + "loss": 2.7583, + "theoretical_loss": 3.344736811827885, + "tokens_seen": 2734424064 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001744040404040404, + "loss": 2.5833, + "theoretical_loss": 3.3447305317114884, + "tokens_seen": 2734489600 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001743838383838384, + "loss": 2.463, + "theoretical_loss": 3.3447242517877447, + "tokens_seen": 2734555136 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017436363636363637, + "loss": 2.6402, + "theoretical_loss": 3.3447179720566425, + "tokens_seen": 2734620672 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017434343434343435, + "loss": 2.646, + "theoretical_loss": 3.344711692518172, + "tokens_seen": 2734686208 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1499499, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.72967791557312, + "objective/train/theoretical_loss": 3.3447054131723224, + "objective/train/tokens_used": 1093610976, + "theoretical_loss": 3.3447054131723224, + "tokens_seen": 2734751744 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017432323232323234, + "loss": 2.5635, + "theoretical_loss": 3.3447054131723224, + "tokens_seen": 2734751744 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001743030303030303, + "loss": 2.3556, + "theoretical_loss": 3.3446991340190833, + "tokens_seen": 2734817280 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017428282828282828, + "loss": 2.4384, + "theoretical_loss": 3.344692855058444, + "tokens_seen": 2734882816 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017426262626262626, + "loss": 2.5683, + "theoretical_loss": 3.344686576290394, + "tokens_seen": 2734948352 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017424242424242425, + "loss": 2.5014, + "theoretical_loss": 3.344680297714923, + "tokens_seen": 2735013888 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017422222222222223, + "loss": 2.3371, + "theoretical_loss": 3.34467401933202, + "tokens_seen": 2735079424 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001742020202020202, + "loss": 2.5409, + "theoretical_loss": 3.344667741141675, + "tokens_seen": 2735144960 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001741818181818182, + "loss": 2.3913, + "theoretical_loss": 3.3446614631438774, + "tokens_seen": 2735210496 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017416161616161616, + "loss": 2.5409, + "theoretical_loss": 3.3446551853386164, + "tokens_seen": 2735276032 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017414141414141417, + "loss": 2.3571, + "theoretical_loss": 3.3446489077258814, + "tokens_seen": 2735341568 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017412121212121212, + "loss": 2.3962, + "theoretical_loss": 3.3446426303056622, + "tokens_seen": 2735407104 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017410101010101008, + "loss": 2.4922, + "theoretical_loss": 3.3446363530779486, + "tokens_seen": 2735472640 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001740808080808081, + "loss": 2.5545, + "theoretical_loss": 3.3446300760427294, + "tokens_seen": 2735538176 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017406060606060605, + "loss": 2.3725, + "theoretical_loss": 3.3446237991999945, + "tokens_seen": 2735603712 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017404040404040406, + "loss": 2.4952, + "theoretical_loss": 3.344617522549733, + "tokens_seen": 2735669248 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017402020202020202, + "loss": 2.3943, + "theoretical_loss": 3.344611246091935, + "tokens_seen": 2735734784 + }, + { + "epoch": 0.66, + "learning_rate": 0.000174, + "loss": 2.4569, + "theoretical_loss": 3.3446049698265896, + "tokens_seen": 2735800320 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017397979797979798, + "loss": 2.66, + "theoretical_loss": 3.344598693753686, + "tokens_seen": 2735865856 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017395959595959597, + "loss": 2.4489, + "theoretical_loss": 3.3445924178732143, + "tokens_seen": 2735931392 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017393939393939395, + "loss": 2.5073, + "theoretical_loss": 3.3445861421851637, + "tokens_seen": 2735996928 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001739191919191919, + "loss": 2.286, + "theoretical_loss": 3.3445798666895237, + "tokens_seen": 2736062464 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001738989898989899, + "loss": 2.637, + "theoretical_loss": 3.3445735913862835, + "tokens_seen": 2736128000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017387878787878788, + "loss": 2.4915, + "theoretical_loss": 3.344567316275433, + "tokens_seen": 2736193536 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017385858585858586, + "loss": 2.3633, + "theoretical_loss": 3.344561041356962, + "tokens_seen": 2736259072 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017383838383838385, + "loss": 2.3622, + "theoretical_loss": 3.344554766630859, + "tokens_seen": 2736324608 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1499930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.625838041305542, + "objective/train/theoretical_loss": 3.3445484920971142, + "objective/train/tokens_used": 1095249376, + "theoretical_loss": 3.3445484920971142, + "tokens_seen": 2736390144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017381818181818183, + "loss": 2.6424, + "theoretical_loss": 3.3445484920971142, + "tokens_seen": 2736390144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017379797979797979, + "loss": 2.6782, + "theoretical_loss": 3.344542217755717, + "tokens_seen": 2736455680 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001737777777777778, + "loss": 2.5903, + "theoretical_loss": 3.344535943606657, + "tokens_seen": 2736521216 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017375757575757575, + "loss": 2.3787, + "theoretical_loss": 3.3445296696499236, + "tokens_seen": 2736586752 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017373737373737374, + "loss": 2.501, + "theoretical_loss": 3.344523395885506, + "tokens_seen": 2736652288 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017371717171717172, + "loss": 2.377, + "theoretical_loss": 3.344517122313394, + "tokens_seen": 2736717824 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001736969696969697, + "loss": 2.3833, + "theoretical_loss": 3.3445108489335773, + "tokens_seen": 2736783360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001736767676767677, + "loss": 2.6227, + "theoretical_loss": 3.3445045757460448, + "tokens_seen": 2736848896 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017365656565656565, + "loss": 2.5532, + "theoretical_loss": 3.3444983027507864, + "tokens_seen": 2736914432 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017363636363636366, + "loss": 2.3314, + "theoretical_loss": 3.3444920299477916, + "tokens_seen": 2736979968 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017361616161616161, + "loss": 2.7198, + "theoretical_loss": 3.34448575733705, + "tokens_seen": 2737045504 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017359595959595963, + "loss": 2.587, + "theoretical_loss": 3.3444794849185513, + "tokens_seen": 2737111040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017357575757575758, + "loss": 2.4204, + "theoretical_loss": 3.344473212692284, + "tokens_seen": 2737176576 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017355555555555554, + "loss": 2.3673, + "theoretical_loss": 3.344466940658239, + "tokens_seen": 2737242112 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017353535353535355, + "loss": 2.6659, + "theoretical_loss": 3.3444606688164047, + "tokens_seen": 2737307648 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001735151515151515, + "loss": 2.4983, + "theoretical_loss": 3.344454397166771, + "tokens_seen": 2737373184 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017349494949494952, + "loss": 2.3814, + "theoretical_loss": 3.3444481257093273, + "tokens_seen": 2737438720 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017347474747474748, + "loss": 2.6721, + "theoretical_loss": 3.3444418544440633, + "tokens_seen": 2737504256 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017345454545454546, + "loss": 2.4244, + "theoretical_loss": 3.3444355833709682, + "tokens_seen": 2737569792 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017343434343434344, + "loss": 2.5724, + "theoretical_loss": 3.344429312490032, + "tokens_seen": 2737635328 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001734141414141414, + "loss": 2.68, + "theoretical_loss": 3.344423041801244, + "tokens_seen": 2737700864 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001733939393939394, + "loss": 2.5312, + "theoretical_loss": 3.3444167713045934, + "tokens_seen": 2737766400 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017337373737373737, + "loss": 2.45, + "theoretical_loss": 3.34441050100007, + "tokens_seen": 2737831936 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017335353535353535, + "loss": 2.4462, + "theoretical_loss": 3.3444042308876636, + "tokens_seen": 2737897472 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017333333333333334, + "loss": 2.4034, + "theoretical_loss": 3.3443979609673633, + "tokens_seen": 2737963008 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1501408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4893600940704346, + "objective/train/theoretical_loss": 3.3443916912391587, + "objective/train/tokens_used": 1096887776, + "theoretical_loss": 3.3443916912391587, + "tokens_seen": 2738028544 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017331313131313132, + "loss": 2.4473, + "theoretical_loss": 3.3443916912391587, + "tokens_seen": 2738028544 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001732929292929293, + "loss": 2.4524, + "theoretical_loss": 3.3443854217030395, + "tokens_seen": 2738094080 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001732727272727273, + "loss": 2.5657, + "theoretical_loss": 3.3443791523589947, + "tokens_seen": 2738159616 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017325252525252524, + "loss": 2.4131, + "theoretical_loss": 3.3443728832070145, + "tokens_seen": 2738225152 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017323232323232323, + "loss": 2.3343, + "theoretical_loss": 3.344366614247088, + "tokens_seen": 2738290688 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001732121212121212, + "loss": 2.3796, + "theoretical_loss": 3.344360345479205, + "tokens_seen": 2738356224 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001731919191919192, + "loss": 2.4422, + "theoretical_loss": 3.3443540769033544, + "tokens_seen": 2738421760 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017317171717171718, + "loss": 2.5798, + "theoretical_loss": 3.344347808519526, + "tokens_seen": 2738487296 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017315151515151514, + "loss": 2.3749, + "theoretical_loss": 3.34434154032771, + "tokens_seen": 2738552832 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017313131313131315, + "loss": 2.1612, + "theoretical_loss": 3.3443352723278954, + "tokens_seen": 2738618368 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001731111111111111, + "loss": 2.4693, + "theoretical_loss": 3.3443290045200715, + "tokens_seen": 2738683904 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017309090909090912, + "loss": 2.4712, + "theoretical_loss": 3.344322736904228, + "tokens_seen": 2738749440 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017307070707070707, + "loss": 2.4421, + "theoretical_loss": 3.3443164694803547, + "tokens_seen": 2738814976 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017305050505050503, + "loss": 2.4397, + "theoretical_loss": 3.3443102022484408, + "tokens_seen": 2738880512 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017303030303030304, + "loss": 2.688, + "theoretical_loss": 3.344303935208476, + "tokens_seen": 2738946048 + }, + { + "epoch": 0.66, + "learning_rate": 0.000173010101010101, + "loss": 2.6083, + "theoretical_loss": 3.3442976683604493, + "tokens_seen": 2739011584 + }, + { + "epoch": 0.66, + "learning_rate": 0.000172989898989899, + "loss": 2.6289, + "theoretical_loss": 3.3442914017043512, + "tokens_seen": 2739077120 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017296969696969697, + "loss": 2.5174, + "theoretical_loss": 3.344285135240171, + "tokens_seen": 2739142656 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017294949494949495, + "loss": 2.6495, + "theoretical_loss": 3.3442788689678973, + "tokens_seen": 2739208192 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017292929292929293, + "loss": 2.3474, + "theoretical_loss": 3.3442726028875205, + "tokens_seen": 2739273728 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017290909090909092, + "loss": 2.4873, + "theoretical_loss": 3.3442663369990298, + "tokens_seen": 2739339264 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001728888888888889, + "loss": 2.3001, + "theoretical_loss": 3.344260071302415, + "tokens_seen": 2739404800 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017286868686868686, + "loss": 2.6948, + "theoretical_loss": 3.3442538057976656, + "tokens_seen": 2739470336 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017284848484848484, + "loss": 2.3724, + "theoretical_loss": 3.344247540484771, + "tokens_seen": 2739535872 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017282828282828283, + "loss": 2.3583, + "theoretical_loss": 3.3442412753637205, + "tokens_seen": 2739601408 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1502052, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.483609199523926, + "objective/train/theoretical_loss": 3.344235010434504, + "objective/train/tokens_used": 1098526176, + "theoretical_loss": 3.344235010434504, + "tokens_seen": 2739666944 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001728080808080808, + "loss": 2.5029, + "theoretical_loss": 3.344235010434504, + "tokens_seen": 2739666944 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001727878787878788, + "loss": 2.5748, + "theoretical_loss": 3.3442287456971114, + "tokens_seen": 2739732480 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017276767676767678, + "loss": 2.5983, + "theoretical_loss": 3.344222481151531, + "tokens_seen": 2739798016 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017274747474747476, + "loss": 2.5335, + "theoretical_loss": 3.3442162167977534, + "tokens_seen": 2739863552 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017272727272727272, + "loss": 2.4976, + "theoretical_loss": 3.344209952635768, + "tokens_seen": 2739929088 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001727070707070707, + "loss": 2.6703, + "theoretical_loss": 3.344203688665564, + "tokens_seen": 2739994624 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001726868686868687, + "loss": 2.4644, + "theoretical_loss": 3.344197424887131, + "tokens_seen": 2740060160 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017266666666666667, + "loss": 2.4428, + "theoretical_loss": 3.344191161300459, + "tokens_seen": 2740125696 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017264646464646466, + "loss": 2.5132, + "theoretical_loss": 3.3441848979055373, + "tokens_seen": 2740191232 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017262626262626264, + "loss": 2.4459, + "theoretical_loss": 3.344178634702355, + "tokens_seen": 2740256768 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001726060606060606, + "loss": 2.3425, + "theoretical_loss": 3.3441723716909024, + "tokens_seen": 2740322304 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001725858585858586, + "loss": 2.5537, + "theoretical_loss": 3.3441661088711685, + "tokens_seen": 2740387840 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017256565656565656, + "loss": 2.3677, + "theoretical_loss": 3.3441598462431426, + "tokens_seen": 2740453376 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017254545454545455, + "loss": 2.4816, + "theoretical_loss": 3.344153583806815, + "tokens_seen": 2740518912 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017252525252525253, + "loss": 2.6452, + "theoretical_loss": 3.3441473215621746, + "tokens_seen": 2740584448 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001725050505050505, + "loss": 2.4854, + "theoretical_loss": 3.344141059509212, + "tokens_seen": 2740649984 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001724848484848485, + "loss": 2.5062, + "theoretical_loss": 3.344134797647915, + "tokens_seen": 2740715520 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017246464646464646, + "loss": 2.3549, + "theoretical_loss": 3.344128535978275, + "tokens_seen": 2740781056 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017244444444444447, + "loss": 2.2427, + "theoretical_loss": 3.3441222745002803, + "tokens_seen": 2740846592 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017242424242424242, + "loss": 2.5639, + "theoretical_loss": 3.3441160132139207, + "tokens_seen": 2740912128 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001724040404040404, + "loss": 2.1027, + "theoretical_loss": 3.344109752119186, + "tokens_seen": 2740977664 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001723838383838384, + "loss": 2.4572, + "theoretical_loss": 3.3441034912160656, + "tokens_seen": 2741043200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017236363636363635, + "loss": 2.3577, + "theoretical_loss": 3.3440972305045493, + "tokens_seen": 2741108736 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017234343434343436, + "loss": 2.4761, + "theoretical_loss": 3.3440909699846264, + "tokens_seen": 2741174272 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017232323232323232, + "loss": 2.251, + "theoretical_loss": 3.3440847096562867, + "tokens_seen": 2741239808 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1503417, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7360618114471436, + "objective/train/theoretical_loss": 3.344078449519519, + "objective/train/tokens_used": 1100164576, + "theoretical_loss": 3.344078449519519, + "tokens_seen": 2741305344 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001723030303030303, + "loss": 2.5625, + "theoretical_loss": 3.344078449519519, + "tokens_seen": 2741305344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017228282828282829, + "loss": 2.4135, + "theoretical_loss": 3.344072189574314, + "tokens_seen": 2741370880 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017226262626262627, + "loss": 2.5926, + "theoretical_loss": 3.3440659298206605, + "tokens_seen": 2741436416 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017224242424242425, + "loss": 2.6506, + "theoretical_loss": 3.3440596702585483, + "tokens_seen": 2741501952 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017222222222222224, + "loss": 2.3075, + "theoretical_loss": 3.344053410887967, + "tokens_seen": 2741567488 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001722020202020202, + "loss": 2.4401, + "theoretical_loss": 3.3440471517089057, + "tokens_seen": 2741633024 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017218181818181818, + "loss": 2.5614, + "theoretical_loss": 3.3440408927213547, + "tokens_seen": 2741698560 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017216161616161616, + "loss": 2.707, + "theoretical_loss": 3.344034633925303, + "tokens_seen": 2741764096 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017214141414141415, + "loss": 2.4705, + "theoretical_loss": 3.3440283753207405, + "tokens_seen": 2741829632 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017212121212121213, + "loss": 2.3959, + "theoretical_loss": 3.3440221169076563, + "tokens_seen": 2741895168 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001721010101010101, + "loss": 2.4262, + "theoretical_loss": 3.3440158586860407, + "tokens_seen": 2741960704 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001720808080808081, + "loss": 2.6377, + "theoretical_loss": 3.3440096006558826, + "tokens_seen": 2742026240 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017206060606060605, + "loss": 2.6157, + "theoretical_loss": 3.344003342817172, + "tokens_seen": 2742091776 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017204040404040407, + "loss": 2.3218, + "theoretical_loss": 3.343997085169898, + "tokens_seen": 2742157312 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017202020202020202, + "loss": 2.3688, + "theoretical_loss": 3.343990827714051, + "tokens_seen": 2742222848 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017199999999999998, + "loss": 2.7244, + "theoretical_loss": 3.3439845704496194, + "tokens_seen": 2742288384 + }, + { + "epoch": 0.66, + "learning_rate": 0.000171979797979798, + "loss": 2.4522, + "theoretical_loss": 3.343978313376594, + "tokens_seen": 2742353920 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017195959595959595, + "loss": 2.3829, + "theoretical_loss": 3.3439720564949633, + "tokens_seen": 2742419456 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017193939393939396, + "loss": 2.5424, + "theoretical_loss": 3.3439657998047174, + "tokens_seen": 2742484992 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017191919191919192, + "loss": 2.4937, + "theoretical_loss": 3.343959543305846, + "tokens_seen": 2742550528 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017189898989898993, + "loss": 2.4347, + "theoretical_loss": 3.3439532869983384, + "tokens_seen": 2742616064 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017187878787878788, + "loss": 2.6582, + "theoretical_loss": 3.3439470308821844, + "tokens_seen": 2742681600 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017185858585858584, + "loss": 2.4595, + "theoretical_loss": 3.3439407749573733, + "tokens_seen": 2742747136 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017183838383838385, + "loss": 2.5899, + "theoretical_loss": 3.343934519223895, + "tokens_seen": 2742812672 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001718181818181818, + "loss": 2.5765, + "theoretical_loss": 3.3439282636817387, + "tokens_seen": 2742878208 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1504564, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4868407249450684, + "objective/train/theoretical_loss": 3.3439220083308943, + "objective/train/tokens_used": 1101802976, + "theoretical_loss": 3.3439220083308943, + "tokens_seen": 2742943744 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017179797979797982, + "loss": 2.295, + "theoretical_loss": 3.3439220083308943, + "tokens_seen": 2742943744 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017177777777777778, + "loss": 2.4951, + "theoretical_loss": 3.343915753171351, + "tokens_seen": 2743009280 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017175757575757576, + "loss": 2.4665, + "theoretical_loss": 3.343909498203099, + "tokens_seen": 2743074816 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017173737373737374, + "loss": 2.5766, + "theoretical_loss": 3.3439032434261273, + "tokens_seen": 2743140352 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017171717171717173, + "loss": 2.5329, + "theoretical_loss": 3.3438969888404255, + "tokens_seen": 2743205888 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001716969696969697, + "loss": 2.4222, + "theoretical_loss": 3.3438907344459836, + "tokens_seen": 2743271424 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017167676767676767, + "loss": 2.3349, + "theoretical_loss": 3.343884480242791, + "tokens_seen": 2743336960 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017165656565656565, + "loss": 2.3954, + "theoretical_loss": 3.343878226230837, + "tokens_seen": 2743402496 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017163636363636364, + "loss": 2.4457, + "theoretical_loss": 3.3438719724101116, + "tokens_seen": 2743468032 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017161616161616162, + "loss": 2.7806, + "theoretical_loss": 3.343865718780604, + "tokens_seen": 2743533568 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001715959595959596, + "loss": 2.3346, + "theoretical_loss": 3.3438594653423044, + "tokens_seen": 2743599104 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001715757575757576, + "loss": 2.4174, + "theoretical_loss": 3.3438532120952016, + "tokens_seen": 2743664640 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017155555555555555, + "loss": 2.2891, + "theoretical_loss": 3.3438469590392854, + "tokens_seen": 2743730176 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017153535353535356, + "loss": 2.4593, + "theoretical_loss": 3.343840706174546, + "tokens_seen": 2743795712 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001715151515151515, + "loss": 2.2727, + "theoretical_loss": 3.3438344535009725, + "tokens_seen": 2743861248 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001714949494949495, + "loss": 2.6239, + "theoretical_loss": 3.3438282010185545, + "tokens_seen": 2743926784 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017147474747474748, + "loss": 2.3674, + "theoretical_loss": 3.343821948727282, + "tokens_seen": 2743992320 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017145454545454544, + "loss": 2.4576, + "theoretical_loss": 3.3438156966271433, + "tokens_seen": 2744057856 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017143434343434345, + "loss": 2.4897, + "theoretical_loss": 3.3438094447181292, + "tokens_seen": 2744123392 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001714141414141414, + "loss": 2.5255, + "theoretical_loss": 3.3438031930002294, + "tokens_seen": 2744188928 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017139393939393942, + "loss": 2.5069, + "theoretical_loss": 3.3437969414734328, + "tokens_seen": 2744254464 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017137373737373737, + "loss": 2.5154, + "theoretical_loss": 3.3437906901377294, + "tokens_seen": 2744320000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017135353535353536, + "loss": 2.5421, + "theoretical_loss": 3.343784438993109, + "tokens_seen": 2744385536 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017133333333333334, + "loss": 2.4949, + "theoretical_loss": 3.3437781880395603, + "tokens_seen": 2744451072 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001713131313131313, + "loss": 2.6241, + "theoretical_loss": 3.3437719372770736, + "tokens_seen": 2744516608 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1505281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.427666664123535, + "objective/train/theoretical_loss": 3.3437656867056385, + "objective/train/tokens_used": 1103441376, + "theoretical_loss": 3.3437656867056385, + "tokens_seen": 2744582144 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001712929292929293, + "loss": 2.2352, + "theoretical_loss": 3.3437656867056385, + "tokens_seen": 2744582144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017127272727272727, + "loss": 2.6694, + "theoretical_loss": 3.3437594363252443, + "tokens_seen": 2744647680 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017125252525252525, + "loss": 2.4521, + "theoretical_loss": 3.343753186135881, + "tokens_seen": 2744713216 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017123232323232323, + "loss": 2.6045, + "theoretical_loss": 3.343746936137538, + "tokens_seen": 2744778752 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017121212121212122, + "loss": 2.4601, + "theoretical_loss": 3.3437406863302046, + "tokens_seen": 2744844288 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001711919191919192, + "loss": 2.2838, + "theoretical_loss": 3.343734436713871, + "tokens_seen": 2744909824 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017117171717171716, + "loss": 2.4298, + "theoretical_loss": 3.3437281872885265, + "tokens_seen": 2744975360 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017115151515151514, + "loss": 2.377, + "theoretical_loss": 3.3437219380541605, + "tokens_seen": 2745040896 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017113131313131313, + "loss": 2.4791, + "theoretical_loss": 3.3437156890107627, + "tokens_seen": 2745106432 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001711111111111111, + "loss": 2.6776, + "theoretical_loss": 3.3437094401583227, + "tokens_seen": 2745171968 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001710909090909091, + "loss": 2.3895, + "theoretical_loss": 3.3437031914968305, + "tokens_seen": 2745237504 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017107070707070708, + "loss": 2.4561, + "theoretical_loss": 3.3436969430262753, + "tokens_seen": 2745303040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017105050505050506, + "loss": 2.4472, + "theoretical_loss": 3.343690694746647, + "tokens_seen": 2745368576 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017103030303030305, + "loss": 2.3924, + "theoretical_loss": 3.3436844466579347, + "tokens_seen": 2745434112 + }, + { + "epoch": 0.66, + "learning_rate": 0.000171010101010101, + "loss": 2.4105, + "theoretical_loss": 3.3436781987601285, + "tokens_seen": 2745499648 + }, + { + "epoch": 0.66, + "learning_rate": 0.000170989898989899, + "loss": 2.4297, + "theoretical_loss": 3.343671951053218, + "tokens_seen": 2745565184 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017096969696969697, + "loss": 2.2014, + "theoretical_loss": 3.3436657035371926, + "tokens_seen": 2745630720 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017094949494949496, + "loss": 2.3567, + "theoretical_loss": 3.3436594562120416, + "tokens_seen": 2745696256 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017092929292929294, + "loss": 2.5734, + "theoretical_loss": 3.3436532090777553, + "tokens_seen": 2745761792 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001709090909090909, + "loss": 2.378, + "theoretical_loss": 3.343646962134323, + "tokens_seen": 2745827328 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001708888888888889, + "loss": 2.5957, + "theoretical_loss": 3.343640715381734, + "tokens_seen": 2745892864 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017086868686868686, + "loss": 2.3871, + "theoretical_loss": 3.3436344688199786, + "tokens_seen": 2745958400 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017084848484848488, + "loss": 2.5008, + "theoretical_loss": 3.343628222449046, + "tokens_seen": 2746023936 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017082828282828283, + "loss": 2.4392, + "theoretical_loss": 3.343621976268926, + "tokens_seen": 2746089472 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001708080808080808, + "loss": 2.4469, + "theoretical_loss": 3.3436157302796072, + "tokens_seen": 2746155008 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1506636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.252068519592285, + "objective/train/theoretical_loss": 3.343609484481081, + "objective/train/tokens_used": 1105079776, + "theoretical_loss": 3.343609484481081, + "tokens_seen": 2746220544 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001707878787878788, + "loss": 2.4955, + "theoretical_loss": 3.343609484481081, + "tokens_seen": 2746220544 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017076767676767676, + "loss": 2.8094, + "theoretical_loss": 3.343603238873336, + "tokens_seen": 2746286080 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017074747474747477, + "loss": 2.642, + "theoretical_loss": 3.3435969934563614, + "tokens_seen": 2746351616 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017072727272727273, + "loss": 2.5326, + "theoretical_loss": 3.343590748230148, + "tokens_seen": 2746417152 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001707070707070707, + "loss": 2.5533, + "theoretical_loss": 3.3435845031946845, + "tokens_seen": 2746482688 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001706868686868687, + "loss": 2.4508, + "theoretical_loss": 3.343578258349961, + "tokens_seen": 2746548224 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017066666666666668, + "loss": 2.5825, + "theoretical_loss": 3.3435720136959666, + "tokens_seen": 2746613760 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017064646464646466, + "loss": 2.2389, + "theoretical_loss": 3.343565769232691, + "tokens_seen": 2746679296 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017062626262626262, + "loss": 2.4324, + "theoretical_loss": 3.3435595249601247, + "tokens_seen": 2746744832 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001706060606060606, + "loss": 2.5639, + "theoretical_loss": 3.343553280878256, + "tokens_seen": 2746810368 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017058585858585859, + "loss": 2.59, + "theoretical_loss": 3.3435470369870757, + "tokens_seen": 2746875904 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017056565656565657, + "loss": 2.6188, + "theoretical_loss": 3.3435407932865733, + "tokens_seen": 2746941440 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017054545454545455, + "loss": 2.5078, + "theoretical_loss": 3.3435345497767375, + "tokens_seen": 2747006976 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017052525252525254, + "loss": 2.3648, + "theoretical_loss": 3.3435283064575585, + "tokens_seen": 2747072512 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001705050505050505, + "loss": 2.3924, + "theoretical_loss": 3.343522063329026, + "tokens_seen": 2747138048 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001704848484848485, + "loss": 2.5121, + "theoretical_loss": 3.3435158203911297, + "tokens_seen": 2747203584 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017046464646464646, + "loss": 2.4281, + "theoretical_loss": 3.343509577643859, + "tokens_seen": 2747269120 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017044444444444445, + "loss": 2.6118, + "theoretical_loss": 3.3435033350872034, + "tokens_seen": 2747334656 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017042424242424243, + "loss": 2.5611, + "theoretical_loss": 3.343497092721153, + "tokens_seen": 2747400192 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001704040404040404, + "loss": 2.527, + "theoretical_loss": 3.3434908505456975, + "tokens_seen": 2747465728 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001703838383838384, + "loss": 2.4903, + "theoretical_loss": 3.343484608560826, + "tokens_seen": 2747531264 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017036363636363636, + "loss": 2.4251, + "theoretical_loss": 3.3434783667665284, + "tokens_seen": 2747596800 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017034343434343437, + "loss": 2.7304, + "theoretical_loss": 3.343472125162794, + "tokens_seen": 2747662336 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017032323232323232, + "loss": 2.4272, + "theoretical_loss": 3.343465883749613, + "tokens_seen": 2747727872 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017030303030303028, + "loss": 2.3809, + "theoretical_loss": 3.343459642526975, + "tokens_seen": 2747793408 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1507074, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4767045974731445, + "objective/train/theoretical_loss": 3.3434534014948687, + "objective/train/tokens_used": 1106718176, + "theoretical_loss": 3.3434534014948687, + "tokens_seen": 2747858944 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001702828282828283, + "loss": 2.473, + "theoretical_loss": 3.3434534014948687, + "tokens_seen": 2747858944 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017026262626262625, + "loss": 2.5404, + "theoretical_loss": 3.343447160653285, + "tokens_seen": 2747924480 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017024242424242426, + "loss": 2.6679, + "theoretical_loss": 3.343440920002213, + "tokens_seen": 2747990016 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017022222222222222, + "loss": 2.5049, + "theoretical_loss": 3.343434679541642, + "tokens_seen": 2748055552 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017020202020202023, + "loss": 2.6454, + "theoretical_loss": 3.3434284392715625, + "tokens_seen": 2748121088 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017018181818181818, + "loss": 2.4172, + "theoretical_loss": 3.3434221991919633, + "tokens_seen": 2748186624 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017016161616161617, + "loss": 2.694, + "theoretical_loss": 3.343415959302834, + "tokens_seen": 2748252160 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017014141414141415, + "loss": 2.4003, + "theoretical_loss": 3.343409719604165, + "tokens_seen": 2748317696 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001701212121212121, + "loss": 2.5977, + "theoretical_loss": 3.3434034800959456, + "tokens_seen": 2748383232 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017010101010101012, + "loss": 2.5354, + "theoretical_loss": 3.3433972407781654, + "tokens_seen": 2748448768 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017008080808080808, + "loss": 2.3547, + "theoretical_loss": 3.343391001650814, + "tokens_seen": 2748514304 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017006060606060606, + "loss": 2.4886, + "theoretical_loss": 3.343384762713881, + "tokens_seen": 2748579840 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017004040404040404, + "loss": 2.3441, + "theoretical_loss": 3.343378523967356, + "tokens_seen": 2748645376 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017002020202020203, + "loss": 2.563, + "theoretical_loss": 3.3433722854112293, + "tokens_seen": 2748710912 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017, + "loss": 2.5884, + "theoretical_loss": 3.34336604704549, + "tokens_seen": 2748776448 + }, + { + "epoch": 0.66, + "learning_rate": 0.000169979797979798, + "loss": 2.2748, + "theoretical_loss": 3.3433598088701273, + "tokens_seen": 2748841984 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016995959595959595, + "loss": 2.4044, + "theoretical_loss": 3.3433535708851316, + "tokens_seen": 2748907520 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016993939393939394, + "loss": 2.5009, + "theoretical_loss": 3.343347333090492, + "tokens_seen": 2748973056 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016991919191919192, + "loss": 2.6287, + "theoretical_loss": 3.3433410954861986, + "tokens_seen": 2749038592 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001698989898989899, + "loss": 2.5195, + "theoretical_loss": 3.3433348580722413, + "tokens_seen": 2749104128 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001698787878787879, + "loss": 2.4808, + "theoretical_loss": 3.343328620848609, + "tokens_seen": 2749169664 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016985858585858585, + "loss": 2.5336, + "theoretical_loss": 3.3433223838152917, + "tokens_seen": 2749235200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016983838383838386, + "loss": 2.5648, + "theoretical_loss": 3.343316146972279, + "tokens_seen": 2749300736 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016981818181818181, + "loss": 2.565, + "theoretical_loss": 3.3433099103195607, + "tokens_seen": 2749366272 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016979797979797982, + "loss": 2.3842, + "theoretical_loss": 3.3433036738571267, + "tokens_seen": 2749431808 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1508309, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5114078521728516, + "objective/train/theoretical_loss": 3.3432974375849662, + "objective/train/tokens_used": 1108356576, + "theoretical_loss": 3.3432974375849662, + "tokens_seen": 2749497344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016977777777777778, + "loss": 2.4407, + "theoretical_loss": 3.3432974375849662, + "tokens_seen": 2749497344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016975757575757574, + "loss": 2.3782, + "theoretical_loss": 3.3432912015030687, + "tokens_seen": 2749562880 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016973737373737375, + "loss": 2.4324, + "theoretical_loss": 3.3432849656114243, + "tokens_seen": 2749628416 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001697171717171717, + "loss": 2.5037, + "theoretical_loss": 3.343278729910023, + "tokens_seen": 2749693952 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016969696969696972, + "loss": 2.3937, + "theoretical_loss": 3.343272494398853, + "tokens_seen": 2749759488 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016967676767676767, + "loss": 2.5391, + "theoretical_loss": 3.3432662590779056, + "tokens_seen": 2749825024 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016965656565656566, + "loss": 2.518, + "theoretical_loss": 3.34326002394717, + "tokens_seen": 2749890560 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016963636363636364, + "loss": 2.5049, + "theoretical_loss": 3.3432537890066354, + "tokens_seen": 2749956096 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016961616161616163, + "loss": 2.5801, + "theoretical_loss": 3.343247554256292, + "tokens_seen": 2750021632 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001695959595959596, + "loss": 2.4574, + "theoretical_loss": 3.3432413196961286, + "tokens_seen": 2750087168 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016957575757575757, + "loss": 2.5269, + "theoretical_loss": 3.343235085326136, + "tokens_seen": 2750152704 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016955555555555555, + "loss": 2.3649, + "theoretical_loss": 3.343228851146303, + "tokens_seen": 2750218240 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016953535353535353, + "loss": 2.583, + "theoretical_loss": 3.34322261715662, + "tokens_seen": 2750283776 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016951515151515152, + "loss": 2.5673, + "theoretical_loss": 3.343216383357076, + "tokens_seen": 2750349312 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001694949494949495, + "loss": 2.4754, + "theoretical_loss": 3.343210149747661, + "tokens_seen": 2750414848 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001694747474747475, + "loss": 2.6453, + "theoretical_loss": 3.3432039163283647, + "tokens_seen": 2750480384 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016945454545454544, + "loss": 2.4965, + "theoretical_loss": 3.3431976830991768, + "tokens_seen": 2750545920 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016943434343434343, + "loss": 2.4504, + "theoretical_loss": 3.3431914500600866, + "tokens_seen": 2750611456 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001694141414141414, + "loss": 2.5478, + "theoretical_loss": 3.343185217211084, + "tokens_seen": 2750676992 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001693939393939394, + "loss": 2.5805, + "theoretical_loss": 3.343178984552159, + "tokens_seen": 2750742528 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016937373737373738, + "loss": 2.542, + "theoretical_loss": 3.3431727520833006, + "tokens_seen": 2750808064 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016935353535353536, + "loss": 2.4412, + "theoretical_loss": 3.343166519804499, + "tokens_seen": 2750873600 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016933333333333335, + "loss": 2.3483, + "theoretical_loss": 3.3431602877157442, + "tokens_seen": 2750939136 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001693131313131313, + "loss": 2.4029, + "theoretical_loss": 3.343154055817025, + "tokens_seen": 2751004672 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016929292929292932, + "loss": 2.6504, + "theoretical_loss": 3.343147824108332, + "tokens_seen": 2751070208 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1509529, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8860881328582764, + "objective/train/theoretical_loss": 3.343141592589654, + "objective/train/tokens_used": 1109994976, + "theoretical_loss": 3.343141592589654, + "tokens_seen": 2751135744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016927272727272727, + "loss": 2.7355, + "theoretical_loss": 3.343141592589654, + "tokens_seen": 2751135744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016925252525252526, + "loss": 2.5975, + "theoretical_loss": 3.3431353612609804, + "tokens_seen": 2751201280 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016923232323232324, + "loss": 2.6238, + "theoretical_loss": 3.3431291301223025, + "tokens_seen": 2751266816 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001692121212121212, + "loss": 2.5902, + "theoretical_loss": 3.3431228991736086, + "tokens_seen": 2751332352 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001691919191919192, + "loss": 2.5332, + "theoretical_loss": 3.343116668414889, + "tokens_seen": 2751397888 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016917171717171716, + "loss": 2.3857, + "theoretical_loss": 3.343110437846133, + "tokens_seen": 2751463424 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016915151515151518, + "loss": 2.5115, + "theoretical_loss": 3.3431042074673307, + "tokens_seen": 2751528960 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016913131313131313, + "loss": 2.6071, + "theoretical_loss": 3.3430979772784712, + "tokens_seen": 2751594496 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016911111111111112, + "loss": 2.8059, + "theoretical_loss": 3.3430917472795447, + "tokens_seen": 2751660032 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001690909090909091, + "loss": 2.4418, + "theoretical_loss": 3.343085517470541, + "tokens_seen": 2751725568 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016907070707070706, + "loss": 2.4796, + "theoretical_loss": 3.3430792878514493, + "tokens_seen": 2751791104 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016905050505050507, + "loss": 2.3931, + "theoretical_loss": 3.3430730584222594, + "tokens_seen": 2751856640 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016903030303030303, + "loss": 2.5562, + "theoretical_loss": 3.343066829182961, + "tokens_seen": 2751922176 + }, + { + "epoch": 0.67, + "learning_rate": 0.000169010101010101, + "loss": 2.6501, + "theoretical_loss": 3.343060600133544, + "tokens_seen": 2751987712 + }, + { + "epoch": 0.67, + "learning_rate": 0.000168989898989899, + "loss": 2.4794, + "theoretical_loss": 3.343054371273998, + "tokens_seen": 2752053248 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016896969696969698, + "loss": 2.339, + "theoretical_loss": 3.343048142604313, + "tokens_seen": 2752118784 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016894949494949496, + "loss": 2.6137, + "theoretical_loss": 3.343041914124478, + "tokens_seen": 2752184320 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016892929292929295, + "loss": 2.7218, + "theoretical_loss": 3.343035685834483, + "tokens_seen": 2752249856 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001689090909090909, + "loss": 2.4793, + "theoretical_loss": 3.3430294577343176, + "tokens_seen": 2752315392 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016888888888888889, + "loss": 2.5701, + "theoretical_loss": 3.343023229823972, + "tokens_seen": 2752380928 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016886868686868687, + "loss": 2.5495, + "theoretical_loss": 3.3430170021034353, + "tokens_seen": 2752446464 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016884848484848485, + "loss": 2.6591, + "theoretical_loss": 3.3430107745726976, + "tokens_seen": 2752512000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016882828282828284, + "loss": 2.4868, + "theoretical_loss": 3.3430045472317484, + "tokens_seen": 2752577536 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001688080808080808, + "loss": 2.3851, + "theoretical_loss": 3.3429983200805773, + "tokens_seen": 2752643072 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001687878787878788, + "loss": 2.2933, + "theoretical_loss": 3.342992093119174, + "tokens_seen": 2752708608 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1510029, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6230833530426025, + "objective/train/theoretical_loss": 3.3429858663475285, + "objective/train/tokens_used": 1111633376, + "theoretical_loss": 3.3429858663475285, + "tokens_seen": 2752774144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016876767676767676, + "loss": 2.4983, + "theoretical_loss": 3.3429858663475285, + "tokens_seen": 2752774144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016874747474747477, + "loss": 2.4781, + "theoretical_loss": 3.3429796397656304, + "tokens_seen": 2752839680 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016872727272727273, + "loss": 2.6614, + "theoretical_loss": 3.342973413373469, + "tokens_seen": 2752905216 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001687070707070707, + "loss": 2.4224, + "theoretical_loss": 3.3429671871710345, + "tokens_seen": 2752970752 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001686868686868687, + "loss": 2.4826, + "theoretical_loss": 3.342960961158316, + "tokens_seen": 2753036288 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016866666666666666, + "loss": 2.6242, + "theoretical_loss": 3.3429547353353044, + "tokens_seen": 2753101824 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016864646464646467, + "loss": 2.275, + "theoretical_loss": 3.3429485097019884, + "tokens_seen": 2753167360 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016862626262626262, + "loss": 2.1682, + "theoretical_loss": 3.342942284258358, + "tokens_seen": 2753232896 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001686060606060606, + "loss": 2.2587, + "theoretical_loss": 3.3429360590044026, + "tokens_seen": 2753298432 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001685858585858586, + "loss": 2.676, + "theoretical_loss": 3.3429298339401123, + "tokens_seen": 2753363968 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016856565656565655, + "loss": 2.6414, + "theoretical_loss": 3.342923609065476, + "tokens_seen": 2753429504 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016854545454545456, + "loss": 2.5263, + "theoretical_loss": 3.342917384380485, + "tokens_seen": 2753495040 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016852525252525252, + "loss": 2.3916, + "theoretical_loss": 3.3429111598851278, + "tokens_seen": 2753560576 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016850505050505053, + "loss": 2.6404, + "theoretical_loss": 3.3429049355793943, + "tokens_seen": 2753626112 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016848484848484848, + "loss": 2.7259, + "theoretical_loss": 3.342898711463274, + "tokens_seen": 2753691648 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016846464646464647, + "loss": 2.4837, + "theoretical_loss": 3.342892487536757, + "tokens_seen": 2753757184 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016844444444444445, + "loss": 2.5165, + "theoretical_loss": 3.3428862637998336, + "tokens_seen": 2753822720 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016842424242424244, + "loss": 2.3796, + "theoretical_loss": 3.342880040252492, + "tokens_seen": 2753888256 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016840404040404042, + "loss": 2.4224, + "theoretical_loss": 3.3428738168947234, + "tokens_seen": 2753953792 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016838383838383838, + "loss": 2.3061, + "theoretical_loss": 3.342867593726516, + "tokens_seen": 2754019328 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016836363636363636, + "loss": 2.2304, + "theoretical_loss": 3.342861370747861, + "tokens_seen": 2754084864 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016834343434343434, + "loss": 2.3142, + "theoretical_loss": 3.3428551479587476, + "tokens_seen": 2754150400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016832323232323233, + "loss": 2.635, + "theoretical_loss": 3.342848925359165, + "tokens_seen": 2754215936 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001683030303030303, + "loss": 2.4569, + "theoretical_loss": 3.3428427029491035, + "tokens_seen": 2754281472 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001682828282828283, + "loss": 2.5827, + "theoretical_loss": 3.342836480728552, + "tokens_seen": 2754347008 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1510960, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.503450632095337, + "objective/train/theoretical_loss": 3.342830258697502, + "objective/train/tokens_used": 1113271776, + "theoretical_loss": 3.342830258697502, + "tokens_seen": 2754412544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016826262626262625, + "loss": 2.4446, + "theoretical_loss": 3.342830258697502, + "tokens_seen": 2754412544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016824242424242426, + "loss": 2.5433, + "theoretical_loss": 3.3428240368559416, + "tokens_seen": 2754478080 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016822222222222222, + "loss": 2.6198, + "theoretical_loss": 3.342817815203861, + "tokens_seen": 2754543616 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001682020202020202, + "loss": 2.5607, + "theoretical_loss": 3.3428115937412497, + "tokens_seen": 2754609152 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001681818181818182, + "loss": 2.3436, + "theoretical_loss": 3.3428053724680975, + "tokens_seen": 2754674688 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016816161616161615, + "loss": 2.6481, + "theoretical_loss": 3.342799151384395, + "tokens_seen": 2754740224 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016814141414141416, + "loss": 2.458, + "theoretical_loss": 3.3427929304901305, + "tokens_seen": 2754805760 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016812121212121211, + "loss": 2.4262, + "theoretical_loss": 3.342786709785295, + "tokens_seen": 2754871296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016810101010101013, + "loss": 2.4641, + "theoretical_loss": 3.3427804892698774, + "tokens_seen": 2754936832 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016808080808080808, + "loss": 2.5883, + "theoretical_loss": 3.342774268943867, + "tokens_seen": 2755002368 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016806060606060607, + "loss": 2.5174, + "theoretical_loss": 3.342768048807255, + "tokens_seen": 2755067904 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016804040404040405, + "loss": 2.5805, + "theoretical_loss": 3.3427618288600303, + "tokens_seen": 2755133440 + }, + { + "epoch": 0.67, + "learning_rate": 0.000168020202020202, + "loss": 2.6488, + "theoretical_loss": 3.3427556091021824, + "tokens_seen": 2755198976 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016800000000000002, + "loss": 2.5924, + "theoretical_loss": 3.3427493895337013, + "tokens_seen": 2755264512 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016797979797979797, + "loss": 2.6395, + "theoretical_loss": 3.3427431701545767, + "tokens_seen": 2755330048 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016795959595959596, + "loss": 2.675, + "theoretical_loss": 3.3427369509647984, + "tokens_seen": 2755395584 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016793939393939394, + "loss": 2.2821, + "theoretical_loss": 3.342730731964356, + "tokens_seen": 2755461120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016791919191919193, + "loss": 2.7047, + "theoretical_loss": 3.3427245131532395, + "tokens_seen": 2755526656 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001678989898989899, + "loss": 2.3648, + "theoretical_loss": 3.342718294531438, + "tokens_seen": 2755592192 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016787878787878787, + "loss": 2.3327, + "theoretical_loss": 3.342712076098942, + "tokens_seen": 2755657728 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016785858585858585, + "loss": 2.4148, + "theoretical_loss": 3.342705857855741, + "tokens_seen": 2755723264 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016783838383838384, + "loss": 2.4149, + "theoretical_loss": 3.3426996398018245, + "tokens_seen": 2755788800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016781818181818182, + "loss": 2.5137, + "theoretical_loss": 3.3426934219371827, + "tokens_seen": 2755854336 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001677979797979798, + "loss": 2.6958, + "theoretical_loss": 3.3426872042618045, + "tokens_seen": 2755919872 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001677777777777778, + "loss": 2.3498, + "theoretical_loss": 3.3426809867756804, + "tokens_seen": 2755985408 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1512146, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4062039852142334, + "objective/train/theoretical_loss": 3.3426747694788, + "objective/train/tokens_used": 1114910176, + "theoretical_loss": 3.3426747694788, + "tokens_seen": 2756050944 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016775757575757574, + "loss": 2.5385, + "theoretical_loss": 3.3426747694788, + "tokens_seen": 2756050944 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016773737373737376, + "loss": 2.3607, + "theoretical_loss": 3.342668552371153, + "tokens_seen": 2756116480 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001677171717171717, + "loss": 2.3892, + "theoretical_loss": 3.342662335452729, + "tokens_seen": 2756182016 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001676969696969697, + "loss": 2.364, + "theoretical_loss": 3.3426561187235175, + "tokens_seen": 2756247552 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016767676767676768, + "loss": 2.49, + "theoretical_loss": 3.342649902183509, + "tokens_seen": 2756313088 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016765656565656566, + "loss": 2.6073, + "theoretical_loss": 3.3426436858326927, + "tokens_seen": 2756378624 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016763636363636365, + "loss": 2.5562, + "theoretical_loss": 3.3426374696710583, + "tokens_seen": 2756444160 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001676161616161616, + "loss": 2.6286, + "theoretical_loss": 3.3426312536985954, + "tokens_seen": 2756509696 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016759595959595962, + "loss": 2.6142, + "theoretical_loss": 3.342625037915295, + "tokens_seen": 2756575232 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016757575757575757, + "loss": 2.6647, + "theoretical_loss": 3.3426188223211453, + "tokens_seen": 2756640768 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016755555555555558, + "loss": 2.4265, + "theoretical_loss": 3.3426126069161364, + "tokens_seen": 2756706304 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016753535353535354, + "loss": 2.3922, + "theoretical_loss": 3.3426063917002584, + "tokens_seen": 2756771840 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001675151515151515, + "loss": 2.6707, + "theoretical_loss": 3.342600176673501, + "tokens_seen": 2756837376 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001674949494949495, + "loss": 2.6063, + "theoretical_loss": 3.342593961835854, + "tokens_seen": 2756902912 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016747474747474747, + "loss": 2.5734, + "theoretical_loss": 3.342587747187307, + "tokens_seen": 2756968448 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016745454545454548, + "loss": 2.535, + "theoretical_loss": 3.34258153272785, + "tokens_seen": 2757033984 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016743434343434343, + "loss": 2.7146, + "theoretical_loss": 3.342575318457472, + "tokens_seen": 2757099520 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016741414141414142, + "loss": 2.4252, + "theoretical_loss": 3.342569104376164, + "tokens_seen": 2757165056 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001673939393939394, + "loss": 2.3517, + "theoretical_loss": 3.3425628904839146, + "tokens_seen": 2757230592 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016737373737373739, + "loss": 2.6101, + "theoretical_loss": 3.342556676780714, + "tokens_seen": 2757296128 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016735353535353537, + "loss": 2.4752, + "theoretical_loss": 3.342550463266552, + "tokens_seen": 2757361664 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016733333333333333, + "loss": 2.4862, + "theoretical_loss": 3.3425442499414184, + "tokens_seen": 2757427200 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001673131313131313, + "loss": 2.7177, + "theoretical_loss": 3.3425380368053026, + "tokens_seen": 2757492736 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001672929292929293, + "loss": 2.7104, + "theoretical_loss": 3.342531823858195, + "tokens_seen": 2757558272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016727272727272728, + "loss": 2.5433, + "theoretical_loss": 3.3425256111000845, + "tokens_seen": 2757623808 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1512703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.699195384979248, + "objective/train/theoretical_loss": 3.342519398530962, + "objective/train/tokens_used": 1116548576, + "theoretical_loss": 3.342519398530962, + "tokens_seen": 2757689344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016725252525252526, + "loss": 2.6129, + "theoretical_loss": 3.342519398530962, + "tokens_seen": 2757689344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016723232323232325, + "loss": 2.3816, + "theoretical_loss": 3.342513186150816, + "tokens_seen": 2757754880 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001672121212121212, + "loss": 2.4668, + "theoretical_loss": 3.342506973959637, + "tokens_seen": 2757820416 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016719191919191921, + "loss": 2.5441, + "theoretical_loss": 3.3425007619574147, + "tokens_seen": 2757885952 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016717171717171717, + "loss": 2.574, + "theoretical_loss": 3.342494550144139, + "tokens_seen": 2757951488 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016715151515151515, + "loss": 2.6698, + "theoretical_loss": 3.342488338519799, + "tokens_seen": 2758017024 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016713131313131314, + "loss": 2.493, + "theoretical_loss": 3.342482127084385, + "tokens_seen": 2758082560 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001671111111111111, + "loss": 2.5522, + "theoretical_loss": 3.3424759158378876, + "tokens_seen": 2758148096 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001670909090909091, + "loss": 2.6004, + "theoretical_loss": 3.342469704780295, + "tokens_seen": 2758213632 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016707070707070706, + "loss": 2.4535, + "theoretical_loss": 3.3424634939115974, + "tokens_seen": 2758279168 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016705050505050507, + "loss": 2.6645, + "theoretical_loss": 3.3424572832317847, + "tokens_seen": 2758344704 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016703030303030303, + "loss": 2.3888, + "theoretical_loss": 3.342451072740847, + "tokens_seen": 2758410240 + }, + { + "epoch": 0.67, + "learning_rate": 0.000167010101010101, + "loss": 2.5406, + "theoretical_loss": 3.3424448624387737, + "tokens_seen": 2758475776 + }, + { + "epoch": 0.67, + "learning_rate": 0.000166989898989899, + "loss": 2.5109, + "theoretical_loss": 3.342438652325555, + "tokens_seen": 2758541312 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016696969696969696, + "loss": 2.5602, + "theoretical_loss": 3.34243244240118, + "tokens_seen": 2758606848 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016694949494949497, + "loss": 2.466, + "theoretical_loss": 3.342426232665639, + "tokens_seen": 2758672384 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016692929292929292, + "loss": 2.5179, + "theoretical_loss": 3.3424200231189216, + "tokens_seen": 2758737920 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001669090909090909, + "loss": 2.7382, + "theoretical_loss": 3.3424138137610173, + "tokens_seen": 2758803456 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001668888888888889, + "loss": 2.6637, + "theoretical_loss": 3.3424076045919167, + "tokens_seen": 2758868992 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016686868686868688, + "loss": 2.4876, + "theoretical_loss": 3.3424013956116085, + "tokens_seen": 2758934528 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016684848484848486, + "loss": 2.655, + "theoretical_loss": 3.342395186820083, + "tokens_seen": 2759000064 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016682828282828282, + "loss": 2.5795, + "theoretical_loss": 3.3423889782173304, + "tokens_seen": 2759065600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016680808080808083, + "loss": 2.7132, + "theoretical_loss": 3.34238276980334, + "tokens_seen": 2759131136 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016678787878787878, + "loss": 2.669, + "theoretical_loss": 3.342376561578101, + "tokens_seen": 2759196672 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016676767676767677, + "loss": 2.724, + "theoretical_loss": 3.3423703535416043, + "tokens_seen": 2759262208 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1514070, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4820828437805176, + "objective/train/theoretical_loss": 3.342364145693839, + "objective/train/tokens_used": 1118186976, + "theoretical_loss": 3.342364145693839, + "tokens_seen": 2759327744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016674747474747475, + "loss": 2.5302, + "theoretical_loss": 3.342364145693839, + "tokens_seen": 2759327744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016672727272727274, + "loss": 2.3592, + "theoretical_loss": 3.3423579380347954, + "tokens_seen": 2759393280 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016670707070707072, + "loss": 2.5375, + "theoretical_loss": 3.342351730564463, + "tokens_seen": 2759458816 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001666868686868687, + "loss": 2.4038, + "theoretical_loss": 3.342345523282831, + "tokens_seen": 2759524352 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016666666666666666, + "loss": 2.3906, + "theoretical_loss": 3.34233931618989, + "tokens_seen": 2759589888 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016664646464646465, + "loss": 2.5902, + "theoretical_loss": 3.3423331092856294, + "tokens_seen": 2759655424 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016662626262626263, + "loss": 2.6271, + "theoretical_loss": 3.342326902570039, + "tokens_seen": 2759720960 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001666060606060606, + "loss": 2.3666, + "theoretical_loss": 3.342320696043109, + "tokens_seen": 2759786496 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001665858585858586, + "loss": 2.484, + "theoretical_loss": 3.3423144897048287, + "tokens_seen": 2759852032 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016656565656565655, + "loss": 2.3156, + "theoretical_loss": 3.342308283555188, + "tokens_seen": 2759917568 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016654545454545456, + "loss": 2.7336, + "theoretical_loss": 3.3423020775941765, + "tokens_seen": 2759983104 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016652525252525252, + "loss": 2.3339, + "theoretical_loss": 3.3422958718217846, + "tokens_seen": 2760048640 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016650505050505053, + "loss": 2.6066, + "theoretical_loss": 3.3422896662380017, + "tokens_seen": 2760114176 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001664848484848485, + "loss": 2.5801, + "theoretical_loss": 3.3422834608428174, + "tokens_seen": 2760179712 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016646464646464645, + "loss": 2.7557, + "theoretical_loss": 3.3422772556362217, + "tokens_seen": 2760245248 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016644444444444446, + "loss": 2.4409, + "theoretical_loss": 3.342271050618204, + "tokens_seen": 2760310784 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016642424242424241, + "loss": 2.6982, + "theoretical_loss": 3.3422648457887547, + "tokens_seen": 2760376320 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016640404040404043, + "loss": 2.5672, + "theoretical_loss": 3.342258641147864, + "tokens_seen": 2760441856 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016638383838383838, + "loss": 2.2897, + "theoretical_loss": 3.3422524366955204, + "tokens_seen": 2760507392 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016636363636363637, + "loss": 2.6074, + "theoretical_loss": 3.342246232431714, + "tokens_seen": 2760572928 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016634343434343435, + "loss": 2.4495, + "theoretical_loss": 3.3422400283564357, + "tokens_seen": 2760638464 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016632323232323233, + "loss": 2.5237, + "theoretical_loss": 3.342233824469674, + "tokens_seen": 2760704000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016630303030303032, + "loss": 2.4297, + "theoretical_loss": 3.34222762077142, + "tokens_seen": 2760769536 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016628282828282828, + "loss": 2.5756, + "theoretical_loss": 3.342221417261662, + "tokens_seen": 2760835072 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016626262626262626, + "loss": 2.6041, + "theoretical_loss": 3.3422152139403907, + "tokens_seen": 2760900608 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1514349, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5181102752685547, + "objective/train/theoretical_loss": 3.3422090108075957, + "objective/train/tokens_used": 1119825376, + "theoretical_loss": 3.3422090108075957, + "tokens_seen": 2760966144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016624242424242424, + "loss": 2.3828, + "theoretical_loss": 3.3422090108075957, + "tokens_seen": 2760966144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016622222222222223, + "loss": 2.3228, + "theoretical_loss": 3.342202807863267, + "tokens_seen": 2761031680 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001662020202020202, + "loss": 2.522, + "theoretical_loss": 3.342196605107394, + "tokens_seen": 2761097216 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001661818181818182, + "loss": 2.6477, + "theoretical_loss": 3.3421904025399667, + "tokens_seen": 2761162752 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016616161616161615, + "loss": 2.4636, + "theoretical_loss": 3.342184200160975, + "tokens_seen": 2761228288 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016614141414141414, + "loss": 2.6654, + "theoretical_loss": 3.342177997970409, + "tokens_seen": 2761293824 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016612121212121212, + "loss": 2.58, + "theoretical_loss": 3.3421717959682575, + "tokens_seen": 2761359360 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001661010101010101, + "loss": 2.6119, + "theoretical_loss": 3.3421655941545114, + "tokens_seen": 2761424896 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001660808080808081, + "loss": 2.5517, + "theoretical_loss": 3.34215939252916, + "tokens_seen": 2761490432 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016606060606060604, + "loss": 2.8304, + "theoretical_loss": 3.342153191092193, + "tokens_seen": 2761555968 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016604040404040406, + "loss": 2.4527, + "theoretical_loss": 3.3421469898436005, + "tokens_seen": 2761621504 + }, + { + "epoch": 0.67, + "learning_rate": 0.000166020202020202, + "loss": 2.5967, + "theoretical_loss": 3.3421407887833716, + "tokens_seen": 2761687040 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016600000000000002, + "loss": 2.6755, + "theoretical_loss": 3.3421345879114974, + "tokens_seen": 2761752576 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016597979797979798, + "loss": 2.5588, + "theoretical_loss": 3.342128387227967, + "tokens_seen": 2761818112 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016595959595959594, + "loss": 2.6735, + "theoretical_loss": 3.3421221867327695, + "tokens_seen": 2761883648 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016593939393939395, + "loss": 2.3662, + "theoretical_loss": 3.3421159864258962, + "tokens_seen": 2761949184 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001659191919191919, + "loss": 2.3604, + "theoretical_loss": 3.3421097863073355, + "tokens_seen": 2762014720 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016589898989898992, + "loss": 2.4748, + "theoretical_loss": 3.3421035863770783, + "tokens_seen": 2762080256 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016587878787878787, + "loss": 2.4228, + "theoretical_loss": 3.3420973866351136, + "tokens_seen": 2762145792 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016585858585858588, + "loss": 2.5678, + "theoretical_loss": 3.3420911870814316, + "tokens_seen": 2762211328 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016583838383838384, + "loss": 2.4854, + "theoretical_loss": 3.3420849877160226, + "tokens_seen": 2762276864 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016581818181818182, + "loss": 2.4906, + "theoretical_loss": 3.342078788538875, + "tokens_seen": 2762342400 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001657979797979798, + "loss": 2.6658, + "theoretical_loss": 3.3420725895499803, + "tokens_seen": 2762407936 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016577777777777777, + "loss": 2.5736, + "theoretical_loss": 3.342066390749327, + "tokens_seen": 2762473472 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016575757575757578, + "loss": 2.6851, + "theoretical_loss": 3.3420601921369055, + "tokens_seen": 2762539008 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1514959, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.672292470932007, + "objective/train/theoretical_loss": 3.342053993712706, + "objective/train/tokens_used": 1121463776, + "theoretical_loss": 3.342053993712706, + "tokens_seen": 2762604544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016573737373737373, + "loss": 2.7192, + "theoretical_loss": 3.342053993712706, + "tokens_seen": 2762604544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016571717171717172, + "loss": 2.548, + "theoretical_loss": 3.3420477954767174, + "tokens_seen": 2762670080 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001656969696969697, + "loss": 2.6235, + "theoretical_loss": 3.3420415974289304, + "tokens_seen": 2762735616 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016567676767676769, + "loss": 2.3474, + "theoretical_loss": 3.342035399569334, + "tokens_seen": 2762801152 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016565656565656567, + "loss": 2.6931, + "theoretical_loss": 3.3420292018979185, + "tokens_seen": 2762866688 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016563636363636365, + "loss": 2.6854, + "theoretical_loss": 3.3420230044146737, + "tokens_seen": 2762932224 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001656161616161616, + "loss": 2.3871, + "theoretical_loss": 3.34201680711959, + "tokens_seen": 2762997760 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001655959595959596, + "loss": 2.4446, + "theoretical_loss": 3.3420106100126556, + "tokens_seen": 2763063296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016557575757575758, + "loss": 2.4995, + "theoretical_loss": 3.342004413093862, + "tokens_seen": 2763128832 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016555555555555556, + "loss": 2.5297, + "theoretical_loss": 3.3419982163631983, + "tokens_seen": 2763194368 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016553535353535355, + "loss": 2.6865, + "theoretical_loss": 3.3419920198206543, + "tokens_seen": 2763259904 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001655151515151515, + "loss": 2.5231, + "theoretical_loss": 3.34198582346622, + "tokens_seen": 2763325440 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016549494949494951, + "loss": 2.6046, + "theoretical_loss": 3.341979627299885, + "tokens_seen": 2763390976 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016547474747474747, + "loss": 2.5949, + "theoretical_loss": 3.341973431321639, + "tokens_seen": 2763456512 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016545454545454548, + "loss": 2.5555, + "theoretical_loss": 3.341967235531473, + "tokens_seen": 2763522048 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016543434343434344, + "loss": 2.6034, + "theoretical_loss": 3.3419610399293753, + "tokens_seen": 2763587584 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001654141414141414, + "loss": 2.5834, + "theoretical_loss": 3.341954844515336, + "tokens_seen": 2763653120 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001653939393939394, + "loss": 2.4433, + "theoretical_loss": 3.341948649289346, + "tokens_seen": 2763718656 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016537373737373736, + "loss": 2.4306, + "theoretical_loss": 3.3419424542513942, + "tokens_seen": 2763784192 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016535353535353537, + "loss": 2.8165, + "theoretical_loss": 3.3419362594014705, + "tokens_seen": 2763849728 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016533333333333333, + "loss": 2.4275, + "theoretical_loss": 3.3419300647395653, + "tokens_seen": 2763915264 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016531313131313132, + "loss": 2.3649, + "theoretical_loss": 3.341923870265668, + "tokens_seen": 2763980800 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001652929292929293, + "loss": 2.444, + "theoretical_loss": 3.341917675979768, + "tokens_seen": 2764046336 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016527272727272726, + "loss": 2.6089, + "theoretical_loss": 3.3419114818818563, + "tokens_seen": 2764111872 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016525252525252527, + "loss": 2.6381, + "theoretical_loss": 3.3419052879719215, + "tokens_seen": 2764177408 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1515566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4727749824523926, + "objective/train/theoretical_loss": 3.3418990942499542, + "objective/train/tokens_used": 1123102176, + "theoretical_loss": 3.3418990942499542, + "tokens_seen": 2764242944 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016523232323232322, + "loss": 2.4772, + "theoretical_loss": 3.3418990942499542, + "tokens_seen": 2764242944 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001652121212121212, + "loss": 2.5884, + "theoretical_loss": 3.341892900715944, + "tokens_seen": 2764308480 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001651919191919192, + "loss": 2.4037, + "theoretical_loss": 3.3418867073698806, + "tokens_seen": 2764374016 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016517171717171718, + "loss": 2.359, + "theoretical_loss": 3.3418805142117547, + "tokens_seen": 2764439552 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016515151515151516, + "loss": 2.5857, + "theoretical_loss": 3.341874321241555, + "tokens_seen": 2764505088 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016513131313131314, + "loss": 2.4537, + "theoretical_loss": 3.3418681284592715, + "tokens_seen": 2764570624 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001651111111111111, + "loss": 2.5181, + "theoretical_loss": 3.3418619358648947, + "tokens_seen": 2764636160 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016509090909090908, + "loss": 2.3551, + "theoretical_loss": 3.3418557434584137, + "tokens_seen": 2764701696 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016507070707070707, + "loss": 2.6376, + "theoretical_loss": 3.341849551239819, + "tokens_seen": 2764767232 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016505050505050505, + "loss": 2.2745, + "theoretical_loss": 3.3418433592091, + "tokens_seen": 2764832768 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016503030303030304, + "loss": 2.5105, + "theoretical_loss": 3.341837167366247, + "tokens_seen": 2764898304 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016501010101010102, + "loss": 2.54, + "theoretical_loss": 3.3418309757112494, + "tokens_seen": 2764963840 + }, + { + "epoch": 0.67, + "learning_rate": 0.000164989898989899, + "loss": 2.6592, + "theoretical_loss": 3.3418247842440976, + "tokens_seen": 2765029376 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016496969696969696, + "loss": 2.7491, + "theoretical_loss": 3.341818592964781, + "tokens_seen": 2765094912 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016494949494949497, + "loss": 2.3027, + "theoretical_loss": 3.341812401873289, + "tokens_seen": 2765160448 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016492929292929293, + "loss": 2.5257, + "theoretical_loss": 3.341806210969612, + "tokens_seen": 2765225984 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001649090909090909, + "loss": 2.3164, + "theoretical_loss": 3.34180002025374, + "tokens_seen": 2765291520 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001648888888888889, + "loss": 2.6219, + "theoretical_loss": 3.341793829725663, + "tokens_seen": 2765357056 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016486868686868685, + "loss": 2.6914, + "theoretical_loss": 3.34178763938537, + "tokens_seen": 2765422592 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016484848484848487, + "loss": 2.5494, + "theoretical_loss": 3.341781449232852, + "tokens_seen": 2765488128 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016482828282828282, + "loss": 2.6226, + "theoretical_loss": 3.341775259268098, + "tokens_seen": 2765553664 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016480808080808083, + "loss": 2.5238, + "theoretical_loss": 3.3417690694910975, + "tokens_seen": 2765619200 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001647878787878788, + "loss": 2.291, + "theoretical_loss": 3.3417628799018413, + "tokens_seen": 2765684736 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016476767676767677, + "loss": 2.5589, + "theoretical_loss": 3.3417566905003193, + "tokens_seen": 2765750272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016474747474747476, + "loss": 2.4972, + "theoretical_loss": 3.3417505012865205, + "tokens_seen": 2765815808 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1516731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.072603702545166, + "objective/train/theoretical_loss": 3.3417443122604356, + "objective/train/tokens_used": 1124740576, + "theoretical_loss": 3.3417443122604356, + "tokens_seen": 2765881344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016472727272727271, + "loss": 2.5783, + "theoretical_loss": 3.3417443122604356, + "tokens_seen": 2765881344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016470707070707073, + "loss": 2.5429, + "theoretical_loss": 3.3417381234220533, + "tokens_seen": 2765946880 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016468686868686868, + "loss": 2.6406, + "theoretical_loss": 3.341731934771365, + "tokens_seen": 2766012416 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016466666666666667, + "loss": 2.4154, + "theoretical_loss": 3.3417257463083594, + "tokens_seen": 2766077952 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016464646464646465, + "loss": 2.4299, + "theoretical_loss": 3.3417195580330272, + "tokens_seen": 2766143488 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016462626262626263, + "loss": 2.4882, + "theoretical_loss": 3.3417133699453574, + "tokens_seen": 2766209024 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016460606060606062, + "loss": 2.6944, + "theoretical_loss": 3.34170718204534, + "tokens_seen": 2766274560 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016458585858585858, + "loss": 2.5948, + "theoretical_loss": 3.341700994332966, + "tokens_seen": 2766340096 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016456565656565656, + "loss": 2.5758, + "theoretical_loss": 3.341694806808224, + "tokens_seen": 2766405632 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016454545454545454, + "loss": 2.4174, + "theoretical_loss": 3.341688619471104, + "tokens_seen": 2766471168 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016452525252525253, + "loss": 2.4039, + "theoretical_loss": 3.341682432321597, + "tokens_seen": 2766536704 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001645050505050505, + "loss": 2.2379, + "theoretical_loss": 3.3416762453596913, + "tokens_seen": 2766602240 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001644848484848485, + "loss": 2.605, + "theoretical_loss": 3.341670058585377, + "tokens_seen": 2766667776 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016446464646464645, + "loss": 2.6538, + "theoretical_loss": 3.3416638719986453, + "tokens_seen": 2766733312 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016444444444444446, + "loss": 2.69, + "theoretical_loss": 3.3416576855994853, + "tokens_seen": 2766798848 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016442424242424242, + "loss": 2.8116, + "theoretical_loss": 3.3416514993878863, + "tokens_seen": 2766864384 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001644040404040404, + "loss": 2.4402, + "theoretical_loss": 3.3416453133638386, + "tokens_seen": 2766929920 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001643838383838384, + "loss": 2.383, + "theoretical_loss": 3.3416391275273325, + "tokens_seen": 2766995456 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016436363636363634, + "loss": 2.5483, + "theoretical_loss": 3.3416329418783572, + "tokens_seen": 2767060992 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016434343434343436, + "loss": 2.5297, + "theoretical_loss": 3.341626756416903, + "tokens_seen": 2767126528 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001643232323232323, + "loss": 2.5741, + "theoretical_loss": 3.34162057114296, + "tokens_seen": 2767192064 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016430303030303032, + "loss": 2.4842, + "theoretical_loss": 3.3416143860565173, + "tokens_seen": 2767257600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016428282828282828, + "loss": 2.4207, + "theoretical_loss": 3.341608201157565, + "tokens_seen": 2767323136 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016426262626262626, + "loss": 2.5274, + "theoretical_loss": 3.341602016446094, + "tokens_seen": 2767388672 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016424242424242425, + "loss": 2.4424, + "theoretical_loss": 3.3415958319220924, + "tokens_seen": 2767454208 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1517810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.459321975708008, + "objective/train/theoretical_loss": 3.3415896475855513, + "objective/train/tokens_used": 1126378976, + "theoretical_loss": 3.3415896475855513, + "tokens_seen": 2767519744 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001642222222222222, + "loss": 2.6265, + "theoretical_loss": 3.3415896475855513, + "tokens_seen": 2767519744 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016420202020202022, + "loss": 2.5564, + "theoretical_loss": 3.3415834634364607, + "tokens_seen": 2767585280 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016418181818181817, + "loss": 2.4528, + "theoretical_loss": 3.34157727947481, + "tokens_seen": 2767650816 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016416161616161618, + "loss": 2.5266, + "theoretical_loss": 3.341571095700589, + "tokens_seen": 2767716352 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016414141414141414, + "loss": 2.5455, + "theoretical_loss": 3.341564912113788, + "tokens_seen": 2767781888 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016412121212121213, + "loss": 2.5253, + "theoretical_loss": 3.3415587287143964, + "tokens_seen": 2767847424 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001641010101010101, + "loss": 2.6848, + "theoretical_loss": 3.341552545502404, + "tokens_seen": 2767912960 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001640808080808081, + "loss": 2.5037, + "theoretical_loss": 3.3415463624778017, + "tokens_seen": 2767978496 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016406060606060608, + "loss": 2.5202, + "theoretical_loss": 3.341540179640578, + "tokens_seen": 2768044032 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016404040404040403, + "loss": 2.5081, + "theoretical_loss": 3.341533996990724, + "tokens_seen": 2768109568 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016402020202020202, + "loss": 2.4229, + "theoretical_loss": 3.341527814528229, + "tokens_seen": 2768175104 + }, + { + "epoch": 0.68, + "learning_rate": 0.000164, + "loss": 2.5212, + "theoretical_loss": 3.3415216322530825, + "tokens_seen": 2768240640 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016397979797979799, + "loss": 2.3994, + "theoretical_loss": 3.341515450165275, + "tokens_seen": 2768306176 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016395959595959597, + "loss": 2.5431, + "theoretical_loss": 3.3415092682647964, + "tokens_seen": 2768371712 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016393939393939395, + "loss": 2.5385, + "theoretical_loss": 3.3415030865516364, + "tokens_seen": 2768437248 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001639191919191919, + "loss": 2.497, + "theoretical_loss": 3.3414969050257852, + "tokens_seen": 2768502784 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016389898989898992, + "loss": 2.5348, + "theoretical_loss": 3.341490723687232, + "tokens_seen": 2768568320 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016387878787878788, + "loss": 2.6457, + "theoretical_loss": 3.341484542535967, + "tokens_seen": 2768633856 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016385858585858586, + "loss": 2.5542, + "theoretical_loss": 3.3414783615719807, + "tokens_seen": 2768699392 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016383838383838385, + "loss": 2.4564, + "theoretical_loss": 3.341472180795262, + "tokens_seen": 2768764928 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001638181818181818, + "loss": 2.3231, + "theoretical_loss": 3.341466000205801, + "tokens_seen": 2768830464 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016379797979797981, + "loss": 2.6891, + "theoretical_loss": 3.3414598198035885, + "tokens_seen": 2768896000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016377777777777777, + "loss": 2.6295, + "theoretical_loss": 3.3414536395886136, + "tokens_seen": 2768961536 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016375757575757578, + "loss": 2.6315, + "theoretical_loss": 3.3414474595608663, + "tokens_seen": 2769027072 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016373737373737374, + "loss": 2.4431, + "theoretical_loss": 3.3414412797203363, + "tokens_seen": 2769092608 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1518443, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.257474660873413, + "objective/train/theoretical_loss": 3.341435100067014, + "objective/train/tokens_used": 1128017376, + "theoretical_loss": 3.341435100067014, + "tokens_seen": 2769158144 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001637171717171717, + "loss": 2.4977, + "theoretical_loss": 3.341435100067014, + "tokens_seen": 2769158144 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001636969696969697, + "loss": 2.6552, + "theoretical_loss": 3.3414289206008885, + "tokens_seen": 2769223680 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016367676767676766, + "loss": 2.2741, + "theoretical_loss": 3.341422741321951, + "tokens_seen": 2769289216 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016365656565656568, + "loss": 2.4318, + "theoretical_loss": 3.34141656223019, + "tokens_seen": 2769354752 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016363636363636363, + "loss": 2.5306, + "theoretical_loss": 3.341410383325597, + "tokens_seen": 2769420288 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016361616161616162, + "loss": 2.3239, + "theoretical_loss": 3.34140420460816, + "tokens_seen": 2769485824 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001635959595959596, + "loss": 2.6397, + "theoretical_loss": 3.34139802607787, + "tokens_seen": 2769551360 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016357575757575758, + "loss": 2.5651, + "theoretical_loss": 3.341391847734717, + "tokens_seen": 2769616896 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016355555555555557, + "loss": 2.5907, + "theoretical_loss": 3.3413856695786905, + "tokens_seen": 2769682432 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016353535353535352, + "loss": 2.3563, + "theoretical_loss": 3.3413794916097808, + "tokens_seen": 2769747968 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001635151515151515, + "loss": 2.4961, + "theoretical_loss": 3.341373313827977, + "tokens_seen": 2769813504 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001634949494949495, + "loss": 2.501, + "theoretical_loss": 3.34136713623327, + "tokens_seen": 2769879040 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016347474747474748, + "loss": 2.2876, + "theoretical_loss": 3.3413609588256494, + "tokens_seen": 2769944576 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016345454545454546, + "loss": 2.6528, + "theoretical_loss": 3.341354781605105, + "tokens_seen": 2770010112 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016343434343434344, + "loss": 2.4382, + "theoretical_loss": 3.341348604571626, + "tokens_seen": 2770075648 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001634141414141414, + "loss": 2.7224, + "theoretical_loss": 3.3413424277252037, + "tokens_seen": 2770141184 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001633939393939394, + "loss": 2.5647, + "theoretical_loss": 3.3413362510658273, + "tokens_seen": 2770206720 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016337373737373737, + "loss": 2.5841, + "theoretical_loss": 3.341330074593486, + "tokens_seen": 2770272256 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016335353535353535, + "loss": 2.6594, + "theoretical_loss": 3.3413238983081714, + "tokens_seen": 2770337792 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016333333333333334, + "loss": 2.5856, + "theoretical_loss": 3.341317722209872, + "tokens_seen": 2770403328 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016331313131313132, + "loss": 2.5069, + "theoretical_loss": 3.341311546298578, + "tokens_seen": 2770468864 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001632929292929293, + "loss": 2.6656, + "theoretical_loss": 3.34130537057428, + "tokens_seen": 2770534400 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016327272727272726, + "loss": 2.6297, + "theoretical_loss": 3.341299195036967, + "tokens_seen": 2770599936 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016325252525252527, + "loss": 2.7451, + "theoretical_loss": 3.3412930196866295, + "tokens_seen": 2770665472 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016323232323232323, + "loss": 2.5948, + "theoretical_loss": 3.341286844523257, + "tokens_seen": 2770731008 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1519063, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.488872528076172, + "objective/train/theoretical_loss": 3.3412806695468396, + "objective/train/tokens_used": 1129655776, + "theoretical_loss": 3.3412806695468396, + "tokens_seen": 2770796544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016321212121212124, + "loss": 2.547, + "theoretical_loss": 3.3412806695468396, + "tokens_seen": 2770796544 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001631919191919192, + "loss": 2.6067, + "theoretical_loss": 3.3412744947573674, + "tokens_seen": 2770862080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016317171717171715, + "loss": 2.4995, + "theoretical_loss": 3.34126832015483, + "tokens_seen": 2770927616 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016315151515151517, + "loss": 2.5637, + "theoretical_loss": 3.341262145739218, + "tokens_seen": 2770993152 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016313131313131312, + "loss": 2.6142, + "theoretical_loss": 3.3412559715105203, + "tokens_seen": 2771058688 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016311111111111113, + "loss": 2.4756, + "theoretical_loss": 3.3412497974687274, + "tokens_seen": 2771124224 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001630909090909091, + "loss": 2.3863, + "theoretical_loss": 3.3412436236138294, + "tokens_seen": 2771189760 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016307070707070707, + "loss": 2.3899, + "theoretical_loss": 3.3412374499458157, + "tokens_seen": 2771255296 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016305050505050506, + "loss": 2.64, + "theoretical_loss": 3.3412312764646765, + "tokens_seen": 2771320832 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016303030303030304, + "loss": 2.3047, + "theoretical_loss": 3.3412251031704026, + "tokens_seen": 2771386368 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016301010101010103, + "loss": 2.431, + "theoretical_loss": 3.3412189300629818, + "tokens_seen": 2771451904 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016298989898989898, + "loss": 2.5177, + "theoretical_loss": 3.341212757142406, + "tokens_seen": 2771517440 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016296969696969697, + "loss": 2.6589, + "theoretical_loss": 3.3412065844086642, + "tokens_seen": 2771582976 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016294949494949495, + "loss": 2.618, + "theoretical_loss": 3.3412004118617467, + "tokens_seen": 2771648512 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016292929292929294, + "loss": 2.543, + "theoretical_loss": 3.341194239501643, + "tokens_seen": 2771714048 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016290909090909092, + "loss": 2.6723, + "theoretical_loss": 3.3411880673283436, + "tokens_seen": 2771779584 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001628888888888889, + "loss": 2.6841, + "theoretical_loss": 3.341181895341838, + "tokens_seen": 2771845120 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016286868686868686, + "loss": 2.5804, + "theoretical_loss": 3.3411757235421162, + "tokens_seen": 2771910656 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016284848484848484, + "loss": 2.3863, + "theoretical_loss": 3.3411695519291684, + "tokens_seen": 2771976192 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016282828282828283, + "loss": 2.529, + "theoretical_loss": 3.341163380502984, + "tokens_seen": 2772041728 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001628080808080808, + "loss": 2.6733, + "theoretical_loss": 3.3411572092635535, + "tokens_seen": 2772107264 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001627878787878788, + "loss": 2.6142, + "theoretical_loss": 3.3411510382108665, + "tokens_seen": 2772172800 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016276767676767675, + "loss": 2.6541, + "theoretical_loss": 3.341144867344913, + "tokens_seen": 2772238336 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016274747474747476, + "loss": 2.3849, + "theoretical_loss": 3.3411386966656833, + "tokens_seen": 2772303872 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016272727272727272, + "loss": 2.7101, + "theoretical_loss": 3.3411325261731664, + "tokens_seen": 2772369408 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1519644, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.670851945877075, + "objective/train/theoretical_loss": 3.341126355867353, + "objective/train/tokens_used": 1131294176, + "theoretical_loss": 3.341126355867353, + "tokens_seen": 2772434944 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016270707070707073, + "loss": 2.6129, + "theoretical_loss": 3.341126355867353, + "tokens_seen": 2772434944 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001626868686868687, + "loss": 2.63, + "theoretical_loss": 3.341120185748233, + "tokens_seen": 2772500480 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016266666666666665, + "loss": 2.8461, + "theoretical_loss": 3.341114015815796, + "tokens_seen": 2772566016 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016264646464646466, + "loss": 2.7287, + "theoretical_loss": 3.3411078460700327, + "tokens_seen": 2772631552 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001626262626262626, + "loss": 2.5764, + "theoretical_loss": 3.341101676510932, + "tokens_seen": 2772697088 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016260606060606062, + "loss": 2.5869, + "theoretical_loss": 3.3410955071384842, + "tokens_seen": 2772762624 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016258585858585858, + "loss": 2.5042, + "theoretical_loss": 3.3410893379526794, + "tokens_seen": 2772828160 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016256565656565656, + "loss": 2.6667, + "theoretical_loss": 3.341083168953508, + "tokens_seen": 2772893696 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016254545454545455, + "loss": 2.2917, + "theoretical_loss": 3.341077000140959, + "tokens_seen": 2772959232 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016252525252525253, + "loss": 2.5474, + "theoretical_loss": 3.341070831515023, + "tokens_seen": 2773024768 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016250505050505052, + "loss": 2.5906, + "theoretical_loss": 3.3410646630756897, + "tokens_seen": 2773090304 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016248484848484847, + "loss": 2.651, + "theoretical_loss": 3.341058494822949, + "tokens_seen": 2773155840 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016246464646464648, + "loss": 2.4476, + "theoretical_loss": 3.341052326756791, + "tokens_seen": 2773221376 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016244444444444444, + "loss": 2.5894, + "theoretical_loss": 3.3410461588772056, + "tokens_seen": 2773286912 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016242424242424243, + "loss": 2.5081, + "theoretical_loss": 3.341039991184183, + "tokens_seen": 2773352448 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001624040404040404, + "loss": 2.4556, + "theoretical_loss": 3.341033823677712, + "tokens_seen": 2773417984 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001623838383838384, + "loss": 2.5536, + "theoretical_loss": 3.3410276563577845, + "tokens_seen": 2773483520 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016236363636363638, + "loss": 2.6117, + "theoretical_loss": 3.3410214892243886, + "tokens_seen": 2773549056 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016234343434343436, + "loss": 2.6355, + "theoretical_loss": 3.3410153222775154, + "tokens_seen": 2773614592 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016232323232323232, + "loss": 2.4232, + "theoretical_loss": 3.3410091555171544, + "tokens_seen": 2773680128 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001623030303030303, + "loss": 2.5252, + "theoretical_loss": 3.3410029889432957, + "tokens_seen": 2773745664 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016228282828282829, + "loss": 2.5139, + "theoretical_loss": 3.3409968225559292, + "tokens_seen": 2773811200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016226262626262627, + "loss": 2.6772, + "theoretical_loss": 3.3409906563550447, + "tokens_seen": 2773876736 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016224242424242425, + "loss": 2.3691, + "theoretical_loss": 3.3409844903406323, + "tokens_seen": 2773942272 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001622222222222222, + "loss": 2.5815, + "theoretical_loss": 3.340978324512682, + "tokens_seen": 2774007808 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1520668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0536701679229736, + "objective/train/theoretical_loss": 3.3409721588711836, + "objective/train/tokens_used": 1132932576, + "theoretical_loss": 3.3409721588711836, + "tokens_seen": 2774073344 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016220202020202022, + "loss": 2.8458, + "theoretical_loss": 3.3409721588711836, + "tokens_seen": 2774073344 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016218181818181818, + "loss": 2.7143, + "theoretical_loss": 3.3409659934161273, + "tokens_seen": 2774138880 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001621616161616162, + "loss": 2.5732, + "theoretical_loss": 3.340959828147503, + "tokens_seen": 2774204416 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016214141414141415, + "loss": 2.7027, + "theoretical_loss": 3.3409536630653003, + "tokens_seen": 2774269952 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001621212121212121, + "loss": 2.4665, + "theoretical_loss": 3.3409474981695095, + "tokens_seen": 2774335488 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016210101010101011, + "loss": 2.4467, + "theoretical_loss": 3.340941333460121, + "tokens_seen": 2774401024 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016208080808080807, + "loss": 2.326, + "theoretical_loss": 3.3409351689371234, + "tokens_seen": 2774466560 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016206060606060608, + "loss": 2.5686, + "theoretical_loss": 3.340929004600508, + "tokens_seen": 2774532096 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016204040404040404, + "loss": 2.549, + "theoretical_loss": 3.340922840450264, + "tokens_seen": 2774597632 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016202020202020202, + "loss": 2.3004, + "theoretical_loss": 3.340916676486382, + "tokens_seen": 2774663168 + }, + { + "epoch": 0.68, + "learning_rate": 0.000162, + "loss": 2.6201, + "theoretical_loss": 3.3409105127088514, + "tokens_seen": 2774728704 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016197979797979796, + "loss": 2.7913, + "theoretical_loss": 3.340904349117663, + "tokens_seen": 2774794240 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016195959595959598, + "loss": 2.3966, + "theoretical_loss": 3.3408981857128053, + "tokens_seen": 2774859776 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016193939393939393, + "loss": 2.5628, + "theoretical_loss": 3.3408920224942693, + "tokens_seen": 2774925312 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016191919191919192, + "loss": 2.6146, + "theoretical_loss": 3.3408858594620447, + "tokens_seen": 2774990848 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001618989898989899, + "loss": 2.554, + "theoretical_loss": 3.3408796966161223, + "tokens_seen": 2775056384 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016187878787878788, + "loss": 2.4501, + "theoretical_loss": 3.3408735339564903, + "tokens_seen": 2775121920 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016185858585858587, + "loss": 2.5027, + "theoretical_loss": 3.3408673714831405, + "tokens_seen": 2775187456 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016183838383838385, + "loss": 2.6133, + "theoretical_loss": 3.3408612091960617, + "tokens_seen": 2775252992 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001618181818181818, + "loss": 2.4547, + "theoretical_loss": 3.340855047095244, + "tokens_seen": 2775318528 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001617979797979798, + "loss": 2.2018, + "theoretical_loss": 3.340848885180678, + "tokens_seen": 2775384064 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016177777777777778, + "loss": 2.567, + "theoretical_loss": 3.340842723452353, + "tokens_seen": 2775449600 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016175757575757576, + "loss": 2.5154, + "theoretical_loss": 3.340836561910259, + "tokens_seen": 2775515136 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016173737373737374, + "loss": 2.4335, + "theoretical_loss": 3.340830400554387, + "tokens_seen": 2775580672 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001617171717171717, + "loss": 2.6162, + "theoretical_loss": 3.340824239384726, + "tokens_seen": 2775646208 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1521377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4955875873565674, + "objective/train/theoretical_loss": 3.3408180784012655, + "objective/train/tokens_used": 1134570976, + "theoretical_loss": 3.3408180784012655, + "tokens_seen": 2775711744 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001616969696969697, + "loss": 2.6075, + "theoretical_loss": 3.3408180784012655, + "tokens_seen": 2775711744 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016167676767676767, + "loss": 2.5113, + "theoretical_loss": 3.3408119176039963, + "tokens_seen": 2775777280 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016165656565656568, + "loss": 2.5171, + "theoretical_loss": 3.340805756992909, + "tokens_seen": 2775842816 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016163636363636364, + "loss": 2.3704, + "theoretical_loss": 3.340799596567992, + "tokens_seen": 2775908352 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016161616161616162, + "loss": 2.3761, + "theoretical_loss": 3.3407934363292364, + "tokens_seen": 2775973888 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001615959595959596, + "loss": 2.4156, + "theoretical_loss": 3.3407872762766315, + "tokens_seen": 2776039424 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016157575757575756, + "loss": 2.4299, + "theoretical_loss": 3.340781116410168, + "tokens_seen": 2776104960 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016155555555555557, + "loss": 2.4115, + "theoretical_loss": 3.3407749567298355, + "tokens_seen": 2776170496 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016153535353535353, + "loss": 2.2878, + "theoretical_loss": 3.340768797235624, + "tokens_seen": 2776236032 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016151515151515154, + "loss": 2.4185, + "theoretical_loss": 3.3407626379275235, + "tokens_seen": 2776301568 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001614949494949495, + "loss": 2.601, + "theoretical_loss": 3.340756478805524, + "tokens_seen": 2776367104 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016147474747474748, + "loss": 2.4135, + "theoretical_loss": 3.340750319869615, + "tokens_seen": 2776432640 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016145454545454547, + "loss": 2.4905, + "theoretical_loss": 3.3407441611197872, + "tokens_seen": 2776498176 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016143434343434342, + "loss": 2.6558, + "theoretical_loss": 3.3407380025560305, + "tokens_seen": 2776563712 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016141414141414143, + "loss": 2.7102, + "theoretical_loss": 3.3407318441783347, + "tokens_seen": 2776629248 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001613939393939394, + "loss": 2.383, + "theoretical_loss": 3.3407256859866896, + "tokens_seen": 2776694784 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016137373737373737, + "loss": 2.2967, + "theoretical_loss": 3.3407195279810855, + "tokens_seen": 2776760320 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016135353535353536, + "loss": 2.6323, + "theoretical_loss": 3.340713370161512, + "tokens_seen": 2776825856 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016133333333333334, + "loss": 2.6789, + "theoretical_loss": 3.3407072125279598, + "tokens_seen": 2776891392 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016131313131313133, + "loss": 2.4848, + "theoretical_loss": 3.3407010550804186, + "tokens_seen": 2776956928 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016129292929292928, + "loss": 2.6104, + "theoretical_loss": 3.340694897818878, + "tokens_seen": 2777022464 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016127272727272727, + "loss": 2.5611, + "theoretical_loss": 3.340688740743328, + "tokens_seen": 2777088000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016125252525252525, + "loss": 2.5374, + "theoretical_loss": 3.340682583853759, + "tokens_seen": 2777153536 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016123232323232324, + "loss": 2.3621, + "theoretical_loss": 3.3406764271501608, + "tokens_seen": 2777219072 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016121212121212122, + "loss": 2.4859, + "theoretical_loss": 3.3406702706325233, + "tokens_seen": 2777284608 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1522539, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6646552085876465, + "objective/train/theoretical_loss": 3.3406641143008367, + "objective/train/tokens_used": 1136209376, + "theoretical_loss": 3.3406641143008367, + "tokens_seen": 2777350144 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001611919191919192, + "loss": 2.5536, + "theoretical_loss": 3.3406641143008367, + "tokens_seen": 2777350144 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016117171717171716, + "loss": 2.4847, + "theoretical_loss": 3.340657958155091, + "tokens_seen": 2777415680 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016115151515151517, + "loss": 2.5023, + "theoretical_loss": 3.3406518021952762, + "tokens_seen": 2777481216 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016113131313131313, + "loss": 2.723, + "theoretical_loss": 3.340645646421382, + "tokens_seen": 2777546752 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001611111111111111, + "loss": 2.3839, + "theoretical_loss": 3.3406394908333987, + "tokens_seen": 2777612288 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001610909090909091, + "loss": 2.5015, + "theoretical_loss": 3.340633335431316, + "tokens_seen": 2777677824 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016107070707070705, + "loss": 2.6068, + "theoretical_loss": 3.340627180215124, + "tokens_seen": 2777743360 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016105050505050506, + "loss": 2.6152, + "theoretical_loss": 3.3406210251848134, + "tokens_seen": 2777808896 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016103030303030302, + "loss": 2.6447, + "theoretical_loss": 3.3406148703403735, + "tokens_seen": 2777874432 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016101010101010103, + "loss": 2.7729, + "theoretical_loss": 3.3406087156817943, + "tokens_seen": 2777939968 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160989898989899, + "loss": 2.3859, + "theoretical_loss": 3.3406025612090655, + "tokens_seen": 2778005504 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016096969696969697, + "loss": 2.4838, + "theoretical_loss": 3.340596406922178, + "tokens_seen": 2778071040 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016094949494949496, + "loss": 2.4317, + "theoretical_loss": 3.340590252821121, + "tokens_seen": 2778136576 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001609292929292929, + "loss": 2.475, + "theoretical_loss": 3.340584098905885, + "tokens_seen": 2778202112 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016090909090909092, + "loss": 2.2991, + "theoretical_loss": 3.3405779451764595, + "tokens_seen": 2778267648 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016088888888888888, + "loss": 2.6116, + "theoretical_loss": 3.3405717916328355, + "tokens_seen": 2778333184 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016086868686868687, + "loss": 2.597, + "theoretical_loss": 3.3405656382750015, + "tokens_seen": 2778398720 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016084848484848485, + "loss": 2.5592, + "theoretical_loss": 3.3405594851029488, + "tokens_seen": 2778464256 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016082828282828283, + "loss": 2.4637, + "theoretical_loss": 3.3405533321166674, + "tokens_seen": 2778529792 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016080808080808082, + "loss": 2.4627, + "theoretical_loss": 3.3405471793161463, + "tokens_seen": 2778595328 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001607878787878788, + "loss": 2.4488, + "theoretical_loss": 3.340541026701376, + "tokens_seen": 2778660864 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016076767676767679, + "loss": 2.5985, + "theoretical_loss": 3.340534874272347, + "tokens_seen": 2778726400 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016074747474747474, + "loss": 2.5007, + "theoretical_loss": 3.3405287220290485, + "tokens_seen": 2778791936 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016072727272727273, + "loss": 2.3902, + "theoretical_loss": 3.3405225699714713, + "tokens_seen": 2778857472 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001607070707070707, + "loss": 2.6528, + "theoretical_loss": 3.3405164180996048, + "tokens_seen": 2778923008 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1523044, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.768937587738037, + "objective/train/theoretical_loss": 3.340510266413439, + "objective/train/tokens_used": 1137847776, + "theoretical_loss": 3.340510266413439, + "tokens_seen": 2778988544 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001606868686868687, + "loss": 2.6315, + "theoretical_loss": 3.340510266413439, + "tokens_seen": 2778988544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016066666666666668, + "loss": 2.4005, + "theoretical_loss": 3.3405041149129646, + "tokens_seen": 2779054080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016064646464646466, + "loss": 2.6206, + "theoretical_loss": 3.340497963598171, + "tokens_seen": 2779119616 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016062626262626262, + "loss": 2.4944, + "theoretical_loss": 3.3404918124690486, + "tokens_seen": 2779185152 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016060606060606063, + "loss": 2.5336, + "theoretical_loss": 3.340485661525587, + "tokens_seen": 2779250688 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001605858585858586, + "loss": 2.4623, + "theoretical_loss": 3.3404795107677763, + "tokens_seen": 2779316224 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016056565656565657, + "loss": 2.4734, + "theoretical_loss": 3.340473360195607, + "tokens_seen": 2779381760 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016054545454545455, + "loss": 2.6917, + "theoretical_loss": 3.340467209809068, + "tokens_seen": 2779447296 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001605252525252525, + "loss": 2.6046, + "theoretical_loss": 3.3404610596081508, + "tokens_seen": 2779512832 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016050505050505052, + "loss": 2.5243, + "theoretical_loss": 3.340454909592845, + "tokens_seen": 2779578368 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016048484848484848, + "loss": 2.5576, + "theoretical_loss": 3.34044875976314, + "tokens_seen": 2779643904 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001604646464646465, + "loss": 2.3988, + "theoretical_loss": 3.340442610119026, + "tokens_seen": 2779709440 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016044444444444445, + "loss": 2.6069, + "theoretical_loss": 3.340436460660493, + "tokens_seen": 2779774976 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001604242424242424, + "loss": 2.463, + "theoretical_loss": 3.3404303113875313, + "tokens_seen": 2779840512 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016040404040404042, + "loss": 2.4132, + "theoretical_loss": 3.340424162300131, + "tokens_seen": 2779906048 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016038383838383837, + "loss": 2.3336, + "theoretical_loss": 3.340418013398282, + "tokens_seen": 2779971584 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016036363636363638, + "loss": 2.483, + "theoretical_loss": 3.340411864681974, + "tokens_seen": 2780037120 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016034343434343434, + "loss": 2.5414, + "theoretical_loss": 3.3404057161511975, + "tokens_seen": 2780102656 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016032323232323232, + "loss": 2.5594, + "theoretical_loss": 3.3403995678059424, + "tokens_seen": 2780168192 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001603030303030303, + "loss": 2.6549, + "theoretical_loss": 3.340393419646199, + "tokens_seen": 2780233728 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001602828282828283, + "loss": 2.7012, + "theoretical_loss": 3.3403872716719563, + "tokens_seen": 2780299264 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016026262626262628, + "loss": 2.8683, + "theoretical_loss": 3.3403811238832053, + "tokens_seen": 2780364800 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016024242424242423, + "loss": 2.6444, + "theoretical_loss": 3.3403749762799357, + "tokens_seen": 2780430336 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016022222222222222, + "loss": 2.652, + "theoretical_loss": 3.340368828862138, + "tokens_seen": 2780495872 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001602020202020202, + "loss": 2.4963, + "theoretical_loss": 3.3403626816298013, + "tokens_seen": 2780561408 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1524063, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6973204612731934, + "objective/train/theoretical_loss": 3.3403565345829165, + "objective/train/tokens_used": 1139486176, + "theoretical_loss": 3.3403565345829165, + "tokens_seen": 2780626944 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016018181818181818, + "loss": 2.5187, + "theoretical_loss": 3.3403565345829165, + "tokens_seen": 2780626944 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016016161616161617, + "loss": 2.5263, + "theoretical_loss": 3.340350387721473, + "tokens_seen": 2780692480 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016014141414141415, + "loss": 2.6961, + "theoretical_loss": 3.3403442410454613, + "tokens_seen": 2780758016 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001601212121212121, + "loss": 2.5047, + "theoretical_loss": 3.3403380945548715, + "tokens_seen": 2780823552 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016010101010101012, + "loss": 2.5578, + "theoretical_loss": 3.340331948249693, + "tokens_seen": 2780889088 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016008080808080808, + "loss": 2.6112, + "theoretical_loss": 3.3403258021299163, + "tokens_seen": 2780954624 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016006060606060606, + "loss": 2.7148, + "theoretical_loss": 3.3403196561955317, + "tokens_seen": 2781020160 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016004040404040405, + "loss": 2.5043, + "theoretical_loss": 3.3403135104465287, + "tokens_seen": 2781085696 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160020202020202, + "loss": 2.4761, + "theoretical_loss": 3.3403073648828974, + "tokens_seen": 2781151232 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016, + "loss": 2.3614, + "theoretical_loss": 3.340301219504628, + "tokens_seen": 2781216768 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015997979797979797, + "loss": 2.5628, + "theoretical_loss": 3.340295074311711, + "tokens_seen": 2781282304 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015995959595959598, + "loss": 2.5824, + "theoretical_loss": 3.340288929304135, + "tokens_seen": 2781347840 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015993939393939394, + "loss": 2.4464, + "theoretical_loss": 3.340282784481892, + "tokens_seen": 2781413376 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015991919191919192, + "loss": 2.4653, + "theoretical_loss": 3.340276639844971, + "tokens_seen": 2781478912 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001598989898989899, + "loss": 2.5715, + "theoretical_loss": 3.3402704953933617, + "tokens_seen": 2781544448 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015987878787878786, + "loss": 2.5535, + "theoretical_loss": 3.340264351127055, + "tokens_seen": 2781609984 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015985858585858587, + "loss": 2.611, + "theoretical_loss": 3.34025820704604, + "tokens_seen": 2781675520 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015983838383838383, + "loss": 2.6541, + "theoretical_loss": 3.3402520631503076, + "tokens_seen": 2781741056 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015981818181818184, + "loss": 2.5694, + "theoretical_loss": 3.3402459194398473, + "tokens_seen": 2781806592 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001597979797979798, + "loss": 2.7202, + "theoretical_loss": 3.3402397759146494, + "tokens_seen": 2781872128 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015977777777777778, + "loss": 2.389, + "theoretical_loss": 3.3402336325747037, + "tokens_seen": 2781937664 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015975757575757577, + "loss": 2.4132, + "theoretical_loss": 3.340227489420001, + "tokens_seen": 2782003200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015973737373737372, + "loss": 2.7229, + "theoretical_loss": 3.34022134645053, + "tokens_seen": 2782068736 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015971717171717173, + "loss": 2.52, + "theoretical_loss": 3.340215203666282, + "tokens_seen": 2782134272 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001596969696969697, + "loss": 2.5388, + "theoretical_loss": 3.340209061067247, + "tokens_seen": 2782199808 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1525440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.696449041366577, + "objective/train/theoretical_loss": 3.340202918653414, + "objective/train/tokens_used": 1141124576, + "theoretical_loss": 3.340202918653414, + "tokens_seen": 2782265344 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015967676767676768, + "loss": 2.4945, + "theoretical_loss": 3.340202918653414, + "tokens_seen": 2782265344 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015965656565656566, + "loss": 2.4624, + "theoretical_loss": 3.340196776424774, + "tokens_seen": 2782330880 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015963636363636364, + "loss": 2.5951, + "theoretical_loss": 3.3401906343813166, + "tokens_seen": 2782396416 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015961616161616163, + "loss": 2.6573, + "theoretical_loss": 3.3401844925230324, + "tokens_seen": 2782461952 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001595959595959596, + "loss": 2.4068, + "theoretical_loss": 3.3401783508499108, + "tokens_seen": 2782527488 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015957575757575757, + "loss": 2.5661, + "theoretical_loss": 3.340172209361942, + "tokens_seen": 2782593024 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015955555555555555, + "loss": 2.7027, + "theoretical_loss": 3.3401660680591165, + "tokens_seen": 2782658560 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015953535353535354, + "loss": 2.6991, + "theoretical_loss": 3.3401599269414235, + "tokens_seen": 2782724096 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015951515151515152, + "loss": 2.5601, + "theoretical_loss": 3.3401537860088544, + "tokens_seen": 2782789632 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001594949494949495, + "loss": 2.575, + "theoretical_loss": 3.340147645261398, + "tokens_seen": 2782855168 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015947474747474746, + "loss": 2.3982, + "theoretical_loss": 3.340141504699045, + "tokens_seen": 2782920704 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015945454545454547, + "loss": 2.5263, + "theoretical_loss": 3.340135364321785, + "tokens_seen": 2782986240 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015943434343434343, + "loss": 2.2968, + "theoretical_loss": 3.3401292241296083, + "tokens_seen": 2783051776 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015941414141414144, + "loss": 2.5574, + "theoretical_loss": 3.340123084122505, + "tokens_seen": 2783117312 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001593939393939394, + "loss": 2.6271, + "theoretical_loss": 3.3401169443004655, + "tokens_seen": 2783182848 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015937373737373735, + "loss": 2.7247, + "theoretical_loss": 3.3401108046634795, + "tokens_seen": 2783248384 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015935353535353536, + "loss": 2.3931, + "theoretical_loss": 3.340104665211537, + "tokens_seen": 2783313920 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015933333333333332, + "loss": 2.7295, + "theoretical_loss": 3.3400985259446285, + "tokens_seen": 2783379456 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015931313131313133, + "loss": 2.501, + "theoretical_loss": 3.340092386862743, + "tokens_seen": 2783444992 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001592929292929293, + "loss": 2.4324, + "theoretical_loss": 3.340086247965872, + "tokens_seen": 2783510528 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015927272727272727, + "loss": 2.421, + "theoretical_loss": 3.340080109254005, + "tokens_seen": 2783576064 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015925252525252526, + "loss": 2.8095, + "theoretical_loss": 3.3400739707271314, + "tokens_seen": 2783641600 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015923232323232324, + "loss": 2.4783, + "theoretical_loss": 3.340067832385242, + "tokens_seen": 2783707136 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015921212121212122, + "loss": 2.5353, + "theoretical_loss": 3.3400616942283268, + "tokens_seen": 2783772672 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015919191919191918, + "loss": 2.395, + "theoretical_loss": 3.3400555562563756, + "tokens_seen": 2783838208 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1526156, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.48842716217041, + "objective/train/theoretical_loss": 3.3400494184693788, + "objective/train/tokens_used": 1142762976, + "theoretical_loss": 3.3400494184693788, + "tokens_seen": 2783903744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015917171717171717, + "loss": 2.645, + "theoretical_loss": 3.3400494184693788, + "tokens_seen": 2783903744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015915151515151515, + "loss": 2.726, + "theoretical_loss": 3.340043280867326, + "tokens_seen": 2783969280 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015913131313131313, + "loss": 2.6242, + "theoretical_loss": 3.340037143450208, + "tokens_seen": 2784034816 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015911111111111112, + "loss": 2.4032, + "theoretical_loss": 3.340031006218014, + "tokens_seen": 2784100352 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001590909090909091, + "loss": 2.56, + "theoretical_loss": 3.3400248691707346, + "tokens_seen": 2784165888 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015907070707070706, + "loss": 2.7339, + "theoretical_loss": 3.34001873230836, + "tokens_seen": 2784231424 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015905050505050507, + "loss": 2.6934, + "theoretical_loss": 3.34001259563088, + "tokens_seen": 2784296960 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015903030303030303, + "loss": 2.6446, + "theoretical_loss": 3.3400064591382845, + "tokens_seen": 2784362496 + }, + { + "epoch": 0.69, + "learning_rate": 0.000159010101010101, + "loss": 2.6008, + "theoretical_loss": 3.340000322830564, + "tokens_seen": 2784428032 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158989898989899, + "loss": 2.4076, + "theoretical_loss": 3.3399941867077088, + "tokens_seen": 2784493568 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015896969696969698, + "loss": 2.6223, + "theoretical_loss": 3.3399880507697084, + "tokens_seen": 2784559104 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015894949494949496, + "loss": 2.6277, + "theoretical_loss": 3.3399819150165526, + "tokens_seen": 2784624640 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015892929292929292, + "loss": 2.5007, + "theoretical_loss": 3.3399757794482325, + "tokens_seen": 2784690176 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015890909090909093, + "loss": 2.3649, + "theoretical_loss": 3.3399696440647375, + "tokens_seen": 2784755712 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001588888888888889, + "loss": 2.6687, + "theoretical_loss": 3.3399635088660578, + "tokens_seen": 2784821248 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001588686868686869, + "loss": 2.5197, + "theoretical_loss": 3.3399573738521835, + "tokens_seen": 2784886784 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015884848484848485, + "loss": 2.5765, + "theoretical_loss": 3.3399512390231045, + "tokens_seen": 2784952320 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001588282828282828, + "loss": 2.4472, + "theoretical_loss": 3.3399451043788115, + "tokens_seen": 2785017856 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015880808080808082, + "loss": 2.5299, + "theoretical_loss": 3.339938969919294, + "tokens_seen": 2785083392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015878787878787878, + "loss": 2.6906, + "theoretical_loss": 3.3399328356445417, + "tokens_seen": 2785148928 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001587676767676768, + "loss": 2.3887, + "theoretical_loss": 3.3399267015545457, + "tokens_seen": 2785214464 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015874747474747475, + "loss": 2.6104, + "theoretical_loss": 3.339920567649296, + "tokens_seen": 2785280000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015872727272727273, + "loss": 2.5876, + "theoretical_loss": 3.3399144339287816, + "tokens_seen": 2785345536 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015870707070707072, + "loss": 2.454, + "theoretical_loss": 3.339908300392994, + "tokens_seen": 2785411072 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015868686868686867, + "loss": 2.5383, + "theoretical_loss": 3.339902167041922, + "tokens_seen": 2785476608 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1527495, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.147812843322754, + "objective/train/theoretical_loss": 3.339896033875556, + "objective/train/tokens_used": 1144401376, + "theoretical_loss": 3.339896033875556, + "tokens_seen": 2785542144 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015866666666666668, + "loss": 2.5366, + "theoretical_loss": 3.339896033875556, + "tokens_seen": 2785542144 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015864646464646464, + "loss": 2.5483, + "theoretical_loss": 3.3398899008938874, + "tokens_seen": 2785607680 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015862626262626262, + "loss": 2.4007, + "theoretical_loss": 3.3398837680969047, + "tokens_seen": 2785673216 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001586060606060606, + "loss": 2.5627, + "theoretical_loss": 3.3398776354845987, + "tokens_seen": 2785738752 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001585858585858586, + "loss": 2.5758, + "theoretical_loss": 3.3398715030569592, + "tokens_seen": 2785804288 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015856565656565658, + "loss": 2.533, + "theoretical_loss": 3.3398653708139765, + "tokens_seen": 2785869824 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015854545454545456, + "loss": 2.517, + "theoretical_loss": 3.3398592387556407, + "tokens_seen": 2785935360 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015852525252525252, + "loss": 2.5876, + "theoretical_loss": 3.3398531068819417, + "tokens_seen": 2786000896 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001585050505050505, + "loss": 2.3341, + "theoretical_loss": 3.33984697519287, + "tokens_seen": 2786066432 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015848484848484848, + "loss": 2.4933, + "theoretical_loss": 3.3398408436884153, + "tokens_seen": 2786131968 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015846464646464647, + "loss": 2.5171, + "theoretical_loss": 3.3398347123685674, + "tokens_seen": 2786197504 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015844444444444445, + "loss": 2.5912, + "theoretical_loss": 3.3398285812333173, + "tokens_seen": 2786263040 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001584242424242424, + "loss": 2.4985, + "theoretical_loss": 3.3398224502826546, + "tokens_seen": 2786328576 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015840404040404042, + "loss": 2.5726, + "theoretical_loss": 3.3398163195165695, + "tokens_seen": 2786394112 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015838383838383838, + "loss": 2.3558, + "theoretical_loss": 3.339810188935052, + "tokens_seen": 2786459648 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001583636363636364, + "loss": 2.3956, + "theoretical_loss": 3.3398040585380926, + "tokens_seen": 2786525184 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015834343434343435, + "loss": 2.5117, + "theoretical_loss": 3.3397979283256802, + "tokens_seen": 2786590720 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001583232323232323, + "loss": 2.4924, + "theoretical_loss": 3.3397917982978065, + "tokens_seen": 2786656256 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001583030303030303, + "loss": 2.5968, + "theoretical_loss": 3.3397856684544607, + "tokens_seen": 2786721792 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015828282828282827, + "loss": 2.5655, + "theoretical_loss": 3.339779538795633, + "tokens_seen": 2786787328 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015826262626262628, + "loss": 2.6844, + "theoretical_loss": 3.3397734093213134, + "tokens_seen": 2786852864 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015824242424242424, + "loss": 2.4677, + "theoretical_loss": 3.3397672800314924, + "tokens_seen": 2786918400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015822222222222222, + "loss": 2.6896, + "theoretical_loss": 3.3397611509261598, + "tokens_seen": 2786983936 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001582020202020202, + "loss": 2.5562, + "theoretical_loss": 3.3397550220053054, + "tokens_seen": 2787049472 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001581818181818182, + "loss": 2.5334, + "theoretical_loss": 3.3397488932689203, + "tokens_seen": 2787115008 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1527941, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6329824924468994, + "objective/train/theoretical_loss": 3.339742764716994, + "objective/train/tokens_used": 1146039776, + "theoretical_loss": 3.339742764716994, + "tokens_seen": 2787180544 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015816161616161617, + "loss": 2.5974, + "theoretical_loss": 3.339742764716994, + "tokens_seen": 2787180544 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015814141414141413, + "loss": 2.4753, + "theoretical_loss": 3.3397366363495165, + "tokens_seen": 2787246080 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015812121212121214, + "loss": 2.6397, + "theoretical_loss": 3.3397305081664777, + "tokens_seen": 2787311616 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001581010101010101, + "loss": 2.4848, + "theoretical_loss": 3.339724380167868, + "tokens_seen": 2787377152 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015808080808080808, + "loss": 2.4093, + "theoretical_loss": 3.3397182523536784, + "tokens_seen": 2787442688 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015806060606060607, + "loss": 2.4799, + "theoretical_loss": 3.3397121247238974, + "tokens_seen": 2787508224 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015804040404040405, + "loss": 2.6244, + "theoretical_loss": 3.339705997278516, + "tokens_seen": 2787573760 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015802020202020203, + "loss": 2.6077, + "theoretical_loss": 3.3396998700175247, + "tokens_seen": 2787639296 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158, + "loss": 2.4154, + "theoretical_loss": 3.339693742940913, + "tokens_seen": 2787704832 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015797979797979798, + "loss": 2.7696, + "theoretical_loss": 3.339687616048671, + "tokens_seen": 2787770368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015795959595959596, + "loss": 2.5866, + "theoretical_loss": 3.339681489340789, + "tokens_seen": 2787835904 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015793939393939394, + "loss": 2.5651, + "theoretical_loss": 3.339675362817257, + "tokens_seen": 2787901440 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015791919191919193, + "loss": 2.5116, + "theoretical_loss": 3.339669236478065, + "tokens_seen": 2787966976 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001578989898989899, + "loss": 2.578, + "theoretical_loss": 3.3396631103232033, + "tokens_seen": 2788032512 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015787878787878787, + "loss": 2.6013, + "theoretical_loss": 3.3396569843526622, + "tokens_seen": 2788098048 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015785858585858588, + "loss": 2.4345, + "theoretical_loss": 3.339650858566432, + "tokens_seen": 2788163584 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015783838383838384, + "loss": 2.5461, + "theoretical_loss": 3.3396447329645023, + "tokens_seen": 2788229120 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015781818181818182, + "loss": 2.4301, + "theoretical_loss": 3.3396386075468634, + "tokens_seen": 2788294656 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001577979797979798, + "loss": 2.5665, + "theoretical_loss": 3.339632482313505, + "tokens_seen": 2788360192 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015777777777777776, + "loss": 2.6078, + "theoretical_loss": 3.339626357264418, + "tokens_seen": 2788425728 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015775757575757577, + "loss": 2.4667, + "theoretical_loss": 3.339620232399592, + "tokens_seen": 2788491264 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015773737373737373, + "loss": 2.3245, + "theoretical_loss": 3.339614107719018, + "tokens_seen": 2788556800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015771717171717174, + "loss": 2.7024, + "theoretical_loss": 3.3396079832226846, + "tokens_seen": 2788622336 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001576969696969697, + "loss": 2.3591, + "theoretical_loss": 3.339601858910583, + "tokens_seen": 2788687872 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015767676767676768, + "loss": 2.2236, + "theoretical_loss": 3.339595734782703, + "tokens_seen": 2788753408 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1529074, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.408823013305664, + "objective/train/theoretical_loss": 3.3395896108390355, + "objective/train/tokens_used": 1147678176, + "theoretical_loss": 3.3395896108390355, + "tokens_seen": 2788818944 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015765656565656566, + "loss": 2.5882, + "theoretical_loss": 3.3395896108390355, + "tokens_seen": 2788818944 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015763636363636362, + "loss": 2.4902, + "theoretical_loss": 3.3395834870795698, + "tokens_seen": 2788884480 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015761616161616163, + "loss": 2.5388, + "theoretical_loss": 3.339577363504296, + "tokens_seen": 2788950016 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001575959595959596, + "loss": 2.6776, + "theoretical_loss": 3.339571240113204, + "tokens_seen": 2789015552 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015757575757575757, + "loss": 2.5946, + "theoretical_loss": 3.3395651169062845, + "tokens_seen": 2789081088 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015755555555555556, + "loss": 2.6231, + "theoretical_loss": 3.3395589938835277, + "tokens_seen": 2789146624 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015753535353535354, + "loss": 2.6282, + "theoretical_loss": 3.3395528710449236, + "tokens_seen": 2789212160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015751515151515153, + "loss": 2.602, + "theoretical_loss": 3.339546748390462, + "tokens_seen": 2789277696 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001574949494949495, + "loss": 2.6143, + "theoretical_loss": 3.3395406259201335, + "tokens_seen": 2789343232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015747474747474747, + "loss": 2.4692, + "theoretical_loss": 3.339534503633928, + "tokens_seen": 2789408768 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015745454545454545, + "loss": 2.7083, + "theoretical_loss": 3.339528381531835, + "tokens_seen": 2789474304 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015743434343434343, + "loss": 2.4526, + "theoretical_loss": 3.339522259613846, + "tokens_seen": 2789539840 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015741414141414142, + "loss": 2.602, + "theoretical_loss": 3.3395161378799507, + "tokens_seen": 2789605376 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001573939393939394, + "loss": 2.4026, + "theoretical_loss": 3.3395100163301383, + "tokens_seen": 2789670912 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015737373737373736, + "loss": 2.3998, + "theoretical_loss": 3.3395038949644, + "tokens_seen": 2789736448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015735353535353537, + "loss": 2.3635, + "theoretical_loss": 3.3394977737827256, + "tokens_seen": 2789801984 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015733333333333333, + "loss": 2.5457, + "theoretical_loss": 3.339491652785105, + "tokens_seen": 2789867520 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015731313131313134, + "loss": 2.4846, + "theoretical_loss": 3.3394855319715284, + "tokens_seen": 2789933056 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001572929292929293, + "loss": 2.5002, + "theoretical_loss": 3.3394794113419866, + "tokens_seen": 2789998592 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015727272727272728, + "loss": 2.5573, + "theoretical_loss": 3.3394732908964686, + "tokens_seen": 2790064128 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015725252525252526, + "loss": 2.4706, + "theoretical_loss": 3.3394671706349657, + "tokens_seen": 2790129664 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015723232323232322, + "loss": 2.5603, + "theoretical_loss": 3.3394610505574676, + "tokens_seen": 2790195200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015721212121212123, + "loss": 2.6589, + "theoretical_loss": 3.3394549306639636, + "tokens_seen": 2790260736 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001571919191919192, + "loss": 2.4695, + "theoretical_loss": 3.3394488109544453, + "tokens_seen": 2790326272 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001571717171717172, + "loss": 2.5342, + "theoretical_loss": 3.3394426914289017, + "tokens_seen": 2790391808 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1530340, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0540807247161865, + "objective/train/theoretical_loss": 3.3394365720873243, + "objective/train/tokens_used": 1149316576, + "theoretical_loss": 3.3394365720873243, + "tokens_seen": 2790457344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015715151515151516, + "loss": 2.6073, + "theoretical_loss": 3.3394365720873243, + "tokens_seen": 2790457344 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001571313131313131, + "loss": 2.4533, + "theoretical_loss": 3.3394304529297014, + "tokens_seen": 2790522880 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015711111111111112, + "loss": 2.5406, + "theoretical_loss": 3.3394243339560243, + "tokens_seen": 2790588416 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015709090909090908, + "loss": 2.5347, + "theoretical_loss": 3.3394182151662832, + "tokens_seen": 2790653952 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001570707070707071, + "loss": 2.4503, + "theoretical_loss": 3.339412096560468, + "tokens_seen": 2790719488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015705050505050505, + "loss": 2.5308, + "theoretical_loss": 3.3394059781385685, + "tokens_seen": 2790785024 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015703030303030303, + "loss": 2.4758, + "theoretical_loss": 3.339399859900576, + "tokens_seen": 2790850560 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015701010101010102, + "loss": 2.5824, + "theoretical_loss": 3.339393741846479, + "tokens_seen": 2790916096 + }, + { + "epoch": 0.69, + "learning_rate": 0.000156989898989899, + "loss": 2.3199, + "theoretical_loss": 3.339387623976269, + "tokens_seen": 2790981632 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015696969696969698, + "loss": 2.7757, + "theoretical_loss": 3.3393815062899357, + "tokens_seen": 2791047168 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015694949494949494, + "loss": 2.3719, + "theoretical_loss": 3.3393753887874693, + "tokens_seen": 2791112704 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015692929292929292, + "loss": 2.4635, + "theoretical_loss": 3.3393692714688594, + "tokens_seen": 2791178240 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001569090909090909, + "loss": 2.5353, + "theoretical_loss": 3.339363154334097, + "tokens_seen": 2791243776 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001568888888888889, + "loss": 2.4945, + "theoretical_loss": 3.339357037383172, + "tokens_seen": 2791309312 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015686868686868688, + "loss": 2.5727, + "theoretical_loss": 3.339350920616074, + "tokens_seen": 2791374848 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015684848484848486, + "loss": 2.4194, + "theoretical_loss": 3.339344804032794, + "tokens_seen": 2791440384 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015682828282828282, + "loss": 2.4938, + "theoretical_loss": 3.339338687633322, + "tokens_seen": 2791505920 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015680808080808083, + "loss": 2.5597, + "theoretical_loss": 3.3393325714176476, + "tokens_seen": 2791571456 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015678787878787879, + "loss": 2.2846, + "theoretical_loss": 3.3393264553857613, + "tokens_seen": 2791636992 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015676767676767677, + "loss": 2.485, + "theoretical_loss": 3.3393203395376534, + "tokens_seen": 2791702528 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015674747474747475, + "loss": 2.6209, + "theoretical_loss": 3.339314223873314, + "tokens_seen": 2791768064 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001567272727272727, + "loss": 2.6967, + "theoretical_loss": 3.339308108392733, + "tokens_seen": 2791833600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015670707070707072, + "loss": 2.6042, + "theoretical_loss": 3.339301993095901, + "tokens_seen": 2791899136 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015668686868686868, + "loss": 2.4045, + "theoretical_loss": 3.3392958779828077, + "tokens_seen": 2791964672 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001566666666666667, + "loss": 2.5268, + "theoretical_loss": 3.339289763053444, + "tokens_seen": 2792030208 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1530963, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8091771602630615, + "objective/train/theoretical_loss": 3.339283648307799, + "objective/train/tokens_used": 1150954976, + "theoretical_loss": 3.339283648307799, + "tokens_seen": 2792095744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015664646464646465, + "loss": 2.4426, + "theoretical_loss": 3.339283648307799, + "tokens_seen": 2792095744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015662626262626263, + "loss": 2.7038, + "theoretical_loss": 3.3392775337458636, + "tokens_seen": 2792161280 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015660606060606061, + "loss": 2.4854, + "theoretical_loss": 3.339271419367628, + "tokens_seen": 2792226816 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015658585858585857, + "loss": 2.5044, + "theoretical_loss": 3.3392653051730816, + "tokens_seen": 2792292352 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015656565656565658, + "loss": 2.4555, + "theoretical_loss": 3.339259191162216, + "tokens_seen": 2792357888 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015654545454545454, + "loss": 2.2044, + "theoretical_loss": 3.33925307733502, + "tokens_seen": 2792423424 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015652525252525252, + "loss": 2.4351, + "theoretical_loss": 3.339246963691484, + "tokens_seen": 2792488960 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001565050505050505, + "loss": 2.4081, + "theoretical_loss": 3.339240850231599, + "tokens_seen": 2792554496 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001564848484848485, + "loss": 2.4825, + "theoretical_loss": 3.3392347369553548, + "tokens_seen": 2792620032 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015646464646464647, + "loss": 2.4679, + "theoretical_loss": 3.339228623862741, + "tokens_seen": 2792685568 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015644444444444443, + "loss": 2.6429, + "theoretical_loss": 3.339222510953748, + "tokens_seen": 2792751104 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015642424242424244, + "loss": 2.5189, + "theoretical_loss": 3.3392163982283667, + "tokens_seen": 2792816640 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001564040404040404, + "loss": 2.6314, + "theoretical_loss": 3.3392102856865864, + "tokens_seen": 2792882176 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015638383838383838, + "loss": 2.5986, + "theoretical_loss": 3.3392041733283975, + "tokens_seen": 2792947712 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015636363636363637, + "loss": 2.3353, + "theoretical_loss": 3.3391980611537906, + "tokens_seen": 2793013248 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015634343434343435, + "loss": 2.5565, + "theoretical_loss": 3.3391919491627555, + "tokens_seen": 2793078784 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015632323232323234, + "loss": 2.4338, + "theoretical_loss": 3.3391858373552825, + "tokens_seen": 2793144320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015630303030303032, + "loss": 2.3473, + "theoretical_loss": 3.3391797257313613, + "tokens_seen": 2793209856 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015628282828282828, + "loss": 2.6957, + "theoretical_loss": 3.339173614290983, + "tokens_seen": 2793275392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015626262626262626, + "loss": 2.6184, + "theoretical_loss": 3.3391675030341372, + "tokens_seen": 2793340928 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015624242424242424, + "loss": 2.2866, + "theoretical_loss": 3.339161391960814, + "tokens_seen": 2793406464 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015622222222222223, + "loss": 2.6464, + "theoretical_loss": 3.3391552810710037, + "tokens_seen": 2793472000 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001562020202020202, + "loss": 2.4466, + "theoretical_loss": 3.339149170364697, + "tokens_seen": 2793537536 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015618181818181817, + "loss": 2.5138, + "theoretical_loss": 3.3391430598418834, + "tokens_seen": 2793603072 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015616161616161618, + "loss": 2.5405, + "theoretical_loss": 3.339136949502553, + "tokens_seen": 2793668608 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1531861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0454108715057373, + "objective/train/theoretical_loss": 3.3391308393466965, + "objective/train/tokens_used": 1152593376, + "theoretical_loss": 3.3391308393466965, + "tokens_seen": 2793734144 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015614141414141414, + "loss": 2.6523, + "theoretical_loss": 3.3391308393466965, + "tokens_seen": 2793734144 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015612121212121215, + "loss": 2.6731, + "theoretical_loss": 3.339124729374304, + "tokens_seen": 2793799680 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001561010101010101, + "loss": 2.6244, + "theoretical_loss": 3.3391186195853657, + "tokens_seen": 2793865216 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015608080808080806, + "loss": 2.426, + "theoretical_loss": 3.3391125099798713, + "tokens_seen": 2793930752 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015606060606060607, + "loss": 2.6614, + "theoretical_loss": 3.3391064005578115, + "tokens_seen": 2793996288 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015604040404040403, + "loss": 2.6031, + "theoretical_loss": 3.3391002913191765, + "tokens_seen": 2794061824 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015602020202020204, + "loss": 2.5909, + "theoretical_loss": 3.339094182263956, + "tokens_seen": 2794127360 + }, + { + "epoch": 0.69, + "learning_rate": 0.000156, + "loss": 2.8263, + "theoretical_loss": 3.339088073392141, + "tokens_seen": 2794192896 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015597979797979798, + "loss": 2.5441, + "theoretical_loss": 3.339081964703721, + "tokens_seen": 2794258432 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015595959595959597, + "loss": 2.6077, + "theoretical_loss": 3.3390758561986864, + "tokens_seen": 2794323968 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015593939393939395, + "loss": 2.5086, + "theoretical_loss": 3.3390697478770273, + "tokens_seen": 2794389504 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015591919191919193, + "loss": 2.5239, + "theoretical_loss": 3.339063639738734, + "tokens_seen": 2794455040 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001558989898989899, + "loss": 2.5868, + "theoretical_loss": 3.339057531783797, + "tokens_seen": 2794520576 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015587878787878787, + "loss": 2.601, + "theoretical_loss": 3.339051424012206, + "tokens_seen": 2794586112 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015585858585858586, + "loss": 2.372, + "theoretical_loss": 3.339045316423951, + "tokens_seen": 2794651648 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015583838383838384, + "loss": 2.5286, + "theoretical_loss": 3.339039209019023, + "tokens_seen": 2794717184 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015581818181818183, + "loss": 2.4864, + "theoretical_loss": 3.339033101797412, + "tokens_seen": 2794782720 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001557979797979798, + "loss": 2.4939, + "theoretical_loss": 3.3390269947591076, + "tokens_seen": 2794848256 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015577777777777777, + "loss": 2.6545, + "theoretical_loss": 3.3390208879041006, + "tokens_seen": 2794913792 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015575757575757578, + "loss": 2.4191, + "theoretical_loss": 3.3390147812323807, + "tokens_seen": 2794979328 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015573737373737373, + "loss": 2.5194, + "theoretical_loss": 3.3390086747439387, + "tokens_seen": 2795044864 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015571717171717172, + "loss": 2.4851, + "theoretical_loss": 3.3390025684387643, + "tokens_seen": 2795110400 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001556969696969697, + "loss": 2.3099, + "theoretical_loss": 3.3389964623168478, + "tokens_seen": 2795175936 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015567676767676766, + "loss": 2.6324, + "theoretical_loss": 3.3389903563781798, + "tokens_seen": 2795241472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015565656565656567, + "loss": 2.547, + "theoretical_loss": 3.33898425062275, + "tokens_seen": 2795307008 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1532487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.366612434387207, + "objective/train/theoretical_loss": 3.338978145050549, + "objective/train/tokens_used": 1154231776, + "theoretical_loss": 3.338978145050549, + "tokens_seen": 2795372544 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015563636363636363, + "loss": 2.4968, + "theoretical_loss": 3.338978145050549, + "tokens_seen": 2795372544 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015561616161616164, + "loss": 2.2641, + "theoretical_loss": 3.3389720396615665, + "tokens_seen": 2795438080 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001555959595959596, + "loss": 2.6109, + "theoretical_loss": 3.338965934455793, + "tokens_seen": 2795503616 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001555757575757576, + "loss": 2.5505, + "theoretical_loss": 3.338959829433219, + "tokens_seen": 2795569152 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015555555555555556, + "loss": 2.5516, + "theoretical_loss": 3.3389537245938343, + "tokens_seen": 2795634688 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015553535353535352, + "loss": 2.4606, + "theoretical_loss": 3.3389476199376293, + "tokens_seen": 2795700224 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015551515151515153, + "loss": 2.5946, + "theoretical_loss": 3.3389415154645943, + "tokens_seen": 2795765760 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001554949494949495, + "loss": 2.554, + "theoretical_loss": 3.3389354111747194, + "tokens_seen": 2795831296 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001554747474747475, + "loss": 2.4172, + "theoretical_loss": 3.3389293070679944, + "tokens_seen": 2795896832 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015545454545454546, + "loss": 2.5793, + "theoretical_loss": 3.33892320314441, + "tokens_seen": 2795962368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015543434343434344, + "loss": 2.4952, + "theoretical_loss": 3.338917099403956, + "tokens_seen": 2796027904 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015541414141414142, + "loss": 2.5795, + "theoretical_loss": 3.3389109958466237, + "tokens_seen": 2796093440 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015539393939393938, + "loss": 2.5075, + "theoretical_loss": 3.338904892472402, + "tokens_seen": 2796158976 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001553737373737374, + "loss": 2.4455, + "theoretical_loss": 3.3388987892812816, + "tokens_seen": 2796224512 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015535353535353535, + "loss": 2.5127, + "theoretical_loss": 3.338892686273253, + "tokens_seen": 2796290048 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015533333333333333, + "loss": 2.7631, + "theoretical_loss": 3.338886583448306, + "tokens_seen": 2796355584 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015531313131313132, + "loss": 2.5081, + "theoretical_loss": 3.3388804808064307, + "tokens_seen": 2796421120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001552929292929293, + "loss": 2.4577, + "theoretical_loss": 3.3388743783476182, + "tokens_seen": 2796486656 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015527272727272728, + "loss": 2.4207, + "theoretical_loss": 3.338868276071858, + "tokens_seen": 2796552192 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015525252525252527, + "loss": 2.5342, + "theoretical_loss": 3.33886217397914, + "tokens_seen": 2796617728 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015523232323232323, + "loss": 2.7969, + "theoretical_loss": 3.3388560720694556, + "tokens_seen": 2796683264 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001552121212121212, + "loss": 2.5009, + "theoretical_loss": 3.338849970342794, + "tokens_seen": 2796748800 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001551919191919192, + "loss": 2.5744, + "theoretical_loss": 3.3388438687991453, + "tokens_seen": 2796814336 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015517171717171718, + "loss": 2.5787, + "theoretical_loss": 3.3388377674385006, + "tokens_seen": 2796879872 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015515151515151516, + "loss": 2.4585, + "theoretical_loss": 3.3388316662608495, + "tokens_seen": 2796945408 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1533684, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3293659687042236, + "objective/train/theoretical_loss": 3.3388255652661822, + "objective/train/tokens_used": 1155870176, + "theoretical_loss": 3.3388255652661822, + "tokens_seen": 2797010944 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015513131313131312, + "loss": 2.3592, + "theoretical_loss": 3.3388255652661822, + "tokens_seen": 2797010944 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015511111111111113, + "loss": 2.5073, + "theoretical_loss": 3.338819464454489, + "tokens_seen": 2797076480 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015509090909090909, + "loss": 2.6182, + "theoretical_loss": 3.3388133638257607, + "tokens_seen": 2797142016 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001550707070707071, + "loss": 2.6404, + "theoretical_loss": 3.3388072633799872, + "tokens_seen": 2797207552 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015505050505050505, + "loss": 2.5199, + "theoretical_loss": 3.338801163117158, + "tokens_seen": 2797273088 + }, + { + "epoch": 0.69, + "learning_rate": 0.000155030303030303, + "loss": 2.5114, + "theoretical_loss": 3.338795063037264, + "tokens_seen": 2797338624 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015501010101010102, + "loss": 2.5406, + "theoretical_loss": 3.3387889631402956, + "tokens_seen": 2797404160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015498989898989898, + "loss": 2.4086, + "theoretical_loss": 3.3387828634262426, + "tokens_seen": 2797469696 + }, + { + "epoch": 0.69, + "learning_rate": 0.000154969696969697, + "loss": 2.6467, + "theoretical_loss": 3.338776763895096, + "tokens_seen": 2797535232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015494949494949495, + "loss": 2.659, + "theoretical_loss": 3.3387706645468445, + "tokens_seen": 2797600768 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015492929292929293, + "loss": 2.4455, + "theoretical_loss": 3.3387645653814797, + "tokens_seen": 2797666304 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015490909090909091, + "loss": 2.6915, + "theoretical_loss": 3.3387584663989913, + "tokens_seen": 2797731840 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001548888888888889, + "loss": 2.75, + "theoretical_loss": 3.3387523675993696, + "tokens_seen": 2797797376 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015486868686868688, + "loss": 2.5955, + "theoretical_loss": 3.338746268982605, + "tokens_seen": 2797862912 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015484848484848484, + "loss": 2.4127, + "theoretical_loss": 3.3387401705486877, + "tokens_seen": 2797928448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015482828282828282, + "loss": 2.4378, + "theoretical_loss": 3.3387340722976075, + "tokens_seen": 2797993984 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001548080808080808, + "loss": 2.6867, + "theoretical_loss": 3.338727974229355, + "tokens_seen": 2798059520 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001547878787878788, + "loss": 2.5887, + "theoretical_loss": 3.3387218763439206, + "tokens_seen": 2798125056 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015476767676767677, + "loss": 2.6311, + "theoretical_loss": 3.338715778641294, + "tokens_seen": 2798190592 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015474747474747476, + "loss": 2.3161, + "theoretical_loss": 3.3387096811214665, + "tokens_seen": 2798256128 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015472727272727274, + "loss": 2.7239, + "theoretical_loss": 3.3387035837844268, + "tokens_seen": 2798321664 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001547070707070707, + "loss": 2.698, + "theoretical_loss": 3.3386974866301666, + "tokens_seen": 2798387200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015468686868686868, + "loss": 2.5643, + "theoretical_loss": 3.338691389658675, + "tokens_seen": 2798452736 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015466666666666667, + "loss": 2.5428, + "theoretical_loss": 3.3386852928699433, + "tokens_seen": 2798518272 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015464646464646465, + "loss": 2.2315, + "theoretical_loss": 3.3386791962639606, + "tokens_seen": 2798583808 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1534363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5379207134246826, + "objective/train/theoretical_loss": 3.338673099840718, + "objective/train/tokens_used": 1157508576, + "theoretical_loss": 3.338673099840718, + "tokens_seen": 2798649344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015462626262626264, + "loss": 2.4977, + "theoretical_loss": 3.338673099840718, + "tokens_seen": 2798649344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015460606060606062, + "loss": 2.412, + "theoretical_loss": 3.3386670036002055, + "tokens_seen": 2798714880 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015458585858585858, + "loss": 2.5595, + "theoretical_loss": 3.338660907542413, + "tokens_seen": 2798780416 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001545656565656566, + "loss": 2.5129, + "theoretical_loss": 3.3386548116673316, + "tokens_seen": 2798845952 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015454545454545454, + "loss": 2.634, + "theoretical_loss": 3.3386487159749505, + "tokens_seen": 2798911488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015452525252525253, + "loss": 2.633, + "theoretical_loss": 3.3386426204652606, + "tokens_seen": 2798977024 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001545050505050505, + "loss": 2.4416, + "theoretical_loss": 3.338636525138252, + "tokens_seen": 2799042560 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015448484848484847, + "loss": 2.4152, + "theoretical_loss": 3.338630429993915, + "tokens_seen": 2799108096 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015446464646464648, + "loss": 2.4567, + "theoretical_loss": 3.3386243350322395, + "tokens_seen": 2799173632 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015444444444444444, + "loss": 2.3606, + "theoretical_loss": 3.3386182402532167, + "tokens_seen": 2799239168 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015442424242424245, + "loss": 2.6451, + "theoretical_loss": 3.338612145656836, + "tokens_seen": 2799304704 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001544040404040404, + "loss": 2.7099, + "theoretical_loss": 3.3386060512430875, + "tokens_seen": 2799370240 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001543838383838384, + "loss": 2.5981, + "theoretical_loss": 3.338599957011962, + "tokens_seen": 2799435776 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015436363636363637, + "loss": 2.6463, + "theoretical_loss": 3.3385938629634495, + "tokens_seen": 2799501312 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015434343434343433, + "loss": 2.4691, + "theoretical_loss": 3.3385877690975403, + "tokens_seen": 2799566848 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015432323232323234, + "loss": 2.4918, + "theoretical_loss": 3.3385816754142246, + "tokens_seen": 2799632384 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001543030303030303, + "loss": 2.4971, + "theoretical_loss": 3.3385755819134926, + "tokens_seen": 2799697920 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015428282828282828, + "loss": 2.3587, + "theoretical_loss": 3.338569488595335, + "tokens_seen": 2799763456 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015426262626262627, + "loss": 2.6274, + "theoretical_loss": 3.3385633954597416, + "tokens_seen": 2799828992 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015424242424242425, + "loss": 2.6756, + "theoretical_loss": 3.3385573025067026, + "tokens_seen": 2799894528 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015422222222222223, + "loss": 2.6012, + "theoretical_loss": 3.3385512097362087, + "tokens_seen": 2799960064 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015420202020202022, + "loss": 2.4313, + "theoretical_loss": 3.3385451171482496, + "tokens_seen": 2800025600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015418181818181817, + "loss": 2.5133, + "theoretical_loss": 3.338539024742816, + "tokens_seen": 2800091136 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015416161616161616, + "loss": 2.3476, + "theoretical_loss": 3.338532932519898, + "tokens_seen": 2800156672 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015414141414141414, + "loss": 2.5692, + "theoretical_loss": 3.338526840479486, + "tokens_seen": 2800222208 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1535268, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.515688419342041, + "objective/train/theoretical_loss": 3.33852074862157, + "objective/train/tokens_used": 1159146976, + "theoretical_loss": 3.33852074862157, + "tokens_seen": 2800287744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015412121212121213, + "loss": 2.4144, + "theoretical_loss": 3.33852074862157, + "tokens_seen": 2800287744 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001541010101010101, + "loss": 2.7267, + "theoretical_loss": 3.3385146569461406, + "tokens_seen": 2800353280 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015408080808080807, + "loss": 2.6239, + "theoretical_loss": 3.338508565453188, + "tokens_seen": 2800418816 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015406060606060608, + "loss": 2.6105, + "theoretical_loss": 3.3385024741427016, + "tokens_seen": 2800484352 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015404040404040403, + "loss": 2.5347, + "theoretical_loss": 3.338496383014673, + "tokens_seen": 2800549888 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015402020202020205, + "loss": 2.5925, + "theoretical_loss": 3.3384902920690918, + "tokens_seen": 2800615424 + }, + { + "epoch": 0.7, + "learning_rate": 0.000154, + "loss": 2.5823, + "theoretical_loss": 3.338484201305948, + "tokens_seen": 2800680960 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015397979797979796, + "loss": 2.5441, + "theoretical_loss": 3.3384781107252324, + "tokens_seen": 2800746496 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015395959595959597, + "loss": 2.5492, + "theoretical_loss": 3.338472020326935, + "tokens_seen": 2800812032 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015393939393939393, + "loss": 2.554, + "theoretical_loss": 3.338465930111046, + "tokens_seen": 2800877568 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015391919191919194, + "loss": 2.6539, + "theoretical_loss": 3.3384598400775563, + "tokens_seen": 2800943104 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001538989898989899, + "loss": 2.3872, + "theoretical_loss": 3.338453750226455, + "tokens_seen": 2801008640 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001538787878787879, + "loss": 2.3886, + "theoretical_loss": 3.3384476605577333, + "tokens_seen": 2801074176 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015385858585858586, + "loss": 2.5951, + "theoretical_loss": 3.3384415710713817, + "tokens_seen": 2801139712 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015383838383838382, + "loss": 2.5277, + "theoretical_loss": 3.338435481767389, + "tokens_seen": 2801205248 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015381818181818183, + "loss": 2.4389, + "theoretical_loss": 3.3384293926457476, + "tokens_seen": 2801270784 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001537979797979798, + "loss": 2.5033, + "theoretical_loss": 3.3384233037064455, + "tokens_seen": 2801336320 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001537777777777778, + "loss": 2.4545, + "theoretical_loss": 3.338417214949475, + "tokens_seen": 2801401856 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015375757575757576, + "loss": 2.4545, + "theoretical_loss": 3.338411126374825, + "tokens_seen": 2801467392 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015373737373737374, + "loss": 2.5227, + "theoretical_loss": 3.3384050379824863, + "tokens_seen": 2801532928 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015371717171717172, + "loss": 2.7292, + "theoretical_loss": 3.3383989497724493, + "tokens_seen": 2801598464 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001536969696969697, + "loss": 2.3962, + "theoretical_loss": 3.338392861744704, + "tokens_seen": 2801664000 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001536767676767677, + "loss": 2.3383, + "theoretical_loss": 3.3383867738992405, + "tokens_seen": 2801729536 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015365656565656565, + "loss": 2.6677, + "theoretical_loss": 3.3383806862360497, + "tokens_seen": 2801795072 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015363636363636363, + "loss": 2.5149, + "theoretical_loss": 3.3383745987551214, + "tokens_seen": 2801860608 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1536573, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4081475734710693, + "objective/train/theoretical_loss": 3.338368511456446, + "objective/train/tokens_used": 1160785376, + "theoretical_loss": 3.338368511456446, + "tokens_seen": 2801926144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015361616161616162, + "loss": 2.5285, + "theoretical_loss": 3.338368511456446, + "tokens_seen": 2801926144 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001535959595959596, + "loss": 2.3222, + "theoretical_loss": 3.338362424340014, + "tokens_seen": 2801991680 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015357575757575758, + "loss": 2.6336, + "theoretical_loss": 3.338356337405815, + "tokens_seen": 2802057216 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015355555555555557, + "loss": 2.6195, + "theoretical_loss": 3.33835025065384, + "tokens_seen": 2802122752 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015353535353535353, + "loss": 2.627, + "theoretical_loss": 3.3383441640840794, + "tokens_seen": 2802188288 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015351515151515154, + "loss": 2.4161, + "theoretical_loss": 3.338338077696523, + "tokens_seen": 2802253824 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001534949494949495, + "loss": 2.6368, + "theoretical_loss": 3.338331991491161, + "tokens_seen": 2802319360 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015347474747474748, + "loss": 2.5341, + "theoretical_loss": 3.338325905467984, + "tokens_seen": 2802384896 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015345454545454546, + "loss": 2.5303, + "theoretical_loss": 3.338319819626982, + "tokens_seen": 2802450432 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015343434343434342, + "loss": 2.579, + "theoretical_loss": 3.3383137339681457, + "tokens_seen": 2802515968 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015341414141414143, + "loss": 2.6865, + "theoretical_loss": 3.338307648491465, + "tokens_seen": 2802581504 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015339393939393939, + "loss": 2.5492, + "theoretical_loss": 3.3383015631969304, + "tokens_seen": 2802647040 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001533737373737374, + "loss": 2.6026, + "theoretical_loss": 3.338295478084532, + "tokens_seen": 2802712576 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015335353535353535, + "loss": 2.618, + "theoretical_loss": 3.3382893931542603, + "tokens_seen": 2802778112 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015333333333333334, + "loss": 2.6311, + "theoretical_loss": 3.3382833084061057, + "tokens_seen": 2802843648 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015331313131313132, + "loss": 2.4723, + "theoretical_loss": 3.338277223840058, + "tokens_seen": 2802909184 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015329292929292928, + "loss": 2.6152, + "theoretical_loss": 3.338271139456108, + "tokens_seen": 2802974720 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001532727272727273, + "loss": 2.5175, + "theoretical_loss": 3.3382650552542463, + "tokens_seen": 2803040256 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015325252525252525, + "loss": 2.5136, + "theoretical_loss": 3.338258971234462, + "tokens_seen": 2803105792 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015323232323232323, + "loss": 2.4572, + "theoretical_loss": 3.338252887396746, + "tokens_seen": 2803171328 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015321212121212121, + "loss": 2.7119, + "theoretical_loss": 3.3382468037410895, + "tokens_seen": 2803236864 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001531919191919192, + "loss": 2.3428, + "theoretical_loss": 3.338240720267481, + "tokens_seen": 2803302400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015317171717171718, + "loss": 2.4571, + "theoretical_loss": 3.3382346369759124, + "tokens_seen": 2803367936 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015315151515151514, + "loss": 2.6215, + "theoretical_loss": 3.338228553866373, + "tokens_seen": 2803433472 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015313131313131312, + "loss": 2.3747, + "theoretical_loss": 3.3382224709388537, + "tokens_seen": 2803499008 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1537291, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.69579815864563, + "objective/train/theoretical_loss": 3.3382163881933447, + "objective/train/tokens_used": 1162423776, + "theoretical_loss": 3.3382163881933447, + "tokens_seen": 2803564544 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001531111111111111, + "loss": 2.5256, + "theoretical_loss": 3.3382163881933447, + "tokens_seen": 2803564544 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001530909090909091, + "loss": 2.414, + "theoretical_loss": 3.338210305629836, + "tokens_seen": 2803630080 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015307070707070708, + "loss": 2.3806, + "theoretical_loss": 3.338204223248318, + "tokens_seen": 2803695616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015305050505050506, + "loss": 2.6628, + "theoretical_loss": 3.338198141048781, + "tokens_seen": 2803761152 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015303030303030302, + "loss": 2.2473, + "theoretical_loss": 3.3381920590312153, + "tokens_seen": 2803826688 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015301010101010103, + "loss": 2.4544, + "theoretical_loss": 3.3381859771956113, + "tokens_seen": 2803892224 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015298989898989898, + "loss": 2.5157, + "theoretical_loss": 3.3381798955419595, + "tokens_seen": 2803957760 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015296969696969697, + "loss": 2.2781, + "theoretical_loss": 3.3381738140702497, + "tokens_seen": 2804023296 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015294949494949495, + "loss": 2.5778, + "theoretical_loss": 3.3381677327804726, + "tokens_seen": 2804088832 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015292929292929294, + "loss": 2.5348, + "theoretical_loss": 3.3381616516726185, + "tokens_seen": 2804154368 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015290909090909092, + "loss": 2.4686, + "theoretical_loss": 3.3381555707466775, + "tokens_seen": 2804219904 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015288888888888888, + "loss": 2.725, + "theoretical_loss": 3.33814949000264, + "tokens_seen": 2804285440 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001528686868686869, + "loss": 2.4045, + "theoretical_loss": 3.3381434094404963, + "tokens_seen": 2804350976 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015284848484848484, + "loss": 2.6955, + "theoretical_loss": 3.338137329060237, + "tokens_seen": 2804416512 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015282828282828286, + "loss": 2.335, + "theoretical_loss": 3.3381312488618518, + "tokens_seen": 2804482048 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001528080808080808, + "loss": 2.606, + "theoretical_loss": 3.3381251688453313, + "tokens_seen": 2804547584 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015278787878787877, + "loss": 2.3848, + "theoretical_loss": 3.338119089010666, + "tokens_seen": 2804613120 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015276767676767678, + "loss": 2.5601, + "theoretical_loss": 3.338113009357846, + "tokens_seen": 2804678656 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015274747474747474, + "loss": 2.8272, + "theoretical_loss": 3.3381069298868615, + "tokens_seen": 2804744192 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015272727272727275, + "loss": 2.4516, + "theoretical_loss": 3.3381008505977032, + "tokens_seen": 2804809728 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001527070707070707, + "loss": 2.6024, + "theoretical_loss": 3.3380947714903613, + "tokens_seen": 2804875264 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001526868686868687, + "loss": 2.5488, + "theoretical_loss": 3.338088692564826, + "tokens_seen": 2804940800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015266666666666667, + "loss": 2.4285, + "theoretical_loss": 3.3380826138210873, + "tokens_seen": 2805006336 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015264646464646466, + "loss": 2.5808, + "theoretical_loss": 3.3380765352591357, + "tokens_seen": 2805071872 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015262626262626264, + "loss": 2.5374, + "theoretical_loss": 3.3380704568789623, + "tokens_seen": 2805137408 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1538491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.382876396179199, + "objective/train/theoretical_loss": 3.338064378680556, + "objective/train/tokens_used": 1164062176, + "theoretical_loss": 3.338064378680556, + "tokens_seen": 2805202944 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001526060606060606, + "loss": 2.4924, + "theoretical_loss": 3.338064378680556, + "tokens_seen": 2805202944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015258585858585858, + "loss": 2.5221, + "theoretical_loss": 3.3380583006639086, + "tokens_seen": 2805268480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015256565656565657, + "loss": 2.4082, + "theoretical_loss": 3.3380522228290093, + "tokens_seen": 2805334016 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015254545454545455, + "loss": 2.2068, + "theoretical_loss": 3.338046145175849, + "tokens_seen": 2805399552 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015252525252525253, + "loss": 2.5177, + "theoretical_loss": 3.3380400677044175, + "tokens_seen": 2805465088 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015250505050505052, + "loss": 2.7238, + "theoretical_loss": 3.338033990414706, + "tokens_seen": 2805530624 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015248484848484847, + "loss": 2.5127, + "theoretical_loss": 3.3380279133067043, + "tokens_seen": 2805596160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015246464646464649, + "loss": 2.3417, + "theoretical_loss": 3.338021836380402, + "tokens_seen": 2805661696 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015244444444444444, + "loss": 2.3691, + "theoretical_loss": 3.338015759635791, + "tokens_seen": 2805727232 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015242424242424243, + "loss": 2.6362, + "theoretical_loss": 3.33800968307286, + "tokens_seen": 2805792768 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001524040404040404, + "loss": 2.5807, + "theoretical_loss": 3.3380036066916006, + "tokens_seen": 2805858304 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015238383838383837, + "loss": 2.4407, + "theoretical_loss": 3.337997530492003, + "tokens_seen": 2805923840 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015236363636363638, + "loss": 2.5439, + "theoretical_loss": 3.3379914544740563, + "tokens_seen": 2805989376 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015234343434343434, + "loss": 2.594, + "theoretical_loss": 3.337985378637752, + "tokens_seen": 2806054912 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015232323232323235, + "loss": 2.724, + "theoretical_loss": 3.33797930298308, + "tokens_seen": 2806120448 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001523030303030303, + "loss": 2.4991, + "theoretical_loss": 3.337973227510031, + "tokens_seen": 2806185984 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015228282828282826, + "loss": 2.5294, + "theoretical_loss": 3.337967152218595, + "tokens_seen": 2806251520 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015226262626262627, + "loss": 2.2893, + "theoretical_loss": 3.337961077108762, + "tokens_seen": 2806317056 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015224242424242423, + "loss": 2.2974, + "theoretical_loss": 3.337955002180523, + "tokens_seen": 2806382592 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015222222222222224, + "loss": 2.5174, + "theoretical_loss": 3.337948927433868, + "tokens_seen": 2806448128 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001522020202020202, + "loss": 2.6392, + "theoretical_loss": 3.3379428528687876, + "tokens_seen": 2806513664 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015218181818181818, + "loss": 2.4954, + "theoretical_loss": 3.3379367784852714, + "tokens_seen": 2806579200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015216161616161616, + "loss": 2.4398, + "theoretical_loss": 3.337930704283311, + "tokens_seen": 2806644736 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015214141414141415, + "loss": 2.5839, + "theoretical_loss": 3.3379246302628953, + "tokens_seen": 2806710272 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015212121212121213, + "loss": 2.4893, + "theoretical_loss": 3.3379185564240155, + "tokens_seen": 2806775808 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1539064, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.295426845550537, + "objective/train/theoretical_loss": 3.3379124827666615, + "objective/train/tokens_used": 1165700576, + "theoretical_loss": 3.3379124827666615, + "tokens_seen": 2806841344 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001521010101010101, + "loss": 2.3355, + "theoretical_loss": 3.3379124827666615, + "tokens_seen": 2806841344 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001520808080808081, + "loss": 2.3514, + "theoretical_loss": 3.337906409290824, + "tokens_seen": 2806906880 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015206060606060606, + "loss": 2.4208, + "theoretical_loss": 3.3379003359964936, + "tokens_seen": 2806972416 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015204040404040404, + "loss": 2.525, + "theoretical_loss": 3.33789426288366, + "tokens_seen": 2807037952 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015202020202020202, + "loss": 2.7146, + "theoretical_loss": 3.337888189952314, + "tokens_seen": 2807103488 + }, + { + "epoch": 0.7, + "learning_rate": 0.000152, + "loss": 2.4362, + "theoretical_loss": 3.3378821172024455, + "tokens_seen": 2807169024 + }, + { + "epoch": 0.7, + "learning_rate": 0.000151979797979798, + "loss": 2.4429, + "theoretical_loss": 3.337876044634045, + "tokens_seen": 2807234560 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015195959595959598, + "loss": 2.5753, + "theoretical_loss": 3.3378699722471032, + "tokens_seen": 2807300096 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015193939393939393, + "loss": 2.5767, + "theoretical_loss": 3.33786390004161, + "tokens_seen": 2807365632 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015191919191919192, + "loss": 2.6383, + "theoretical_loss": 3.337857828017556, + "tokens_seen": 2807431168 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001518989898989899, + "loss": 2.5414, + "theoretical_loss": 3.3378517561749312, + "tokens_seen": 2807496704 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015187878787878789, + "loss": 2.3931, + "theoretical_loss": 3.3378456845137263, + "tokens_seen": 2807562240 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015185858585858587, + "loss": 2.5273, + "theoretical_loss": 3.337839613033932, + "tokens_seen": 2807627776 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015183838383838383, + "loss": 2.4399, + "theoretical_loss": 3.3378335417355376, + "tokens_seen": 2807693312 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015181818181818184, + "loss": 2.5798, + "theoretical_loss": 3.3378274706185342, + "tokens_seen": 2807758848 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001517979797979798, + "loss": 2.5198, + "theoretical_loss": 3.337821399682912, + "tokens_seen": 2807824384 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001517777777777778, + "loss": 2.539, + "theoretical_loss": 3.3378153289286616, + "tokens_seen": 2807889920 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015175757575757576, + "loss": 2.4885, + "theoretical_loss": 3.337809258355773, + "tokens_seen": 2807955456 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015173737373737372, + "loss": 2.4359, + "theoretical_loss": 3.3378031879642363, + "tokens_seen": 2808020992 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015171717171717173, + "loss": 2.3851, + "theoretical_loss": 3.337797117754042, + "tokens_seen": 2808086528 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001516969696969697, + "loss": 2.3875, + "theoretical_loss": 3.337791047725181, + "tokens_seen": 2808152064 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001516767676767677, + "loss": 2.6053, + "theoretical_loss": 3.337784977877643, + "tokens_seen": 2808217600 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015165656565656565, + "loss": 2.5705, + "theoretical_loss": 3.337778908211419, + "tokens_seen": 2808283136 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015163636363636364, + "loss": 2.3152, + "theoretical_loss": 3.3377728387264987, + "tokens_seen": 2808348672 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015161616161616162, + "loss": 2.4822, + "theoretical_loss": 3.3377667694228728, + "tokens_seen": 2808414208 + }, + { + "debugging/Self-BLEU-5": 0.4089085537148741, + "debugging/distinct-1-grams": 0.7963924931509507, + "debugging/distinct-2-grams": 0.9664141286579346, + "debugging/entropy-1-grams": 5.66123923722335, + "debugging/entropy-2-grams": 6.274232850805275, + "debugging/length": 520.625, + "debugging/num_segments": 8, + "epoch": 0.7, + "objective/train/docs_used": 1540274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8786096572875977, + "objective/train/theoretical_loss": 3.3377607003005316, + "objective/train/tokens_used": 1167338976, + "theoretical_loss": 3.3377607003005316, + "tokens_seen": 2808479744 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001515959595959596, + "loss": 2.6427, + "theoretical_loss": 3.3377607003005316, + "tokens_seen": 2808479744 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001515757575757576, + "loss": 2.2885, + "theoretical_loss": 3.3377546313594655, + "tokens_seen": 2808545280 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015155555555555555, + "loss": 2.5785, + "theoretical_loss": 3.3377485625996646, + "tokens_seen": 2808610816 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015153535353535353, + "loss": 2.5783, + "theoretical_loss": 3.3377424940211196, + "tokens_seen": 2808676352 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015151515151515152, + "loss": 2.3863, + "theoretical_loss": 3.337736425623821, + "tokens_seen": 2808741888 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001514949494949495, + "loss": 2.6048, + "theoretical_loss": 3.337730357407758, + "tokens_seen": 2808807424 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015147474747474748, + "loss": 2.3878, + "theoretical_loss": 3.3377242893729226, + "tokens_seen": 2808872960 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015145454545454547, + "loss": 2.7406, + "theoretical_loss": 3.3377182215193044, + "tokens_seen": 2808938496 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015143434343434342, + "loss": 2.5364, + "theoretical_loss": 3.3377121538468932, + "tokens_seen": 2809004032 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001514141414141414, + "loss": 2.6531, + "theoretical_loss": 3.3377060863556807, + "tokens_seen": 2809069568 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001513939393939394, + "loss": 2.5639, + "theoretical_loss": 3.3377000190456556, + "tokens_seen": 2809135104 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015137373737373738, + "loss": 2.6856, + "theoretical_loss": 3.3376939519168096, + "tokens_seen": 2809200640 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015135353535353536, + "loss": 2.28, + "theoretical_loss": 3.3376878849691325, + "tokens_seen": 2809266176 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015133333333333332, + "loss": 2.6922, + "theoretical_loss": 3.337681818202615, + "tokens_seen": 2809331712 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015131313131313133, + "loss": 2.6139, + "theoretical_loss": 3.337675751617247, + "tokens_seen": 2809397248 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015129292929292928, + "loss": 2.586, + "theoretical_loss": 3.337669685213019, + "tokens_seen": 2809462784 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001512727272727273, + "loss": 2.35, + "theoretical_loss": 3.3376636189899216, + "tokens_seen": 2809528320 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015125252525252525, + "loss": 2.6281, + "theoretical_loss": 3.3376575529479453, + "tokens_seen": 2809593856 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015123232323232324, + "loss": 2.4651, + "theoretical_loss": 3.3376514870870797, + "tokens_seen": 2809659392 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015121212121212122, + "loss": 2.7367, + "theoretical_loss": 3.3376454214073155, + "tokens_seen": 2809724928 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015119191919191918, + "loss": 2.2225, + "theoretical_loss": 3.3376393559086437, + "tokens_seen": 2809790464 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001511717171717172, + "loss": 2.6578, + "theoretical_loss": 3.337633290591054, + "tokens_seen": 2809856000 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015115151515151515, + "loss": 2.4453, + "theoretical_loss": 3.337627225454537, + "tokens_seen": 2809921536 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015113131313131316, + "loss": 2.6169, + "theoretical_loss": 3.3376211604990833, + "tokens_seen": 2809987072 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001511111111111111, + "loss": 2.4065, + "theoretical_loss": 3.337615095724683, + "tokens_seen": 2810052608 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1540907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3177528381347656, + "objective/train/theoretical_loss": 3.337609031131326, + "objective/train/tokens_used": 1168977376, + "theoretical_loss": 3.337609031131326, + "tokens_seen": 2810118144 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001510909090909091, + "loss": 2.373, + "theoretical_loss": 3.337609031131326, + "tokens_seen": 2810118144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015107070707070708, + "loss": 2.4572, + "theoretical_loss": 3.3376029667190035, + "tokens_seen": 2810183680 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015105050505050504, + "loss": 2.4451, + "theoretical_loss": 3.3375969024877055, + "tokens_seen": 2810249216 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015103030303030305, + "loss": 2.3305, + "theoretical_loss": 3.337590838437422, + "tokens_seen": 2810314752 + }, + { + "epoch": 0.7, + "learning_rate": 0.000151010101010101, + "loss": 2.4307, + "theoretical_loss": 3.3375847745681444, + "tokens_seen": 2810380288 + }, + { + "epoch": 0.7, + "learning_rate": 0.000150989898989899, + "loss": 2.3383, + "theoretical_loss": 3.3375787108798622, + "tokens_seen": 2810445824 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015096969696969697, + "loss": 2.2622, + "theoretical_loss": 3.337572647372566, + "tokens_seen": 2810511360 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015094949494949496, + "loss": 2.6768, + "theoretical_loss": 3.337566584046246, + "tokens_seen": 2810576896 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015092929292929294, + "loss": 2.6204, + "theoretical_loss": 3.337560520900893, + "tokens_seen": 2810642432 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015090909090909093, + "loss": 2.3181, + "theoretical_loss": 3.3375544579364975, + "tokens_seen": 2810707968 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015088888888888888, + "loss": 2.2021, + "theoretical_loss": 3.337548395153049, + "tokens_seen": 2810773504 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015086868686868687, + "loss": 2.452, + "theoretical_loss": 3.3375423325505387, + "tokens_seen": 2810839040 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015084848484848485, + "loss": 2.446, + "theoretical_loss": 3.3375362701289566, + "tokens_seen": 2810904576 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015082828282828283, + "loss": 2.4306, + "theoretical_loss": 3.3375302078882934, + "tokens_seen": 2810970112 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015080808080808082, + "loss": 2.7435, + "theoretical_loss": 3.337524145828539, + "tokens_seen": 2811035648 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015078787878787877, + "loss": 2.3333, + "theoretical_loss": 3.3375180839496847, + "tokens_seen": 2811101184 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015076767676767679, + "loss": 2.5902, + "theoretical_loss": 3.33751202225172, + "tokens_seen": 2811166720 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015074747474747474, + "loss": 2.5236, + "theoretical_loss": 3.337505960734635, + "tokens_seen": 2811232256 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015072727272727275, + "loss": 2.4051, + "theoretical_loss": 3.337499899398421, + "tokens_seen": 2811297792 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001507070707070707, + "loss": 2.5784, + "theoretical_loss": 3.337493838243068, + "tokens_seen": 2811363328 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015068686868686867, + "loss": 2.574, + "theoretical_loss": 3.3374877772685663, + "tokens_seen": 2811428864 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015066666666666668, + "loss": 2.6748, + "theoretical_loss": 3.3374817164749064, + "tokens_seen": 2811494400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015064646464646464, + "loss": 2.3427, + "theoretical_loss": 3.337475655862079, + "tokens_seen": 2811559936 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015062626262626265, + "loss": 2.6201, + "theoretical_loss": 3.3374695954300737, + "tokens_seen": 2811625472 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001506060606060606, + "loss": 2.5978, + "theoretical_loss": 3.3374635351788813, + "tokens_seen": 2811691008 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1542005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.391789436340332, + "objective/train/theoretical_loss": 3.3374574751084927, + "objective/train/tokens_used": 1170615776, + "theoretical_loss": 3.3374574751084927, + "tokens_seen": 2811756544 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001505858585858586, + "loss": 2.4917, + "theoretical_loss": 3.3374574751084927, + "tokens_seen": 2811756544 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015056565656565657, + "loss": 2.3723, + "theoretical_loss": 3.337451415218897, + "tokens_seen": 2811822080 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015054545454545453, + "loss": 2.4739, + "theoretical_loss": 3.3374453555100865, + "tokens_seen": 2811887616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015052525252525254, + "loss": 2.6674, + "theoretical_loss": 3.33743929598205, + "tokens_seen": 2811953152 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001505050505050505, + "loss": 2.5012, + "theoretical_loss": 3.3374332366347783, + "tokens_seen": 2812018688 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015048484848484848, + "loss": 2.3637, + "theoretical_loss": 3.3374271774682622, + "tokens_seen": 2812084224 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015046464646464646, + "loss": 2.6741, + "theoretical_loss": 3.337421118482492, + "tokens_seen": 2812149760 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015044444444444445, + "loss": 2.6175, + "theoretical_loss": 3.337415059677457, + "tokens_seen": 2812215296 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015042424242424243, + "loss": 2.3771, + "theoretical_loss": 3.337409001053149, + "tokens_seen": 2812280832 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015040404040404042, + "loss": 2.5683, + "theoretical_loss": 3.3374029426095584, + "tokens_seen": 2812346368 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001503838383838384, + "loss": 2.5182, + "theoretical_loss": 3.3373968843466746, + "tokens_seen": 2812411904 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015036363636363636, + "loss": 2.6133, + "theoretical_loss": 3.3373908262644885, + "tokens_seen": 2812477440 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015034343434343434, + "loss": 2.2606, + "theoretical_loss": 3.3373847683629903, + "tokens_seen": 2812542976 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015032323232323232, + "loss": 2.5747, + "theoretical_loss": 3.337378710642171, + "tokens_seen": 2812608512 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001503030303030303, + "loss": 2.5645, + "theoretical_loss": 3.3373726531020207, + "tokens_seen": 2812674048 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001502828282828283, + "loss": 2.5088, + "theoretical_loss": 3.3373665957425294, + "tokens_seen": 2812739584 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015026262626262628, + "loss": 2.6207, + "theoretical_loss": 3.3373605385636878, + "tokens_seen": 2812805120 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015024242424242423, + "loss": 2.6206, + "theoretical_loss": 3.337354481565486, + "tokens_seen": 2812870656 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015022222222222224, + "loss": 2.6419, + "theoretical_loss": 3.3373484247479155, + "tokens_seen": 2812936192 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001502020202020202, + "loss": 2.3964, + "theoretical_loss": 3.3373423681109653, + "tokens_seen": 2813001728 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015018181818181819, + "loss": 2.5811, + "theoretical_loss": 3.3373363116546266, + "tokens_seen": 2813067264 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015016161616161617, + "loss": 2.6422, + "theoretical_loss": 3.3373302553788897, + "tokens_seen": 2813132800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015014141414141413, + "loss": 2.439, + "theoretical_loss": 3.337324199283745, + "tokens_seen": 2813198336 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015012121212121214, + "loss": 2.6012, + "theoretical_loss": 3.3373181433691825, + "tokens_seen": 2813263872 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001501010101010101, + "loss": 2.5056, + "theoretical_loss": 3.337312087635193, + "tokens_seen": 2813329408 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1543381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4076125621795654, + "objective/train/theoretical_loss": 3.337306032081767, + "objective/train/tokens_used": 1172254176, + "theoretical_loss": 3.337306032081767, + "tokens_seen": 2813394944 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001500808080808081, + "loss": 2.4433, + "theoretical_loss": 3.337306032081767, + "tokens_seen": 2813394944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015006060606060606, + "loss": 2.5988, + "theoretical_loss": 3.337299976708895, + "tokens_seen": 2813460480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015004040404040405, + "loss": 2.6104, + "theoretical_loss": 3.337293921516567, + "tokens_seen": 2813526016 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015002020202020203, + "loss": 2.6658, + "theoretical_loss": 3.337287866504773, + "tokens_seen": 2813591552 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015, + "loss": 2.4219, + "theoretical_loss": 3.337281811673505, + "tokens_seen": 2813657088 + }, + { + "epoch": 0.7, + "learning_rate": 0.000149979797979798, + "loss": 2.6402, + "theoretical_loss": 3.337275757022752, + "tokens_seen": 2813722624 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014995959595959595, + "loss": 2.3286, + "theoretical_loss": 3.3372697025525047, + "tokens_seen": 2813788160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014993939393939394, + "loss": 2.5936, + "theoretical_loss": 3.337263648262754, + "tokens_seen": 2813853696 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014991919191919192, + "loss": 2.3746, + "theoretical_loss": 3.337257594153489, + "tokens_seen": 2813919232 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001498989898989899, + "loss": 2.3566, + "theoretical_loss": 3.3372515402247025, + "tokens_seen": 2813984768 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001498787878787879, + "loss": 2.3936, + "theoretical_loss": 3.3372454864763825, + "tokens_seen": 2814050304 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014985858585858585, + "loss": 2.5389, + "theoretical_loss": 3.337239432908521, + "tokens_seen": 2814115840 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014983838383838383, + "loss": 2.5794, + "theoretical_loss": 3.3372333795211073, + "tokens_seen": 2814181376 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014981818181818182, + "loss": 2.7035, + "theoretical_loss": 3.3372273263141325, + "tokens_seen": 2814246912 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001497979797979798, + "loss": 2.5967, + "theoretical_loss": 3.337221273287587, + "tokens_seen": 2814312448 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014977777777777778, + "loss": 2.4073, + "theoretical_loss": 3.337215220441461, + "tokens_seen": 2814377984 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014975757575757577, + "loss": 2.2951, + "theoretical_loss": 3.337209167775745, + "tokens_seen": 2814443520 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014973737373737372, + "loss": 2.4842, + "theoretical_loss": 3.33720311529043, + "tokens_seen": 2814509056 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014971717171717174, + "loss": 2.3849, + "theoretical_loss": 3.337197062985505, + "tokens_seen": 2814574592 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001496969696969697, + "loss": 2.5229, + "theoretical_loss": 3.3371910108609617, + "tokens_seen": 2814640128 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014967676767676768, + "loss": 2.4882, + "theoretical_loss": 3.33718495891679, + "tokens_seen": 2814705664 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014965656565656566, + "loss": 2.5955, + "theoretical_loss": 3.3371789071529805, + "tokens_seen": 2814771200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014963636363636362, + "loss": 2.4903, + "theoretical_loss": 3.3371728555695235, + "tokens_seen": 2814836736 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014961616161616163, + "loss": 2.4997, + "theoretical_loss": 3.3371668041664098, + "tokens_seen": 2814902272 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014959595959595958, + "loss": 2.5782, + "theoretical_loss": 3.337160752943629, + "tokens_seen": 2814967808 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1543761, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.297109603881836, + "objective/train/theoretical_loss": 3.3371547019011722, + "objective/train/tokens_used": 1173892576, + "theoretical_loss": 3.3371547019011722, + "tokens_seen": 2815033344 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001495757575757576, + "loss": 2.5068, + "theoretical_loss": 3.3371547019011722, + "tokens_seen": 2815033344 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014955555555555555, + "loss": 2.6634, + "theoretical_loss": 3.33714865103903, + "tokens_seen": 2815098880 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014953535353535356, + "loss": 2.3954, + "theoretical_loss": 3.337142600357192, + "tokens_seen": 2815164416 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014951515151515152, + "loss": 2.5692, + "theoretical_loss": 3.337136549855649, + "tokens_seen": 2815229952 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014949494949494948, + "loss": 2.332, + "theoretical_loss": 3.3371304995343922, + "tokens_seen": 2815295488 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001494747474747475, + "loss": 2.5497, + "theoretical_loss": 3.337124449393411, + "tokens_seen": 2815361024 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014945454545454545, + "loss": 2.5498, + "theoretical_loss": 3.3371183994326965, + "tokens_seen": 2815426560 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014943434343434346, + "loss": 2.3472, + "theoretical_loss": 3.3371123496522386, + "tokens_seen": 2815492096 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001494141414141414, + "loss": 2.5596, + "theoretical_loss": 3.337106300052028, + "tokens_seen": 2815557632 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001493939393939394, + "loss": 2.4906, + "theoretical_loss": 3.337100250632055, + "tokens_seen": 2815623168 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014937373737373738, + "loss": 2.4941, + "theoretical_loss": 3.3370942013923104, + "tokens_seen": 2815688704 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014935353535353537, + "loss": 2.505, + "theoretical_loss": 3.337088152332784, + "tokens_seen": 2815754240 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014933333333333335, + "loss": 2.7451, + "theoretical_loss": 3.3370821034534672, + "tokens_seen": 2815819776 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001493131313131313, + "loss": 2.4954, + "theoretical_loss": 3.3370760547543497, + "tokens_seen": 2815885312 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001492929292929293, + "loss": 2.5513, + "theoretical_loss": 3.337070006235422, + "tokens_seen": 2815950848 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014927272727272727, + "loss": 2.3103, + "theoretical_loss": 3.3370639578966745, + "tokens_seen": 2816016384 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014925252525252526, + "loss": 2.507, + "theoretical_loss": 3.337057909738098, + "tokens_seen": 2816081920 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014923232323232324, + "loss": 2.6062, + "theoretical_loss": 3.337051861759683, + "tokens_seen": 2816147456 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014921212121212123, + "loss": 2.5351, + "theoretical_loss": 3.337045813961419, + "tokens_seen": 2816212992 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014919191919191918, + "loss": 2.4436, + "theoretical_loss": 3.3370397663432976, + "tokens_seen": 2816278528 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001491717171717172, + "loss": 2.5447, + "theoretical_loss": 3.3370337189053085, + "tokens_seen": 2816344064 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014915151515151515, + "loss": 2.1893, + "theoretical_loss": 3.3370276716474425, + "tokens_seen": 2816409600 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014913131313131313, + "loss": 2.4392, + "theoretical_loss": 3.3370216245696898, + "tokens_seen": 2816475136 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014911111111111112, + "loss": 2.5965, + "theoretical_loss": 3.337015577672041, + "tokens_seen": 2816540672 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014909090909090908, + "loss": 2.3938, + "theoretical_loss": 3.3370095309544867, + "tokens_seen": 2816606208 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1544909, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0402820110321045, + "objective/train/theoretical_loss": 3.337003484417017, + "objective/train/tokens_used": 1175530976, + "theoretical_loss": 3.337003484417017, + "tokens_seen": 2816671744 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001490707070707071, + "loss": 2.4325, + "theoretical_loss": 3.337003484417017, + "tokens_seen": 2816671744 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014905050505050504, + "loss": 2.6141, + "theoretical_loss": 3.3369974380596226, + "tokens_seen": 2816737280 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014903030303030305, + "loss": 2.5239, + "theoretical_loss": 3.336991391882294, + "tokens_seen": 2816802816 + }, + { + "epoch": 0.71, + "learning_rate": 0.000149010101010101, + "loss": 2.5857, + "theoretical_loss": 3.336985345885021, + "tokens_seen": 2816868352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014898989898989897, + "loss": 2.3817, + "theoretical_loss": 3.336979300067795, + "tokens_seen": 2816933888 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014896969696969698, + "loss": 2.3698, + "theoretical_loss": 3.336973254430606, + "tokens_seen": 2816999424 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014894949494949494, + "loss": 2.6252, + "theoretical_loss": 3.3369672089734443, + "tokens_seen": 2817064960 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014892929292929295, + "loss": 2.6003, + "theoretical_loss": 3.3369611636963006, + "tokens_seen": 2817130496 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001489090909090909, + "loss": 2.3109, + "theoretical_loss": 3.3369551185991653, + "tokens_seen": 2817196032 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001488888888888889, + "loss": 2.5003, + "theoretical_loss": 3.3369490736820286, + "tokens_seen": 2817261568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014886868686868687, + "loss": 2.5494, + "theoretical_loss": 3.3369430289448814, + "tokens_seen": 2817327104 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014884848484848486, + "loss": 2.548, + "theoretical_loss": 3.336936984387714, + "tokens_seen": 2817392640 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014882828282828284, + "loss": 2.5223, + "theoretical_loss": 3.3369309400105163, + "tokens_seen": 2817458176 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001488080808080808, + "loss": 2.5994, + "theoretical_loss": 3.33692489581328, + "tokens_seen": 2817523712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014878787878787878, + "loss": 2.4584, + "theoretical_loss": 3.3369188517959945, + "tokens_seen": 2817589248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014876767676767676, + "loss": 2.3667, + "theoretical_loss": 3.33691280795865, + "tokens_seen": 2817654784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014874747474747475, + "loss": 2.5087, + "theoretical_loss": 3.3369067643012382, + "tokens_seen": 2817720320 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014872727272727273, + "loss": 2.5249, + "theoretical_loss": 3.3369007208237482, + "tokens_seen": 2817785856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014870707070707072, + "loss": 2.5113, + "theoretical_loss": 3.3368946775261716, + "tokens_seen": 2817851392 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001486868686868687, + "loss": 2.6231, + "theoretical_loss": 3.3368886344084983, + "tokens_seen": 2817916928 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014866666666666668, + "loss": 2.4657, + "theoretical_loss": 3.3368825914707188, + "tokens_seen": 2817982464 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014864646464646464, + "loss": 2.452, + "theoretical_loss": 3.336876548712824, + "tokens_seen": 2818048000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014862626262626263, + "loss": 2.5397, + "theoretical_loss": 3.3368705061348036, + "tokens_seen": 2818113536 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001486060606060606, + "loss": 2.6348, + "theoretical_loss": 3.336864463736648, + "tokens_seen": 2818179072 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001485858585858586, + "loss": 2.6421, + "theoretical_loss": 3.3368584215183485, + "tokens_seen": 2818244608 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1545329, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4824910163879395, + "objective/train/theoretical_loss": 3.3368523794798954, + "objective/train/tokens_used": 1177169376, + "theoretical_loss": 3.3368523794798954, + "tokens_seen": 2818310144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014856565656565658, + "loss": 2.4511, + "theoretical_loss": 3.3368523794798954, + "tokens_seen": 2818310144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014854545454545453, + "loss": 2.5684, + "theoretical_loss": 3.336846337621279, + "tokens_seen": 2818375680 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014852525252525254, + "loss": 2.6883, + "theoretical_loss": 3.3368402959424888, + "tokens_seen": 2818441216 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001485050505050505, + "loss": 2.5521, + "theoretical_loss": 3.336834254443517, + "tokens_seen": 2818506752 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001484848484848485, + "loss": 2.7295, + "theoretical_loss": 3.336828213124353, + "tokens_seen": 2818572288 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014846464646464647, + "loss": 2.3323, + "theoretical_loss": 3.3368221719849873, + "tokens_seen": 2818637824 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014844444444444443, + "loss": 2.6035, + "theoretical_loss": 3.3368161310254107, + "tokens_seen": 2818703360 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014842424242424244, + "loss": 2.4436, + "theoretical_loss": 3.3368100902456135, + "tokens_seen": 2818768896 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001484040404040404, + "loss": 2.4129, + "theoretical_loss": 3.3368040496455866, + "tokens_seen": 2818834432 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001483838383838384, + "loss": 2.5415, + "theoretical_loss": 3.3367980092253196, + "tokens_seen": 2818899968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014836363636363636, + "loss": 2.2387, + "theoretical_loss": 3.3367919689848033, + "tokens_seen": 2818965504 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014834343434343435, + "loss": 2.6121, + "theoretical_loss": 3.3367859289240287, + "tokens_seen": 2819031040 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014832323232323233, + "loss": 2.6852, + "theoretical_loss": 3.336779889042986, + "tokens_seen": 2819096576 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001483030303030303, + "loss": 2.681, + "theoretical_loss": 3.3367738493416654, + "tokens_seen": 2819162112 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001482828282828283, + "loss": 2.3518, + "theoretical_loss": 3.3367678098200573, + "tokens_seen": 2819227648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014826262626262626, + "loss": 2.4424, + "theoretical_loss": 3.3367617704781525, + "tokens_seen": 2819293184 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014824242424242424, + "loss": 2.6784, + "theoretical_loss": 3.3367557313159413, + "tokens_seen": 2819358720 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014822222222222222, + "loss": 2.4099, + "theoretical_loss": 3.336749692333415, + "tokens_seen": 2819424256 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001482020202020202, + "loss": 2.6596, + "theoretical_loss": 3.3367436535305623, + "tokens_seen": 2819489792 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001481818181818182, + "loss": 2.4806, + "theoretical_loss": 3.336737614907375, + "tokens_seen": 2819555328 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014816161616161617, + "loss": 2.7067, + "theoretical_loss": 3.3367315764638437, + "tokens_seen": 2819620864 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014814141414141413, + "loss": 2.4387, + "theoretical_loss": 3.336725538199958, + "tokens_seen": 2819686400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014812121212121212, + "loss": 2.3936, + "theoretical_loss": 3.3367195001157093, + "tokens_seen": 2819751936 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001481010101010101, + "loss": 2.6026, + "theoretical_loss": 3.3367134622110877, + "tokens_seen": 2819817472 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014808080808080808, + "loss": 2.331, + "theoretical_loss": 3.3367074244860833, + "tokens_seen": 2819883008 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1546518, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9841865301132202, + "objective/train/theoretical_loss": 3.3367013869406867, + "objective/train/tokens_used": 1178807776, + "theoretical_loss": 3.3367013869406867, + "tokens_seen": 2819948544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014806060606060607, + "loss": 2.4183, + "theoretical_loss": 3.3367013869406867, + "tokens_seen": 2819948544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014804040404040402, + "loss": 2.4696, + "theoretical_loss": 3.336695349574889, + "tokens_seen": 2820014080 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014802020202020204, + "loss": 2.5295, + "theoretical_loss": 3.33668931238868, + "tokens_seen": 2820079616 + }, + { + "epoch": 0.71, + "learning_rate": 0.000148, + "loss": 2.4702, + "theoretical_loss": 3.3366832753820503, + "tokens_seen": 2820145152 + }, + { + "epoch": 0.71, + "learning_rate": 0.000147979797979798, + "loss": 2.3708, + "theoretical_loss": 3.3366772385549908, + "tokens_seen": 2820210688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014795959595959596, + "loss": 2.46, + "theoretical_loss": 3.336671201907492, + "tokens_seen": 2820276224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014793939393939392, + "loss": 2.492, + "theoretical_loss": 3.336665165439544, + "tokens_seen": 2820341760 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014791919191919193, + "loss": 2.5065, + "theoretical_loss": 3.336659129151137, + "tokens_seen": 2820407296 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014789898989898989, + "loss": 2.718, + "theoretical_loss": 3.336653093042262, + "tokens_seen": 2820472832 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001478787878787879, + "loss": 2.4638, + "theoretical_loss": 3.3366470571129097, + "tokens_seen": 2820538368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014785858585858585, + "loss": 2.3556, + "theoretical_loss": 3.3366410213630697, + "tokens_seen": 2820603904 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014783838383838386, + "loss": 2.583, + "theoretical_loss": 3.3366349857927333, + "tokens_seen": 2820669440 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014781818181818182, + "loss": 2.4604, + "theoretical_loss": 3.336628950401891, + "tokens_seen": 2820734976 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001477979797979798, + "loss": 2.3676, + "theoretical_loss": 3.336622915190533, + "tokens_seen": 2820800512 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001477777777777778, + "loss": 2.3986, + "theoretical_loss": 3.3366168801586498, + "tokens_seen": 2820866048 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014775757575757575, + "loss": 2.561, + "theoretical_loss": 3.3366108453062315, + "tokens_seen": 2820931584 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014773737373737376, + "loss": 2.5619, + "theoretical_loss": 3.3366048106332697, + "tokens_seen": 2820997120 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014771717171717171, + "loss": 2.4221, + "theoretical_loss": 3.3365987761397538, + "tokens_seen": 2821062656 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001476969696969697, + "loss": 2.4276, + "theoretical_loss": 3.3365927418256747, + "tokens_seen": 2821128192 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014767676767676768, + "loss": 2.5575, + "theoretical_loss": 3.336586707691023, + "tokens_seen": 2821193728 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014765656565656567, + "loss": 2.6101, + "theoretical_loss": 3.336580673735789, + "tokens_seen": 2821259264 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014763636363636365, + "loss": 2.2437, + "theoretical_loss": 3.3365746399599634, + "tokens_seen": 2821324800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014761616161616163, + "loss": 2.4829, + "theoretical_loss": 3.336568606363537, + "tokens_seen": 2821390336 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001475959595959596, + "loss": 2.4354, + "theoretical_loss": 3.3365625729464994, + "tokens_seen": 2821455872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014757575757575757, + "loss": 2.5635, + "theoretical_loss": 3.3365565397088415, + "tokens_seen": 2821521408 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1547222, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7162699699401855, + "objective/train/theoretical_loss": 3.336550506650554, + "objective/train/tokens_used": 1180446176, + "theoretical_loss": 3.336550506650554, + "tokens_seen": 2821586944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014755555555555556, + "loss": 2.6892, + "theoretical_loss": 3.336550506650554, + "tokens_seen": 2821586944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014753535353535354, + "loss": 2.4847, + "theoretical_loss": 3.336544473771627, + "tokens_seen": 2821652480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014751515151515153, + "loss": 2.577, + "theoretical_loss": 3.336538441072052, + "tokens_seen": 2821718016 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014749494949494948, + "loss": 2.6168, + "theoretical_loss": 3.3365324085518187, + "tokens_seen": 2821783552 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001474747474747475, + "loss": 2.5693, + "theoretical_loss": 3.3365263762109176, + "tokens_seen": 2821849088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014745454545454545, + "loss": 2.4566, + "theoretical_loss": 3.336520344049339, + "tokens_seen": 2821914624 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014743434343434346, + "loss": 2.4969, + "theoretical_loss": 3.3365143120670737, + "tokens_seen": 2821980160 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014741414141414142, + "loss": 2.471, + "theoretical_loss": 3.3365082802641126, + "tokens_seen": 2822045696 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014739393939393938, + "loss": 2.349, + "theoretical_loss": 3.3365022486404454, + "tokens_seen": 2822111232 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001473737373737374, + "loss": 2.4419, + "theoretical_loss": 3.3364962171960633, + "tokens_seen": 2822176768 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014735353535353534, + "loss": 2.6028, + "theoretical_loss": 3.336490185930957, + "tokens_seen": 2822242304 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014733333333333335, + "loss": 2.4462, + "theoretical_loss": 3.336484154845116, + "tokens_seen": 2822307840 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001473131313131313, + "loss": 2.3973, + "theoretical_loss": 3.336478123938531, + "tokens_seen": 2822373376 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001472929292929293, + "loss": 2.2181, + "theoretical_loss": 3.3364720932111935, + "tokens_seen": 2822438912 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014727272727272728, + "loss": 2.4296, + "theoretical_loss": 3.336466062663093, + "tokens_seen": 2822504448 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014725252525252524, + "loss": 2.5094, + "theoretical_loss": 3.336460032294221, + "tokens_seen": 2822569984 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014723232323232325, + "loss": 2.5197, + "theoretical_loss": 3.336454002104567, + "tokens_seen": 2822635520 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001472121212121212, + "loss": 2.3919, + "theoretical_loss": 3.336447972094122, + "tokens_seen": 2822701056 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001471919191919192, + "loss": 2.4516, + "theoretical_loss": 3.336441942262876, + "tokens_seen": 2822766592 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014717171717171717, + "loss": 2.5135, + "theoretical_loss": 3.3364359126108205, + "tokens_seen": 2822832128 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014715151515151516, + "loss": 2.459, + "theoretical_loss": 3.3364298831379453, + "tokens_seen": 2822897664 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014713131313131314, + "loss": 2.2841, + "theoretical_loss": 3.336423853844241, + "tokens_seen": 2822963200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014711111111111112, + "loss": 2.418, + "theoretical_loss": 3.3364178247296983, + "tokens_seen": 2823028736 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014709090909090908, + "loss": 2.448, + "theoretical_loss": 3.3364117957943074, + "tokens_seen": 2823094272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014707070707070706, + "loss": 2.5341, + "theoretical_loss": 3.336405767038059, + "tokens_seen": 2823159808 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1548263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.856104612350464, + "objective/train/theoretical_loss": 3.336399738460944, + "objective/train/tokens_used": 1182084576, + "theoretical_loss": 3.336399738460944, + "tokens_seen": 2823225344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014705050505050505, + "loss": 2.5434, + "theoretical_loss": 3.336399738460944, + "tokens_seen": 2823225344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014703030303030303, + "loss": 2.5174, + "theoretical_loss": 3.336393710062952, + "tokens_seen": 2823290880 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014701010101010102, + "loss": 2.4975, + "theoretical_loss": 3.3363876818440747, + "tokens_seen": 2823356416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014698989898989897, + "loss": 2.649, + "theoretical_loss": 3.3363816538043016, + "tokens_seen": 2823421952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014696969696969698, + "loss": 2.6029, + "theoretical_loss": 3.336375625943624, + "tokens_seen": 2823487488 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014694949494949494, + "loss": 2.4688, + "theoretical_loss": 3.336369598262032, + "tokens_seen": 2823553024 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014692929292929295, + "loss": 2.4812, + "theoretical_loss": 3.3363635707595156, + "tokens_seen": 2823618560 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001469090909090909, + "loss": 2.4266, + "theoretical_loss": 3.3363575434360664, + "tokens_seen": 2823684096 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001468888888888889, + "loss": 2.5619, + "theoretical_loss": 3.3363515162916744, + "tokens_seen": 2823749632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014686868686868688, + "loss": 2.3878, + "theoretical_loss": 3.3363454893263302, + "tokens_seen": 2823815168 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014684848484848483, + "loss": 2.4137, + "theoretical_loss": 3.3363394625400242, + "tokens_seen": 2823880704 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014682828282828285, + "loss": 2.51, + "theoretical_loss": 3.3363334359327466, + "tokens_seen": 2823946240 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001468080808080808, + "loss": 2.3203, + "theoretical_loss": 3.336327409504489, + "tokens_seen": 2824011776 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001467878787878788, + "loss": 2.5331, + "theoretical_loss": 3.3363213832552407, + "tokens_seen": 2824077312 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014676767676767677, + "loss": 2.484, + "theoretical_loss": 3.336315357184993, + "tokens_seen": 2824142848 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014674747474747475, + "loss": 2.5866, + "theoretical_loss": 3.3363093312937364, + "tokens_seen": 2824208384 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014672727272727274, + "loss": 2.4145, + "theoretical_loss": 3.336303305581461, + "tokens_seen": 2824273920 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001467070707070707, + "loss": 2.4557, + "theoretical_loss": 3.3362972800481576, + "tokens_seen": 2824339456 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001466868686868687, + "loss": 2.5438, + "theoretical_loss": 3.336291254693817, + "tokens_seen": 2824404992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014666666666666666, + "loss": 2.5279, + "theoretical_loss": 3.336285229518429, + "tokens_seen": 2824470528 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014664646464646465, + "loss": 2.4785, + "theoretical_loss": 3.3362792045219845, + "tokens_seen": 2824536064 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014662626262626263, + "loss": 2.7389, + "theoretical_loss": 3.3362731797044747, + "tokens_seen": 2824601600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014660606060606061, + "loss": 2.4726, + "theoretical_loss": 3.336267155065889, + "tokens_seen": 2824667136 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001465858585858586, + "loss": 2.2595, + "theoretical_loss": 3.3362611306062187, + "tokens_seen": 2824732672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014656565656565656, + "loss": 2.4656, + "theoretical_loss": 3.3362551063254537, + "tokens_seen": 2824798208 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1549182, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4828224182128906, + "objective/train/theoretical_loss": 3.3362490822235857, + "objective/train/tokens_used": 1183722976, + "theoretical_loss": 3.3362490822235857, + "tokens_seen": 2824863744 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014654545454545454, + "loss": 2.4916, + "theoretical_loss": 3.3362490822235857, + "tokens_seen": 2824863744 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014652525252525252, + "loss": 2.5541, + "theoretical_loss": 3.336243058300604, + "tokens_seen": 2824929280 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001465050505050505, + "loss": 2.4892, + "theoretical_loss": 3.3362370345564996, + "tokens_seen": 2824994816 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001464848484848485, + "loss": 2.5957, + "theoretical_loss": 3.336231010991263, + "tokens_seen": 2825060352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014646464646464648, + "loss": 2.6118, + "theoretical_loss": 3.3362249876048855, + "tokens_seen": 2825125888 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014644444444444443, + "loss": 2.6803, + "theoretical_loss": 3.3362189643973563, + "tokens_seen": 2825191424 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014642424242424244, + "loss": 2.5285, + "theoretical_loss": 3.3362129413686663, + "tokens_seen": 2825256960 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001464040404040404, + "loss": 2.4544, + "theoretical_loss": 3.336206918518807, + "tokens_seen": 2825322496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014638383838383838, + "loss": 2.4611, + "theoretical_loss": 3.3362008958477682, + "tokens_seen": 2825388032 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014636363636363637, + "loss": 2.4181, + "theoretical_loss": 3.3361948733555398, + "tokens_seen": 2825453568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014634343434343432, + "loss": 2.5487, + "theoretical_loss": 3.3361888510421136, + "tokens_seen": 2825519104 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014632323232323234, + "loss": 2.5548, + "theoretical_loss": 3.3361828289074795, + "tokens_seen": 2825584640 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001463030303030303, + "loss": 2.4938, + "theoretical_loss": 3.336176806951628, + "tokens_seen": 2825650176 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001462828282828283, + "loss": 2.6433, + "theoretical_loss": 3.3361707851745503, + "tokens_seen": 2825715712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014626262626262626, + "loss": 2.5351, + "theoretical_loss": 3.336164763576236, + "tokens_seen": 2825781248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014624242424242424, + "loss": 2.6836, + "theoretical_loss": 3.3361587421566763, + "tokens_seen": 2825846784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014622222222222223, + "loss": 2.4458, + "theoretical_loss": 3.3361527209158615, + "tokens_seen": 2825912320 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014620202020202019, + "loss": 2.4625, + "theoretical_loss": 3.3361466998537823, + "tokens_seen": 2825977856 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001461818181818182, + "loss": 2.6263, + "theoretical_loss": 3.3361406789704287, + "tokens_seen": 2826043392 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014616161616161615, + "loss": 2.1992, + "theoretical_loss": 3.336134658265792, + "tokens_seen": 2826108928 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014614141414141414, + "loss": 2.4944, + "theoretical_loss": 3.3361286377398622, + "tokens_seen": 2826174464 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014612121212121212, + "loss": 2.3674, + "theoretical_loss": 3.33612261739263, + "tokens_seen": 2826240000 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001461010101010101, + "loss": 2.8067, + "theoretical_loss": 3.3361165972240867, + "tokens_seen": 2826305536 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001460808080808081, + "loss": 2.6022, + "theoretical_loss": 3.3361105772342214, + "tokens_seen": 2826371072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014606060606060607, + "loss": 2.3892, + "theoretical_loss": 3.3361045574230257, + "tokens_seen": 2826436608 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1549852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3939852714538574, + "objective/train/theoretical_loss": 3.33609853779049, + "objective/train/tokens_used": 1185361376, + "theoretical_loss": 3.33609853779049, + "tokens_seen": 2826502144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014604040404040406, + "loss": 2.3817, + "theoretical_loss": 3.33609853779049, + "tokens_seen": 2826502144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014602020202020201, + "loss": 2.5027, + "theoretical_loss": 3.336092518336605, + "tokens_seen": 2826567680 + }, + { + "epoch": 0.71, + "learning_rate": 0.000146, + "loss": 2.6148, + "theoretical_loss": 3.3360864990613606, + "tokens_seen": 2826633216 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014597979797979798, + "loss": 2.8313, + "theoretical_loss": 3.336080479964748, + "tokens_seen": 2826698752 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014595959595959597, + "loss": 2.5257, + "theoretical_loss": 3.336074461046757, + "tokens_seen": 2826764288 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014593939393939395, + "loss": 2.4959, + "theoretical_loss": 3.336068442307379, + "tokens_seen": 2826829824 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014591919191919193, + "loss": 2.3403, + "theoretical_loss": 3.336062423746605, + "tokens_seen": 2826895360 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001458989898989899, + "loss": 2.6225, + "theoretical_loss": 3.336056405364424, + "tokens_seen": 2826960896 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001458787878787879, + "loss": 2.58, + "theoretical_loss": 3.336050387160827, + "tokens_seen": 2827026432 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014585858585858586, + "loss": 2.5773, + "theoretical_loss": 3.3360443691358057, + "tokens_seen": 2827091968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014583838383838384, + "loss": 2.5914, + "theoretical_loss": 3.33603835128935, + "tokens_seen": 2827157504 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014581818181818183, + "loss": 2.567, + "theoretical_loss": 3.3360323336214495, + "tokens_seen": 2827223040 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014579797979797978, + "loss": 2.5559, + "theoretical_loss": 3.336026316132096, + "tokens_seen": 2827288576 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001457777777777778, + "loss": 2.4291, + "theoretical_loss": 3.33602029882128, + "tokens_seen": 2827354112 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014575757575757575, + "loss": 2.4238, + "theoretical_loss": 3.3360142816889913, + "tokens_seen": 2827419648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014573737373737376, + "loss": 2.4672, + "theoretical_loss": 3.336008264735221, + "tokens_seen": 2827485184 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014571717171717172, + "loss": 2.6942, + "theoretical_loss": 3.33600224795996, + "tokens_seen": 2827550720 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014569696969696968, + "loss": 2.6527, + "theoretical_loss": 3.3359962313631977, + "tokens_seen": 2827616256 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001456767676767677, + "loss": 2.4777, + "theoretical_loss": 3.3359902149449256, + "tokens_seen": 2827681792 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014565656565656564, + "loss": 2.3061, + "theoretical_loss": 3.3359841987051344, + "tokens_seen": 2827747328 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014563636363636366, + "loss": 2.6427, + "theoretical_loss": 3.335978182643814, + "tokens_seen": 2827812864 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001456161616161616, + "loss": 2.3856, + "theoretical_loss": 3.3359721667609556, + "tokens_seen": 2827878400 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001455959595959596, + "loss": 2.7118, + "theoretical_loss": 3.3359661510565495, + "tokens_seen": 2827943936 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014557575757575758, + "loss": 2.6451, + "theoretical_loss": 3.335960135530586, + "tokens_seen": 2828009472 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014555555555555556, + "loss": 2.5695, + "theoretical_loss": 3.335954120183056, + "tokens_seen": 2828075008 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1550895, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7013211250305176, + "objective/train/theoretical_loss": 3.33594810501395, + "objective/train/tokens_used": 1186999776, + "theoretical_loss": 3.33594810501395, + "tokens_seen": 2828140544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014553535353535355, + "loss": 2.788, + "theoretical_loss": 3.33594810501395, + "tokens_seen": 2828140544 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001455151515151515, + "loss": 2.5634, + "theoretical_loss": 3.3359420900232584, + "tokens_seen": 2828206080 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001454949494949495, + "loss": 2.3282, + "theoretical_loss": 3.3359360752109724, + "tokens_seen": 2828271616 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014547474747474747, + "loss": 2.6087, + "theoretical_loss": 3.335930060577082, + "tokens_seen": 2828337152 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014545454545454546, + "loss": 2.4711, + "theoretical_loss": 3.3359240461215776, + "tokens_seen": 2828402688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014543434343434344, + "loss": 2.3771, + "theoretical_loss": 3.33591803184445, + "tokens_seen": 2828468224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014541414141414142, + "loss": 2.392, + "theoretical_loss": 3.3359120177456902, + "tokens_seen": 2828533760 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014539393939393938, + "loss": 2.7174, + "theoretical_loss": 3.335906003825288, + "tokens_seen": 2828599296 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001453737373737374, + "loss": 2.3878, + "theoretical_loss": 3.335899990083235, + "tokens_seen": 2828664832 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014535353535353535, + "loss": 2.4091, + "theoretical_loss": 3.335893976519521, + "tokens_seen": 2828730368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014533333333333333, + "loss": 2.4785, + "theoretical_loss": 3.3358879631341365, + "tokens_seen": 2828795904 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014531313131313132, + "loss": 2.7032, + "theoretical_loss": 3.3358819499270727, + "tokens_seen": 2828861440 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014529292929292927, + "loss": 2.6466, + "theoretical_loss": 3.3358759368983195, + "tokens_seen": 2828926976 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014527272727272729, + "loss": 2.8232, + "theoretical_loss": 3.3358699240478678, + "tokens_seen": 2828992512 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014525252525252524, + "loss": 2.3487, + "theoretical_loss": 3.335863911375708, + "tokens_seen": 2829058048 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014523232323232325, + "loss": 2.6393, + "theoretical_loss": 3.335857898881831, + "tokens_seen": 2829123584 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001452121212121212, + "loss": 2.5982, + "theoretical_loss": 3.3358518865662274, + "tokens_seen": 2829189120 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014519191919191922, + "loss": 2.3298, + "theoretical_loss": 3.335845874428888, + "tokens_seen": 2829254656 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014517171717171718, + "loss": 2.509, + "theoretical_loss": 3.3358398624698027, + "tokens_seen": 2829320192 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014515151515151513, + "loss": 2.5478, + "theoretical_loss": 3.335833850688962, + "tokens_seen": 2829385728 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014513131313131315, + "loss": 2.4575, + "theoretical_loss": 3.3358278390863574, + "tokens_seen": 2829451264 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001451111111111111, + "loss": 2.5393, + "theoretical_loss": 3.335821827661979, + "tokens_seen": 2829516800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014509090909090911, + "loss": 2.5332, + "theoretical_loss": 3.335815816415817, + "tokens_seen": 2829582336 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014507070707070707, + "loss": 2.7377, + "theoretical_loss": 3.335809805347863, + "tokens_seen": 2829647872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014505050505050505, + "loss": 2.3647, + "theoretical_loss": 3.3358037944581063, + "tokens_seen": 2829713408 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1551292, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.010021448135376, + "objective/train/theoretical_loss": 3.3357977837465382, + "objective/train/tokens_used": 1188638176, + "theoretical_loss": 3.3357977837465382, + "tokens_seen": 2829778944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014503030303030304, + "loss": 2.372, + "theoretical_loss": 3.3357977837465382, + "tokens_seen": 2829778944 + }, + { + "epoch": 0.71, + "learning_rate": 0.000145010101010101, + "loss": 2.619, + "theoretical_loss": 3.3357917732131495, + "tokens_seen": 2829844480 + }, + { + "epoch": 0.71, + "learning_rate": 0.000144989898989899, + "loss": 2.5035, + "theoretical_loss": 3.3357857628579306, + "tokens_seen": 2829910016 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014496969696969696, + "loss": 2.4966, + "theoretical_loss": 3.335779752680872, + "tokens_seen": 2829975552 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014494949494949495, + "loss": 2.5172, + "theoretical_loss": 3.335773742681964, + "tokens_seen": 2830041088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014492929292929293, + "loss": 2.6471, + "theoretical_loss": 3.3357677328611977, + "tokens_seen": 2830106624 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014490909090909092, + "loss": 2.4552, + "theoretical_loss": 3.335761723218563, + "tokens_seen": 2830172160 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001448888888888889, + "loss": 2.4797, + "theoretical_loss": 3.3357557137540517, + "tokens_seen": 2830237696 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014486868686868688, + "loss": 2.3638, + "theoretical_loss": 3.3357497044676534, + "tokens_seen": 2830303232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014484848484848484, + "loss": 2.7106, + "theoretical_loss": 3.3357436953593593, + "tokens_seen": 2830368768 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014482828282828282, + "loss": 2.5898, + "theoretical_loss": 3.3357376864291592, + "tokens_seen": 2830434304 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001448080808080808, + "loss": 2.3486, + "theoretical_loss": 3.3357316776770443, + "tokens_seen": 2830499840 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001447878787878788, + "loss": 2.4524, + "theoretical_loss": 3.3357256691030055, + "tokens_seen": 2830565376 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014476767676767678, + "loss": 2.5324, + "theoretical_loss": 3.3357196607070323, + "tokens_seen": 2830630912 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014474747474747473, + "loss": 2.6281, + "theoretical_loss": 3.3357136524891167, + "tokens_seen": 2830696448 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014472727272727274, + "loss": 2.5811, + "theoretical_loss": 3.335707644449248, + "tokens_seen": 2830761984 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001447070707070707, + "loss": 2.6378, + "theoretical_loss": 3.3357016365874177, + "tokens_seen": 2830827520 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001446868686868687, + "loss": 2.6912, + "theoretical_loss": 3.3356956289036157, + "tokens_seen": 2830893056 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014466666666666667, + "loss": 2.3259, + "theoretical_loss": 3.3356896213978335, + "tokens_seen": 2830958592 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014464646464646463, + "loss": 2.6109, + "theoretical_loss": 3.335683614070061, + "tokens_seen": 2831024128 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014462626262626264, + "loss": 2.6496, + "theoretical_loss": 3.335677606920289, + "tokens_seen": 2831089664 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001446060606060606, + "loss": 2.4065, + "theoretical_loss": 3.3356715999485083, + "tokens_seen": 2831155200 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001445858585858586, + "loss": 2.4639, + "theoretical_loss": 3.335665593154709, + "tokens_seen": 2831220736 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014456565656565656, + "loss": 2.498, + "theoretical_loss": 3.335659586538882, + "tokens_seen": 2831286272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014454545454545455, + "loss": 2.5483, + "theoretical_loss": 3.335653580101018, + "tokens_seen": 2831351808 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1552402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.707756757736206, + "objective/train/theoretical_loss": 3.3356475738411078, + "objective/train/tokens_used": 1190276576, + "theoretical_loss": 3.3356475738411078, + "tokens_seen": 2831417344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014452525252525253, + "loss": 2.5595, + "theoretical_loss": 3.3356475738411078, + "tokens_seen": 2831417344 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001445050505050505, + "loss": 2.6068, + "theoretical_loss": 3.3356415677591413, + "tokens_seen": 2831482880 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001444848484848485, + "loss": 2.5212, + "theoretical_loss": 3.33563556185511, + "tokens_seen": 2831548416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014446464646464645, + "loss": 2.4911, + "theoretical_loss": 3.3356295561290037, + "tokens_seen": 2831613952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014444444444444444, + "loss": 2.3585, + "theoretical_loss": 3.335623550580814, + "tokens_seen": 2831679488 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014442424242424242, + "loss": 2.5722, + "theoretical_loss": 3.33561754521053, + "tokens_seen": 2831745024 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001444040404040404, + "loss": 2.5328, + "theoretical_loss": 3.3356115400181436, + "tokens_seen": 2831810560 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001443838383838384, + "loss": 2.5772, + "theoretical_loss": 3.335605535003645, + "tokens_seen": 2831876096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014436363636363637, + "loss": 2.5951, + "theoretical_loss": 3.335599530167025, + "tokens_seen": 2831941632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014434343434343436, + "loss": 2.522, + "theoretical_loss": 3.335593525508274, + "tokens_seen": 2832007168 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014432323232323234, + "loss": 2.403, + "theoretical_loss": 3.3355875210273824, + "tokens_seen": 2832072704 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001443030303030303, + "loss": 2.6755, + "theoretical_loss": 3.3355815167243414, + "tokens_seen": 2832138240 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014428282828282828, + "loss": 2.6878, + "theoretical_loss": 3.335575512599141, + "tokens_seen": 2832203776 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014426262626262627, + "loss": 2.6354, + "theoretical_loss": 3.3355695086517723, + "tokens_seen": 2832269312 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014424242424242425, + "loss": 2.4917, + "theoretical_loss": 3.3355635048822254, + "tokens_seen": 2832334848 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014422222222222223, + "loss": 2.6982, + "theoretical_loss": 3.335557501290492, + "tokens_seen": 2832400384 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001442020202020202, + "loss": 2.3125, + "theoretical_loss": 3.335551497876561, + "tokens_seen": 2832465920 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001441818181818182, + "loss": 2.5098, + "theoretical_loss": 3.335545494640425, + "tokens_seen": 2832531456 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014416161616161616, + "loss": 2.4983, + "theoretical_loss": 3.3355394915820726, + "tokens_seen": 2832596992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014414141414141417, + "loss": 2.4279, + "theoretical_loss": 3.335533488701496, + "tokens_seen": 2832662528 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014412121212121213, + "loss": 2.3771, + "theoretical_loss": 3.335527485998685, + "tokens_seen": 2832728064 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014410101010101008, + "loss": 2.4398, + "theoretical_loss": 3.335521483473631, + "tokens_seen": 2832793600 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001440808080808081, + "loss": 2.4479, + "theoretical_loss": 3.3355154811263237, + "tokens_seen": 2832859136 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014406060606060605, + "loss": 2.4095, + "theoretical_loss": 3.335509478956754, + "tokens_seen": 2832924672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014404040404040406, + "loss": 2.5803, + "theoretical_loss": 3.3355034769649126, + "tokens_seen": 2832990208 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1552972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.235657215118408, + "objective/train/theoretical_loss": 3.3354974751507904, + "objective/train/tokens_used": 1191914976, + "theoretical_loss": 3.3354974751507904, + "tokens_seen": 2833055744 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014402020202020202, + "loss": 2.5574, + "theoretical_loss": 3.3354974751507904, + "tokens_seen": 2833055744 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014402020202020202, + "loss": 2.4768, + "theoretical_loss": 3.3354914735143777, + "tokens_seen": 2833121280 + }, + { + "epoch": 0.72, + "learning_rate": 0.000144, + "loss": 2.4725, + "theoretical_loss": 3.3354854720556655, + "tokens_seen": 2833186816 + }, + { + "epoch": 0.72, + "learning_rate": 0.000143979797979798, + "loss": 2.3878, + "theoretical_loss": 3.335479470774644, + "tokens_seen": 2833252352 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014395959595959594, + "loss": 2.5209, + "theoretical_loss": 3.335473469671304, + "tokens_seen": 2833317888 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014393939393939396, + "loss": 2.4284, + "theoretical_loss": 3.3354674687456356, + "tokens_seen": 2833383424 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001439191919191919, + "loss": 2.5053, + "theoretical_loss": 3.3354614679976304, + "tokens_seen": 2833448960 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001438989898989899, + "loss": 2.4506, + "theoretical_loss": 3.3354554674272787, + "tokens_seen": 2833514496 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014387878787878788, + "loss": 2.5875, + "theoretical_loss": 3.3354494670345707, + "tokens_seen": 2833580032 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014385858585858586, + "loss": 2.5171, + "theoretical_loss": 3.335443466819498, + "tokens_seen": 2833645568 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014383838383838385, + "loss": 2.5279, + "theoretical_loss": 3.33543746678205, + "tokens_seen": 2833711104 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014381818181818183, + "loss": 2.5539, + "theoretical_loss": 3.335431466922218, + "tokens_seen": 2833776640 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001437979797979798, + "loss": 2.6066, + "theoretical_loss": 3.3354254672399923, + "tokens_seen": 2833842176 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014377777777777777, + "loss": 2.4157, + "theoretical_loss": 3.335419467735364, + "tokens_seen": 2833907712 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014375757575757576, + "loss": 2.4145, + "theoretical_loss": 3.3354134684083236, + "tokens_seen": 2833973248 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014373737373737374, + "loss": 2.5155, + "theoretical_loss": 3.3354074692588616, + "tokens_seen": 2834038784 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014371717171717172, + "loss": 2.4012, + "theoretical_loss": 3.3354014702869685, + "tokens_seen": 2834104320 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014369696969696968, + "loss": 2.5255, + "theoretical_loss": 3.3353954714926353, + "tokens_seen": 2834169856 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001436767676767677, + "loss": 2.3643, + "theoretical_loss": 3.335389472875853, + "tokens_seen": 2834235392 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014365656565656565, + "loss": 2.555, + "theoretical_loss": 3.335383474436611, + "tokens_seen": 2834300928 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014363636363636366, + "loss": 2.5568, + "theoretical_loss": 3.3353774761749007, + "tokens_seen": 2834366464 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014361616161616162, + "loss": 2.6271, + "theoretical_loss": 3.3353714780907127, + "tokens_seen": 2834432000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014359595959595957, + "loss": 2.453, + "theoretical_loss": 3.335365480184038, + "tokens_seen": 2834497536 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014357575757575759, + "loss": 2.5937, + "theoretical_loss": 3.3353594824548662, + "tokens_seen": 2834563072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014355555555555554, + "loss": 2.4757, + "theoretical_loss": 3.335353484903189, + "tokens_seen": 2834628608 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1554194, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1688873767852783, + "objective/train/theoretical_loss": 3.335347487528997, + "objective/train/tokens_used": 1193553376, + "theoretical_loss": 3.335347487528997, + "tokens_seen": 2834694144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014353535353535355, + "loss": 2.4749, + "theoretical_loss": 3.335347487528997, + "tokens_seen": 2834694144 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001435151515151515, + "loss": 2.6339, + "theoretical_loss": 3.3353414903322802, + "tokens_seen": 2834759680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014349494949494952, + "loss": 2.6223, + "theoretical_loss": 3.3353354933130293, + "tokens_seen": 2834825216 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014347474747474748, + "loss": 2.5533, + "theoretical_loss": 3.3353294964712354, + "tokens_seen": 2834890752 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014345454545454546, + "loss": 2.4313, + "theoretical_loss": 3.335323499806889, + "tokens_seen": 2834956288 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014343434343434345, + "loss": 2.5763, + "theoretical_loss": 3.3353175033199807, + "tokens_seen": 2835021824 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001434141414141414, + "loss": 2.4139, + "theoretical_loss": 3.335311507010501, + "tokens_seen": 2835087360 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014339393939393941, + "loss": 2.6672, + "theoretical_loss": 3.335305510878441, + "tokens_seen": 2835152896 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014337373737373737, + "loss": 2.5906, + "theoretical_loss": 3.3352995149237907, + "tokens_seen": 2835218432 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014335353535353535, + "loss": 2.4722, + "theoretical_loss": 3.335293519146541, + "tokens_seen": 2835283968 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014333333333333334, + "loss": 2.5756, + "theoretical_loss": 3.335287523546683, + "tokens_seen": 2835349504 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014331313131313132, + "loss": 2.5238, + "theoretical_loss": 3.3352815281242068, + "tokens_seen": 2835415040 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001432929292929293, + "loss": 2.6506, + "theoretical_loss": 3.3352755328791033, + "tokens_seen": 2835480576 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014327272727272726, + "loss": 2.3047, + "theoretical_loss": 3.335269537811363, + "tokens_seen": 2835546112 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014325252525252525, + "loss": 2.5072, + "theoretical_loss": 3.335263542920977, + "tokens_seen": 2835611648 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014323232323232323, + "loss": 2.6078, + "theoretical_loss": 3.335257548207935, + "tokens_seen": 2835677184 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014321212121212122, + "loss": 2.3894, + "theoretical_loss": 3.3352515536722285, + "tokens_seen": 2835742720 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001431919191919192, + "loss": 2.6999, + "theoretical_loss": 3.335245559313848, + "tokens_seen": 2835808256 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014317171717171718, + "loss": 2.4213, + "theoretical_loss": 3.3352395651327837, + "tokens_seen": 2835873792 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014315151515151514, + "loss": 2.4762, + "theoretical_loss": 3.3352335711290273, + "tokens_seen": 2835939328 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014313131313131315, + "loss": 2.2899, + "theoretical_loss": 3.335227577302568, + "tokens_seen": 2836004864 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001431111111111111, + "loss": 2.5306, + "theoretical_loss": 3.3352215836533983, + "tokens_seen": 2836070400 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001430909090909091, + "loss": 2.4065, + "theoretical_loss": 3.335215590181507, + "tokens_seen": 2836135936 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014307070707070708, + "loss": 2.4866, + "theoretical_loss": 3.3352095968868856, + "tokens_seen": 2836201472 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014305050505050503, + "loss": 2.5253, + "theoretical_loss": 3.335203603769525, + "tokens_seen": 2836267008 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1555190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.309720277786255, + "objective/train/theoretical_loss": 3.3351976108294155, + "objective/train/tokens_used": 1195191776, + "theoretical_loss": 3.3351976108294155, + "tokens_seen": 2836332544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014303030303030304, + "loss": 2.3882, + "theoretical_loss": 3.3351976108294155, + "tokens_seen": 2836332544 + }, + { + "epoch": 0.72, + "learning_rate": 0.000143010101010101, + "loss": 2.4869, + "theoretical_loss": 3.3351916180665477, + "tokens_seen": 2836398080 + }, + { + "epoch": 0.72, + "learning_rate": 0.000142989898989899, + "loss": 2.458, + "theoretical_loss": 3.3351856254809125, + "tokens_seen": 2836463616 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014296969696969697, + "loss": 2.5357, + "theoretical_loss": 3.3351796330725003, + "tokens_seen": 2836529152 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014294949494949495, + "loss": 2.4477, + "theoretical_loss": 3.3351736408413024, + "tokens_seen": 2836594688 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014292929292929294, + "loss": 2.4855, + "theoretical_loss": 3.3351676487873085, + "tokens_seen": 2836660224 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001429090909090909, + "loss": 2.6121, + "theoretical_loss": 3.3351616569105103, + "tokens_seen": 2836725760 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001428888888888889, + "loss": 2.5937, + "theoretical_loss": 3.3351556652108973, + "tokens_seen": 2836791296 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014286868686868686, + "loss": 2.5649, + "theoretical_loss": 3.3351496736884614, + "tokens_seen": 2836856832 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014284848484848485, + "loss": 2.3154, + "theoretical_loss": 3.335143682343192, + "tokens_seen": 2836922368 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014282828282828283, + "loss": 2.6822, + "theoretical_loss": 3.335137691175081, + "tokens_seen": 2836987904 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001428080808080808, + "loss": 2.6328, + "theoretical_loss": 3.3351317001841183, + "tokens_seen": 2837053440 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001427878787878788, + "loss": 2.6216, + "theoretical_loss": 3.335125709370295, + "tokens_seen": 2837118976 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014276767676767678, + "loss": 2.7282, + "theoretical_loss": 3.3351197187336012, + "tokens_seen": 2837184512 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014274747474747474, + "loss": 2.4341, + "theoretical_loss": 3.3351137282740284, + "tokens_seen": 2837250048 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014272727272727272, + "loss": 2.444, + "theoretical_loss": 3.3351077379915663, + "tokens_seen": 2837315584 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001427070707070707, + "loss": 2.5763, + "theoretical_loss": 3.335101747886206, + "tokens_seen": 2837381120 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001426868686868687, + "loss": 2.2932, + "theoretical_loss": 3.335095757957939, + "tokens_seen": 2837446656 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014266666666666667, + "loss": 2.4584, + "theoretical_loss": 3.3350897682067546, + "tokens_seen": 2837512192 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014264646464646466, + "loss": 2.6073, + "theoretical_loss": 3.335083778632644, + "tokens_seen": 2837577728 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014262626262626264, + "loss": 2.3988, + "theoretical_loss": 3.3350777892355983, + "tokens_seen": 2837643264 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001426060606060606, + "loss": 2.3342, + "theoretical_loss": 3.3350718000156077, + "tokens_seen": 2837708800 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001425858585858586, + "loss": 2.5598, + "theoretical_loss": 3.3350658109726634, + "tokens_seen": 2837774336 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014256565656565657, + "loss": 2.5447, + "theoretical_loss": 3.335059822106755, + "tokens_seen": 2837839872 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014254545454545455, + "loss": 2.6259, + "theoretical_loss": 3.3350538334178745, + "tokens_seen": 2837905408 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1555824, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.966188669204712, + "objective/train/theoretical_loss": 3.3350478449060112, + "objective/train/tokens_used": 1196830176, + "theoretical_loss": 3.3350478449060112, + "tokens_seen": 2837970944 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014252525252525253, + "loss": 2.6614, + "theoretical_loss": 3.3350478449060112, + "tokens_seen": 2837970944 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001425050505050505, + "loss": 2.4787, + "theoretical_loss": 3.3350418565711575, + "tokens_seen": 2838036480 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001424848484848485, + "loss": 2.7117, + "theoretical_loss": 3.3350358684133026, + "tokens_seen": 2838102016 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014246464646464646, + "loss": 2.3986, + "theoretical_loss": 3.3350298804324376, + "tokens_seen": 2838167552 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014244444444444447, + "loss": 2.4161, + "theoretical_loss": 3.3350238926285534, + "tokens_seen": 2838233088 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014242424242424243, + "loss": 2.4758, + "theoretical_loss": 3.3350179050016404, + "tokens_seen": 2838298624 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014240404040404038, + "loss": 2.4409, + "theoretical_loss": 3.3350119175516895, + "tokens_seen": 2838364160 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001423838383838384, + "loss": 2.3875, + "theoretical_loss": 3.3350059302786916, + "tokens_seen": 2838429696 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014236363636363635, + "loss": 2.5824, + "theoretical_loss": 3.3349999431826367, + "tokens_seen": 2838495232 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014234343434343436, + "loss": 2.4269, + "theoretical_loss": 3.3349939562635162, + "tokens_seen": 2838560768 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014232323232323232, + "loss": 2.4625, + "theoretical_loss": 3.3349879695213205, + "tokens_seen": 2838626304 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001423030303030303, + "loss": 2.5004, + "theoretical_loss": 3.33498198295604, + "tokens_seen": 2838691840 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001422828282828283, + "loss": 2.5947, + "theoretical_loss": 3.3349759965676657, + "tokens_seen": 2838757376 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014226262626262627, + "loss": 2.6393, + "theoretical_loss": 3.3349700103561886, + "tokens_seen": 2838822912 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014224242424242426, + "loss": 2.7206, + "theoretical_loss": 3.334964024321599, + "tokens_seen": 2838888448 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001422222222222222, + "loss": 2.4285, + "theoretical_loss": 3.3349580384638875, + "tokens_seen": 2838953984 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001422020202020202, + "loss": 2.7003, + "theoretical_loss": 3.334952052783045, + "tokens_seen": 2839019520 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014218181818181818, + "loss": 2.3876, + "theoretical_loss": 3.334946067279062, + "tokens_seen": 2839085056 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014216161616161616, + "loss": 2.513, + "theoretical_loss": 3.3349400819519293, + "tokens_seen": 2839150592 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014214141414141415, + "loss": 2.4524, + "theoretical_loss": 3.3349340968016374, + "tokens_seen": 2839216128 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014212121212121213, + "loss": 2.4981, + "theoretical_loss": 3.3349281118281775, + "tokens_seen": 2839281664 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001421010101010101, + "loss": 2.5551, + "theoretical_loss": 3.3349221270315397, + "tokens_seen": 2839347200 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001420808080808081, + "loss": 2.4768, + "theoretical_loss": 3.334916142411715, + "tokens_seen": 2839412736 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014206060606060606, + "loss": 2.3724, + "theoretical_loss": 3.3349101579686944, + "tokens_seen": 2839478272 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014204040404040404, + "loss": 2.5245, + "theoretical_loss": 3.3349041737024683, + "tokens_seen": 2839543808 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1556752, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3337228298187256, + "objective/train/theoretical_loss": 3.334898189613027, + "objective/train/tokens_used": 1198468576, + "theoretical_loss": 3.334898189613027, + "tokens_seen": 2839609344 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014202020202020203, + "loss": 2.4768, + "theoretical_loss": 3.334898189613027, + "tokens_seen": 2839609344 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014199999999999998, + "loss": 2.5858, + "theoretical_loss": 3.3348922057003616, + "tokens_seen": 2839674880 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141979797979798, + "loss": 2.5207, + "theoretical_loss": 3.334886221964463, + "tokens_seen": 2839740416 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014195959595959595, + "loss": 2.6307, + "theoretical_loss": 3.3348802384053213, + "tokens_seen": 2839805952 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014193939393939396, + "loss": 2.5675, + "theoretical_loss": 3.334874255022928, + "tokens_seen": 2839871488 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014191919191919192, + "loss": 2.6299, + "theoretical_loss": 3.334868271817273, + "tokens_seen": 2839937024 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001418989898989899, + "loss": 2.615, + "theoretical_loss": 3.3348622887883472, + "tokens_seen": 2840002560 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014187878787878789, + "loss": 2.5269, + "theoretical_loss": 3.334856305936142, + "tokens_seen": 2840068096 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014185858585858584, + "loss": 2.4811, + "theoretical_loss": 3.3348503232606475, + "tokens_seen": 2840133632 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014183838383838385, + "loss": 2.4898, + "theoretical_loss": 3.334844340761854, + "tokens_seen": 2840199168 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001418181818181818, + "loss": 2.3763, + "theoretical_loss": 3.334838358439753, + "tokens_seen": 2840264704 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014179797979797982, + "loss": 2.6123, + "theoretical_loss": 3.334832376294335, + "tokens_seen": 2840330240 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014177777777777778, + "loss": 2.5652, + "theoretical_loss": 3.3348263943255905, + "tokens_seen": 2840395776 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014175757575757576, + "loss": 2.7354, + "theoretical_loss": 3.33482041253351, + "tokens_seen": 2840461312 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014173737373737375, + "loss": 2.3827, + "theoretical_loss": 3.334814430918085, + "tokens_seen": 2840526848 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001417171717171717, + "loss": 2.6727, + "theoretical_loss": 3.334808449479305, + "tokens_seen": 2840592384 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014169696969696971, + "loss": 2.4044, + "theoretical_loss": 3.334802468217162, + "tokens_seen": 2840657920 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014167676767676767, + "loss": 2.4341, + "theoretical_loss": 3.334796487131646, + "tokens_seen": 2840723456 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014165656565656566, + "loss": 2.4694, + "theoretical_loss": 3.334790506222748, + "tokens_seen": 2840788992 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014163636363636364, + "loss": 2.5832, + "theoretical_loss": 3.3347845254904582, + "tokens_seen": 2840854528 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014161616161616162, + "loss": 2.6315, + "theoretical_loss": 3.3347785449347676, + "tokens_seen": 2840920064 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001415959595959596, + "loss": 2.5023, + "theoretical_loss": 3.334772564555667, + "tokens_seen": 2840985600 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001415757575757576, + "loss": 2.4461, + "theoretical_loss": 3.3347665843531473, + "tokens_seen": 2841051136 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014155555555555555, + "loss": 2.5689, + "theoretical_loss": 3.334760604327199, + "tokens_seen": 2841116672 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014153535353535353, + "loss": 2.5216, + "theoretical_loss": 3.334754624477813, + "tokens_seen": 2841182208 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1557330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3897173404693604, + "objective/train/theoretical_loss": 3.3347486448049795, + "objective/train/tokens_used": 1200106976, + "theoretical_loss": 3.3347486448049795, + "tokens_seen": 2841247744 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014151515151515152, + "loss": 2.5539, + "theoretical_loss": 3.3347486448049795, + "tokens_seen": 2841247744 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001414949494949495, + "loss": 2.4999, + "theoretical_loss": 3.3347426653086893, + "tokens_seen": 2841313280 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014147474747474748, + "loss": 2.6079, + "theoretical_loss": 3.3347366859889336, + "tokens_seen": 2841378816 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014145454545454544, + "loss": 2.5075, + "theoretical_loss": 3.334730706845703, + "tokens_seen": 2841444352 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014143434343434345, + "loss": 2.678, + "theoretical_loss": 3.3347247278789878, + "tokens_seen": 2841509888 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001414141414141414, + "loss": 2.584, + "theoretical_loss": 3.334718749088779, + "tokens_seen": 2841575424 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014139393939393942, + "loss": 2.5166, + "theoretical_loss": 3.3347127704750674, + "tokens_seen": 2841640960 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014137373737373738, + "loss": 2.431, + "theoretical_loss": 3.334706792037844, + "tokens_seen": 2841706496 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014135353535353533, + "loss": 2.4946, + "theoretical_loss": 3.334700813777099, + "tokens_seen": 2841772032 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014133333333333334, + "loss": 2.5424, + "theoretical_loss": 3.3346948356928228, + "tokens_seen": 2841837568 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001413131313131313, + "loss": 2.5664, + "theoretical_loss": 3.334688857785007, + "tokens_seen": 2841903104 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001412929292929293, + "loss": 2.5174, + "theoretical_loss": 3.334682880053642, + "tokens_seen": 2841968640 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014127272727272727, + "loss": 2.5579, + "theoretical_loss": 3.334676902498718, + "tokens_seen": 2842034176 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014125252525252525, + "loss": 2.3266, + "theoretical_loss": 3.3346709251202267, + "tokens_seen": 2842099712 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014123232323232324, + "loss": 2.462, + "theoretical_loss": 3.334664947918158, + "tokens_seen": 2842165248 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014121212121212122, + "loss": 2.6227, + "theoretical_loss": 3.334658970892503, + "tokens_seen": 2842230784 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001411919191919192, + "loss": 2.2097, + "theoretical_loss": 3.3346529940432523, + "tokens_seen": 2842296320 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014117171717171716, + "loss": 2.4468, + "theoretical_loss": 3.3346470173703966, + "tokens_seen": 2842361856 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014115151515151515, + "loss": 2.2659, + "theoretical_loss": 3.334641040873927, + "tokens_seen": 2842427392 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014113131313131313, + "loss": 2.5181, + "theoretical_loss": 3.3346350645538334, + "tokens_seen": 2842492928 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014111111111111111, + "loss": 2.332, + "theoretical_loss": 3.3346290884101073, + "tokens_seen": 2842558464 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001410909090909091, + "loss": 2.4146, + "theoretical_loss": 3.3346231124427392, + "tokens_seen": 2842624000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014107070707070708, + "loss": 2.6289, + "theoretical_loss": 3.33461713665172, + "tokens_seen": 2842689536 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014105050505050504, + "loss": 2.5433, + "theoretical_loss": 3.33461116103704, + "tokens_seen": 2842755072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014103030303030305, + "loss": 2.5434, + "theoretical_loss": 3.33460518559869, + "tokens_seen": 2842820608 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1558696, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.670680046081543, + "objective/train/theoretical_loss": 3.3345992103366613, + "objective/train/tokens_used": 1201745376, + "theoretical_loss": 3.3345992103366613, + "tokens_seen": 2842886144 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141010101010101, + "loss": 2.6843, + "theoretical_loss": 3.3345992103366613, + "tokens_seen": 2842886144 + }, + { + "epoch": 0.72, + "learning_rate": 0.000140989898989899, + "loss": 2.4631, + "theoretical_loss": 3.334593235250944, + "tokens_seen": 2842951680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014096969696969697, + "loss": 2.542, + "theoretical_loss": 3.334587260341529, + "tokens_seen": 2843017216 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014094949494949493, + "loss": 2.3007, + "theoretical_loss": 3.3345812856084076, + "tokens_seen": 2843082752 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014092929292929294, + "loss": 2.4853, + "theoretical_loss": 3.3345753110515695, + "tokens_seen": 2843148288 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001409090909090909, + "loss": 2.4291, + "theoretical_loss": 3.3345693366710063, + "tokens_seen": 2843213824 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001408888888888889, + "loss": 2.5848, + "theoretical_loss": 3.3345633624667084, + "tokens_seen": 2843279360 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014086868686868687, + "loss": 2.6197, + "theoretical_loss": 3.3345573884386663, + "tokens_seen": 2843344896 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014084848484848488, + "loss": 2.4311, + "theoretical_loss": 3.3345514145868713, + "tokens_seen": 2843410432 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014082828282828284, + "loss": 2.8766, + "theoretical_loss": 3.3345454409113136, + "tokens_seen": 2843475968 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001408080808080808, + "loss": 2.6169, + "theoretical_loss": 3.334539467411984, + "tokens_seen": 2843541504 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001407878787878788, + "loss": 2.5019, + "theoretical_loss": 3.334533494088874, + "tokens_seen": 2843607040 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014076767676767676, + "loss": 2.6675, + "theoretical_loss": 3.3345275209419736, + "tokens_seen": 2843672576 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014074747474747477, + "loss": 2.3048, + "theoretical_loss": 3.3345215479712733, + "tokens_seen": 2843738112 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014072727272727273, + "loss": 2.4104, + "theoretical_loss": 3.334515575176764, + "tokens_seen": 2843803648 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001407070707070707, + "loss": 2.7196, + "theoretical_loss": 3.334509602558437, + "tokens_seen": 2843869184 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001406868686868687, + "loss": 2.3145, + "theoretical_loss": 3.334503630116283, + "tokens_seen": 2843934720 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014066666666666665, + "loss": 2.6657, + "theoretical_loss": 3.3344976578502923, + "tokens_seen": 2844000256 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014064646464646466, + "loss": 2.5249, + "theoretical_loss": 3.334491685760456, + "tokens_seen": 2844065792 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014062626262626262, + "loss": 2.6714, + "theoretical_loss": 3.334485713846764, + "tokens_seen": 2844131328 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001406060606060606, + "loss": 2.4354, + "theoretical_loss": 3.3344797421092083, + "tokens_seen": 2844196864 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001405858585858586, + "loss": 2.5217, + "theoretical_loss": 3.3344737705477785, + "tokens_seen": 2844262400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014056565656565657, + "loss": 2.5779, + "theoretical_loss": 3.3344677991624665, + "tokens_seen": 2844327936 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014054545454545456, + "loss": 2.7826, + "theoretical_loss": 3.334461827953262, + "tokens_seen": 2844393472 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014052525252525254, + "loss": 2.5736, + "theoretical_loss": 3.3344558569201563, + "tokens_seen": 2844459008 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1559218, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3371129035949707, + "objective/train/theoretical_loss": 3.33444988606314, + "objective/train/tokens_used": 1203383776, + "theoretical_loss": 3.33444988606314, + "tokens_seen": 2844524544 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001405050505050505, + "loss": 2.6232, + "theoretical_loss": 3.33444988606314, + "tokens_seen": 2844524544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014048484848484848, + "loss": 2.6202, + "theoretical_loss": 3.334443915382204, + "tokens_seen": 2844590080 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014046464646464647, + "loss": 2.5525, + "theoretical_loss": 3.334437944877339, + "tokens_seen": 2844655616 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014044444444444445, + "loss": 2.5496, + "theoretical_loss": 3.3344319745485356, + "tokens_seen": 2844721152 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014042424242424243, + "loss": 2.5567, + "theoretical_loss": 3.3344260043957843, + "tokens_seen": 2844786688 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001404040404040404, + "loss": 2.6344, + "theoretical_loss": 3.3344200344190766, + "tokens_seen": 2844852224 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001403838383838384, + "loss": 2.4549, + "theoretical_loss": 3.334414064618403, + "tokens_seen": 2844917760 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014036363636363636, + "loss": 2.5288, + "theoretical_loss": 3.3344080949937536, + "tokens_seen": 2844983296 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014034343434343437, + "loss": 2.4969, + "theoretical_loss": 3.33440212554512, + "tokens_seen": 2845048832 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014032323232323233, + "loss": 2.5372, + "theoretical_loss": 3.3343961562724926, + "tokens_seen": 2845114368 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014030303030303028, + "loss": 2.4704, + "theoretical_loss": 3.334390187175862, + "tokens_seen": 2845179904 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001402828282828283, + "loss": 2.5812, + "theoretical_loss": 3.3343842182552192, + "tokens_seen": 2845245440 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014026262626262625, + "loss": 2.4435, + "theoretical_loss": 3.3343782495105545, + "tokens_seen": 2845310976 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014024242424242426, + "loss": 2.5505, + "theoretical_loss": 3.3343722809418597, + "tokens_seen": 2845376512 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014022222222222222, + "loss": 2.51, + "theoretical_loss": 3.3343663125491245, + "tokens_seen": 2845442048 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001402020202020202, + "loss": 2.4887, + "theoretical_loss": 3.33436034433234, + "tokens_seen": 2845507584 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014018181818181819, + "loss": 2.2729, + "theoretical_loss": 3.3343543762914973, + "tokens_seen": 2845573120 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014016161616161614, + "loss": 2.4593, + "theoretical_loss": 3.3343484084265866, + "tokens_seen": 2845638656 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014014141414141415, + "loss": 2.3958, + "theoretical_loss": 3.3343424407375992, + "tokens_seen": 2845704192 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001401212121212121, + "loss": 2.672, + "theoretical_loss": 3.3343364732245258, + "tokens_seen": 2845769728 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001401010101010101, + "loss": 2.3868, + "theoretical_loss": 3.3343305058873565, + "tokens_seen": 2845835264 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014008080808080808, + "loss": 2.5368, + "theoretical_loss": 3.334324538726083, + "tokens_seen": 2845900800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014006060606060606, + "loss": 2.5591, + "theoretical_loss": 3.3343185717406953, + "tokens_seen": 2845966336 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014004040404040405, + "loss": 2.363, + "theoretical_loss": 3.3343126049311844, + "tokens_seen": 2846031872 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014002020202020203, + "loss": 2.5754, + "theoretical_loss": 3.3343066382975413, + "tokens_seen": 2846097408 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1560564, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.548229217529297, + "objective/train/theoretical_loss": 3.334300671839756, + "objective/train/tokens_used": 1205022176, + "theoretical_loss": 3.334300671839756, + "tokens_seen": 2846162944 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014000000000000001, + "loss": 2.5829, + "theoretical_loss": 3.334300671839756, + "tokens_seen": 2846162944 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013997979797979797, + "loss": 2.7019, + "theoretical_loss": 3.334294705557821, + "tokens_seen": 2846228480 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013995959595959596, + "loss": 2.4604, + "theoretical_loss": 3.334288739451725, + "tokens_seen": 2846294016 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013993939393939394, + "loss": 2.5809, + "theoretical_loss": 3.33428277352146, + "tokens_seen": 2846359552 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013991919191919192, + "loss": 2.5123, + "theoretical_loss": 3.334276807767017, + "tokens_seen": 2846425088 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001398989898989899, + "loss": 2.5439, + "theoretical_loss": 3.3342708421883853, + "tokens_seen": 2846490624 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001398787878787879, + "loss": 2.6115, + "theoretical_loss": 3.334264876785557, + "tokens_seen": 2846556160 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013985858585858585, + "loss": 2.3239, + "theoretical_loss": 3.3342589115585226, + "tokens_seen": 2846621696 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013983838383838386, + "loss": 2.5194, + "theoretical_loss": 3.3342529465072728, + "tokens_seen": 2846687232 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013981818181818182, + "loss": 2.5582, + "theoretical_loss": 3.3342469816317983, + "tokens_seen": 2846752768 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001397979797979798, + "loss": 2.5173, + "theoretical_loss": 3.3342410169320895, + "tokens_seen": 2846818304 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013977777777777778, + "loss": 2.6122, + "theoretical_loss": 3.334235052408138, + "tokens_seen": 2846883840 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013975757575757574, + "loss": 2.5622, + "theoretical_loss": 3.334229088059934, + "tokens_seen": 2846949376 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013973737373737375, + "loss": 2.5143, + "theoretical_loss": 3.334223123887469, + "tokens_seen": 2847014912 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001397171717171717, + "loss": 2.3804, + "theoretical_loss": 3.3342171598907324, + "tokens_seen": 2847080448 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013969696969696972, + "loss": 2.6695, + "theoretical_loss": 3.334211196069716, + "tokens_seen": 2847145984 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013967676767676768, + "loss": 2.2176, + "theoretical_loss": 3.3342052324244107, + "tokens_seen": 2847211520 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013965656565656566, + "loss": 2.4515, + "theoretical_loss": 3.3341992689548063, + "tokens_seen": 2847277056 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013963636363636364, + "loss": 2.4974, + "theoretical_loss": 3.334193305660895, + "tokens_seen": 2847342592 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001396161616161616, + "loss": 2.5767, + "theoretical_loss": 3.3341873425426662, + "tokens_seen": 2847408128 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001395959595959596, + "loss": 2.5739, + "theoretical_loss": 3.3341813796001114, + "tokens_seen": 2847473664 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013957575757575757, + "loss": 2.3423, + "theoretical_loss": 3.3341754168332214, + "tokens_seen": 2847539200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013955555555555555, + "loss": 2.3189, + "theoretical_loss": 3.3341694542419873, + "tokens_seen": 2847604736 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013953535353535354, + "loss": 2.4985, + "theoretical_loss": 3.3341634918263985, + "tokens_seen": 2847670272 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013951515151515152, + "loss": 2.6473, + "theoretical_loss": 3.3341575295864474, + "tokens_seen": 2847735808 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1561701, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.759636163711548, + "objective/train/theoretical_loss": 3.334151567522124, + "objective/train/tokens_used": 1206660576, + "theoretical_loss": 3.334151567522124, + "tokens_seen": 2847801344 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001394949494949495, + "loss": 2.5707, + "theoretical_loss": 3.334151567522124, + "tokens_seen": 2847801344 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001394747474747475, + "loss": 2.5228, + "theoretical_loss": 3.334145605633419, + "tokens_seen": 2847866880 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013945454545454545, + "loss": 2.2764, + "theoretical_loss": 3.3341396439203232, + "tokens_seen": 2847932416 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013943434343434343, + "loss": 2.6174, + "theoretical_loss": 3.334133682382828, + "tokens_seen": 2847997952 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013941414141414141, + "loss": 2.5689, + "theoretical_loss": 3.3341277210209235, + "tokens_seen": 2848063488 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001393939393939394, + "loss": 2.3053, + "theoretical_loss": 3.3341217598346007, + "tokens_seen": 2848129024 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013937373737373738, + "loss": 2.4259, + "theoretical_loss": 3.3341157988238503, + "tokens_seen": 2848194560 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013935353535353534, + "loss": 2.5446, + "theoretical_loss": 3.3341098379886636, + "tokens_seen": 2848260096 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013933333333333335, + "loss": 2.4465, + "theoretical_loss": 3.3341038773290306, + "tokens_seen": 2848325632 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001393131313131313, + "loss": 2.6108, + "theoretical_loss": 3.334097916844943, + "tokens_seen": 2848391168 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013929292929292932, + "loss": 2.572, + "theoretical_loss": 3.3340919565363905, + "tokens_seen": 2848456704 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013927272727272727, + "loss": 2.5145, + "theoretical_loss": 3.3340859964033647, + "tokens_seen": 2848522240 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013925252525252523, + "loss": 2.7747, + "theoretical_loss": 3.3340800364458563, + "tokens_seen": 2848587776 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013923232323232324, + "loss": 2.3782, + "theoretical_loss": 3.334074076663856, + "tokens_seen": 2848653312 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001392121212121212, + "loss": 2.6013, + "theoretical_loss": 3.3340681170573543, + "tokens_seen": 2848718848 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001391919191919192, + "loss": 2.2213, + "theoretical_loss": 3.3340621576263425, + "tokens_seen": 2848784384 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013917171717171717, + "loss": 2.5462, + "theoretical_loss": 3.334056198370811, + "tokens_seen": 2848849920 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013915151515151518, + "loss": 2.4305, + "theoretical_loss": 3.3340502392907503, + "tokens_seen": 2848915456 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013913131313131314, + "loss": 2.4422, + "theoretical_loss": 3.334044280386152, + "tokens_seen": 2848980992 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001391111111111111, + "loss": 2.4569, + "theoretical_loss": 3.334038321657007, + "tokens_seen": 2849046528 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001390909090909091, + "loss": 2.4297, + "theoretical_loss": 3.3340323631033053, + "tokens_seen": 2849112064 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013907070707070706, + "loss": 2.3842, + "theoretical_loss": 3.334026404725038, + "tokens_seen": 2849177600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013905050505050507, + "loss": 2.4761, + "theoretical_loss": 3.334020446522196, + "tokens_seen": 2849243136 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013903030303030303, + "loss": 2.5751, + "theoretical_loss": 3.33401448849477, + "tokens_seen": 2849308672 + }, + { + "epoch": 0.72, + "learning_rate": 0.000139010101010101, + "loss": 2.3838, + "theoretical_loss": 3.3340085306427505, + "tokens_seen": 2849374208 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1562417, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.22106671333313, + "objective/train/theoretical_loss": 3.3340025729661287, + "objective/train/tokens_used": 1208298976, + "theoretical_loss": 3.3340025729661287, + "tokens_seen": 2849439744 + }, + { + "epoch": 0.72, + "learning_rate": 0.000138989898989899, + "loss": 2.4476, + "theoretical_loss": 3.3340025729661287, + "tokens_seen": 2849439744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013896969696969698, + "loss": 2.338, + "theoretical_loss": 3.333996615464896, + "tokens_seen": 2849505280 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013894949494949496, + "loss": 2.3535, + "theoretical_loss": 3.333990658139042, + "tokens_seen": 2849570816 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013892929292929292, + "loss": 2.4375, + "theoretical_loss": 3.333984700988558, + "tokens_seen": 2849636352 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001389090909090909, + "loss": 2.328, + "theoretical_loss": 3.333978744013435, + "tokens_seen": 2849701888 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001388888888888889, + "loss": 2.7163, + "theoretical_loss": 3.3339727872136633, + "tokens_seen": 2849767424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013886868686868687, + "loss": 2.5646, + "theoretical_loss": 3.3339668305892345, + "tokens_seen": 2849832960 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013884848484848486, + "loss": 2.5299, + "theoretical_loss": 3.3339608741401388, + "tokens_seen": 2849898496 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013882828282828284, + "loss": 2.3088, + "theoretical_loss": 3.333954917866367, + "tokens_seen": 2849964032 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001388080808080808, + "loss": 2.4623, + "theoretical_loss": 3.3339489617679106, + "tokens_seen": 2850029568 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001387878787878788, + "loss": 2.455, + "theoretical_loss": 3.3339430058447594, + "tokens_seen": 2850095104 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013876767676767677, + "loss": 2.3624, + "theoretical_loss": 3.333937050096905, + "tokens_seen": 2850160640 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013874747474747475, + "loss": 2.4189, + "theoretical_loss": 3.333931094524338, + "tokens_seen": 2850226176 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013872727272727273, + "loss": 2.6246, + "theoretical_loss": 3.333925139127049, + "tokens_seen": 2850291712 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001387070707070707, + "loss": 2.6423, + "theoretical_loss": 3.333919183905029, + "tokens_seen": 2850357248 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001386868686868687, + "loss": 2.6026, + "theoretical_loss": 3.3339132288582682, + "tokens_seen": 2850422784 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013866666666666666, + "loss": 2.4815, + "theoretical_loss": 3.3339072739867586, + "tokens_seen": 2850488320 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013864646464646467, + "loss": 2.6394, + "theoretical_loss": 3.33390131929049, + "tokens_seen": 2850553856 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013862626262626263, + "loss": 2.8337, + "theoretical_loss": 3.333895364769454, + "tokens_seen": 2850619392 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001386060606060606, + "loss": 2.5351, + "theoretical_loss": 3.33388941042364, + "tokens_seen": 2850684928 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001385858585858586, + "loss": 2.5256, + "theoretical_loss": 3.333883456253041, + "tokens_seen": 2850750464 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013856565656565655, + "loss": 2.5466, + "theoretical_loss": 3.333877502257646, + "tokens_seen": 2850816000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013854545454545456, + "loss": 2.5292, + "theoretical_loss": 3.333871548437447, + "tokens_seen": 2850881536 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013852525252525252, + "loss": 2.5348, + "theoretical_loss": 3.3338655947924334, + "tokens_seen": 2850947072 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001385050505050505, + "loss": 2.4299, + "theoretical_loss": 3.333859641322597, + "tokens_seen": 2851012608 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1563803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0461678504943848, + "objective/train/theoretical_loss": 3.3338536880279293, + "objective/train/tokens_used": 1209937376, + "theoretical_loss": 3.3338536880279293, + "tokens_seen": 2851078144 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001384848484848485, + "loss": 2.3824, + "theoretical_loss": 3.3338536880279293, + "tokens_seen": 2851078144 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013846464646464647, + "loss": 2.4229, + "theoretical_loss": 3.33384773490842, + "tokens_seen": 2851143680 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013844444444444445, + "loss": 2.281, + "theoretical_loss": 3.33384178196406, + "tokens_seen": 2851209216 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001384242424242424, + "loss": 2.5803, + "theoretical_loss": 3.3338358291948404, + "tokens_seen": 2851274752 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001384040404040404, + "loss": 2.4804, + "theoretical_loss": 3.3338298766007517, + "tokens_seen": 2851340288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013838383838383838, + "loss": 2.3015, + "theoretical_loss": 3.3338239241817855, + "tokens_seen": 2851405824 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013836363636363636, + "loss": 2.5867, + "theoretical_loss": 3.333817971937932, + "tokens_seen": 2851471360 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013834343434343435, + "loss": 2.3013, + "theoretical_loss": 3.3338120198691823, + "tokens_seen": 2851536896 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013832323232323233, + "loss": 2.6673, + "theoretical_loss": 3.333806067975527, + "tokens_seen": 2851602432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013830303030303032, + "loss": 2.5615, + "theoretical_loss": 3.333800116256957, + "tokens_seen": 2851667968 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001382828282828283, + "loss": 2.5732, + "theoretical_loss": 3.333794164713463, + "tokens_seen": 2851733504 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013826262626262626, + "loss": 2.5204, + "theoretical_loss": 3.333788213345036, + "tokens_seen": 2851799040 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013824242424242424, + "loss": 2.5109, + "theoretical_loss": 3.333782262151667, + "tokens_seen": 2851864576 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013822222222222222, + "loss": 2.5164, + "theoretical_loss": 3.333776311133346, + "tokens_seen": 2851930112 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001382020202020202, + "loss": 2.5812, + "theoretical_loss": 3.333770360290065, + "tokens_seen": 2851995648 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001381818181818182, + "loss": 2.3813, + "theoretical_loss": 3.333764409621814, + "tokens_seen": 2852061184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013816161616161615, + "loss": 2.409, + "theoretical_loss": 3.3337584591285845, + "tokens_seen": 2852126720 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013814141414141416, + "loss": 2.7831, + "theoretical_loss": 3.3337525088103668, + "tokens_seen": 2852192256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013812121212121212, + "loss": 2.5639, + "theoretical_loss": 3.3337465586671517, + "tokens_seen": 2852257792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013810101010101013, + "loss": 2.5358, + "theoretical_loss": 3.33374060869893, + "tokens_seen": 2852323328 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013808080808080808, + "loss": 2.3671, + "theoretical_loss": 3.3337346589056933, + "tokens_seen": 2852388864 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013806060606060604, + "loss": 2.4856, + "theoretical_loss": 3.3337287092874313, + "tokens_seen": 2852454400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013804040404040405, + "loss": 2.34, + "theoretical_loss": 3.333722759844136, + "tokens_seen": 2852519936 + }, + { + "epoch": 0.73, + "learning_rate": 0.000138020202020202, + "loss": 2.4847, + "theoretical_loss": 3.333716810575797, + "tokens_seen": 2852585472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013800000000000002, + "loss": 2.3329, + "theoretical_loss": 3.3337108614824063, + "tokens_seen": 2852651008 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1564513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1430630683898926, + "objective/train/theoretical_loss": 3.3337049125639537, + "objective/train/tokens_used": 1211575776, + "theoretical_loss": 3.3337049125639537, + "tokens_seen": 2852716544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013797979797979798, + "loss": 2.3523, + "theoretical_loss": 3.3337049125639537, + "tokens_seen": 2852716544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013795959595959596, + "loss": 2.6716, + "theoretical_loss": 3.3336989638204306, + "tokens_seen": 2852782080 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013793939393939395, + "loss": 2.4775, + "theoretical_loss": 3.333693015251828, + "tokens_seen": 2852847616 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013791919191919193, + "loss": 2.615, + "theoretical_loss": 3.3336870668581366, + "tokens_seen": 2852913152 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001378989898989899, + "loss": 2.4869, + "theoretical_loss": 3.333681118639347, + "tokens_seen": 2852978688 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013787878787878787, + "loss": 2.5369, + "theoretical_loss": 3.33367517059545, + "tokens_seen": 2853044224 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013785858585858585, + "loss": 2.5702, + "theoretical_loss": 3.3336692227264373, + "tokens_seen": 2853109760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013783838383838384, + "loss": 2.4221, + "theoretical_loss": 3.3336632750322983, + "tokens_seen": 2853175296 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013781818181818182, + "loss": 2.3637, + "theoretical_loss": 3.3336573275130252, + "tokens_seen": 2853240832 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001377979797979798, + "loss": 2.5007, + "theoretical_loss": 3.333651380168608, + "tokens_seen": 2853306368 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001377777777777778, + "loss": 2.3365, + "theoretical_loss": 3.3336454329990377, + "tokens_seen": 2853371904 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013775757575757575, + "loss": 2.6668, + "theoretical_loss": 3.3336394860043055, + "tokens_seen": 2853437440 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013773737373737376, + "loss": 2.5958, + "theoretical_loss": 3.333633539184402, + "tokens_seen": 2853502976 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013771717171717171, + "loss": 2.6205, + "theoretical_loss": 3.333627592539318, + "tokens_seen": 2853568512 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001376969696969697, + "loss": 2.4288, + "theoretical_loss": 3.333621646069044, + "tokens_seen": 2853634048 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013767676767676768, + "loss": 2.6513, + "theoretical_loss": 3.3336156997735715, + "tokens_seen": 2853699584 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013765656565656564, + "loss": 2.3775, + "theoretical_loss": 3.3336097536528912, + "tokens_seen": 2853765120 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013763636363636365, + "loss": 2.4498, + "theoretical_loss": 3.333603807706994, + "tokens_seen": 2853830656 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001376161616161616, + "loss": 2.4516, + "theoretical_loss": 3.3335978619358704, + "tokens_seen": 2853896192 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013759595959595962, + "loss": 2.3586, + "theoretical_loss": 3.333591916339511, + "tokens_seen": 2853961728 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013757575757575758, + "loss": 2.4369, + "theoretical_loss": 3.3335859709179076, + "tokens_seen": 2854027264 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013755555555555553, + "loss": 2.6579, + "theoretical_loss": 3.33358002567105, + "tokens_seen": 2854092800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013753535353535354, + "loss": 2.6855, + "theoretical_loss": 3.3335740805989302, + "tokens_seen": 2854158336 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001375151515151515, + "loss": 2.5251, + "theoretical_loss": 3.333568135701538, + "tokens_seen": 2854223872 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001374949494949495, + "loss": 2.5206, + "theoretical_loss": 3.3335621909788653, + "tokens_seen": 2854289408 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1565714, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.424259662628174, + "objective/train/theoretical_loss": 3.3335562464309016, + "objective/train/tokens_used": 1213214176, + "theoretical_loss": 3.3335562464309016, + "tokens_seen": 2854354944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013747474747474747, + "loss": 2.4821, + "theoretical_loss": 3.3335562464309016, + "tokens_seen": 2854354944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013745454545454548, + "loss": 2.5864, + "theoretical_loss": 3.3335503020576387, + "tokens_seen": 2854420480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013743434343434344, + "loss": 2.325, + "theoretical_loss": 3.3335443578590676, + "tokens_seen": 2854486016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013741414141414142, + "loss": 2.3606, + "theoretical_loss": 3.3335384138351785, + "tokens_seen": 2854551552 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001373939393939394, + "loss": 2.4032, + "theoretical_loss": 3.3335324699859625, + "tokens_seen": 2854617088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013737373737373736, + "loss": 2.5654, + "theoretical_loss": 3.3335265263114104, + "tokens_seen": 2854682624 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013735353535353537, + "loss": 2.5511, + "theoretical_loss": 3.3335205828115138, + "tokens_seen": 2854748160 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013733333333333333, + "loss": 2.5371, + "theoretical_loss": 3.3335146394862623, + "tokens_seen": 2854813696 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001373131313131313, + "loss": 2.4084, + "theoretical_loss": 3.3335086963356475, + "tokens_seen": 2854879232 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001372929292929293, + "loss": 2.5337, + "theoretical_loss": 3.33350275335966, + "tokens_seen": 2854944768 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013727272727272728, + "loss": 2.5702, + "theoretical_loss": 3.3334968105582914, + "tokens_seen": 2855010304 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013725252525252526, + "loss": 2.7723, + "theoretical_loss": 3.333490867931532, + "tokens_seen": 2855075840 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013723232323232325, + "loss": 2.5147, + "theoretical_loss": 3.3334849254793717, + "tokens_seen": 2855141376 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001372121212121212, + "loss": 2.5039, + "theoretical_loss": 3.333478983201803, + "tokens_seen": 2855206912 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001371919191919192, + "loss": 2.3213, + "theoretical_loss": 3.333473041098816, + "tokens_seen": 2855272448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013717171717171717, + "loss": 2.4009, + "theoretical_loss": 3.3334670991704014, + "tokens_seen": 2855337984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013715151515151516, + "loss": 2.5065, + "theoretical_loss": 3.3334611574165507, + "tokens_seen": 2855403520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013713131313131314, + "loss": 2.6119, + "theoretical_loss": 3.333455215837254, + "tokens_seen": 2855469056 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001371111111111111, + "loss": 2.586, + "theoretical_loss": 3.3334492744325024, + "tokens_seen": 2855534592 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001370909090909091, + "loss": 2.4284, + "theoretical_loss": 3.333443333202287, + "tokens_seen": 2855600128 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013707070707070707, + "loss": 2.4897, + "theoretical_loss": 3.3334373921465987, + "tokens_seen": 2855665664 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013705050505050508, + "loss": 2.3995, + "theoretical_loss": 3.3334314512654286, + "tokens_seen": 2855731200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013703030303030303, + "loss": 2.3331, + "theoretical_loss": 3.333425510558767, + "tokens_seen": 2855796736 + }, + { + "epoch": 0.73, + "learning_rate": 0.000137010101010101, + "loss": 2.2934, + "theoretical_loss": 3.3334195700266043, + "tokens_seen": 2855862272 + }, + { + "epoch": 0.73, + "learning_rate": 0.000136989898989899, + "loss": 2.4431, + "theoretical_loss": 3.3334136296689327, + "tokens_seen": 2855927808 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1566259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.127732515335083, + "objective/train/theoretical_loss": 3.333407689485742, + "objective/train/tokens_used": 1214852576, + "theoretical_loss": 3.333407689485742, + "tokens_seen": 2855993344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013696969696969696, + "loss": 2.31, + "theoretical_loss": 3.333407689485742, + "tokens_seen": 2855993344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013694949494949497, + "loss": 2.5374, + "theoretical_loss": 3.333401749477024, + "tokens_seen": 2856058880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013692929292929293, + "loss": 2.398, + "theoretical_loss": 3.3333958096427683, + "tokens_seen": 2856124416 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001369090909090909, + "loss": 2.4491, + "theoretical_loss": 3.333389869982967, + "tokens_seen": 2856189952 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001368888888888889, + "loss": 2.5159, + "theoretical_loss": 3.33338393049761, + "tokens_seen": 2856255488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013686868686868685, + "loss": 2.5834, + "theoretical_loss": 3.333377991186689, + "tokens_seen": 2856321024 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013684848484848486, + "loss": 2.4558, + "theoretical_loss": 3.333372052050195, + "tokens_seen": 2856386560 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013682828282828282, + "loss": 2.5889, + "theoretical_loss": 3.3333661130881183, + "tokens_seen": 2856452096 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001368080808080808, + "loss": 2.5123, + "theoretical_loss": 3.3333601743004495, + "tokens_seen": 2856517632 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001367878787878788, + "loss": 2.3347, + "theoretical_loss": 3.33335423568718, + "tokens_seen": 2856583168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013676767676767677, + "loss": 2.6178, + "theoretical_loss": 3.3333482972483, + "tokens_seen": 2856648704 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013674747474747475, + "loss": 2.6724, + "theoretical_loss": 3.333342358983802, + "tokens_seen": 2856714240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013672727272727274, + "loss": 2.5016, + "theoretical_loss": 3.333336420893675, + "tokens_seen": 2856779776 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001367070707070707, + "loss": 2.7069, + "theoretical_loss": 3.333330482977911, + "tokens_seen": 2856845312 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013668686868686868, + "loss": 2.5048, + "theoretical_loss": 3.3333245452365006, + "tokens_seen": 2856910848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013666666666666666, + "loss": 2.3943, + "theoretical_loss": 3.3333186076694346, + "tokens_seen": 2856976384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013664646464646465, + "loss": 2.531, + "theoretical_loss": 3.3333126702767037, + "tokens_seen": 2857041920 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013662626262626263, + "loss": 2.4677, + "theoretical_loss": 3.3333067330582993, + "tokens_seen": 2857107456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013660606060606062, + "loss": 2.4172, + "theoretical_loss": 3.333300796014212, + "tokens_seen": 2857172992 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001365858585858586, + "loss": 2.505, + "theoretical_loss": 3.3332948591444325, + "tokens_seen": 2857238528 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013656565656565656, + "loss": 2.4429, + "theoretical_loss": 3.3332889224489515, + "tokens_seen": 2857304064 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013654545454545457, + "loss": 2.5377, + "theoretical_loss": 3.333282985927761, + "tokens_seen": 2857369600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013652525252525252, + "loss": 2.4962, + "theoretical_loss": 3.333277049580851, + "tokens_seen": 2857435136 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001365050505050505, + "loss": 2.5142, + "theoretical_loss": 3.333271113408212, + "tokens_seen": 2857500672 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001364848484848485, + "loss": 2.4521, + "theoretical_loss": 3.3332651774098356, + "tokens_seen": 2857566208 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1567479, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.662693738937378, + "objective/train/theoretical_loss": 3.3332592415857127, + "objective/train/tokens_used": 1216490976, + "theoretical_loss": 3.3332592415857127, + "tokens_seen": 2857631744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013646464646464645, + "loss": 2.5072, + "theoretical_loss": 3.3332592415857127, + "tokens_seen": 2857631744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013644444444444446, + "loss": 2.5806, + "theoretical_loss": 3.333253305935834, + "tokens_seen": 2857697280 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013642424242424242, + "loss": 2.4537, + "theoretical_loss": 3.33324737046019, + "tokens_seen": 2857762816 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013640404040404043, + "loss": 2.4725, + "theoretical_loss": 3.3332414351587727, + "tokens_seen": 2857828352 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013638383838383838, + "loss": 2.3638, + "theoretical_loss": 3.3332355000315714, + "tokens_seen": 2857893888 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013636363636363637, + "loss": 2.5221, + "theoretical_loss": 3.3332295650785784, + "tokens_seen": 2857959424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013634343434343435, + "loss": 2.3354, + "theoretical_loss": 3.3332236302997837, + "tokens_seen": 2858024960 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001363232323232323, + "loss": 2.4418, + "theoretical_loss": 3.333217695695179, + "tokens_seen": 2858090496 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013630303030303032, + "loss": 2.256, + "theoretical_loss": 3.3332117612647543, + "tokens_seen": 2858156032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013628282828282828, + "loss": 2.4741, + "theoretical_loss": 3.333205827008501, + "tokens_seen": 2858221568 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013626262626262626, + "loss": 2.4657, + "theoretical_loss": 3.33319989292641, + "tokens_seen": 2858287104 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013624242424242425, + "loss": 2.4469, + "theoretical_loss": 3.3331939590184723, + "tokens_seen": 2858352640 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013622222222222223, + "loss": 2.4276, + "theoretical_loss": 3.333188025284678, + "tokens_seen": 2858418176 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001362020202020202, + "loss": 2.4196, + "theoretical_loss": 3.333182091725019, + "tokens_seen": 2858483712 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001361818181818182, + "loss": 2.4479, + "theoretical_loss": 3.333176158339486, + "tokens_seen": 2858549248 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013616161616161615, + "loss": 2.6181, + "theoretical_loss": 3.333170225128069, + "tokens_seen": 2858614784 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013614141414141414, + "loss": 2.3813, + "theoretical_loss": 3.3331642920907605, + "tokens_seen": 2858680320 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013612121212121212, + "loss": 2.3491, + "theoretical_loss": 3.33315835922755, + "tokens_seen": 2858745856 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001361010101010101, + "loss": 2.6581, + "theoretical_loss": 3.3331524265384287, + "tokens_seen": 2858811392 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001360808080808081, + "loss": 2.5151, + "theoretical_loss": 3.333146494023388, + "tokens_seen": 2858876928 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013606060606060605, + "loss": 2.4668, + "theoretical_loss": 3.3331405616824186, + "tokens_seen": 2858942464 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013604040404040406, + "loss": 2.5168, + "theoretical_loss": 3.333134629515511, + "tokens_seen": 2859008000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013602020202020201, + "loss": 2.2885, + "theoretical_loss": 3.333128697522657, + "tokens_seen": 2859073536 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013600000000000003, + "loss": 2.5849, + "theoretical_loss": 3.333122765703846, + "tokens_seen": 2859139072 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013597979797979798, + "loss": 2.4969, + "theoretical_loss": 3.33311683405907, + "tokens_seen": 2859204608 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1568614, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9926602840423584, + "objective/train/theoretical_loss": 3.33311090258832, + "objective/train/tokens_used": 1218129376, + "theoretical_loss": 3.33311090258832, + "tokens_seen": 2859270144 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013595959595959594, + "loss": 2.4357, + "theoretical_loss": 3.33311090258832, + "tokens_seen": 2859270144 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013593939393939395, + "loss": 2.6162, + "theoretical_loss": 3.333104971291587, + "tokens_seen": 2859335680 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001359191919191919, + "loss": 2.4206, + "theoretical_loss": 3.333099040168861, + "tokens_seen": 2859401216 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013589898989898992, + "loss": 2.5948, + "theoretical_loss": 3.3330931092201332, + "tokens_seen": 2859466752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013587878787878788, + "loss": 2.3369, + "theoretical_loss": 3.3330871784453953, + "tokens_seen": 2859532288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013585858585858586, + "loss": 2.4176, + "theoretical_loss": 3.3330812478446377, + "tokens_seen": 2859597824 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013583838383838384, + "loss": 2.4898, + "theoretical_loss": 3.333075317417851, + "tokens_seen": 2859663360 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001358181818181818, + "loss": 2.3594, + "theoretical_loss": 3.3330693871650263, + "tokens_seen": 2859728896 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001357979797979798, + "loss": 2.4627, + "theoretical_loss": 3.333063457086155, + "tokens_seen": 2859794432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013577777777777777, + "loss": 2.6494, + "theoretical_loss": 3.333057527181227, + "tokens_seen": 2859859968 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013575757575757578, + "loss": 2.4482, + "theoretical_loss": 3.333051597450234, + "tokens_seen": 2859925504 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013573737373737374, + "loss": 2.2451, + "theoretical_loss": 3.333045667893167, + "tokens_seen": 2859991040 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013571717171717172, + "loss": 2.4394, + "theoretical_loss": 3.333039738510016, + "tokens_seen": 2860056576 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001356969696969697, + "loss": 2.4411, + "theoretical_loss": 3.333033809300773, + "tokens_seen": 2860122112 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001356767676767677, + "loss": 2.456, + "theoretical_loss": 3.3330278802654285, + "tokens_seen": 2860187648 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013565656565656567, + "loss": 2.4176, + "theoretical_loss": 3.3330219514039734, + "tokens_seen": 2860253184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013563636363636363, + "loss": 2.6719, + "theoretical_loss": 3.3330160227163983, + "tokens_seen": 2860318720 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001356161616161616, + "loss": 2.3934, + "theoretical_loss": 3.333010094202695, + "tokens_seen": 2860384256 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001355959595959596, + "loss": 2.4527, + "theoretical_loss": 3.333004165862853, + "tokens_seen": 2860449792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013557575757575758, + "loss": 2.4521, + "theoretical_loss": 3.3329982376968643, + "tokens_seen": 2860515328 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013555555555555556, + "loss": 2.4699, + "theoretical_loss": 3.3329923097047196, + "tokens_seen": 2860580864 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013553535353535355, + "loss": 2.5335, + "theoretical_loss": 3.33298638188641, + "tokens_seen": 2860646400 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001355151515151515, + "loss": 2.5247, + "theoretical_loss": 3.3329804542419263, + "tokens_seen": 2860711936 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013549494949494952, + "loss": 2.2427, + "theoretical_loss": 3.332974526771259, + "tokens_seen": 2860777472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013547474747474747, + "loss": 2.5999, + "theoretical_loss": 3.3329685994743996, + "tokens_seen": 2860843008 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1568983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8074564933776855, + "objective/train/theoretical_loss": 3.3329626723513384, + "objective/train/tokens_used": 1219767776, + "theoretical_loss": 3.3329626723513384, + "tokens_seen": 2860908544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013545454545454546, + "loss": 2.4901, + "theoretical_loss": 3.3329626723513384, + "tokens_seen": 2860908544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013543434343434344, + "loss": 2.6952, + "theoretical_loss": 3.332956745402067, + "tokens_seen": 2860974080 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001354141414141414, + "loss": 2.3668, + "theoretical_loss": 3.332950818626576, + "tokens_seen": 2861039616 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001353939393939394, + "loss": 2.592, + "theoretical_loss": 3.332944892024856, + "tokens_seen": 2861105152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013537373737373737, + "loss": 2.5305, + "theoretical_loss": 3.332938965596899, + "tokens_seen": 2861170688 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013535353535353538, + "loss": 2.6361, + "theoretical_loss": 3.3329330393426946, + "tokens_seen": 2861236224 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013533333333333333, + "loss": 2.3571, + "theoretical_loss": 3.3329271132622345, + "tokens_seen": 2861301760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013531313131313132, + "loss": 2.373, + "theoretical_loss": 3.332921187355509, + "tokens_seen": 2861367296 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001352929292929293, + "loss": 2.6123, + "theoretical_loss": 3.33291526162251, + "tokens_seen": 2861432832 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013527272727272726, + "loss": 2.5286, + "theoretical_loss": 3.332909336063228, + "tokens_seen": 2861498368 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013525252525252527, + "loss": 2.4107, + "theoretical_loss": 3.332903410677653, + "tokens_seen": 2861563904 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013523232323232323, + "loss": 2.6097, + "theoretical_loss": 3.3328974854657774, + "tokens_seen": 2861629440 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001352121212121212, + "loss": 2.4375, + "theoretical_loss": 3.3328915604275915, + "tokens_seen": 2861694976 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001351919191919192, + "loss": 2.6304, + "theoretical_loss": 3.332885635563086, + "tokens_seen": 2861760512 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013517171717171718, + "loss": 2.3694, + "theoretical_loss": 3.332879710872252, + "tokens_seen": 2861826048 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013515151515151516, + "loss": 2.5819, + "theoretical_loss": 3.332873786355081, + "tokens_seen": 2861891584 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013513131313131312, + "loss": 2.3419, + "theoretical_loss": 3.332867862011563, + "tokens_seen": 2861957120 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001351111111111111, + "loss": 2.3538, + "theoretical_loss": 3.332861937841689, + "tokens_seen": 2862022656 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001350909090909091, + "loss": 2.3578, + "theoretical_loss": 3.3328560138454506, + "tokens_seen": 2862088192 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013507070707070707, + "loss": 2.463, + "theoretical_loss": 3.3328500900228386, + "tokens_seen": 2862153728 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013505050505050506, + "loss": 2.7465, + "theoretical_loss": 3.3328441663738437, + "tokens_seen": 2862219264 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013503030303030304, + "loss": 2.6068, + "theoretical_loss": 3.3328382428984566, + "tokens_seen": 2862284800 + }, + { + "epoch": 0.73, + "learning_rate": 0.000135010101010101, + "loss": 2.3784, + "theoretical_loss": 3.332832319596669, + "tokens_seen": 2862350336 + }, + { + "epoch": 0.73, + "learning_rate": 0.000134989898989899, + "loss": 2.5182, + "theoretical_loss": 3.3328263964684712, + "tokens_seen": 2862415872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013496969696969696, + "loss": 2.4258, + "theoretical_loss": 3.332820473513854, + "tokens_seen": 2862481408 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1570281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4658923149108887, + "objective/train/theoretical_loss": 3.3328145507328086, + "objective/train/tokens_used": 1221406176, + "theoretical_loss": 3.3328145507328086, + "tokens_seen": 2862546944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013494949494949495, + "loss": 2.5665, + "theoretical_loss": 3.3328145507328086, + "tokens_seen": 2862546944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013492929292929293, + "loss": 2.6223, + "theoretical_loss": 3.3328086281253264, + "tokens_seen": 2862612480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013490909090909092, + "loss": 2.4792, + "theoretical_loss": 3.332802705691398, + "tokens_seen": 2862678016 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001348888888888889, + "loss": 2.3935, + "theoretical_loss": 3.3327967834310135, + "tokens_seen": 2862743552 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013486868686868686, + "loss": 2.2804, + "theoretical_loss": 3.3327908613441655, + "tokens_seen": 2862809088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013484848484848487, + "loss": 2.5192, + "theoretical_loss": 3.3327849394308435, + "tokens_seen": 2862874624 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013482828282828282, + "loss": 2.5347, + "theoretical_loss": 3.3327790176910392, + "tokens_seen": 2862940160 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013480808080808084, + "loss": 2.4147, + "theoretical_loss": 3.3327730961247433, + "tokens_seen": 2863005696 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001347878787878788, + "loss": 2.403, + "theoretical_loss": 3.3327671747319467, + "tokens_seen": 2863071232 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013476767676767675, + "loss": 2.5615, + "theoretical_loss": 3.3327612535126403, + "tokens_seen": 2863136768 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013474747474747476, + "loss": 2.5789, + "theoretical_loss": 3.3327553324668155, + "tokens_seen": 2863202304 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013472727272727272, + "loss": 2.5163, + "theoretical_loss": 3.3327494115944627, + "tokens_seen": 2863267840 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013470707070707073, + "loss": 2.4865, + "theoretical_loss": 3.332743490895573, + "tokens_seen": 2863333376 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013468686868686869, + "loss": 2.4451, + "theoretical_loss": 3.3327375703701376, + "tokens_seen": 2863398912 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013466666666666667, + "loss": 2.5257, + "theoretical_loss": 3.332731650018147, + "tokens_seen": 2863464448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013464646464646465, + "loss": 2.4196, + "theoretical_loss": 3.3327257298395927, + "tokens_seen": 2863529984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013462626262626264, + "loss": 2.5818, + "theoretical_loss": 3.3327198098344653, + "tokens_seen": 2863595520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013460606060606062, + "loss": 2.4526, + "theoretical_loss": 3.332713890002756, + "tokens_seen": 2863661056 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013458585858585858, + "loss": 2.2174, + "theoretical_loss": 3.332707970344455, + "tokens_seen": 2863726592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013456565656565656, + "loss": 2.5702, + "theoretical_loss": 3.332702050859554, + "tokens_seen": 2863792128 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013454545454545455, + "loss": 2.4242, + "theoretical_loss": 3.3326961315480443, + "tokens_seen": 2863857664 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013452525252525253, + "loss": 2.4685, + "theoretical_loss": 3.332690212409916, + "tokens_seen": 2863923200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013450505050505051, + "loss": 2.3678, + "theoretical_loss": 3.33268429344516, + "tokens_seen": 2863988736 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001344848484848485, + "loss": 2.3834, + "theoretical_loss": 3.332678374653768, + "tokens_seen": 2864054272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013446464646464645, + "loss": 2.4702, + "theoretical_loss": 3.3326724560357306, + "tokens_seen": 2864119808 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1570911, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1789724826812744, + "objective/train/theoretical_loss": 3.3326665375910385, + "objective/train/tokens_used": 1223044576, + "theoretical_loss": 3.3326665375910385, + "tokens_seen": 2864185344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013444444444444447, + "loss": 2.3806, + "theoretical_loss": 3.3326665375910385, + "tokens_seen": 2864185344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013442424242424242, + "loss": 2.5591, + "theoretical_loss": 3.3326606193196833, + "tokens_seen": 2864250880 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001344040404040404, + "loss": 2.4116, + "theoretical_loss": 3.332654701221655, + "tokens_seen": 2864316416 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001343838383838384, + "loss": 2.5049, + "theoretical_loss": 3.3326487832969454, + "tokens_seen": 2864381952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013436363636363635, + "loss": 2.6509, + "theoretical_loss": 3.3326428655455453, + "tokens_seen": 2864447488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013434343434343436, + "loss": 2.1558, + "theoretical_loss": 3.3326369479674454, + "tokens_seen": 2864513024 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013432323232323232, + "loss": 2.5679, + "theoretical_loss": 3.332631030562637, + "tokens_seen": 2864578560 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013430303030303033, + "loss": 2.317, + "theoretical_loss": 3.3326251133311104, + "tokens_seen": 2864644096 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013428282828282828, + "loss": 2.4211, + "theoretical_loss": 3.3326191962728573, + "tokens_seen": 2864709632 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013426262626262624, + "loss": 2.4211, + "theoretical_loss": 3.332613279387868, + "tokens_seen": 2864775168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013424242424242425, + "loss": 2.5505, + "theoretical_loss": 3.3326073626761348, + "tokens_seen": 2864840704 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001342222222222222, + "loss": 2.29, + "theoretical_loss": 3.3326014461376468, + "tokens_seen": 2864906240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013420202020202022, + "loss": 2.3606, + "theoretical_loss": 3.332595529772396, + "tokens_seen": 2864971776 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013418181818181818, + "loss": 2.5469, + "theoretical_loss": 3.3325896135803736, + "tokens_seen": 2865037312 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013416161616161616, + "loss": 2.389, + "theoretical_loss": 3.33258369756157, + "tokens_seen": 2865102848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013414141414141414, + "loss": 2.5303, + "theoretical_loss": 3.332577781715976, + "tokens_seen": 2865168384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013412121212121213, + "loss": 2.3831, + "theoretical_loss": 3.3325718660435837, + "tokens_seen": 2865233920 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001341010101010101, + "loss": 2.4298, + "theoretical_loss": 3.3325659505443825, + "tokens_seen": 2865299456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013408080808080807, + "loss": 2.7559, + "theoretical_loss": 3.332560035218365, + "tokens_seen": 2865364992 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013406060606060605, + "loss": 2.5571, + "theoretical_loss": 3.3325541200655207, + "tokens_seen": 2865430528 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013404040404040404, + "loss": 2.2371, + "theoretical_loss": 3.3325482050858417, + "tokens_seen": 2865496064 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013402020202020202, + "loss": 2.4859, + "theoretical_loss": 3.332542290279318, + "tokens_seen": 2865561600 + }, + { + "epoch": 0.73, + "learning_rate": 0.000134, + "loss": 2.602, + "theoretical_loss": 3.332536375645941, + "tokens_seen": 2865627136 + }, + { + "epoch": 0.73, + "learning_rate": 0.000133979797979798, + "loss": 2.504, + "theoretical_loss": 3.3325304611857023, + "tokens_seen": 2865692672 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013395959595959597, + "loss": 2.5187, + "theoretical_loss": 3.3325245468985916, + "tokens_seen": 2865758208 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1572188, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2405028343200684, + "objective/train/theoretical_loss": 3.332518632784601, + "objective/train/tokens_used": 1224682976, + "theoretical_loss": 3.332518632784601, + "tokens_seen": 2865823744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013393939393939396, + "loss": 2.2901, + "theoretical_loss": 3.332518632784601, + "tokens_seen": 2865823744 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001339191919191919, + "loss": 2.2672, + "theoretical_loss": 3.332512718843721, + "tokens_seen": 2865889280 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001338989898989899, + "loss": 2.5067, + "theoretical_loss": 3.3325068050759423, + "tokens_seen": 2865954816 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013387878787878788, + "loss": 2.3992, + "theoretical_loss": 3.3325008914812564, + "tokens_seen": 2866020352 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013385858585858587, + "loss": 2.3715, + "theoretical_loss": 3.3324949780596542, + "tokens_seen": 2866085888 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013383838383838385, + "loss": 2.576, + "theoretical_loss": 3.3324890648111265, + "tokens_seen": 2866151424 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001338181818181818, + "loss": 2.3884, + "theoretical_loss": 3.3324831517356643, + "tokens_seen": 2866216960 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013379797979797982, + "loss": 2.4175, + "theoretical_loss": 3.332477238833259, + "tokens_seen": 2866282496 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013377777777777777, + "loss": 2.5142, + "theoretical_loss": 3.3324713261039003, + "tokens_seen": 2866348032 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013375757575757578, + "loss": 2.5885, + "theoretical_loss": 3.3324654135475806, + "tokens_seen": 2866413568 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013373737373737374, + "loss": 2.5501, + "theoretical_loss": 3.33245950116429, + "tokens_seen": 2866479104 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001337171717171717, + "loss": 2.3897, + "theoretical_loss": 3.33245358895402, + "tokens_seen": 2866544640 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001336969696969697, + "loss": 2.6365, + "theoretical_loss": 3.3324476769167615, + "tokens_seen": 2866610176 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013367676767676767, + "loss": 2.5464, + "theoretical_loss": 3.332441765052505, + "tokens_seen": 2866675712 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013365656565656568, + "loss": 2.6049, + "theoretical_loss": 3.3324358533612424, + "tokens_seen": 2866741248 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013363636363636363, + "loss": 2.6155, + "theoretical_loss": 3.332429941842964, + "tokens_seen": 2866806784 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013361616161616162, + "loss": 2.4292, + "theoretical_loss": 3.332424030497661, + "tokens_seen": 2866872320 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001335959595959596, + "loss": 2.462, + "theoretical_loss": 3.332418119325324, + "tokens_seen": 2866937856 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013357575757575756, + "loss": 2.5407, + "theoretical_loss": 3.332412208325944, + "tokens_seen": 2867003392 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013355555555555557, + "loss": 2.263, + "theoretical_loss": 3.3324062974995132, + "tokens_seen": 2867068928 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013353535353535353, + "loss": 2.5092, + "theoretical_loss": 3.3324003868460212, + "tokens_seen": 2867134464 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001335151515151515, + "loss": 2.5803, + "theoretical_loss": 3.3323944763654594, + "tokens_seen": 2867200000 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001334949494949495, + "loss": 2.6497, + "theoretical_loss": 3.3323885660578187, + "tokens_seen": 2867265536 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013347474747474748, + "loss": 2.3436, + "theoretical_loss": 3.3323826559230905, + "tokens_seen": 2867331072 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013345454545454546, + "loss": 2.3844, + "theoretical_loss": 3.332376745961265, + "tokens_seen": 2867396608 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1572866, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.715240001678467, + "objective/train/theoretical_loss": 3.3323708361723345, + "objective/train/tokens_used": 1226321376, + "theoretical_loss": 3.3323708361723345, + "tokens_seen": 2867462144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013343434343434345, + "loss": 2.5241, + "theoretical_loss": 3.3323708361723345, + "tokens_seen": 2867462144 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001334141414141414, + "loss": 2.3468, + "theoretical_loss": 3.332364926556289, + "tokens_seen": 2867527680 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001333939393939394, + "loss": 2.3709, + "theoretical_loss": 3.3323590171131197, + "tokens_seen": 2867593216 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013337373737373737, + "loss": 2.4239, + "theoretical_loss": 3.3323531078428172, + "tokens_seen": 2867658752 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013335353535353536, + "loss": 2.4824, + "theoretical_loss": 3.3323471987453734, + "tokens_seen": 2867724288 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013333333333333334, + "loss": 2.505, + "theoretical_loss": 3.3323412898207785, + "tokens_seen": 2867789824 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001333131313131313, + "loss": 2.5749, + "theoretical_loss": 3.3323353810690235, + "tokens_seen": 2867855360 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001332929292929293, + "loss": 2.5073, + "theoretical_loss": 3.3323294724901, + "tokens_seen": 2867920896 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013327272727272726, + "loss": 2.4535, + "theoretical_loss": 3.3323235640839988, + "tokens_seen": 2867986432 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013325252525252528, + "loss": 2.5162, + "theoretical_loss": 3.3323176558507104, + "tokens_seen": 2868051968 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013323232323232323, + "loss": 2.5968, + "theoretical_loss": 3.332311747790226, + "tokens_seen": 2868117504 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001332121212121212, + "loss": 2.3643, + "theoretical_loss": 3.332305839902537, + "tokens_seen": 2868183040 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001331919191919192, + "loss": 2.4901, + "theoretical_loss": 3.3322999321876345, + "tokens_seen": 2868248576 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013317171717171716, + "loss": 2.5513, + "theoretical_loss": 3.332294024645509, + "tokens_seen": 2868314112 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013315151515151517, + "loss": 2.4092, + "theoretical_loss": 3.332288117276151, + "tokens_seen": 2868379648 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013313131313131313, + "loss": 2.4969, + "theoretical_loss": 3.3322822100795526, + "tokens_seen": 2868445184 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013311111111111114, + "loss": 2.5785, + "theoretical_loss": 3.3322763030557043, + "tokens_seen": 2868510720 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001330909090909091, + "loss": 2.347, + "theoretical_loss": 3.3322703962045974, + "tokens_seen": 2868576256 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013307070707070708, + "loss": 2.4036, + "theoretical_loss": 3.3322644895262226, + "tokens_seen": 2868641792 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013305050505050506, + "loss": 2.3068, + "theoretical_loss": 3.332258583020571, + "tokens_seen": 2868707328 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013303030303030302, + "loss": 2.5809, + "theoretical_loss": 3.332252676687633, + "tokens_seen": 2868772864 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013301010101010103, + "loss": 2.4207, + "theoretical_loss": 3.3322467705274006, + "tokens_seen": 2868838400 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013298989898989899, + "loss": 2.4907, + "theoretical_loss": 3.3322408645398642, + "tokens_seen": 2868903936 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013296969696969697, + "loss": 2.387, + "theoretical_loss": 3.332234958725015, + "tokens_seen": 2868969472 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013294949494949495, + "loss": 2.5492, + "theoretical_loss": 3.3322290530828442, + "tokens_seen": 2869035008 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1573983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.544900894165039, + "objective/train/theoretical_loss": 3.3322231476133424, + "objective/train/tokens_used": 1227959776, + "theoretical_loss": 3.3322231476133424, + "tokens_seen": 2869100544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013292929292929294, + "loss": 2.5447, + "theoretical_loss": 3.3322231476133424, + "tokens_seen": 2869100544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013290909090909092, + "loss": 2.6805, + "theoretical_loss": 3.3322172423165006, + "tokens_seen": 2869166080 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001328888888888889, + "loss": 2.5507, + "theoretical_loss": 3.3322113371923106, + "tokens_seen": 2869231616 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013286868686868686, + "loss": 2.3299, + "theoretical_loss": 3.332205432240762, + "tokens_seen": 2869297152 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013284848484848485, + "loss": 2.5934, + "theoretical_loss": 3.332199527461847, + "tokens_seen": 2869362688 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013282828282828283, + "loss": 2.4844, + "theoretical_loss": 3.332193622855556, + "tokens_seen": 2869428224 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013280808080808081, + "loss": 2.4689, + "theoretical_loss": 3.332187718421881, + "tokens_seen": 2869493760 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001327878787878788, + "loss": 2.429, + "theoretical_loss": 3.3321818141608115, + "tokens_seen": 2869559296 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013276767676767676, + "loss": 2.5563, + "theoretical_loss": 3.3321759100723396, + "tokens_seen": 2869624832 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013274747474747477, + "loss": 2.5427, + "theoretical_loss": 3.332170006156456, + "tokens_seen": 2869690368 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013272727272727272, + "loss": 2.296, + "theoretical_loss": 3.3321641024131514, + "tokens_seen": 2869755904 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013270707070707073, + "loss": 2.4906, + "theoretical_loss": 3.3321581988424174, + "tokens_seen": 2869821440 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001326868686868687, + "loss": 2.5763, + "theoretical_loss": 3.3321522954442444, + "tokens_seen": 2869886976 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013266666666666665, + "loss": 2.2222, + "theoretical_loss": 3.3321463922186236, + "tokens_seen": 2869952512 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013264646464646466, + "loss": 2.3595, + "theoretical_loss": 3.3321404891655466, + "tokens_seen": 2870018048 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013262626262626262, + "loss": 2.5839, + "theoretical_loss": 3.332134586285004, + "tokens_seen": 2870083584 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013260606060606063, + "loss": 2.4743, + "theoretical_loss": 3.3321286835769865, + "tokens_seen": 2870149120 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013258585858585858, + "loss": 2.5198, + "theoretical_loss": 3.3321227810414857, + "tokens_seen": 2870214656 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013256565656565657, + "loss": 2.762, + "theoretical_loss": 3.332116878678492, + "tokens_seen": 2870280192 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013254545454545455, + "loss": 2.5873, + "theoretical_loss": 3.332110976487997, + "tokens_seen": 2870345728 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001325252525252525, + "loss": 2.343, + "theoretical_loss": 3.3321050744699914, + "tokens_seen": 2870411264 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013250505050505052, + "loss": 2.4587, + "theoretical_loss": 3.332099172624466, + "tokens_seen": 2870476800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013248484848484848, + "loss": 2.5611, + "theoretical_loss": 3.3320932709514124, + "tokens_seen": 2870542336 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013246464646464646, + "loss": 2.3806, + "theoretical_loss": 3.3320873694508215, + "tokens_seen": 2870607872 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013244444444444444, + "loss": 2.6075, + "theoretical_loss": 3.332081468122684, + "tokens_seen": 2870673408 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1575152, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1421406269073486, + "objective/train/theoretical_loss": 3.3320755669669904, + "objective/train/tokens_used": 1229598176, + "theoretical_loss": 3.3320755669669904, + "tokens_seen": 2870738944 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013242424242424243, + "loss": 2.3884, + "theoretical_loss": 3.3320755669669904, + "tokens_seen": 2870738944 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001324040404040404, + "loss": 2.6367, + "theoretical_loss": 3.332069665983733, + "tokens_seen": 2870804480 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001323838383838384, + "loss": 2.5677, + "theoretical_loss": 3.3320637651729026, + "tokens_seen": 2870870016 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013236363636363635, + "loss": 2.4412, + "theoretical_loss": 3.332057864534489, + "tokens_seen": 2870935552 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013234343434343434, + "loss": 2.5299, + "theoretical_loss": 3.3320519640684845, + "tokens_seen": 2871001088 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013232323232323232, + "loss": 2.1899, + "theoretical_loss": 3.33204606377488, + "tokens_seen": 2871066624 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001323030303030303, + "loss": 2.414, + "theoretical_loss": 3.332040163653666, + "tokens_seen": 2871132160 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001322828282828283, + "loss": 2.5591, + "theoretical_loss": 3.3320342637048337, + "tokens_seen": 2871197696 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013226262626262627, + "loss": 2.5943, + "theoretical_loss": 3.3320283639283743, + "tokens_seen": 2871263232 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013224242424242426, + "loss": 2.3007, + "theoretical_loss": 3.3320224643242784, + "tokens_seen": 2871328768 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013222222222222221, + "loss": 2.4784, + "theoretical_loss": 3.3320165648925375, + "tokens_seen": 2871394304 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013220202020202022, + "loss": 2.4735, + "theoretical_loss": 3.3320106656331427, + "tokens_seen": 2871459840 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013218181818181818, + "loss": 2.5194, + "theoretical_loss": 3.332004766546085, + "tokens_seen": 2871525376 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013216161616161617, + "loss": 2.3389, + "theoretical_loss": 3.3319988676313543, + "tokens_seen": 2871590912 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013214141414141415, + "loss": 2.644, + "theoretical_loss": 3.3319929688889434, + "tokens_seen": 2871656448 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001321212121212121, + "loss": 2.4867, + "theoretical_loss": 3.331987070318842, + "tokens_seen": 2871721984 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013210101010101012, + "loss": 2.5721, + "theoretical_loss": 3.331981171921042, + "tokens_seen": 2871787520 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013208080808080807, + "loss": 2.4679, + "theoretical_loss": 3.331975273695534, + "tokens_seen": 2871853056 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013206060606060609, + "loss": 2.5318, + "theoretical_loss": 3.331969375642309, + "tokens_seen": 2871918592 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013204040404040404, + "loss": 2.536, + "theoretical_loss": 3.3319634777613585, + "tokens_seen": 2871984128 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013202020202020203, + "loss": 2.4562, + "theoretical_loss": 3.331957580052673, + "tokens_seen": 2872049664 + }, + { + "epoch": 0.74, + "learning_rate": 0.000132, + "loss": 2.5287, + "theoretical_loss": 3.3319516825162436, + "tokens_seen": 2872115200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013197979797979797, + "loss": 2.4136, + "theoretical_loss": 3.3319457851520617, + "tokens_seen": 2872180736 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013195959595959598, + "loss": 2.4434, + "theoretical_loss": 3.331939887960118, + "tokens_seen": 2872246272 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013193939393939393, + "loss": 2.4236, + "theoretical_loss": 3.3319339909404038, + "tokens_seen": 2872311808 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1575784, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3510477542877197, + "objective/train/theoretical_loss": 3.3319280940929095, + "objective/train/tokens_used": 1231236576, + "theoretical_loss": 3.3319280940929095, + "tokens_seen": 2872377344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013191919191919192, + "loss": 2.2901, + "theoretical_loss": 3.3319280940929095, + "tokens_seen": 2872377344 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001318989898989899, + "loss": 2.5297, + "theoretical_loss": 3.3319221974176267, + "tokens_seen": 2872442880 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001318787878787879, + "loss": 2.5394, + "theoretical_loss": 3.3319163009145467, + "tokens_seen": 2872508416 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013185858585858587, + "loss": 2.2781, + "theoretical_loss": 3.3319104045836605, + "tokens_seen": 2872573952 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013183838383838383, + "loss": 2.4149, + "theoretical_loss": 3.331904508424958, + "tokens_seen": 2872639488 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001318181818181818, + "loss": 2.5337, + "theoretical_loss": 3.3318986124384318, + "tokens_seen": 2872705024 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001317979797979798, + "loss": 2.3317, + "theoretical_loss": 3.331892716624072, + "tokens_seen": 2872770560 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013177777777777778, + "loss": 2.4499, + "theoretical_loss": 3.3318868209818695, + "tokens_seen": 2872836096 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013175757575757576, + "loss": 2.4566, + "theoretical_loss": 3.3318809255118165, + "tokens_seen": 2872901632 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013173737373737375, + "loss": 2.4497, + "theoretical_loss": 3.3318750302139026, + "tokens_seen": 2872967168 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001317171717171717, + "loss": 2.5548, + "theoretical_loss": 3.33186913508812, + "tokens_seen": 2873032704 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013169696969696972, + "loss": 2.6692, + "theoretical_loss": 3.331863240134459, + "tokens_seen": 2873098240 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013167676767676767, + "loss": 2.4525, + "theoretical_loss": 3.331857345352911, + "tokens_seen": 2873163776 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013165656565656566, + "loss": 2.3968, + "theoretical_loss": 3.331851450743467, + "tokens_seen": 2873229312 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013163636363636364, + "loss": 2.4769, + "theoretical_loss": 3.331845556306118, + "tokens_seen": 2873294848 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001316161616161616, + "loss": 2.5502, + "theoretical_loss": 3.331839662040855, + "tokens_seen": 2873360384 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001315959595959596, + "loss": 2.6275, + "theoretical_loss": 3.331833767947669, + "tokens_seen": 2873425920 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013157575757575756, + "loss": 2.5454, + "theoretical_loss": 3.3318278740265512, + "tokens_seen": 2873491456 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013155555555555558, + "loss": 2.328, + "theoretical_loss": 3.331821980277493, + "tokens_seen": 2873556992 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013153535353535353, + "loss": 2.5472, + "theoretical_loss": 3.331816086700485, + "tokens_seen": 2873622528 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013151515151515152, + "loss": 2.4305, + "theoretical_loss": 3.331810193295518, + "tokens_seen": 2873688064 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001314949494949495, + "loss": 2.4458, + "theoretical_loss": 3.3318043000625837, + "tokens_seen": 2873753600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013147474747474746, + "loss": 2.5242, + "theoretical_loss": 3.331798407001673, + "tokens_seen": 2873819136 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013145454545454547, + "loss": 2.3542, + "theoretical_loss": 3.3317925141127764, + "tokens_seen": 2873884672 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013143434343434343, + "loss": 2.5815, + "theoretical_loss": 3.3317866213958856, + "tokens_seen": 2873950208 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1577046, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.629937171936035, + "objective/train/theoretical_loss": 3.3317807288509913, + "objective/train/tokens_used": 1232874976, + "theoretical_loss": 3.3317807288509913, + "tokens_seen": 2874015744 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013141414141414144, + "loss": 2.5408, + "theoretical_loss": 3.3317807288509913, + "tokens_seen": 2874015744 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001313939393939394, + "loss": 2.4838, + "theoretical_loss": 3.3317748364780844, + "tokens_seen": 2874081280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013137373737373738, + "loss": 2.3548, + "theoretical_loss": 3.3317689442771568, + "tokens_seen": 2874146816 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013135353535353536, + "loss": 2.452, + "theoretical_loss": 3.3317630522481987, + "tokens_seen": 2874212352 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013133333333333335, + "loss": 2.4798, + "theoretical_loss": 3.3317571603912013, + "tokens_seen": 2874277888 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013131313131313133, + "loss": 2.6645, + "theoretical_loss": 3.331751268706156, + "tokens_seen": 2874343424 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013129292929292929, + "loss": 2.4697, + "theoretical_loss": 3.3317453771930534, + "tokens_seen": 2874408960 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013127272727272727, + "loss": 2.4213, + "theoretical_loss": 3.3317394858518856, + "tokens_seen": 2874474496 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013125252525252525, + "loss": 2.7602, + "theoretical_loss": 3.331733594682642, + "tokens_seen": 2874540032 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013123232323232324, + "loss": 2.4777, + "theoretical_loss": 3.3317277036853152, + "tokens_seen": 2874605568 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013121212121212122, + "loss": 2.3539, + "theoretical_loss": 3.3317218128598953, + "tokens_seen": 2874671104 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001311919191919192, + "loss": 2.3239, + "theoretical_loss": 3.3317159222063735, + "tokens_seen": 2874736640 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013117171717171716, + "loss": 2.5169, + "theoretical_loss": 3.3317100317247412, + "tokens_seen": 2874802176 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013115151515151517, + "loss": 2.4555, + "theoretical_loss": 3.3317041414149897, + "tokens_seen": 2874867712 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013113131313131313, + "loss": 2.4916, + "theoretical_loss": 3.3316982512771096, + "tokens_seen": 2874933248 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013111111111111111, + "loss": 2.4022, + "theoretical_loss": 3.3316923613110916, + "tokens_seen": 2874998784 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001310909090909091, + "loss": 2.4191, + "theoretical_loss": 3.3316864715169277, + "tokens_seen": 2875064320 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013107070707070706, + "loss": 2.6592, + "theoretical_loss": 3.331680581894608, + "tokens_seen": 2875129856 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013105050505050507, + "loss": 2.5565, + "theoretical_loss": 3.3316746924441243, + "tokens_seen": 2875195392 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013103030303030302, + "loss": 2.5548, + "theoretical_loss": 3.3316688031654675, + "tokens_seen": 2875260928 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013101010101010103, + "loss": 2.4482, + "theoretical_loss": 3.3316629140586285, + "tokens_seen": 2875326464 + }, + { + "epoch": 0.74, + "learning_rate": 0.000130989898989899, + "loss": 2.5593, + "theoretical_loss": 3.3316570251235986, + "tokens_seen": 2875392000 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013096969696969695, + "loss": 2.5321, + "theoretical_loss": 3.3316511363603687, + "tokens_seen": 2875457536 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013094949494949496, + "loss": 2.6465, + "theoretical_loss": 3.33164524776893, + "tokens_seen": 2875523072 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013092929292929292, + "loss": 2.4661, + "theoretical_loss": 3.3316393593492735, + "tokens_seen": 2875588608 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1577600, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4921200275421143, + "objective/train/theoretical_loss": 3.33163347110139, + "objective/train/tokens_used": 1234513376, + "theoretical_loss": 3.33163347110139, + "tokens_seen": 2875654144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013090909090909093, + "loss": 2.6053, + "theoretical_loss": 3.33163347110139, + "tokens_seen": 2875654144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013088888888888888, + "loss": 2.6224, + "theoretical_loss": 3.331627583025271, + "tokens_seen": 2875719680 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013086868686868687, + "loss": 2.4808, + "theoretical_loss": 3.3316216951209072, + "tokens_seen": 2875785216 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013084848484848485, + "loss": 2.3261, + "theoretical_loss": 3.33161580738829, + "tokens_seen": 2875850752 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013082828282828284, + "loss": 2.3344, + "theoretical_loss": 3.3316099198274105, + "tokens_seen": 2875916288 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013080808080808082, + "loss": 2.4982, + "theoretical_loss": 3.3316040324382596, + "tokens_seen": 2875981824 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013078787878787878, + "loss": 2.4575, + "theoretical_loss": 3.3315981452208288, + "tokens_seen": 2876047360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013076767676767676, + "loss": 2.5323, + "theoretical_loss": 3.331592258175108, + "tokens_seen": 2876112896 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013074747474747474, + "loss": 2.4125, + "theoretical_loss": 3.3315863713010896, + "tokens_seen": 2876178432 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013072727272727273, + "loss": 2.5868, + "theoretical_loss": 3.3315804845987644, + "tokens_seen": 2876243968 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001307070707070707, + "loss": 2.4502, + "theoretical_loss": 3.331574598068123, + "tokens_seen": 2876309504 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001306868686868687, + "loss": 2.3878, + "theoretical_loss": 3.3315687117091564, + "tokens_seen": 2876375040 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013066666666666665, + "loss": 2.458, + "theoretical_loss": 3.331562825521856, + "tokens_seen": 2876440576 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013064646464646466, + "loss": 2.7368, + "theoretical_loss": 3.3315569395062132, + "tokens_seen": 2876506112 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013062626262626262, + "loss": 2.5187, + "theoretical_loss": 3.331551053662219, + "tokens_seen": 2876571648 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001306060606060606, + "loss": 2.3792, + "theoretical_loss": 3.3315451679898636, + "tokens_seen": 2876637184 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001305858585858586, + "loss": 2.4001, + "theoretical_loss": 3.331539282489139, + "tokens_seen": 2876702720 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013056565656565657, + "loss": 2.6019, + "theoretical_loss": 3.331533397160036, + "tokens_seen": 2876768256 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013054545454545456, + "loss": 2.5554, + "theoretical_loss": 3.3315275120025456, + "tokens_seen": 2876833792 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013052525252525251, + "loss": 2.5238, + "theoretical_loss": 3.3315216270166594, + "tokens_seen": 2876899328 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013050505050505053, + "loss": 2.4433, + "theoretical_loss": 3.3315157422023676, + "tokens_seen": 2876964864 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013048484848484848, + "loss": 2.4119, + "theoretical_loss": 3.331509857559662, + "tokens_seen": 2877030400 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001304646464646465, + "loss": 2.4319, + "theoretical_loss": 3.331503973088534, + "tokens_seen": 2877095936 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013044444444444445, + "loss": 2.5009, + "theoretical_loss": 3.3314980887889734, + "tokens_seen": 2877161472 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001304242424242424, + "loss": 2.4205, + "theoretical_loss": 3.331492204660972, + "tokens_seen": 2877227008 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1578655, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.570821523666382, + "objective/train/theoretical_loss": 3.3314863207045216, + "objective/train/tokens_used": 1236151776, + "theoretical_loss": 3.3314863207045216, + "tokens_seen": 2877292544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013040404040404042, + "loss": 2.4973, + "theoretical_loss": 3.3314863207045216, + "tokens_seen": 2877292544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013038383838383837, + "loss": 2.3117, + "theoretical_loss": 3.331480436919612, + "tokens_seen": 2877358080 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013036363636363639, + "loss": 2.4647, + "theoretical_loss": 3.331474553306235, + "tokens_seen": 2877423616 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013034343434343434, + "loss": 2.4585, + "theoretical_loss": 3.331468669864382, + "tokens_seen": 2877489152 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013032323232323233, + "loss": 2.2099, + "theoretical_loss": 3.3314627865940434, + "tokens_seen": 2877554688 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001303030303030303, + "loss": 2.5862, + "theoretical_loss": 3.3314569034952104, + "tokens_seen": 2877620224 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013028282828282827, + "loss": 2.3864, + "theoretical_loss": 3.3314510205678745, + "tokens_seen": 2877685760 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013026262626262628, + "loss": 2.4623, + "theoretical_loss": 3.3314451378120267, + "tokens_seen": 2877751296 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013024242424242424, + "loss": 2.7087, + "theoretical_loss": 3.3314392552276577, + "tokens_seen": 2877816832 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013022222222222222, + "loss": 2.6115, + "theoretical_loss": 3.331433372814759, + "tokens_seen": 2877882368 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001302020202020202, + "loss": 2.3419, + "theoretical_loss": 3.3314274905733217, + "tokens_seen": 2877947904 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001301818181818182, + "loss": 2.4222, + "theoretical_loss": 3.331421608503337, + "tokens_seen": 2878013440 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013016161616161617, + "loss": 2.5378, + "theoretical_loss": 3.3314157266047952, + "tokens_seen": 2878078976 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013014141414141416, + "loss": 2.4176, + "theoretical_loss": 3.331409844877688, + "tokens_seen": 2878144512 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001301212121212121, + "loss": 2.3685, + "theoretical_loss": 3.331403963322007, + "tokens_seen": 2878210048 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001301010101010101, + "loss": 2.6251, + "theoretical_loss": 3.3313980819377425, + "tokens_seen": 2878275584 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013008080808080808, + "loss": 2.5271, + "theoretical_loss": 3.3313922007248857, + "tokens_seen": 2878341120 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013006060606060606, + "loss": 2.4349, + "theoretical_loss": 3.331386319683428, + "tokens_seen": 2878406656 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013004040404040405, + "loss": 2.4613, + "theoretical_loss": 3.33138043881336, + "tokens_seen": 2878472192 + }, + { + "epoch": 0.74, + "learning_rate": 0.000130020202020202, + "loss": 2.513, + "theoretical_loss": 3.331374558114674, + "tokens_seen": 2878537728 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013000000000000002, + "loss": 2.5189, + "theoretical_loss": 3.3313686775873594, + "tokens_seen": 2878603264 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012997979797979797, + "loss": 2.4686, + "theoretical_loss": 3.331362797231409, + "tokens_seen": 2878668800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012995959595959598, + "loss": 2.5304, + "theoretical_loss": 3.3313569170468122, + "tokens_seen": 2878734336 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012993939393939394, + "loss": 2.4939, + "theoretical_loss": 3.3313510370335617, + "tokens_seen": 2878799872 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001299191919191919, + "loss": 2.3634, + "theoretical_loss": 3.3313451571916475, + "tokens_seen": 2878865408 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1579013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6680827140808105, + "objective/train/theoretical_loss": 3.3313392775210615, + "objective/train/tokens_used": 1237790176, + "theoretical_loss": 3.3313392775210615, + "tokens_seen": 2878930944 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001298989898989899, + "loss": 2.4812, + "theoretical_loss": 3.3313392775210615, + "tokens_seen": 2878930944 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012987878787878787, + "loss": 2.4309, + "theoretical_loss": 3.331333398021794, + "tokens_seen": 2878996480 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012985858585858588, + "loss": 2.3457, + "theoretical_loss": 3.3313275186938367, + "tokens_seen": 2879062016 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012983838383838383, + "loss": 2.5355, + "theoretical_loss": 3.3313216395371805, + "tokens_seen": 2879127552 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012981818181818182, + "loss": 2.6198, + "theoretical_loss": 3.3313157605518167, + "tokens_seen": 2879193088 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001297979797979798, + "loss": 2.5985, + "theoretical_loss": 3.3313098817377362, + "tokens_seen": 2879258624 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012977777777777779, + "loss": 2.5384, + "theoretical_loss": 3.3313040030949304, + "tokens_seen": 2879324160 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012975757575757577, + "loss": 2.4109, + "theoretical_loss": 3.3312981246233897, + "tokens_seen": 2879389696 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012973737373737373, + "loss": 2.6019, + "theoretical_loss": 3.331292246323106, + "tokens_seen": 2879455232 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012971717171717174, + "loss": 2.4553, + "theoretical_loss": 3.33128636819407, + "tokens_seen": 2879520768 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001296969696969697, + "loss": 2.5529, + "theoretical_loss": 3.331280490236273, + "tokens_seen": 2879586304 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012967676767676768, + "loss": 2.5365, + "theoretical_loss": 3.3312746124497057, + "tokens_seen": 2879651840 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012965656565656566, + "loss": 2.586, + "theoretical_loss": 3.33126873483436, + "tokens_seen": 2879717376 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012963636363636365, + "loss": 2.4713, + "theoretical_loss": 3.331262857390226, + "tokens_seen": 2879782912 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012961616161616163, + "loss": 2.574, + "theoretical_loss": 3.331256980117296, + "tokens_seen": 2879848448 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012959595959595961, + "loss": 2.7233, + "theoretical_loss": 3.3312511030155605, + "tokens_seen": 2879913984 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012957575757575757, + "loss": 2.6158, + "theoretical_loss": 3.3312452260850103, + "tokens_seen": 2879979520 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012955555555555555, + "loss": 2.1771, + "theoretical_loss": 3.331239349325637, + "tokens_seen": 2880045056 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012953535353535354, + "loss": 2.5499, + "theoretical_loss": 3.331233472737431, + "tokens_seen": 2880110592 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012951515151515152, + "loss": 2.5681, + "theoretical_loss": 3.3312275963203843, + "tokens_seen": 2880176128 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001294949494949495, + "loss": 2.4406, + "theoretical_loss": 3.3312217200744882, + "tokens_seen": 2880241664 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012947474747474746, + "loss": 2.5194, + "theoretical_loss": 3.3312158439997326, + "tokens_seen": 2880307200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012945454545454547, + "loss": 2.5268, + "theoretical_loss": 3.33120996809611, + "tokens_seen": 2880372736 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012943434343434343, + "loss": 2.5914, + "theoretical_loss": 3.3312040923636106, + "tokens_seen": 2880438272 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012941414141414142, + "loss": 2.7321, + "theoretical_loss": 3.3311982168022256, + "tokens_seen": 2880503808 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1579013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5685291290283203, + "objective/train/theoretical_loss": 3.3311923414119464, + "objective/train/tokens_used": 1239428576, + "theoretical_loss": 3.3311923414119464, + "tokens_seen": 2880569344 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001293939393939394, + "loss": 2.5504, + "theoretical_loss": 3.3311923414119464, + "tokens_seen": 2880569344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012937373737373736, + "loss": 2.5391, + "theoretical_loss": 3.3311864661927637, + "tokens_seen": 2880634880 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012935353535353537, + "loss": 2.63, + "theoretical_loss": 3.3311805911446695, + "tokens_seen": 2880700416 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012933333333333332, + "loss": 2.4613, + "theoretical_loss": 3.331174716267654, + "tokens_seen": 2880765952 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012931313131313133, + "loss": 2.4308, + "theoretical_loss": 3.331168841561709, + "tokens_seen": 2880831488 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001292929292929293, + "loss": 2.782, + "theoretical_loss": 3.3311629670268252, + "tokens_seen": 2880897024 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012927272727272728, + "loss": 2.6603, + "theoretical_loss": 3.331157092662994, + "tokens_seen": 2880962560 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012925252525252526, + "loss": 2.6064, + "theoretical_loss": 3.331151218470206, + "tokens_seen": 2881028096 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012923232323232322, + "loss": 2.5751, + "theoretical_loss": 3.331145344448453, + "tokens_seen": 2881093632 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012921212121212123, + "loss": 2.6569, + "theoretical_loss": 3.331139470597726, + "tokens_seen": 2881159168 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012919191919191918, + "loss": 2.7374, + "theoretical_loss": 3.331133596918016, + "tokens_seen": 2881224704 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012917171717171717, + "loss": 2.6744, + "theoretical_loss": 3.3311277234093137, + "tokens_seen": 2881290240 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012915151515151515, + "loss": 2.5912, + "theoretical_loss": 3.331121850071611, + "tokens_seen": 2881355776 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012913131313131314, + "loss": 2.5224, + "theoretical_loss": 3.3311159769048984, + "tokens_seen": 2881421312 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012911111111111112, + "loss": 2.6241, + "theoretical_loss": 3.3311101039091673, + "tokens_seen": 2881486848 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001290909090909091, + "loss": 2.8217, + "theoretical_loss": 3.3311042310844092, + "tokens_seen": 2881552384 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012907070707070706, + "loss": 2.563, + "theoretical_loss": 3.3310983584306144, + "tokens_seen": 2881617920 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012905050505050505, + "loss": 2.7898, + "theoretical_loss": 3.331092485947775, + "tokens_seen": 2881683456 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012903030303030303, + "loss": 2.4229, + "theoretical_loss": 3.331086613635881, + "tokens_seen": 2881748992 + }, + { + "epoch": 0.74, + "learning_rate": 0.000129010101010101, + "loss": 2.5876, + "theoretical_loss": 3.331080741494925, + "tokens_seen": 2881814528 + }, + { + "epoch": 0.74, + "learning_rate": 0.000128989898989899, + "loss": 2.5954, + "theoretical_loss": 3.3310748695248966, + "tokens_seen": 2881880064 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012896969696969695, + "loss": 2.599, + "theoretical_loss": 3.3310689977257883, + "tokens_seen": 2881945600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012894949494949496, + "loss": 2.4765, + "theoretical_loss": 3.33106312609759, + "tokens_seen": 2882011136 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012892929292929292, + "loss": 2.6582, + "theoretical_loss": 3.331057254640294, + "tokens_seen": 2882076672 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012890909090909093, + "loss": 2.5969, + "theoretical_loss": 3.3310513833538904, + "tokens_seen": 2882142208 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1579530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.716421127319336, + "objective/train/theoretical_loss": 3.3310455122383713, + "objective/train/tokens_used": 1241066976, + "theoretical_loss": 3.3310455122383713, + "tokens_seen": 2882207744 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001288888888888889, + "loss": 2.5931, + "theoretical_loss": 3.3310455122383713, + "tokens_seen": 2882207744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012886868686868687, + "loss": 2.6591, + "theoretical_loss": 3.331039641293727, + "tokens_seen": 2882273280 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012884848484848486, + "loss": 2.603, + "theoretical_loss": 3.331033770519949, + "tokens_seen": 2882338816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012882828282828281, + "loss": 2.6704, + "theoretical_loss": 3.3310278999170286, + "tokens_seen": 2882404352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012880808080808083, + "loss": 2.3753, + "theoretical_loss": 3.3310220294849566, + "tokens_seen": 2882469888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012878787878787878, + "loss": 2.5352, + "theoretical_loss": 3.3310161592237244, + "tokens_seen": 2882535424 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001287676767676768, + "loss": 2.5545, + "theoretical_loss": 3.331010289133323, + "tokens_seen": 2882600960 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012874747474747475, + "loss": 2.4752, + "theoretical_loss": 3.331004419213744, + "tokens_seen": 2882666496 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001287272727272727, + "loss": 2.7771, + "theoretical_loss": 3.3309985494649776, + "tokens_seen": 2882732032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012870707070707072, + "loss": 2.4701, + "theoretical_loss": 3.3309926798870158, + "tokens_seen": 2882797568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012868686868686868, + "loss": 2.6399, + "theoretical_loss": 3.3309868104798492, + "tokens_seen": 2882863104 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012866666666666669, + "loss": 2.6802, + "theoretical_loss": 3.3309809412434697, + "tokens_seen": 2882928640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012864646464646464, + "loss": 2.3759, + "theoretical_loss": 3.3309750721778677, + "tokens_seen": 2882994176 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012862626262626263, + "loss": 2.6111, + "theoretical_loss": 3.3309692032830345, + "tokens_seen": 2883059712 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001286060606060606, + "loss": 2.7344, + "theoretical_loss": 3.3309633345589615, + "tokens_seen": 2883125248 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001285858585858586, + "loss": 2.5339, + "theoretical_loss": 3.3309574660056396, + "tokens_seen": 2883190784 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012856565656565658, + "loss": 2.4348, + "theoretical_loss": 3.33095159762306, + "tokens_seen": 2883256320 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012854545454545454, + "loss": 2.4636, + "theoretical_loss": 3.330945729411214, + "tokens_seen": 2883321856 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012852525252525252, + "loss": 2.5373, + "theoretical_loss": 3.3309398613700925, + "tokens_seen": 2883387392 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001285050505050505, + "loss": 2.6128, + "theoretical_loss": 3.330933993499687, + "tokens_seen": 2883452928 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001284848484848485, + "loss": 2.6247, + "theoretical_loss": 3.3309281257999883, + "tokens_seen": 2883518464 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012846464646464647, + "loss": 2.5304, + "theoretical_loss": 3.330922258270988, + "tokens_seen": 2883584000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012844444444444446, + "loss": 2.5874, + "theoretical_loss": 3.3309163909126767, + "tokens_seen": 2883649536 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001284242424242424, + "loss": 2.5335, + "theoretical_loss": 3.330910523725046, + "tokens_seen": 2883715072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012840404040404042, + "loss": 2.3832, + "theoretical_loss": 3.330904656708087, + "tokens_seen": 2883780608 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1580723, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.49957275390625, + "objective/train/theoretical_loss": 3.3308987898617906, + "objective/train/tokens_used": 1242705376, + "theoretical_loss": 3.3308987898617906, + "tokens_seen": 2883846144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012838383838383838, + "loss": 2.4458, + "theoretical_loss": 3.3308987898617906, + "tokens_seen": 2883846144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012836363636363636, + "loss": 2.4575, + "theoretical_loss": 3.3308929231861475, + "tokens_seen": 2883911680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012834343434343435, + "loss": 2.5383, + "theoretical_loss": 3.33088705668115, + "tokens_seen": 2883977216 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001283232323232323, + "loss": 2.7408, + "theoretical_loss": 3.3308811903467888, + "tokens_seen": 2884042752 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012830303030303032, + "loss": 2.6895, + "theoretical_loss": 3.330875324183055, + "tokens_seen": 2884108288 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012828282828282827, + "loss": 2.5044, + "theoretical_loss": 3.3308694581899396, + "tokens_seen": 2884173824 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012826262626262628, + "loss": 2.5539, + "theoretical_loss": 3.330863592367434, + "tokens_seen": 2884239360 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012824242424242424, + "loss": 2.4867, + "theoretical_loss": 3.330857726715529, + "tokens_seen": 2884304896 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012822222222222222, + "loss": 2.6183, + "theoretical_loss": 3.3308518612342164, + "tokens_seen": 2884370432 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001282020202020202, + "loss": 2.6634, + "theoretical_loss": 3.3308459959234864, + "tokens_seen": 2884435968 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012818181818181817, + "loss": 2.4357, + "theoretical_loss": 3.330840130783331, + "tokens_seen": 2884501504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012816161616161618, + "loss": 2.5849, + "theoretical_loss": 3.330834265813741, + "tokens_seen": 2884567040 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012814141414141413, + "loss": 2.5148, + "theoretical_loss": 3.3308284010147085, + "tokens_seen": 2884632576 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012812121212121212, + "loss": 2.5642, + "theoretical_loss": 3.330822536386223, + "tokens_seen": 2884698112 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001281010101010101, + "loss": 2.571, + "theoretical_loss": 3.3308166719282766, + "tokens_seen": 2884763648 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012808080808080809, + "loss": 2.5833, + "theoretical_loss": 3.3308108076408605, + "tokens_seen": 2884829184 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012806060606060607, + "loss": 2.5071, + "theoretical_loss": 3.3308049435239657, + "tokens_seen": 2884894720 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012804040404040405, + "loss": 2.3928, + "theoretical_loss": 3.3307990795775835, + "tokens_seen": 2884960256 + }, + { + "epoch": 0.75, + "learning_rate": 0.000128020202020202, + "loss": 2.6011, + "theoretical_loss": 3.330793215801705, + "tokens_seen": 2885025792 + }, + { + "epoch": 0.75, + "learning_rate": 0.000128, + "loss": 2.6392, + "theoretical_loss": 3.330787352196321, + "tokens_seen": 2885091328 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012797979797979798, + "loss": 2.5475, + "theoretical_loss": 3.3307814887614233, + "tokens_seen": 2885156864 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012795959595959596, + "loss": 2.5647, + "theoretical_loss": 3.3307756254970027, + "tokens_seen": 2885222400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012793939393939395, + "loss": 2.4572, + "theoretical_loss": 3.3307697624030506, + "tokens_seen": 2885287936 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012791919191919193, + "loss": 2.5809, + "theoretical_loss": 3.330763899479558, + "tokens_seen": 2885353472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012789898989898991, + "loss": 2.6237, + "theoretical_loss": 3.330758036726516, + "tokens_seen": 2885419008 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1581802, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3045804500579834, + "objective/train/theoretical_loss": 3.3307521741439157, + "objective/train/tokens_used": 1244343776, + "theoretical_loss": 3.3307521741439157, + "tokens_seen": 2885484544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012787878787878787, + "loss": 2.5989, + "theoretical_loss": 3.3307521741439157, + "tokens_seen": 2885484544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012785858585858588, + "loss": 2.2709, + "theoretical_loss": 3.330746311731749, + "tokens_seen": 2885550080 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012783838383838384, + "loss": 2.572, + "theoretical_loss": 3.330740449490006, + "tokens_seen": 2885615616 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012781818181818182, + "loss": 2.5794, + "theoretical_loss": 3.3307345874186787, + "tokens_seen": 2885681152 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001277979797979798, + "loss": 2.6367, + "theoretical_loss": 3.3307287255177576, + "tokens_seen": 2885746688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012777777777777776, + "loss": 2.5196, + "theoretical_loss": 3.3307228637872344, + "tokens_seen": 2885812224 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012775757575757577, + "loss": 2.3319, + "theoretical_loss": 3.3307170022271, + "tokens_seen": 2885877760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012773737373737373, + "loss": 2.6547, + "theoretical_loss": 3.330711140837346, + "tokens_seen": 2885943296 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012771717171717174, + "loss": 2.6076, + "theoretical_loss": 3.3307052796179635, + "tokens_seen": 2886008832 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001276969696969697, + "loss": 2.7076, + "theoretical_loss": 3.330699418568943, + "tokens_seen": 2886074368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012767676767676766, + "loss": 2.5505, + "theoretical_loss": 3.3306935576902763, + "tokens_seen": 2886139904 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012765656565656567, + "loss": 2.6927, + "theoretical_loss": 3.3306876969819545, + "tokens_seen": 2886205440 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012763636363636362, + "loss": 2.3875, + "theoretical_loss": 3.3306818364439685, + "tokens_seen": 2886270976 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012761616161616164, + "loss": 2.6869, + "theoretical_loss": 3.33067597607631, + "tokens_seen": 2886336512 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001275959595959596, + "loss": 2.3189, + "theoretical_loss": 3.33067011587897, + "tokens_seen": 2886402048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012757575757575758, + "loss": 2.4985, + "theoretical_loss": 3.3306642558519393, + "tokens_seen": 2886467584 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012755555555555556, + "loss": 2.5011, + "theoretical_loss": 3.330658395995209, + "tokens_seen": 2886533120 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012753535353535354, + "loss": 2.6026, + "theoretical_loss": 3.330652536308771, + "tokens_seen": 2886598656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012751515151515153, + "loss": 2.6557, + "theoretical_loss": 3.330646676792616, + "tokens_seen": 2886664192 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012749494949494948, + "loss": 2.4972, + "theoretical_loss": 3.330640817446735, + "tokens_seen": 2886729728 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012747474747474747, + "loss": 2.608, + "theoretical_loss": 3.33063495827112, + "tokens_seen": 2886795264 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012745454545454545, + "loss": 2.4411, + "theoretical_loss": 3.3306290992657615, + "tokens_seen": 2886860800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012743434343434344, + "loss": 2.5363, + "theoretical_loss": 3.3306232404306506, + "tokens_seen": 2886926336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012741414141414142, + "loss": 2.7718, + "theoretical_loss": 3.330617381765779, + "tokens_seen": 2886991872 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001273939393939394, + "loss": 2.5615, + "theoretical_loss": 3.3306115232711377, + "tokens_seen": 2887057408 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1582344, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6120378971099854, + "objective/train/theoretical_loss": 3.3306056649467175, + "objective/train/tokens_used": 1245982176, + "theoretical_loss": 3.3306056649467175, + "tokens_seen": 2887122944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012737373737373736, + "loss": 2.4529, + "theoretical_loss": 3.3306056649467175, + "tokens_seen": 2887122944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012735353535353537, + "loss": 2.5494, + "theoretical_loss": 3.33059980679251, + "tokens_seen": 2887188480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012733333333333333, + "loss": 2.5858, + "theoretical_loss": 3.330593948808507, + "tokens_seen": 2887254016 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001273131313131313, + "loss": 2.7183, + "theoretical_loss": 3.3305880909946977, + "tokens_seen": 2887319552 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001272929292929293, + "loss": 2.553, + "theoretical_loss": 3.3305822333510755, + "tokens_seen": 2887385088 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012727272727272725, + "loss": 2.4378, + "theoretical_loss": 3.3305763758776306, + "tokens_seen": 2887450624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012725252525252527, + "loss": 2.5672, + "theoretical_loss": 3.330570518574354, + "tokens_seen": 2887516160 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012723232323232322, + "loss": 2.4419, + "theoretical_loss": 3.330564661441237, + "tokens_seen": 2887581696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012721212121212123, + "loss": 2.6502, + "theoretical_loss": 3.3305588044782715, + "tokens_seen": 2887647232 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001271919191919192, + "loss": 2.5885, + "theoretical_loss": 3.330552947685448, + "tokens_seen": 2887712768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012717171717171717, + "loss": 2.4093, + "theoretical_loss": 3.3305470910627575, + "tokens_seen": 2887778304 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012715151515151516, + "loss": 2.6335, + "theoretical_loss": 3.330541234610192, + "tokens_seen": 2887843840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012713131313131311, + "loss": 2.5169, + "theoretical_loss": 3.330535378327742, + "tokens_seen": 2887909376 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012711111111111113, + "loss": 2.4491, + "theoretical_loss": 3.330529522215399, + "tokens_seen": 2887974912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012709090909090908, + "loss": 2.7864, + "theoretical_loss": 3.3305236662731543, + "tokens_seen": 2888040448 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001270707070707071, + "loss": 2.4938, + "theoretical_loss": 3.3305178105009987, + "tokens_seen": 2888105984 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012705050505050505, + "loss": 2.4621, + "theoretical_loss": 3.330511954898924, + "tokens_seen": 2888171520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012703030303030303, + "loss": 2.5799, + "theoretical_loss": 3.3305060994669207, + "tokens_seen": 2888237056 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012701010101010102, + "loss": 2.3666, + "theoretical_loss": 3.3305002442049805, + "tokens_seen": 2888302592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012698989898989898, + "loss": 2.5718, + "theoretical_loss": 3.330494389113094, + "tokens_seen": 2888368128 + }, + { + "epoch": 0.75, + "learning_rate": 0.000126969696969697, + "loss": 2.6118, + "theoretical_loss": 3.3304885341912533, + "tokens_seen": 2888433664 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012694949494949494, + "loss": 2.6016, + "theoretical_loss": 3.330482679439449, + "tokens_seen": 2888499200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012692929292929293, + "loss": 2.5327, + "theoretical_loss": 3.330476824857673, + "tokens_seen": 2888564736 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001269090909090909, + "loss": 2.3624, + "theoretical_loss": 3.3304709704459152, + "tokens_seen": 2888630272 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001268888888888889, + "loss": 2.4556, + "theoretical_loss": 3.330465116204168, + "tokens_seen": 2888695808 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1583524, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4346120357513428, + "objective/train/theoretical_loss": 3.330459262132422, + "objective/train/tokens_used": 1247620576, + "theoretical_loss": 3.330459262132422, + "tokens_seen": 2888761344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012686868686868688, + "loss": 2.5956, + "theoretical_loss": 3.330459262132422, + "tokens_seen": 2888761344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012684848484848486, + "loss": 2.6182, + "theoretical_loss": 3.3304534082306683, + "tokens_seen": 2888826880 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012682828282828282, + "loss": 2.5778, + "theoretical_loss": 3.3304475544988987, + "tokens_seen": 2888892416 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001268080808080808, + "loss": 2.5715, + "theoretical_loss": 3.3304417009371043, + "tokens_seen": 2888957952 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001267878787878788, + "loss": 2.3892, + "theoretical_loss": 3.330435847545276, + "tokens_seen": 2889023488 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012676767676767677, + "loss": 2.5917, + "theoretical_loss": 3.330429994323405, + "tokens_seen": 2889089024 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012674747474747476, + "loss": 2.367, + "theoretical_loss": 3.330424141271483, + "tokens_seen": 2889154560 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001267272727272727, + "loss": 2.6833, + "theoretical_loss": 3.3304182883895, + "tokens_seen": 2889220096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012670707070707072, + "loss": 2.4373, + "theoretical_loss": 3.3304124356774487, + "tokens_seen": 2889285632 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012668686868686868, + "loss": 2.5892, + "theoretical_loss": 3.3304065831353196, + "tokens_seen": 2889351168 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001266666666666667, + "loss": 2.5289, + "theoretical_loss": 3.330400730763104, + "tokens_seen": 2889416704 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012664646464646465, + "loss": 2.5073, + "theoretical_loss": 3.330394878560793, + "tokens_seen": 2889482240 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001266262626262626, + "loss": 2.641, + "theoretical_loss": 3.3303890265283775, + "tokens_seen": 2889547776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012660606060606062, + "loss": 2.5748, + "theoretical_loss": 3.3303831746658497, + "tokens_seen": 2889613312 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012658585858585857, + "loss": 2.4354, + "theoretical_loss": 3.3303773229731997, + "tokens_seen": 2889678848 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012656565656565658, + "loss": 2.354, + "theoretical_loss": 3.3303714714504196, + "tokens_seen": 2889744384 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012654545454545454, + "loss": 2.5986, + "theoretical_loss": 3.3303656200975005, + "tokens_seen": 2889809920 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012652525252525253, + "loss": 2.7758, + "theoretical_loss": 3.330359768914433, + "tokens_seen": 2889875456 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001265050505050505, + "loss": 2.5902, + "theoretical_loss": 3.330353917901209, + "tokens_seen": 2889940992 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001264848484848485, + "loss": 2.5372, + "theoretical_loss": 3.330348067057819, + "tokens_seen": 2890006528 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012646464646464648, + "loss": 2.4706, + "theoretical_loss": 3.330342216384255, + "tokens_seen": 2890072064 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012644444444444443, + "loss": 2.5858, + "theoretical_loss": 3.3303363658805076, + "tokens_seen": 2890137600 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012642424242424242, + "loss": 2.7072, + "theoretical_loss": 3.3303305155465686, + "tokens_seen": 2890203136 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001264040404040404, + "loss": 2.7154, + "theoretical_loss": 3.3303246653824283, + "tokens_seen": 2890268672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012638383838383839, + "loss": 2.5102, + "theoretical_loss": 3.330318815388079, + "tokens_seen": 2890334208 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1584612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.49517822265625, + "objective/train/theoretical_loss": 3.3303129655635115, + "objective/train/tokens_used": 1249258976, + "theoretical_loss": 3.3303129655635115, + "tokens_seen": 2890399744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012636363636363637, + "loss": 2.5534, + "theoretical_loss": 3.3303129655635115, + "tokens_seen": 2890399744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012634343434343435, + "loss": 2.7334, + "theoretical_loss": 3.3303071159087168, + "tokens_seen": 2890465280 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001263232323232323, + "loss": 2.6828, + "theoretical_loss": 3.330301266423686, + "tokens_seen": 2890530816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012630303030303032, + "loss": 2.8244, + "theoretical_loss": 3.3302954171084114, + "tokens_seen": 2890596352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012628282828282828, + "loss": 2.502, + "theoretical_loss": 3.3302895679628826, + "tokens_seen": 2890661888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012626262626262626, + "loss": 2.7659, + "theoretical_loss": 3.330283718987092, + "tokens_seen": 2890727424 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012624242424242425, + "loss": 2.8123, + "theoretical_loss": 3.3302778701810305, + "tokens_seen": 2890792960 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012622222222222223, + "loss": 2.3651, + "theoretical_loss": 3.3302720215446895, + "tokens_seen": 2890858496 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012620202020202021, + "loss": 2.3899, + "theoretical_loss": 3.33026617307806, + "tokens_seen": 2890924032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012618181818181817, + "loss": 2.7415, + "theoretical_loss": 3.3302603247811327, + "tokens_seen": 2890989568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012616161616161618, + "loss": 2.3822, + "theoretical_loss": 3.3302544766539, + "tokens_seen": 2891055104 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012614141414141414, + "loss": 2.8228, + "theoretical_loss": 3.3302486286963524, + "tokens_seen": 2891120640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012612121212121212, + "loss": 2.4043, + "theoretical_loss": 3.330242780908481, + "tokens_seen": 2891186176 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001261010101010101, + "loss": 2.4807, + "theoretical_loss": 3.3302369332902777, + "tokens_seen": 2891251712 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012608080808080806, + "loss": 2.4392, + "theoretical_loss": 3.330231085841733, + "tokens_seen": 2891317248 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012606060606060608, + "loss": 2.5825, + "theoretical_loss": 3.3302252385628384, + "tokens_seen": 2891382784 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012604040404040403, + "loss": 2.464, + "theoretical_loss": 3.3302193914535856, + "tokens_seen": 2891448320 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012602020202020204, + "loss": 2.4951, + "theoretical_loss": 3.330213544513965, + "tokens_seen": 2891513856 + }, + { + "epoch": 0.75, + "learning_rate": 0.000126, + "loss": 2.6029, + "theoretical_loss": 3.3302076977439685, + "tokens_seen": 2891579392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012597979797979798, + "loss": 2.6289, + "theoretical_loss": 3.3302018511435874, + "tokens_seen": 2891644928 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012595959595959597, + "loss": 2.671, + "theoretical_loss": 3.330196004712812, + "tokens_seen": 2891710464 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012593939393939392, + "loss": 2.4029, + "theoretical_loss": 3.3301901584516345, + "tokens_seen": 2891776000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012591919191919194, + "loss": 2.5978, + "theoretical_loss": 3.330184312360046, + "tokens_seen": 2891841536 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001258989898989899, + "loss": 2.5886, + "theoretical_loss": 3.3301784664380367, + "tokens_seen": 2891907072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012587878787878788, + "loss": 2.4996, + "theoretical_loss": 3.3301726206855995, + "tokens_seen": 2891972608 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1585177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3358516693115234, + "objective/train/theoretical_loss": 3.3301667751027244, + "objective/train/tokens_used": 1250897376, + "theoretical_loss": 3.3301667751027244, + "tokens_seen": 2892038144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012585858585858586, + "loss": 2.3557, + "theoretical_loss": 3.3301667751027244, + "tokens_seen": 2892038144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012583838383838384, + "loss": 2.3841, + "theoretical_loss": 3.3301609296894035, + "tokens_seen": 2892103680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012581818181818183, + "loss": 2.5414, + "theoretical_loss": 3.330155084445627, + "tokens_seen": 2892169216 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001257979797979798, + "loss": 2.6892, + "theoretical_loss": 3.330149239371387, + "tokens_seen": 2892234752 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012577777777777777, + "loss": 2.5605, + "theoretical_loss": 3.3301433944666745, + "tokens_seen": 2892300288 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012575757575757575, + "loss": 2.4792, + "theoretical_loss": 3.330137549731481, + "tokens_seen": 2892365824 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012573737373737374, + "loss": 2.5647, + "theoretical_loss": 3.330131705165797, + "tokens_seen": 2892431360 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012571717171717172, + "loss": 2.6976, + "theoretical_loss": 3.3301258607696145, + "tokens_seen": 2892496896 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001256969696969697, + "loss": 2.3855, + "theoretical_loss": 3.3301200165429243, + "tokens_seen": 2892562432 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012567676767676766, + "loss": 2.5059, + "theoretical_loss": 3.3301141724857177, + "tokens_seen": 2892627968 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012565656565656567, + "loss": 2.5914, + "theoretical_loss": 3.3301083285979862, + "tokens_seen": 2892693504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012563636363636363, + "loss": 2.6148, + "theoretical_loss": 3.330102484879721, + "tokens_seen": 2892759040 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012561616161616164, + "loss": 2.6007, + "theoretical_loss": 3.330096641330913, + "tokens_seen": 2892824576 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001255959595959596, + "loss": 2.5319, + "theoretical_loss": 3.330090797951554, + "tokens_seen": 2892890112 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012557575757575755, + "loss": 2.8228, + "theoretical_loss": 3.3300849547416345, + "tokens_seen": 2892955648 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012555555555555557, + "loss": 2.4521, + "theoretical_loss": 3.3300791117011466, + "tokens_seen": 2893021184 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012553535353535352, + "loss": 2.5307, + "theoretical_loss": 3.3300732688300805, + "tokens_seen": 2893086720 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012551515151515153, + "loss": 2.5557, + "theoretical_loss": 3.3300674261284287, + "tokens_seen": 2893152256 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001254949494949495, + "loss": 2.4677, + "theoretical_loss": 3.3300615835961813, + "tokens_seen": 2893217792 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012547474747474747, + "loss": 2.5752, + "theoretical_loss": 3.3300557412333305, + "tokens_seen": 2893283328 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012545454545454546, + "loss": 2.7187, + "theoretical_loss": 3.330049899039867, + "tokens_seen": 2893348864 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012543434343434342, + "loss": 2.5278, + "theoretical_loss": 3.330044057015782, + "tokens_seen": 2893414400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012541414141414143, + "loss": 2.4531, + "theoretical_loss": 3.3300382151610672, + "tokens_seen": 2893479936 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012539393939393938, + "loss": 2.427, + "theoretical_loss": 3.330032373475713, + "tokens_seen": 2893545472 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001253737373737374, + "loss": 2.5871, + "theoretical_loss": 3.330026531959712, + "tokens_seen": 2893611008 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1586411, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4675378799438477, + "objective/train/theoretical_loss": 3.3300206906130545, + "objective/train/tokens_used": 1252535776, + "theoretical_loss": 3.3300206906130545, + "tokens_seen": 2893676544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012535353535353535, + "loss": 2.4176, + "theoretical_loss": 3.3300206906130545, + "tokens_seen": 2893676544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012533333333333334, + "loss": 2.6462, + "theoretical_loss": 3.330014849435732, + "tokens_seen": 2893742080 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012531313131313132, + "loss": 2.4385, + "theoretical_loss": 3.3300090084277354, + "tokens_seen": 2893807616 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001252929292929293, + "loss": 2.6407, + "theoretical_loss": 3.3300031675890565, + "tokens_seen": 2893873152 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001252727272727273, + "loss": 2.2817, + "theoretical_loss": 3.329997326919686, + "tokens_seen": 2893938688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012525252525252524, + "loss": 2.5449, + "theoretical_loss": 3.329991486419616, + "tokens_seen": 2894004224 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012523232323232323, + "loss": 2.5682, + "theoretical_loss": 3.329985646088837, + "tokens_seen": 2894069760 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001252121212121212, + "loss": 2.6317, + "theoretical_loss": 3.32997980592734, + "tokens_seen": 2894135296 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001251919191919192, + "loss": 2.5096, + "theoretical_loss": 3.3299739659351175, + "tokens_seen": 2894200832 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012517171717171718, + "loss": 2.7948, + "theoretical_loss": 3.3299681261121594, + "tokens_seen": 2894266368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012515151515151516, + "loss": 2.4156, + "theoretical_loss": 3.329962286458458, + "tokens_seen": 2894331904 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012513131313131312, + "loss": 2.4449, + "theoretical_loss": 3.329956446974004, + "tokens_seen": 2894397440 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012511111111111113, + "loss": 2.6396, + "theoretical_loss": 3.329950607658789, + "tokens_seen": 2894462976 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001250909090909091, + "loss": 2.4788, + "theoretical_loss": 3.329944768512804, + "tokens_seen": 2894528512 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012507070707070707, + "loss": 2.505, + "theoretical_loss": 3.32993892953604, + "tokens_seen": 2894594048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012505050505050506, + "loss": 2.6566, + "theoretical_loss": 3.3299330907284888, + "tokens_seen": 2894659584 + }, + { + "epoch": 0.75, + "learning_rate": 0.000125030303030303, + "loss": 2.5622, + "theoretical_loss": 3.3299272520901413, + "tokens_seen": 2894725120 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012501010101010102, + "loss": 2.6655, + "theoretical_loss": 3.3299214136209896, + "tokens_seen": 2894790656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012498989898989898, + "loss": 2.5463, + "theoretical_loss": 3.3299155753210234, + "tokens_seen": 2894856192 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012496969696969696, + "loss": 2.4504, + "theoretical_loss": 3.3299097371902358, + "tokens_seen": 2894921728 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012494949494949495, + "loss": 2.3726, + "theoretical_loss": 3.3299038992286163, + "tokens_seen": 2894987264 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012492929292929293, + "loss": 2.4429, + "theoretical_loss": 3.3298980614361575, + "tokens_seen": 2895052800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012490909090909092, + "loss": 2.523, + "theoretical_loss": 3.3298922238128497, + "tokens_seen": 2895118336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012488888888888887, + "loss": 2.5631, + "theoretical_loss": 3.3298863863586847, + "tokens_seen": 2895183872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012486868686868686, + "loss": 2.5908, + "theoretical_loss": 3.3298805490736543, + "tokens_seen": 2895249408 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1586924, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.826359987258911, + "objective/train/theoretical_loss": 3.329874711957749, + "objective/train/tokens_used": 1254174176, + "theoretical_loss": 3.329874711957749, + "tokens_seen": 2895314944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012484848484848484, + "loss": 2.8359, + "theoretical_loss": 3.329874711957749, + "tokens_seen": 2895314944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012482828282828283, + "loss": 2.4196, + "theoretical_loss": 3.32986887501096, + "tokens_seen": 2895380480 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001248080808080808, + "loss": 2.216, + "theoretical_loss": 3.329863038233279, + "tokens_seen": 2895446016 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001247878787878788, + "loss": 2.6644, + "theoretical_loss": 3.3298572016246966, + "tokens_seen": 2895511552 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012476767676767678, + "loss": 2.6605, + "theoretical_loss": 3.329851365185205, + "tokens_seen": 2895577088 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012474747474747476, + "loss": 2.3819, + "theoretical_loss": 3.329845528914795, + "tokens_seen": 2895642624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012472727272727272, + "loss": 2.552, + "theoretical_loss": 3.329839692813458, + "tokens_seen": 2895708160 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001247070707070707, + "loss": 2.6716, + "theoretical_loss": 3.329833856881185, + "tokens_seen": 2895773696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012468686868686869, + "loss": 2.667, + "theoretical_loss": 3.3298280211179674, + "tokens_seen": 2895839232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012466666666666667, + "loss": 2.5174, + "theoretical_loss": 3.329822185523797, + "tokens_seen": 2895904768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012464646464646465, + "loss": 2.4265, + "theoretical_loss": 3.329816350098664, + "tokens_seen": 2895970304 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012462626262626264, + "loss": 2.4937, + "theoretical_loss": 3.3298105148425607, + "tokens_seen": 2896035840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012460606060606062, + "loss": 2.4482, + "theoretical_loss": 3.329804679755478, + "tokens_seen": 2896101376 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001245858585858586, + "loss": 2.5588, + "theoretical_loss": 3.3297988448374074, + "tokens_seen": 2896166912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012456565656565656, + "loss": 2.6058, + "theoretical_loss": 3.3297930100883395, + "tokens_seen": 2896232448 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012454545454545455, + "loss": 2.5103, + "theoretical_loss": 3.329787175508266, + "tokens_seen": 2896297984 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012452525252525253, + "loss": 2.4586, + "theoretical_loss": 3.3297813410971786, + "tokens_seen": 2896363520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012450505050505051, + "loss": 2.6056, + "theoretical_loss": 3.329775506855068, + "tokens_seen": 2896429056 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001244848484848485, + "loss": 2.4938, + "theoretical_loss": 3.3297696727819255, + "tokens_seen": 2896494592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012446464646464646, + "loss": 2.5724, + "theoretical_loss": 3.329763838877743, + "tokens_seen": 2896560128 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012444444444444444, + "loss": 2.4748, + "theoretical_loss": 3.329758005142511, + "tokens_seen": 2896625664 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012442424242424242, + "loss": 2.628, + "theoretical_loss": 3.329752171576221, + "tokens_seen": 2896691200 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001244040404040404, + "loss": 2.4469, + "theoretical_loss": 3.3297463381788646, + "tokens_seen": 2896756736 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001243838383838384, + "loss": 2.32, + "theoretical_loss": 3.3297405049504327, + "tokens_seen": 2896822272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012436363636363635, + "loss": 2.5069, + "theoretical_loss": 3.3297346718909173, + "tokens_seen": 2896887808 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1587902, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.844057083129883, + "objective/train/theoretical_loss": 3.3297288390003086, + "objective/train/tokens_used": 1255812576, + "theoretical_loss": 3.3297288390003086, + "tokens_seen": 2896953344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012434343434343433, + "loss": 2.6647, + "theoretical_loss": 3.3297288390003086, + "tokens_seen": 2896953344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012432323232323232, + "loss": 2.4749, + "theoretical_loss": 3.3297230062785985, + "tokens_seen": 2897018880 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001243030303030303, + "loss": 2.5702, + "theoretical_loss": 3.3297171737257787, + "tokens_seen": 2897084416 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012428282828282828, + "loss": 2.4597, + "theoretical_loss": 3.3297113413418398, + "tokens_seen": 2897149952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012426262626262627, + "loss": 2.62, + "theoretical_loss": 3.3297055091267733, + "tokens_seen": 2897215488 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012424242424242425, + "loss": 2.601, + "theoretical_loss": 3.3296996770805705, + "tokens_seen": 2897281024 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012422222222222224, + "loss": 2.6143, + "theoretical_loss": 3.3296938452032228, + "tokens_seen": 2897346560 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001242020202020202, + "loss": 2.4222, + "theoretical_loss": 3.3296880134947213, + "tokens_seen": 2897412096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012418181818181818, + "loss": 2.7361, + "theoretical_loss": 3.3296821819550573, + "tokens_seen": 2897477632 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012416161616161616, + "loss": 2.5655, + "theoretical_loss": 3.329676350584222, + "tokens_seen": 2897543168 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012414141414141414, + "loss": 2.6643, + "theoretical_loss": 3.3296705193822076, + "tokens_seen": 2897608704 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012412121212121213, + "loss": 2.4182, + "theoretical_loss": 3.329664688349004, + "tokens_seen": 2897674240 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001241010101010101, + "loss": 2.6213, + "theoretical_loss": 3.3296588574846036, + "tokens_seen": 2897739776 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001240808080808081, + "loss": 2.5788, + "theoretical_loss": 3.329653026788997, + "tokens_seen": 2897805312 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012406060606060608, + "loss": 2.6695, + "theoretical_loss": 3.329647196262176, + "tokens_seen": 2897870848 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012404040404040404, + "loss": 2.5378, + "theoretical_loss": 3.3296413659041315, + "tokens_seen": 2897936384 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012402020202020202, + "loss": 2.4129, + "theoretical_loss": 3.329635535714855, + "tokens_seen": 2898001920 + }, + { + "epoch": 0.75, + "learning_rate": 0.000124, + "loss": 2.6171, + "theoretical_loss": 3.329629705694338, + "tokens_seen": 2898067456 + }, + { + "epoch": 0.75, + "learning_rate": 0.000123979797979798, + "loss": 2.5395, + "theoretical_loss": 3.329623875842571, + "tokens_seen": 2898132992 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012395959595959597, + "loss": 2.6542, + "theoretical_loss": 3.329618046159546, + "tokens_seen": 2898198528 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012393939393939393, + "loss": 2.5076, + "theoretical_loss": 3.3296122166452546, + "tokens_seen": 2898264064 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012391919191919191, + "loss": 2.6504, + "theoretical_loss": 3.329606387299687, + "tokens_seen": 2898329600 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001238989898989899, + "loss": 2.548, + "theoretical_loss": 3.3296005581228356, + "tokens_seen": 2898395136 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012387878787878788, + "loss": 2.5327, + "theoretical_loss": 3.3295947291146915, + "tokens_seen": 2898460672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012385858585858587, + "loss": 2.6431, + "theoretical_loss": 3.3295889002752452, + "tokens_seen": 2898526208 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1588590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.681248188018799, + "objective/train/theoretical_loss": 3.329583071604489, + "objective/train/tokens_used": 1257450976, + "theoretical_loss": 3.329583071604489, + "tokens_seen": 2898591744 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012383838383838385, + "loss": 2.6747, + "theoretical_loss": 3.329583071604489, + "tokens_seen": 2898591744 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001238181818181818, + "loss": 2.6345, + "theoretical_loss": 3.3295772431024133, + "tokens_seen": 2898657280 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001237979797979798, + "loss": 2.5725, + "theoretical_loss": 3.32957141476901, + "tokens_seen": 2898722816 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012377777777777777, + "loss": 2.5111, + "theoretical_loss": 3.3295655866042706, + "tokens_seen": 2898788352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012375757575757576, + "loss": 2.5602, + "theoretical_loss": 3.3295597586081858, + "tokens_seen": 2898853888 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012373737373737374, + "loss": 2.4543, + "theoretical_loss": 3.3295539307807474, + "tokens_seen": 2898919424 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012371717171717173, + "loss": 2.617, + "theoretical_loss": 3.3295481031219465, + "tokens_seen": 2898984960 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012369696969696968, + "loss": 2.3812, + "theoretical_loss": 3.3295422756317743, + "tokens_seen": 2899050496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012367676767676767, + "loss": 2.2915, + "theoretical_loss": 3.329536448310222, + "tokens_seen": 2899116032 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012365656565656565, + "loss": 2.5119, + "theoretical_loss": 3.3295306211572813, + "tokens_seen": 2899181568 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012363636363636364, + "loss": 2.4825, + "theoretical_loss": 3.3295247941729436, + "tokens_seen": 2899247104 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012361616161616162, + "loss": 2.662, + "theoretical_loss": 3.3295189673572, + "tokens_seen": 2899312640 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001235959595959596, + "loss": 2.4252, + "theoretical_loss": 3.3295131407100413, + "tokens_seen": 2899378176 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001235757575757576, + "loss": 2.6129, + "theoretical_loss": 3.329507314231459, + "tokens_seen": 2899443712 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012355555555555557, + "loss": 2.3926, + "theoretical_loss": 3.3295014879214455, + "tokens_seen": 2899509248 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012353535353535356, + "loss": 2.4245, + "theoretical_loss": 3.329495661779991, + "tokens_seen": 2899574784 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001235151515151515, + "loss": 2.7297, + "theoretical_loss": 3.329489835807087, + "tokens_seen": 2899640320 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001234949494949495, + "loss": 2.7125, + "theoretical_loss": 3.3294840100027248, + "tokens_seen": 2899705856 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012347474747474748, + "loss": 2.4379, + "theoretical_loss": 3.329478184366896, + "tokens_seen": 2899771392 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012345454545454546, + "loss": 2.7569, + "theoretical_loss": 3.329472358899592, + "tokens_seen": 2899836928 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012343434343434345, + "loss": 2.6384, + "theoretical_loss": 3.3294665336008036, + "tokens_seen": 2899902464 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012341414141414143, + "loss": 2.559, + "theoretical_loss": 3.3294607084705223, + "tokens_seen": 2899968000 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001233939393939394, + "loss": 2.5799, + "theoretical_loss": 3.3294548835087396, + "tokens_seen": 2900033536 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012337373737373737, + "loss": 2.4447, + "theoretical_loss": 3.329449058715447, + "tokens_seen": 2900099072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012335353535353536, + "loss": 2.475, + "theoretical_loss": 3.3294432340906353, + "tokens_seen": 2900164608 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1589771, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4306414127349854, + "objective/train/theoretical_loss": 3.329437409634296, + "objective/train/tokens_used": 1259089376, + "theoretical_loss": 3.329437409634296, + "tokens_seen": 2900230144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012333333333333334, + "loss": 2.492, + "theoretical_loss": 3.329437409634296, + "tokens_seen": 2900230144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012331313131313132, + "loss": 2.5941, + "theoretical_loss": 3.3294315853464207, + "tokens_seen": 2900295680 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012329292929292928, + "loss": 2.5096, + "theoretical_loss": 3.329425761227, + "tokens_seen": 2900361216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012327272727272727, + "loss": 2.3844, + "theoretical_loss": 3.3294199372760263, + "tokens_seen": 2900426752 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012325252525252525, + "loss": 2.272, + "theoretical_loss": 3.32941411349349, + "tokens_seen": 2900492288 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012323232323232323, + "loss": 2.6005, + "theoretical_loss": 3.329408289879383, + "tokens_seen": 2900557824 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012321212121212122, + "loss": 2.4552, + "theoretical_loss": 3.329402466433696, + "tokens_seen": 2900623360 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001231919191919192, + "loss": 2.3619, + "theoretical_loss": 3.3293966431564215, + "tokens_seen": 2900688896 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012317171717171716, + "loss": 2.5482, + "theoretical_loss": 3.329390820047549, + "tokens_seen": 2900754432 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012315151515151514, + "loss": 2.6123, + "theoretical_loss": 3.3293849971070717, + "tokens_seen": 2900819968 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012313131313131313, + "loss": 2.5832, + "theoretical_loss": 3.3293791743349797, + "tokens_seen": 2900885504 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001231111111111111, + "loss": 2.8217, + "theoretical_loss": 3.329373351731265, + "tokens_seen": 2900951040 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001230909090909091, + "loss": 2.6802, + "theoretical_loss": 3.3293675292959186, + "tokens_seen": 2901016576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012307070707070708, + "loss": 2.6926, + "theoretical_loss": 3.329361707028932, + "tokens_seen": 2901082112 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012305050505050506, + "loss": 2.4373, + "theoretical_loss": 3.3293558849302958, + "tokens_seen": 2901147648 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012303030303030305, + "loss": 2.501, + "theoretical_loss": 3.3293500630000024, + "tokens_seen": 2901213184 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012301010101010103, + "loss": 2.5419, + "theoretical_loss": 3.3293442412380427, + "tokens_seen": 2901278720 + }, + { + "epoch": 0.76, + "learning_rate": 0.000122989898989899, + "loss": 2.4127, + "theoretical_loss": 3.329338419644408, + "tokens_seen": 2901344256 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012296969696969697, + "loss": 2.6965, + "theoretical_loss": 3.3293325982190893, + "tokens_seen": 2901409792 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012294949494949495, + "loss": 2.6856, + "theoretical_loss": 3.3293267769620787, + "tokens_seen": 2901475328 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012292929292929294, + "loss": 2.5207, + "theoretical_loss": 3.3293209558733663, + "tokens_seen": 2901540864 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012290909090909092, + "loss": 2.5152, + "theoretical_loss": 3.329315134952945, + "tokens_seen": 2901606400 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001228888888888889, + "loss": 2.5232, + "theoretical_loss": 3.3293093142008052, + "tokens_seen": 2901671936 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012286868686868686, + "loss": 2.7445, + "theoretical_loss": 3.3293034936169383, + "tokens_seen": 2901737472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012284848484848485, + "loss": 2.3886, + "theoretical_loss": 3.329297673201336, + "tokens_seen": 2901803008 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1590456, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.331303596496582, + "objective/train/theoretical_loss": 3.3292918529539888, + "objective/train/tokens_used": 1260727776, + "theoretical_loss": 3.3292918529539888, + "tokens_seen": 2901868544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012282828282828283, + "loss": 2.4623, + "theoretical_loss": 3.3292918529539888, + "tokens_seen": 2901868544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012280808080808082, + "loss": 2.5057, + "theoretical_loss": 3.3292860328748892, + "tokens_seen": 2901934080 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001227878787878788, + "loss": 2.7713, + "theoretical_loss": 3.3292802129640275, + "tokens_seen": 2901999616 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012276767676767676, + "loss": 2.5039, + "theoretical_loss": 3.3292743932213957, + "tokens_seen": 2902065152 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012274747474747474, + "loss": 2.3802, + "theoretical_loss": 3.329268573646985, + "tokens_seen": 2902130688 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012272727272727272, + "loss": 2.6423, + "theoretical_loss": 3.329262754240786, + "tokens_seen": 2902196224 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001227070707070707, + "loss": 2.4852, + "theoretical_loss": 3.3292569350027916, + "tokens_seen": 2902261760 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001226868686868687, + "loss": 2.4954, + "theoretical_loss": 3.329251115932992, + "tokens_seen": 2902327296 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012266666666666668, + "loss": 2.4406, + "theoretical_loss": 3.3292452970313784, + "tokens_seen": 2902392832 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012264646464646463, + "loss": 2.4893, + "theoretical_loss": 3.329239478297943, + "tokens_seen": 2902458368 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012262626262626262, + "loss": 2.6435, + "theoretical_loss": 3.329233659732676, + "tokens_seen": 2902523904 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001226060606060606, + "loss": 2.4991, + "theoretical_loss": 3.32922784133557, + "tokens_seen": 2902589440 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012258585858585858, + "loss": 2.7374, + "theoretical_loss": 3.3292220231066154, + "tokens_seen": 2902654976 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012256565656565657, + "loss": 2.5318, + "theoretical_loss": 3.329216205045804, + "tokens_seen": 2902720512 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012254545454545455, + "loss": 2.6257, + "theoretical_loss": 3.329210387153127, + "tokens_seen": 2902786048 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012252525252525254, + "loss": 2.6579, + "theoretical_loss": 3.3292045694285757, + "tokens_seen": 2902851584 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012250505050505052, + "loss": 2.6904, + "theoretical_loss": 3.3291987518721418, + "tokens_seen": 2902917120 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012248484848484848, + "loss": 2.4053, + "theoretical_loss": 3.329192934483816, + "tokens_seen": 2902982656 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012246464646464646, + "loss": 2.6341, + "theoretical_loss": 3.3291871172635905, + "tokens_seen": 2903048192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012244444444444445, + "loss": 2.5128, + "theoretical_loss": 3.329181300211456, + "tokens_seen": 2903113728 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012242424242424243, + "loss": 2.5996, + "theoretical_loss": 3.3291754833274037, + "tokens_seen": 2903179264 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001224040404040404, + "loss": 2.3205, + "theoretical_loss": 3.3291696666114254, + "tokens_seen": 2903244800 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001223838383838384, + "loss": 2.5337, + "theoretical_loss": 3.3291638500635123, + "tokens_seen": 2903310336 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012236363636363638, + "loss": 2.5555, + "theoretical_loss": 3.329158033683656, + "tokens_seen": 2903375872 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012234343434343434, + "loss": 2.4881, + "theoretical_loss": 3.3291522174718473, + "tokens_seen": 2903441408 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1591431, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2740724086761475, + "objective/train/theoretical_loss": 3.329146401428078, + "objective/train/tokens_used": 1262366176, + "theoretical_loss": 3.329146401428078, + "tokens_seen": 2903506944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012232323232323232, + "loss": 2.4331, + "theoretical_loss": 3.329146401428078, + "tokens_seen": 2903506944 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001223030303030303, + "loss": 2.4371, + "theoretical_loss": 3.3291405855523393, + "tokens_seen": 2903572480 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001222828282828283, + "loss": 2.3844, + "theoretical_loss": 3.3291347698446225, + "tokens_seen": 2903638016 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012226262626262627, + "loss": 2.4248, + "theoretical_loss": 3.3291289543049194, + "tokens_seen": 2903703552 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012224242424242423, + "loss": 2.4436, + "theoretical_loss": 3.3291231389332205, + "tokens_seen": 2903769088 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012222222222222221, + "loss": 2.3047, + "theoretical_loss": 3.329117323729518, + "tokens_seen": 2903834624 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001222020202020202, + "loss": 2.4966, + "theoretical_loss": 3.329111508693803, + "tokens_seen": 2903900160 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012218181818181818, + "loss": 2.4692, + "theoretical_loss": 3.329105693826066, + "tokens_seen": 2903965696 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012216161616161617, + "loss": 2.6205, + "theoretical_loss": 3.3290998791263, + "tokens_seen": 2904031232 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012214141414141415, + "loss": 2.4081, + "theoretical_loss": 3.3290940645944946, + "tokens_seen": 2904096768 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001221212121212121, + "loss": 2.5609, + "theoretical_loss": 3.329088250230643, + "tokens_seen": 2904162304 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001221010101010101, + "loss": 2.3593, + "theoretical_loss": 3.3290824360347346, + "tokens_seen": 2904227840 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012208080808080808, + "loss": 2.643, + "theoretical_loss": 3.3290766220067622, + "tokens_seen": 2904293376 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012206060606060607, + "loss": 2.6744, + "theoretical_loss": 3.329070808146717, + "tokens_seen": 2904358912 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012204040404040404, + "loss": 2.4824, + "theoretical_loss": 3.3290649944545896, + "tokens_seen": 2904424448 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012202020202020201, + "loss": 2.4307, + "theoretical_loss": 3.329059180930372, + "tokens_seen": 2904489984 + }, + { + "epoch": 0.76, + "learning_rate": 0.000122, + "loss": 2.8064, + "theoretical_loss": 3.3290533675740552, + "tokens_seen": 2904555520 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012197979797979798, + "loss": 2.4957, + "theoretical_loss": 3.329047554385631, + "tokens_seen": 2904621056 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012195959595959597, + "loss": 2.4842, + "theoretical_loss": 3.3290417413650903, + "tokens_seen": 2904686592 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012193939393939395, + "loss": 2.5587, + "theoretical_loss": 3.3290359285124245, + "tokens_seen": 2904752128 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012191919191919192, + "loss": 2.4815, + "theoretical_loss": 3.329030115827625, + "tokens_seen": 2904817664 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001218989898989899, + "loss": 2.3543, + "theoretical_loss": 3.329024303310684, + "tokens_seen": 2904883200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012187878787878787, + "loss": 2.5032, + "theoretical_loss": 3.329018490961592, + "tokens_seen": 2904948736 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012185858585858586, + "loss": 2.4741, + "theoretical_loss": 3.32901267878034, + "tokens_seen": 2905014272 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012183838383838384, + "loss": 2.7377, + "theoretical_loss": 3.3290068667669206, + "tokens_seen": 2905079808 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1592590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9145681858062744, + "objective/train/theoretical_loss": 3.3290010549213243, + "objective/train/tokens_used": 1264004576, + "theoretical_loss": 3.3290010549213243, + "tokens_seen": 2905145344 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012181818181818181, + "loss": 2.6975, + "theoretical_loss": 3.3290010549213243, + "tokens_seen": 2905145344 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001217979797979798, + "loss": 2.5275, + "theoretical_loss": 3.3289952432435426, + "tokens_seen": 2905210880 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012177777777777778, + "loss": 2.484, + "theoretical_loss": 3.3289894317335667, + "tokens_seen": 2905276416 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012175757575757576, + "loss": 2.5523, + "theoretical_loss": 3.328983620391388, + "tokens_seen": 2905341952 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012173737373737375, + "loss": 2.4919, + "theoretical_loss": 3.3289778092169984, + "tokens_seen": 2905407488 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012171717171717173, + "loss": 2.5861, + "theoretical_loss": 3.3289719982103887, + "tokens_seen": 2905473024 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012169696969696969, + "loss": 2.544, + "theoretical_loss": 3.3289661873715506, + "tokens_seen": 2905538560 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012167676767676767, + "loss": 2.5105, + "theoretical_loss": 3.328960376700475, + "tokens_seen": 2905604096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012165656565656566, + "loss": 2.4695, + "theoretical_loss": 3.3289545661971545, + "tokens_seen": 2905669632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012163636363636364, + "loss": 2.4647, + "theoretical_loss": 3.328948755861579, + "tokens_seen": 2905735168 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012161616161616162, + "loss": 2.5128, + "theoretical_loss": 3.3289429456937403, + "tokens_seen": 2905800704 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001215959595959596, + "loss": 2.4952, + "theoretical_loss": 3.3289371356936304, + "tokens_seen": 2905866240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012157575757575758, + "loss": 2.4355, + "theoretical_loss": 3.32893132586124, + "tokens_seen": 2905931776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012155555555555556, + "loss": 2.37, + "theoretical_loss": 3.328925516196561, + "tokens_seen": 2905997312 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012153535353535355, + "loss": 2.697, + "theoretical_loss": 3.328919706699584, + "tokens_seen": 2906062848 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012151515151515152, + "loss": 2.6431, + "theoretical_loss": 3.328913897370301, + "tokens_seen": 2906128384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012149494949494949, + "loss": 2.511, + "theoretical_loss": 3.3289080882087037, + "tokens_seen": 2906193920 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012147474747474747, + "loss": 2.3989, + "theoretical_loss": 3.328902279214782, + "tokens_seen": 2906259456 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012145454545454546, + "loss": 2.4707, + "theoretical_loss": 3.3288964703885293, + "tokens_seen": 2906324992 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012143434343434344, + "loss": 2.4636, + "theoretical_loss": 3.3288906617299356, + "tokens_seen": 2906390528 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012141414141414142, + "loss": 2.4085, + "theoretical_loss": 3.3288848532389927, + "tokens_seen": 2906456064 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001213939393939394, + "loss": 2.628, + "theoretical_loss": 3.328879044915692, + "tokens_seen": 2906521600 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012137373737373738, + "loss": 2.6713, + "theoretical_loss": 3.3288732367600247, + "tokens_seen": 2906587136 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012135353535353535, + "loss": 2.6528, + "theoretical_loss": 3.328867428771982, + "tokens_seen": 2906652672 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012133333333333333, + "loss": 2.5797, + "theoretical_loss": 3.3288616209515562, + "tokens_seen": 2906718208 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1593018, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8246383666992188, + "objective/train/theoretical_loss": 3.3288558132987376, + "objective/train/tokens_used": 1265642976, + "theoretical_loss": 3.3288558132987376, + "tokens_seen": 2906783744 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012131313131313132, + "loss": 2.6322, + "theoretical_loss": 3.3288558132987376, + "tokens_seen": 2906783744 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001212929292929293, + "loss": 2.7202, + "theoretical_loss": 3.3288500058135186, + "tokens_seen": 2906849280 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012127272727272727, + "loss": 2.4309, + "theoretical_loss": 3.3288441984958896, + "tokens_seen": 2906914816 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012125252525252525, + "loss": 2.5314, + "theoretical_loss": 3.3288383913458426, + "tokens_seen": 2906980352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012123232323232324, + "loss": 2.6407, + "theoretical_loss": 3.3288325843633686, + "tokens_seen": 2907045888 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012121212121212122, + "loss": 2.5245, + "theoretical_loss": 3.3288267775484592, + "tokens_seen": 2907111424 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012119191919191921, + "loss": 2.5162, + "theoretical_loss": 3.3288209709011056, + "tokens_seen": 2907176960 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012117171717171716, + "loss": 2.6299, + "theoretical_loss": 3.3288151644213, + "tokens_seen": 2907242496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012115151515151515, + "loss": 2.3816, + "theoretical_loss": 3.328809358109033, + "tokens_seen": 2907308032 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012113131313131313, + "loss": 2.4285, + "theoretical_loss": 3.328803551964296, + "tokens_seen": 2907373568 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012111111111111112, + "loss": 2.5926, + "theoretical_loss": 3.3287977459870803, + "tokens_seen": 2907439104 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001210909090909091, + "loss": 2.6533, + "theoretical_loss": 3.3287919401773776, + "tokens_seen": 2907504640 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012107070707070707, + "loss": 2.451, + "theoretical_loss": 3.3287861345351795, + "tokens_seen": 2907570176 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012105050505050505, + "loss": 2.5305, + "theoretical_loss": 3.3287803290604767, + "tokens_seen": 2907635712 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012103030303030304, + "loss": 2.6687, + "theoretical_loss": 3.3287745237532613, + "tokens_seen": 2907701248 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012101010101010101, + "loss": 2.5897, + "theoretical_loss": 3.3287687186135244, + "tokens_seen": 2907766784 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012098989898989899, + "loss": 2.4745, + "theoretical_loss": 3.3287629136412575, + "tokens_seen": 2907832320 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012096969696969696, + "loss": 2.6392, + "theoretical_loss": 3.328757108836452, + "tokens_seen": 2907897856 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012094949494949495, + "loss": 2.7079, + "theoretical_loss": 3.328751304199099, + "tokens_seen": 2907963392 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012092929292929293, + "loss": 2.4448, + "theoretical_loss": 3.3287454997291896, + "tokens_seen": 2908028928 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012090909090909091, + "loss": 2.7077, + "theoretical_loss": 3.3287396954267163, + "tokens_seen": 2908094464 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001208888888888889, + "loss": 2.4325, + "theoretical_loss": 3.3287338912916695, + "tokens_seen": 2908160000 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012086868686868688, + "loss": 2.628, + "theoretical_loss": 3.3287280873240412, + "tokens_seen": 2908225536 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012084848484848485, + "loss": 2.5193, + "theoretical_loss": 3.3287222835238226, + "tokens_seen": 2908291072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012082828282828282, + "loss": 2.6684, + "theoretical_loss": 3.3287164798910047, + "tokens_seen": 2908356608 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1594159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3875834941864014, + "objective/train/theoretical_loss": 3.3287106764255796, + "objective/train/tokens_used": 1267281376, + "theoretical_loss": 3.3287106764255796, + "tokens_seen": 2908422144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012080808080808081, + "loss": 2.6049, + "theoretical_loss": 3.3287106764255796, + "tokens_seen": 2908422144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012078787878787879, + "loss": 2.362, + "theoretical_loss": 3.3287048731275384, + "tokens_seen": 2908487680 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012076767676767678, + "loss": 2.6948, + "theoretical_loss": 3.3286990699968726, + "tokens_seen": 2908553216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012074747474747475, + "loss": 2.5791, + "theoretical_loss": 3.328693267033573, + "tokens_seen": 2908618752 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012072727272727273, + "loss": 2.5486, + "theoretical_loss": 3.328687464237632, + "tokens_seen": 2908684288 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012070707070707071, + "loss": 2.4539, + "theoretical_loss": 3.32868166160904, + "tokens_seen": 2908749824 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001206868686868687, + "loss": 2.7347, + "theoretical_loss": 3.328675859147789, + "tokens_seen": 2908815360 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012066666666666667, + "loss": 2.688, + "theoretical_loss": 3.3286700568538703, + "tokens_seen": 2908880896 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012064646464646464, + "loss": 2.5781, + "theoretical_loss": 3.328664254727275, + "tokens_seen": 2908946432 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012062626262626262, + "loss": 2.4584, + "theoretical_loss": 3.3286584527679954, + "tokens_seen": 2909011968 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001206060606060606, + "loss": 2.3508, + "theoretical_loss": 3.328652650976022, + "tokens_seen": 2909077504 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012058585858585859, + "loss": 2.6705, + "theoretical_loss": 3.3286468493513466, + "tokens_seen": 2909143040 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012056565656565657, + "loss": 2.5976, + "theoretical_loss": 3.32864104789396, + "tokens_seen": 2909208576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012054545454545454, + "loss": 2.3773, + "theoretical_loss": 3.328635246603855, + "tokens_seen": 2909274112 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012052525252525253, + "loss": 2.6437, + "theoretical_loss": 3.3286294454810217, + "tokens_seen": 2909339648 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012050505050505051, + "loss": 2.2538, + "theoretical_loss": 3.328623644525452, + "tokens_seen": 2909405184 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012048484848484848, + "loss": 2.5252, + "theoretical_loss": 3.328617843737137, + "tokens_seen": 2909470720 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012046464646464647, + "loss": 2.5452, + "theoretical_loss": 3.3286120431160686, + "tokens_seen": 2909536256 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012044444444444445, + "loss": 2.6678, + "theoretical_loss": 3.3286062426622376, + "tokens_seen": 2909601792 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012042424242424242, + "loss": 2.5469, + "theoretical_loss": 3.3286004423756363, + "tokens_seen": 2909667328 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001204040404040404, + "loss": 2.589, + "theoretical_loss": 3.328594642256255, + "tokens_seen": 2909732864 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012038383838383839, + "loss": 2.4171, + "theoretical_loss": 3.3285888423040864, + "tokens_seen": 2909798400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012036363636363637, + "loss": 2.6525, + "theoretical_loss": 3.3285830425191207, + "tokens_seen": 2909863936 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012034343434343436, + "loss": 2.6002, + "theoretical_loss": 3.32857724290135, + "tokens_seen": 2909929472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012032323232323231, + "loss": 2.6282, + "theoretical_loss": 3.3285714434507656, + "tokens_seen": 2909995008 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1595279, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3457517623901367, + "objective/train/theoretical_loss": 3.328565644167359, + "objective/train/tokens_used": 1268919776, + "theoretical_loss": 3.328565644167359, + "tokens_seen": 2910060544 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001203030303030303, + "loss": 2.3813, + "theoretical_loss": 3.328565644167359, + "tokens_seen": 2910060544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012028282828282828, + "loss": 2.6384, + "theoretical_loss": 3.328559845051121, + "tokens_seen": 2910126080 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012026262626262627, + "loss": 2.4688, + "theoretical_loss": 3.328554046102044, + "tokens_seen": 2910191616 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012024242424242425, + "loss": 2.4016, + "theoretical_loss": 3.3285482473201187, + "tokens_seen": 2910257152 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012022222222222222, + "loss": 2.5696, + "theoretical_loss": 3.3285424487053366, + "tokens_seen": 2910322688 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001202020202020202, + "loss": 2.5637, + "theoretical_loss": 3.328536650257689, + "tokens_seen": 2910388224 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012018181818181819, + "loss": 2.7225, + "theoretical_loss": 3.3285308519771686, + "tokens_seen": 2910453760 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012016161616161617, + "loss": 2.6432, + "theoretical_loss": 3.328525053863765, + "tokens_seen": 2910519296 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012014141414141414, + "loss": 2.5281, + "theoretical_loss": 3.3285192559174703, + "tokens_seen": 2910584832 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012012121212121211, + "loss": 2.6176, + "theoretical_loss": 3.3285134581382763, + "tokens_seen": 2910650368 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001201010101010101, + "loss": 2.4578, + "theoretical_loss": 3.3285076605261743, + "tokens_seen": 2910715904 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012008080808080808, + "loss": 2.6496, + "theoretical_loss": 3.328501863081155, + "tokens_seen": 2910781440 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012006060606060606, + "loss": 2.7095, + "theoretical_loss": 3.3284960658032112, + "tokens_seen": 2910846976 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012004040404040405, + "loss": 2.669, + "theoretical_loss": 3.328490268692333, + "tokens_seen": 2910912512 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012002020202020203, + "loss": 2.6376, + "theoretical_loss": 3.3284844717485127, + "tokens_seen": 2910978048 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012, + "loss": 2.7225, + "theoretical_loss": 3.328478674971741, + "tokens_seen": 2911043584 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011997979797979799, + "loss": 2.6134, + "theoretical_loss": 3.32847287836201, + "tokens_seen": 2911109120 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011995959595959596, + "loss": 2.5529, + "theoretical_loss": 3.3284670819193103, + "tokens_seen": 2911174656 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011993939393939394, + "loss": 2.5652, + "theoretical_loss": 3.3284612856436344, + "tokens_seen": 2911240192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011991919191919193, + "loss": 2.6119, + "theoretical_loss": 3.328455489534973, + "tokens_seen": 2911305728 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001198989898989899, + "loss": 2.5792, + "theoretical_loss": 3.328449693593318, + "tokens_seen": 2911371264 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011987878787878788, + "loss": 2.4418, + "theoretical_loss": 3.32844389781866, + "tokens_seen": 2911436800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011985858585858586, + "loss": 2.5182, + "theoretical_loss": 3.328438102210991, + "tokens_seen": 2911502336 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011983838383838385, + "loss": 2.8304, + "theoretical_loss": 3.3284323067703028, + "tokens_seen": 2911567872 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011981818181818183, + "loss": 2.6939, + "theoretical_loss": 3.3284265114965863, + "tokens_seen": 2911633408 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1595816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3074779510498047, + "objective/train/theoretical_loss": 3.3284207163898327, + "objective/train/tokens_used": 1270558176, + "theoretical_loss": 3.3284207163898327, + "tokens_seen": 2911698944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011979797979797979, + "loss": 2.5501, + "theoretical_loss": 3.3284207163898327, + "tokens_seen": 2911698944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011977777777777777, + "loss": 2.4931, + "theoretical_loss": 3.328414921450034, + "tokens_seen": 2911764480 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011975757575757576, + "loss": 2.4516, + "theoretical_loss": 3.328409126677182, + "tokens_seen": 2911830016 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011973737373737374, + "loss": 2.6008, + "theoretical_loss": 3.3284033320712667, + "tokens_seen": 2911895552 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011971717171717172, + "loss": 2.3446, + "theoretical_loss": 3.3283975376322807, + "tokens_seen": 2911961088 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001196969696969697, + "loss": 2.3743, + "theoretical_loss": 3.328391743360215, + "tokens_seen": 2912026624 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011967676767676768, + "loss": 2.4994, + "theoretical_loss": 3.328385949255061, + "tokens_seen": 2912092160 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011965656565656566, + "loss": 2.4737, + "theoretical_loss": 3.328380155316811, + "tokens_seen": 2912157696 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011963636363636365, + "loss": 2.5945, + "theoretical_loss": 3.328374361545455, + "tokens_seen": 2912223232 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011961616161616162, + "loss": 2.6905, + "theoretical_loss": 3.328368567940986, + "tokens_seen": 2912288768 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001195959595959596, + "loss": 2.3023, + "theoretical_loss": 3.3283627745033937, + "tokens_seen": 2912354304 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011957575757575757, + "loss": 2.5712, + "theoretical_loss": 3.328356981232671, + "tokens_seen": 2912419840 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011955555555555556, + "loss": 2.5284, + "theoretical_loss": 3.3283511881288086, + "tokens_seen": 2912485376 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011953535353535354, + "loss": 2.532, + "theoretical_loss": 3.328345395191798, + "tokens_seen": 2912550912 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011951515151515152, + "loss": 2.4358, + "theoretical_loss": 3.328339602421631, + "tokens_seen": 2912616448 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011949494949494951, + "loss": 2.4424, + "theoretical_loss": 3.3283338098182984, + "tokens_seen": 2912681984 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011947474747474748, + "loss": 2.4279, + "theoretical_loss": 3.3283280173817924, + "tokens_seen": 2912747520 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011945454545454545, + "loss": 2.5684, + "theoretical_loss": 3.328322225112104, + "tokens_seen": 2912813056 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011943434343434343, + "loss": 2.4895, + "theoretical_loss": 3.328316433009225, + "tokens_seen": 2912878592 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011941414141414142, + "loss": 2.6762, + "theoretical_loss": 3.3283106410731462, + "tokens_seen": 2912944128 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001193939393939394, + "loss": 2.4701, + "theoretical_loss": 3.3283048493038594, + "tokens_seen": 2913009664 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011937373737373737, + "loss": 2.2951, + "theoretical_loss": 3.328299057701356, + "tokens_seen": 2913075200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011935353535353535, + "loss": 2.6867, + "theoretical_loss": 3.3282932662656277, + "tokens_seen": 2913140736 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011933333333333334, + "loss": 2.4113, + "theoretical_loss": 3.328287474996666, + "tokens_seen": 2913206272 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011931313131313132, + "loss": 2.5102, + "theoretical_loss": 3.3282816838944616, + "tokens_seen": 2913271808 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1597120, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7149505615234375, + "objective/train/theoretical_loss": 3.328275892959007, + "objective/train/tokens_used": 1272196576, + "theoretical_loss": 3.328275892959007, + "tokens_seen": 2913337344 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001192929292929293, + "loss": 2.4169, + "theoretical_loss": 3.328275892959007, + "tokens_seen": 2913337344 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011927272727272726, + "loss": 2.6755, + "theoretical_loss": 3.3282701021902925, + "tokens_seen": 2913402880 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011925252525252525, + "loss": 2.4195, + "theoretical_loss": 3.3282643115883106, + "tokens_seen": 2913468416 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011923232323232323, + "loss": 2.6159, + "theoretical_loss": 3.3282585211530518, + "tokens_seen": 2913533952 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011921212121212121, + "loss": 2.6286, + "theoretical_loss": 3.3282527308845085, + "tokens_seen": 2913599488 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001191919191919192, + "loss": 2.3898, + "theoretical_loss": 3.3282469407826714, + "tokens_seen": 2913665024 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011917171717171718, + "loss": 2.6457, + "theoretical_loss": 3.3282411508475325, + "tokens_seen": 2913730560 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011915151515151515, + "loss": 2.7587, + "theoretical_loss": 3.3282353610790825, + "tokens_seen": 2913796096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011913131313131314, + "loss": 2.2708, + "theoretical_loss": 3.328229571477314, + "tokens_seen": 2913861632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011911111111111112, + "loss": 2.5074, + "theoretical_loss": 3.328223782042217, + "tokens_seen": 2913927168 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011909090909090909, + "loss": 2.5825, + "theoretical_loss": 3.3282179927737845, + "tokens_seen": 2913992704 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011907070707070708, + "loss": 2.5882, + "theoretical_loss": 3.328212203672007, + "tokens_seen": 2914058240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011905050505050505, + "loss": 2.6348, + "theoretical_loss": 3.328206414736876, + "tokens_seen": 2914123776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011903030303030303, + "loss": 2.2322, + "theoretical_loss": 3.328200625968383, + "tokens_seen": 2914189312 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011901010101010101, + "loss": 2.5889, + "theoretical_loss": 3.32819483736652, + "tokens_seen": 2914254848 + }, + { + "epoch": 0.76, + "learning_rate": 0.000118989898989899, + "loss": 2.4616, + "theoretical_loss": 3.3281890489312778, + "tokens_seen": 2914320384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011896969696969698, + "loss": 2.519, + "theoretical_loss": 3.328183260662648, + "tokens_seen": 2914385920 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011894949494949495, + "loss": 2.6738, + "theoretical_loss": 3.3281774725606224, + "tokens_seen": 2914451456 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011892929292929292, + "loss": 2.8795, + "theoretical_loss": 3.328171684625192, + "tokens_seen": 2914516992 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001189090909090909, + "loss": 2.6561, + "theoretical_loss": 3.3281658968563486, + "tokens_seen": 2914582528 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011888888888888889, + "loss": 2.5734, + "theoretical_loss": 3.328160109254083, + "tokens_seen": 2914648064 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011886868686868687, + "loss": 2.5141, + "theoretical_loss": 3.3281543218183876, + "tokens_seen": 2914713600 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011884848484848484, + "loss": 2.5193, + "theoretical_loss": 3.3281485345492534, + "tokens_seen": 2914779136 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011882828282828283, + "loss": 2.2462, + "theoretical_loss": 3.3281427474466723, + "tokens_seen": 2914844672 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011880808080808081, + "loss": 2.3822, + "theoretical_loss": 3.328136960510635, + "tokens_seen": 2914910208 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1597547, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3916378021240234, + "objective/train/theoretical_loss": 3.328131173741133, + "objective/train/tokens_used": 1273834976, + "theoretical_loss": 3.328131173741133, + "tokens_seen": 2914975744 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001187878787878788, + "loss": 2.5146, + "theoretical_loss": 3.328131173741133, + "tokens_seen": 2914975744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011876767676767678, + "loss": 2.4867, + "theoretical_loss": 3.3281253871381584, + "tokens_seen": 2915041280 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011874747474747475, + "loss": 2.6786, + "theoretical_loss": 3.328119600701702, + "tokens_seen": 2915106816 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011872727272727272, + "loss": 2.7009, + "theoretical_loss": 3.3281138144317564, + "tokens_seen": 2915172352 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001187070707070707, + "loss": 2.3949, + "theoretical_loss": 3.3281080283283115, + "tokens_seen": 2915237888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011868686868686869, + "loss": 2.6061, + "theoretical_loss": 3.32810224239136, + "tokens_seen": 2915303424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011866666666666667, + "loss": 2.5467, + "theoretical_loss": 3.3280964566208926, + "tokens_seen": 2915368960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011864646464646466, + "loss": 2.5998, + "theoretical_loss": 3.3280906710169016, + "tokens_seen": 2915434496 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011862626262626263, + "loss": 2.5021, + "theoretical_loss": 3.3280848855793774, + "tokens_seen": 2915500032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011860606060606061, + "loss": 2.5798, + "theoretical_loss": 3.3280791003083126, + "tokens_seen": 2915565568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011858585858585858, + "loss": 2.4821, + "theoretical_loss": 3.3280733152036976, + "tokens_seen": 2915631104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011856565656565657, + "loss": 2.4064, + "theoretical_loss": 3.3280675302655247, + "tokens_seen": 2915696640 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011854545454545455, + "loss": 2.4477, + "theoretical_loss": 3.328061745493785, + "tokens_seen": 2915762176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011852525252525252, + "loss": 2.6167, + "theoretical_loss": 3.3280559608884692, + "tokens_seen": 2915827712 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001185050505050505, + "loss": 2.6034, + "theoretical_loss": 3.3280501764495707, + "tokens_seen": 2915893248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011848484848484849, + "loss": 2.619, + "theoretical_loss": 3.328044392177079, + "tokens_seen": 2915958784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011846464646464647, + "loss": 2.4594, + "theoretical_loss": 3.3280386080709867, + "tokens_seen": 2916024320 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011844444444444446, + "loss": 2.5571, + "theoretical_loss": 3.3280328241312853, + "tokens_seen": 2916089856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011842424242424243, + "loss": 2.4897, + "theoretical_loss": 3.3280270403579655, + "tokens_seen": 2916155392 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001184040404040404, + "loss": 2.5958, + "theoretical_loss": 3.3280212567510197, + "tokens_seen": 2916220928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011838383838383838, + "loss": 2.376, + "theoretical_loss": 3.3280154733104386, + "tokens_seen": 2916286464 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011836363636363637, + "loss": 2.7094, + "theoretical_loss": 3.328009690036214, + "tokens_seen": 2916352000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011834343434343435, + "loss": 2.48, + "theoretical_loss": 3.3280039069283376, + "tokens_seen": 2916417536 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011832323232323232, + "loss": 2.6026, + "theoretical_loss": 3.3279981239868004, + "tokens_seen": 2916483072 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001183030303030303, + "loss": 2.5472, + "theoretical_loss": 3.327992341211594, + "tokens_seen": 2916548608 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1598685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.977783203125, + "objective/train/theoretical_loss": 3.3279865586027104, + "objective/train/tokens_used": 1275473376, + "theoretical_loss": 3.3279865586027104, + "tokens_seen": 2916614144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011828282828282829, + "loss": 2.498, + "theoretical_loss": 3.3279865586027104, + "tokens_seen": 2916614144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011826262626262627, + "loss": 2.9222, + "theoretical_loss": 3.3279807761601403, + "tokens_seen": 2916679680 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011824242424242426, + "loss": 2.4962, + "theoretical_loss": 3.327974993883876, + "tokens_seen": 2916745216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011822222222222223, + "loss": 2.6317, + "theoretical_loss": 3.327969211773908, + "tokens_seen": 2916810752 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001182020202020202, + "loss": 2.4927, + "theoretical_loss": 3.3279634298302287, + "tokens_seen": 2916876288 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011818181818181818, + "loss": 2.6417, + "theoretical_loss": 3.327957648052829, + "tokens_seen": 2916941824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011816161616161616, + "loss": 2.465, + "theoretical_loss": 3.3279518664417007, + "tokens_seen": 2917007360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011814141414141415, + "loss": 2.4492, + "theoretical_loss": 3.327946084996835, + "tokens_seen": 2917072896 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011812121212121213, + "loss": 2.5899, + "theoretical_loss": 3.327940303718224, + "tokens_seen": 2917138432 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001181010101010101, + "loss": 2.5915, + "theoretical_loss": 3.327934522605858, + "tokens_seen": 2917203968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011808080808080809, + "loss": 2.3482, + "theoretical_loss": 3.32792874165973, + "tokens_seen": 2917269504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011806060606060606, + "loss": 2.4939, + "theoretical_loss": 3.3279229608798304, + "tokens_seen": 2917335040 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011804040404040404, + "loss": 2.392, + "theoretical_loss": 3.3279171802661507, + "tokens_seen": 2917400576 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011802020202020202, + "loss": 2.4456, + "theoretical_loss": 3.3279113998186833, + "tokens_seen": 2917466112 + }, + { + "epoch": 0.77, + "learning_rate": 0.000118, + "loss": 2.6715, + "theoretical_loss": 3.3279056195374186, + "tokens_seen": 2917531648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011797979797979798, + "loss": 2.6971, + "theoretical_loss": 3.3278998394223485, + "tokens_seen": 2917597184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011795959595959596, + "loss": 2.597, + "theoretical_loss": 3.327894059473465, + "tokens_seen": 2917662720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011793939393939395, + "loss": 2.6964, + "theoretical_loss": 3.3278882796907587, + "tokens_seen": 2917728256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011791919191919193, + "loss": 2.5439, + "theoretical_loss": 3.327882500074222, + "tokens_seen": 2917793792 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001178989898989899, + "loss": 2.3699, + "theoretical_loss": 3.3278767206238453, + "tokens_seen": 2917859328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011787878787878787, + "loss": 2.4614, + "theoretical_loss": 3.327870941339621, + "tokens_seen": 2917924864 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011785858585858586, + "loss": 2.6376, + "theoretical_loss": 3.3278651622215403, + "tokens_seen": 2917990400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011783838383838384, + "loss": 2.413, + "theoretical_loss": 3.327859383269595, + "tokens_seen": 2918055936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011781818181818182, + "loss": 2.551, + "theoretical_loss": 3.327853604483775, + "tokens_seen": 2918121472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011779797979797981, + "loss": 2.3118, + "theoretical_loss": 3.3278478258640742, + "tokens_seen": 2918187008 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1599978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5951783657073975, + "objective/train/theoretical_loss": 3.327842047410483, + "objective/train/tokens_used": 1277111776, + "theoretical_loss": 3.327842047410483, + "tokens_seen": 2918252544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011777777777777778, + "loss": 2.681, + "theoretical_loss": 3.327842047410483, + "tokens_seen": 2918252544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011775757575757576, + "loss": 2.6321, + "theoretical_loss": 3.327836269122993, + "tokens_seen": 2918318080 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011773737373737375, + "loss": 2.5907, + "theoretical_loss": 3.327830491001595, + "tokens_seen": 2918383616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011771717171717172, + "loss": 2.4226, + "theoretical_loss": 3.3278247130462812, + "tokens_seen": 2918449152 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001176969696969697, + "loss": 2.5679, + "theoretical_loss": 3.327818935257043, + "tokens_seen": 2918514688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011767676767676767, + "loss": 2.6199, + "theoretical_loss": 3.327813157633872, + "tokens_seen": 2918580224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011765656565656565, + "loss": 2.4837, + "theoretical_loss": 3.3278073801767594, + "tokens_seen": 2918645760 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011763636363636364, + "loss": 2.5418, + "theoretical_loss": 3.327801602885697, + "tokens_seen": 2918711296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011761616161616162, + "loss": 2.3749, + "theoretical_loss": 3.327795825760676, + "tokens_seen": 2918776832 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001175959595959596, + "loss": 2.6305, + "theoretical_loss": 3.327790048801688, + "tokens_seen": 2918842368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011757575757575758, + "loss": 2.6042, + "theoretical_loss": 3.3277842720087243, + "tokens_seen": 2918907904 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011755555555555556, + "loss": 2.4292, + "theoretical_loss": 3.3277784953817773, + "tokens_seen": 2918973440 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011753535353535353, + "loss": 2.4443, + "theoretical_loss": 3.3277727189208375, + "tokens_seen": 2919038976 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011751515151515152, + "loss": 2.5717, + "theoretical_loss": 3.327766942625897, + "tokens_seen": 2919104512 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001174949494949495, + "loss": 2.5089, + "theoretical_loss": 3.3277611664969466, + "tokens_seen": 2919170048 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011747474747474747, + "loss": 2.7678, + "theoretical_loss": 3.3277553905339783, + "tokens_seen": 2919235584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011745454545454545, + "loss": 2.4643, + "theoretical_loss": 3.327749614736984, + "tokens_seen": 2919301120 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011743434343434344, + "loss": 2.4535, + "theoretical_loss": 3.3277438391059544, + "tokens_seen": 2919366656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011741414141414142, + "loss": 2.5834, + "theoretical_loss": 3.327738063640881, + "tokens_seen": 2919432192 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001173939393939394, + "loss": 2.0865, + "theoretical_loss": 3.3277322883417564, + "tokens_seen": 2919497728 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011737373737373738, + "loss": 2.4174, + "theoretical_loss": 3.3277265132085714, + "tokens_seen": 2919563264 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011735353535353535, + "loss": 2.7476, + "theoretical_loss": 3.327720738241317, + "tokens_seen": 2919628800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011733333333333333, + "loss": 2.6421, + "theoretical_loss": 3.3277149634399854, + "tokens_seen": 2919694336 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011731313131313131, + "loss": 2.5358, + "theoretical_loss": 3.327709188804568, + "tokens_seen": 2919759872 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001172929292929293, + "loss": 2.6426, + "theoretical_loss": 3.327703414335056, + "tokens_seen": 2919825408 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1600503, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.385477066040039, + "objective/train/theoretical_loss": 3.3276976400314418, + "objective/train/tokens_used": 1278750176, + "theoretical_loss": 3.3276976400314418, + "tokens_seen": 2919890944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011727272727272728, + "loss": 2.5637, + "theoretical_loss": 3.3276976400314418, + "tokens_seen": 2919890944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011725252525252525, + "loss": 2.5755, + "theoretical_loss": 3.3276918658937156, + "tokens_seen": 2919956480 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011723232323232324, + "loss": 2.6474, + "theoretical_loss": 3.3276860919218696, + "tokens_seen": 2920022016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011721212121212122, + "loss": 2.4759, + "theoretical_loss": 3.3276803181158954, + "tokens_seen": 2920087552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011719191919191919, + "loss": 2.451, + "theoretical_loss": 3.3276745444757845, + "tokens_seen": 2920153088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011717171717171717, + "loss": 2.6455, + "theoretical_loss": 3.3276687710015276, + "tokens_seen": 2920218624 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011715151515151515, + "loss": 2.7701, + "theoretical_loss": 3.3276629976931176, + "tokens_seen": 2920284160 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011713131313131313, + "loss": 2.5682, + "theoretical_loss": 3.327657224550545, + "tokens_seen": 2920349696 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011711111111111111, + "loss": 2.5741, + "theoretical_loss": 3.327651451573802, + "tokens_seen": 2920415232 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001170909090909091, + "loss": 2.5456, + "theoretical_loss": 3.327645678762879, + "tokens_seen": 2920480768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011707070707070708, + "loss": 2.7049, + "theoretical_loss": 3.3276399061177693, + "tokens_seen": 2920546304 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011705050505050505, + "loss": 2.2341, + "theoretical_loss": 3.3276341336384627, + "tokens_seen": 2920611840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011703030303030302, + "loss": 2.5659, + "theoretical_loss": 3.3276283613249515, + "tokens_seen": 2920677376 + }, + { + "epoch": 0.77, + "learning_rate": 0.000117010101010101, + "loss": 2.5564, + "theoretical_loss": 3.3276225891772273, + "tokens_seen": 2920742912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011698989898989899, + "loss": 2.5986, + "theoretical_loss": 3.327616817195281, + "tokens_seen": 2920808448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011696969696969697, + "loss": 2.4433, + "theoretical_loss": 3.327611045379105, + "tokens_seen": 2920873984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011694949494949496, + "loss": 2.3385, + "theoretical_loss": 3.3276052737286905, + "tokens_seen": 2920939520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011692929292929293, + "loss": 2.5217, + "theoretical_loss": 3.3275995022440283, + "tokens_seen": 2921005056 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011690909090909091, + "loss": 2.4604, + "theoretical_loss": 3.327593730925111, + "tokens_seen": 2921070592 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001168888888888889, + "loss": 2.3598, + "theoretical_loss": 3.3275879597719293, + "tokens_seen": 2921136128 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011686868686868688, + "loss": 2.4304, + "theoretical_loss": 3.327582188784475, + "tokens_seen": 2921201664 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011684848484848485, + "loss": 2.5123, + "theoretical_loss": 3.32757641796274, + "tokens_seen": 2921267200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011682828282828282, + "loss": 2.5172, + "theoretical_loss": 3.3275706473067155, + "tokens_seen": 2921332736 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001168080808080808, + "loss": 2.382, + "theoretical_loss": 3.327564876816393, + "tokens_seen": 2921398272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011678787878787879, + "loss": 2.4381, + "theoretical_loss": 3.327559106491764, + "tokens_seen": 2921463808 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1601766, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7888519763946533, + "objective/train/theoretical_loss": 3.3275533363328202, + "objective/train/tokens_used": 1280388576, + "theoretical_loss": 3.3275533363328202, + "tokens_seen": 2921529344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011676767676767677, + "loss": 2.7322, + "theoretical_loss": 3.3275533363328202, + "tokens_seen": 2921529344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011674747474747476, + "loss": 2.5574, + "theoretical_loss": 3.3275475663395526, + "tokens_seen": 2921594880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011672727272727273, + "loss": 2.565, + "theoretical_loss": 3.3275417965119534, + "tokens_seen": 2921660416 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011670707070707071, + "loss": 2.3505, + "theoretical_loss": 3.327536026850014, + "tokens_seen": 2921725952 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001166868686868687, + "loss": 2.5172, + "theoretical_loss": 3.3275302573537258, + "tokens_seen": 2921791488 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011666666666666667, + "loss": 2.5296, + "theoretical_loss": 3.32752448802308, + "tokens_seen": 2921857024 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011664646464646465, + "loss": 2.7444, + "theoretical_loss": 3.3275187188580686, + "tokens_seen": 2921922560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011662626262626262, + "loss": 2.1633, + "theoretical_loss": 3.3275129498586833, + "tokens_seen": 2921988096 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001166060606060606, + "loss": 2.7763, + "theoretical_loss": 3.327507181024915, + "tokens_seen": 2922053632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011658585858585859, + "loss": 2.4775, + "theoretical_loss": 3.3275014123567552, + "tokens_seen": 2922119168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011656565656565657, + "loss": 2.5396, + "theoretical_loss": 3.3274956438541965, + "tokens_seen": 2922184704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011654545454545456, + "loss": 2.5066, + "theoretical_loss": 3.3274898755172293, + "tokens_seen": 2922250240 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011652525252525254, + "loss": 2.3627, + "theoretical_loss": 3.327484107345845, + "tokens_seen": 2922315776 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001165050505050505, + "loss": 2.5248, + "theoretical_loss": 3.3274783393400367, + "tokens_seen": 2922381312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011648484848484848, + "loss": 2.5963, + "theoretical_loss": 3.3274725714997944, + "tokens_seen": 2922446848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011646464646464646, + "loss": 2.5227, + "theoretical_loss": 3.32746680382511, + "tokens_seen": 2922512384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011644444444444445, + "loss": 2.2998, + "theoretical_loss": 3.3274610363159756, + "tokens_seen": 2922577920 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011642424242424243, + "loss": 2.3676, + "theoretical_loss": 3.3274552689723818, + "tokens_seen": 2922643456 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001164040404040404, + "loss": 2.7323, + "theoretical_loss": 3.327449501794321, + "tokens_seen": 2922708992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011638383838383839, + "loss": 2.5508, + "theoretical_loss": 3.327443734781784, + "tokens_seen": 2922774528 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011636363636363637, + "loss": 2.5658, + "theoretical_loss": 3.327437967934763, + "tokens_seen": 2922840064 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011634343434343435, + "loss": 2.6687, + "theoretical_loss": 3.3274322012532496, + "tokens_seen": 2922905600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011632323232323233, + "loss": 2.6248, + "theoretical_loss": 3.3274264347372347, + "tokens_seen": 2922971136 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001163030303030303, + "loss": 2.402, + "theoretical_loss": 3.3274206683867096, + "tokens_seen": 2923036672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011628282828282828, + "loss": 2.4796, + "theoretical_loss": 3.3274149022016672, + "tokens_seen": 2923102208 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1602847, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.577550172805786, + "objective/train/theoretical_loss": 3.327409136182098, + "objective/train/tokens_used": 1282026976, + "theoretical_loss": 3.327409136182098, + "tokens_seen": 2923167744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011626262626262626, + "loss": 2.7143, + "theoretical_loss": 3.327409136182098, + "tokens_seen": 2923167744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011624242424242425, + "loss": 2.631, + "theoretical_loss": 3.3274033703279935, + "tokens_seen": 2923233280 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011622222222222223, + "loss": 2.4703, + "theoretical_loss": 3.3273976046393456, + "tokens_seen": 2923298816 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001162020202020202, + "loss": 2.4985, + "theoretical_loss": 3.327391839116146, + "tokens_seen": 2923364352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011618181818181819, + "loss": 2.5003, + "theoretical_loss": 3.3273860737583854, + "tokens_seen": 2923429888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011616161616161616, + "loss": 2.7442, + "theoretical_loss": 3.3273803085660565, + "tokens_seen": 2923495424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011614141414141414, + "loss": 2.6213, + "theoretical_loss": 3.32737454353915, + "tokens_seen": 2923560960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011612121212121212, + "loss": 2.3878, + "theoretical_loss": 3.327368778677658, + "tokens_seen": 2923626496 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011610101010101011, + "loss": 2.5908, + "theoretical_loss": 3.3273630139815715, + "tokens_seen": 2923692032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011608080808080808, + "loss": 2.5799, + "theoretical_loss": 3.3273572494508823, + "tokens_seen": 2923757568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011606060606060606, + "loss": 2.638, + "theoretical_loss": 3.327351485085582, + "tokens_seen": 2923823104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011604040404040405, + "loss": 2.6885, + "theoretical_loss": 3.3273457208856625, + "tokens_seen": 2923888640 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011602020202020203, + "loss": 2.4962, + "theoretical_loss": 3.327339956851114, + "tokens_seen": 2923954176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011600000000000001, + "loss": 2.5841, + "theoretical_loss": 3.32733419298193, + "tokens_seen": 2924019712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011597979797979797, + "loss": 2.2721, + "theoretical_loss": 3.3273284292781007, + "tokens_seen": 2924085248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011595959595959596, + "loss": 2.7008, + "theoretical_loss": 3.327322665739618, + "tokens_seen": 2924150784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011593939393939394, + "loss": 2.3429, + "theoretical_loss": 3.3273169023664737, + "tokens_seen": 2924216320 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011591919191919192, + "loss": 2.556, + "theoretical_loss": 3.3273111391586587, + "tokens_seen": 2924281856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011589898989898991, + "loss": 2.2776, + "theoretical_loss": 3.3273053761161653, + "tokens_seen": 2924347392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011587878787878788, + "loss": 2.5994, + "theoretical_loss": 3.3272996132389845, + "tokens_seen": 2924412928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011585858585858586, + "loss": 2.4717, + "theoretical_loss": 3.3272938505271084, + "tokens_seen": 2924478464 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011583838383838385, + "loss": 2.4857, + "theoretical_loss": 3.327288087980528, + "tokens_seen": 2924544000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011581818181818183, + "loss": 2.5649, + "theoretical_loss": 3.327282325599235, + "tokens_seen": 2924609536 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001157979797979798, + "loss": 2.5546, + "theoretical_loss": 3.327276563383221, + "tokens_seen": 2924675072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011577777777777777, + "loss": 2.473, + "theoretical_loss": 3.3272708013324777, + "tokens_seen": 2924740608 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1603512, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.698882579803467, + "objective/train/theoretical_loss": 3.3272650394469965, + "objective/train/tokens_used": 1283665376, + "theoretical_loss": 3.3272650394469965, + "tokens_seen": 2924806144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011575757575757575, + "loss": 2.6131, + "theoretical_loss": 3.3272650394469965, + "tokens_seen": 2924806144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011573737373737374, + "loss": 2.5834, + "theoretical_loss": 3.3272592777267693, + "tokens_seen": 2924871680 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011571717171717172, + "loss": 2.514, + "theoretical_loss": 3.327253516171787, + "tokens_seen": 2924937216 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001156969696969697, + "loss": 2.4985, + "theoretical_loss": 3.3272477547820416, + "tokens_seen": 2925002752 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011567676767676769, + "loss": 2.5397, + "theoretical_loss": 3.3272419935575246, + "tokens_seen": 2925068288 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011565656565656566, + "loss": 2.5118, + "theoretical_loss": 3.327236232498228, + "tokens_seen": 2925133824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011563636363636363, + "loss": 2.3929, + "theoretical_loss": 3.327230471604142, + "tokens_seen": 2925199360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011561616161616161, + "loss": 2.463, + "theoretical_loss": 3.3272247108752597, + "tokens_seen": 2925264896 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001155959595959596, + "loss": 2.6173, + "theoretical_loss": 3.327218950311572, + "tokens_seen": 2925330432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011557575757575758, + "loss": 2.5734, + "theoretical_loss": 3.32721318991307, + "tokens_seen": 2925395968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011555555555555555, + "loss": 2.4801, + "theoretical_loss": 3.327207429679746, + "tokens_seen": 2925461504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011553535353535354, + "loss": 2.4712, + "theoretical_loss": 3.3272016696115916, + "tokens_seen": 2925527040 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011551515151515152, + "loss": 2.6392, + "theoretical_loss": 3.327195909708598, + "tokens_seen": 2925592576 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001154949494949495, + "loss": 2.4991, + "theoretical_loss": 3.3271901499707566, + "tokens_seen": 2925658112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011547474747474749, + "loss": 2.4992, + "theoretical_loss": 3.327184390398059, + "tokens_seen": 2925723648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011545454545454545, + "loss": 2.5803, + "theoretical_loss": 3.3271786309904976, + "tokens_seen": 2925789184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011543434343434343, + "loss": 2.4938, + "theoretical_loss": 3.327172871748063, + "tokens_seen": 2925854720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011541414141414141, + "loss": 2.2613, + "theoretical_loss": 3.327167112670747, + "tokens_seen": 2925920256 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001153939393939394, + "loss": 2.1502, + "theoretical_loss": 3.3271613537585414, + "tokens_seen": 2925985792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011537373737373738, + "loss": 2.6305, + "theoretical_loss": 3.3271555950114378, + "tokens_seen": 2926051328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011535353535353535, + "loss": 2.4141, + "theoretical_loss": 3.3271498364294274, + "tokens_seen": 2926116864 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011533333333333334, + "loss": 2.4604, + "theoretical_loss": 3.327144078012502, + "tokens_seen": 2926182400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011531313131313132, + "loss": 2.5813, + "theoretical_loss": 3.327138319760653, + "tokens_seen": 2926247936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011529292929292929, + "loss": 2.4823, + "theoretical_loss": 3.3271325616738725, + "tokens_seen": 2926313472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011527272727272727, + "loss": 2.4068, + "theoretical_loss": 3.3271268037521518, + "tokens_seen": 2926379008 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1604738, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.182546615600586, + "objective/train/theoretical_loss": 3.327121045995482, + "objective/train/tokens_used": 1285303776, + "theoretical_loss": 3.327121045995482, + "tokens_seen": 2926444544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011525252525252526, + "loss": 2.3755, + "theoretical_loss": 3.327121045995482, + "tokens_seen": 2926444544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011523232323232323, + "loss": 2.4908, + "theoretical_loss": 3.327115288403855, + "tokens_seen": 2926510080 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011521212121212121, + "loss": 2.6332, + "theoretical_loss": 3.3271095309772627, + "tokens_seen": 2926575616 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001151919191919192, + "loss": 2.6729, + "theoretical_loss": 3.327103773715696, + "tokens_seen": 2926641152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011517171717171718, + "loss": 2.5521, + "theoretical_loss": 3.3270980166191473, + "tokens_seen": 2926706688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011515151515151516, + "loss": 2.6592, + "theoretical_loss": 3.3270922596876074, + "tokens_seen": 2926772224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011513131313131313, + "loss": 2.3819, + "theoretical_loss": 3.3270865029210683, + "tokens_seen": 2926837760 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001151111111111111, + "loss": 2.5587, + "theoretical_loss": 3.3270807463195213, + "tokens_seen": 2926903296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011509090909090909, + "loss": 2.4829, + "theoretical_loss": 3.327074989882959, + "tokens_seen": 2926968832 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011507070707070707, + "loss": 2.5193, + "theoretical_loss": 3.327069233611371, + "tokens_seen": 2927034368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011505050505050506, + "loss": 2.3971, + "theoretical_loss": 3.3270634775047507, + "tokens_seen": 2927099904 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011503030303030303, + "loss": 2.4503, + "theoretical_loss": 3.3270577215630888, + "tokens_seen": 2927165440 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011501010101010101, + "loss": 2.5421, + "theoretical_loss": 3.327051965786377, + "tokens_seen": 2927230976 + }, + { + "epoch": 0.77, + "learning_rate": 0.000114989898989899, + "loss": 2.7133, + "theoretical_loss": 3.327046210174607, + "tokens_seen": 2927296512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011496969696969698, + "loss": 2.5248, + "theoretical_loss": 3.3270404547277708, + "tokens_seen": 2927362048 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011494949494949495, + "loss": 2.3771, + "theoretical_loss": 3.327034699445859, + "tokens_seen": 2927427584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011492929292929292, + "loss": 2.5069, + "theoretical_loss": 3.327028944328864, + "tokens_seen": 2927493120 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001149090909090909, + "loss": 2.4855, + "theoretical_loss": 3.327023189376777, + "tokens_seen": 2927558656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011488888888888889, + "loss": 2.6927, + "theoretical_loss": 3.3270174345895893, + "tokens_seen": 2927624192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011486868686868687, + "loss": 2.1362, + "theoretical_loss": 3.3270116799672933, + "tokens_seen": 2927689728 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011484848484848486, + "loss": 2.4207, + "theoretical_loss": 3.32700592550988, + "tokens_seen": 2927755264 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011482828282828284, + "loss": 2.3857, + "theoretical_loss": 3.327000171217341, + "tokens_seen": 2927820800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011480808080808081, + "loss": 2.4977, + "theoretical_loss": 3.326994417089668, + "tokens_seen": 2927886336 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001147878787878788, + "loss": 2.4302, + "theoretical_loss": 3.326988663126853, + "tokens_seen": 2927951872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011476767676767676, + "loss": 2.6456, + "theoretical_loss": 3.326982909328887, + "tokens_seen": 2928017408 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1605386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.860295057296753, + "objective/train/theoretical_loss": 3.3269771556957615, + "objective/train/tokens_used": 1286942176, + "theoretical_loss": 3.3269771556957615, + "tokens_seen": 2928082944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011474747474747475, + "loss": 2.6299, + "theoretical_loss": 3.3269771556957615, + "tokens_seen": 2928082944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011472727272727273, + "loss": 2.7545, + "theoretical_loss": 3.3269714022274686, + "tokens_seen": 2928148480 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001147070707070707, + "loss": 2.5389, + "theoretical_loss": 3.3269656489239994, + "tokens_seen": 2928214016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011468686868686869, + "loss": 2.6982, + "theoretical_loss": 3.326959895785346, + "tokens_seen": 2928279552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011466666666666667, + "loss": 2.4267, + "theoretical_loss": 3.3269541428114997, + "tokens_seen": 2928345088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011464646464646466, + "loss": 2.4689, + "theoretical_loss": 3.326948390002452, + "tokens_seen": 2928410624 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011462626262626264, + "loss": 2.5405, + "theoretical_loss": 3.3269426373581945, + "tokens_seen": 2928476160 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001146060606060606, + "loss": 2.6284, + "theoretical_loss": 3.326936884878719, + "tokens_seen": 2928541696 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011458585858585858, + "loss": 2.6085, + "theoretical_loss": 3.326931132564017, + "tokens_seen": 2928607232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011456565656565656, + "loss": 2.4897, + "theoretical_loss": 3.3269253804140804, + "tokens_seen": 2928672768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011454545454545455, + "loss": 2.3347, + "theoretical_loss": 3.3269196284289, + "tokens_seen": 2928738304 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011452525252525253, + "loss": 2.2845, + "theoretical_loss": 3.3269138766084683, + "tokens_seen": 2928803840 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001145050505050505, + "loss": 2.5911, + "theoretical_loss": 3.3269081249527765, + "tokens_seen": 2928869376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011448484848484849, + "loss": 2.5381, + "theoretical_loss": 3.3269023734618157, + "tokens_seen": 2928934912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011446464646464647, + "loss": 2.7128, + "theoretical_loss": 3.3268966221355782, + "tokens_seen": 2929000448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011444444444444445, + "loss": 2.395, + "theoretical_loss": 3.3268908709740552, + "tokens_seen": 2929065984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011442424242424242, + "loss": 2.6088, + "theoretical_loss": 3.3268851199772387, + "tokens_seen": 2929131520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011440404040404041, + "loss": 2.5313, + "theoretical_loss": 3.32687936914512, + "tokens_seen": 2929197056 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011438383838383838, + "loss": 2.4568, + "theoretical_loss": 3.326873618477691, + "tokens_seen": 2929262592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011436363636363636, + "loss": 2.7564, + "theoretical_loss": 3.3268678679749426, + "tokens_seen": 2929328128 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011434343434343435, + "loss": 2.7042, + "theoretical_loss": 3.3268621176368667, + "tokens_seen": 2929393664 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011432323232323233, + "loss": 2.601, + "theoretical_loss": 3.3268563674634555, + "tokens_seen": 2929459200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011430303030303031, + "loss": 2.334, + "theoretical_loss": 3.3268506174547, + "tokens_seen": 2929524736 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011428282828282829, + "loss": 2.5458, + "theoretical_loss": 3.326844867610592, + "tokens_seen": 2929590272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011426262626262627, + "loss": 2.6105, + "theoretical_loss": 3.326839117931123, + "tokens_seen": 2929655808 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1606097, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.69283390045166, + "objective/train/theoretical_loss": 3.3268333684162847, + "objective/train/tokens_used": 1288580576, + "theoretical_loss": 3.3268333684162847, + "tokens_seen": 2929721344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011424242424242424, + "loss": 2.6226, + "theoretical_loss": 3.3268333684162847, + "tokens_seen": 2929721344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011422222222222222, + "loss": 2.5208, + "theoretical_loss": 3.3268276190660684, + "tokens_seen": 2929786880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011420202020202021, + "loss": 2.5021, + "theoretical_loss": 3.3268218698804666, + "tokens_seen": 2929852416 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011418181818181818, + "loss": 2.6819, + "theoretical_loss": 3.32681612085947, + "tokens_seen": 2929917952 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011416161616161616, + "loss": 2.727, + "theoretical_loss": 3.32681037200307, + "tokens_seen": 2929983488 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011414141414141415, + "loss": 2.3576, + "theoretical_loss": 3.326804623311259, + "tokens_seen": 2930049024 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011412121212121213, + "loss": 2.6268, + "theoretical_loss": 3.3267988747840285, + "tokens_seen": 2930114560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011410101010101011, + "loss": 2.4192, + "theoretical_loss": 3.3267931264213697, + "tokens_seen": 2930180096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011408080808080807, + "loss": 2.5583, + "theoretical_loss": 3.3267873782232744, + "tokens_seen": 2930245632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011406060606060605, + "loss": 2.4905, + "theoretical_loss": 3.326781630189734, + "tokens_seen": 2930311168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011404040404040404, + "loss": 2.5581, + "theoretical_loss": 3.326775882320741, + "tokens_seen": 2930376704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011402020202020202, + "loss": 2.466, + "theoretical_loss": 3.326770134616286, + "tokens_seen": 2930442240 + }, + { + "epoch": 0.77, + "learning_rate": 0.000114, + "loss": 2.5923, + "theoretical_loss": 3.3267643870763606, + "tokens_seen": 2930507776 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011397979797979799, + "loss": 2.4808, + "theoretical_loss": 3.326758639700957, + "tokens_seen": 2930573312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011395959595959596, + "loss": 2.6356, + "theoretical_loss": 3.3267528924900667, + "tokens_seen": 2930638848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011393939393939394, + "loss": 2.5241, + "theoretical_loss": 3.3267471454436808, + "tokens_seen": 2930704384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011391919191919193, + "loss": 2.5298, + "theoretical_loss": 3.3267413985617917, + "tokens_seen": 2930769920 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001138989898989899, + "loss": 2.5722, + "theoretical_loss": 3.3267356518443902, + "tokens_seen": 2930835456 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011387878787878788, + "loss": 2.4083, + "theoretical_loss": 3.3267299052914687, + "tokens_seen": 2930900992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011385858585858585, + "loss": 2.5877, + "theoretical_loss": 3.3267241589030183, + "tokens_seen": 2930966528 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011383838383838384, + "loss": 2.6443, + "theoretical_loss": 3.326718412679031, + "tokens_seen": 2931032064 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011381818181818182, + "loss": 2.4304, + "theoretical_loss": 3.3267126666194975, + "tokens_seen": 2931097600 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001137979797979798, + "loss": 2.4313, + "theoretical_loss": 3.3267069207244107, + "tokens_seen": 2931163136 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011377777777777779, + "loss": 2.4553, + "theoretical_loss": 3.326701174993761, + "tokens_seen": 2931228672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011375757575757576, + "loss": 2.53, + "theoretical_loss": 3.3266954294275415, + "tokens_seen": 2931294208 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1606724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8390579223632812, + "objective/train/theoretical_loss": 3.3266896840257423, + "objective/train/tokens_used": 1290218976, + "theoretical_loss": 3.3266896840257423, + "tokens_seen": 2931359744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011373737373737373, + "loss": 2.6834, + "theoretical_loss": 3.3266896840257423, + "tokens_seen": 2931359744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011371717171717171, + "loss": 2.4689, + "theoretical_loss": 3.3266839387883556, + "tokens_seen": 2931425280 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001136969696969697, + "loss": 2.4046, + "theoretical_loss": 3.3266781937153733, + "tokens_seen": 2931490816 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011367676767676768, + "loss": 2.5723, + "theoretical_loss": 3.3266724488067867, + "tokens_seen": 2931556352 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011365656565656565, + "loss": 2.4819, + "theoretical_loss": 3.3266667040625877, + "tokens_seen": 2931621888 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011363636363636364, + "loss": 2.6697, + "theoretical_loss": 3.3266609594827674, + "tokens_seen": 2931687424 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011361616161616162, + "loss": 2.7537, + "theoretical_loss": 3.3266552150673183, + "tokens_seen": 2931752960 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001135959595959596, + "loss": 2.6939, + "theoretical_loss": 3.326649470816231, + "tokens_seen": 2931818496 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011357575757575759, + "loss": 2.4281, + "theoretical_loss": 3.3266437267294977, + "tokens_seen": 2931884032 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011355555555555556, + "loss": 2.4854, + "theoretical_loss": 3.3266379828071098, + "tokens_seen": 2931949568 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011353535353535353, + "loss": 2.6985, + "theoretical_loss": 3.3266322390490592, + "tokens_seen": 2932015104 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011351515151515151, + "loss": 2.4425, + "theoretical_loss": 3.3266264954553373, + "tokens_seen": 2932080640 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001134949494949495, + "loss": 2.7626, + "theoretical_loss": 3.326620752025936, + "tokens_seen": 2932146176 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011347474747474748, + "loss": 2.5025, + "theoretical_loss": 3.3266150087608466, + "tokens_seen": 2932211712 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011345454545454546, + "loss": 2.5128, + "theoretical_loss": 3.326609265660061, + "tokens_seen": 2932277248 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011343434343434344, + "loss": 2.609, + "theoretical_loss": 3.3266035227235706, + "tokens_seen": 2932342784 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011341414141414142, + "loss": 2.6366, + "theoretical_loss": 3.326597779951367, + "tokens_seen": 2932408320 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001133939393939394, + "loss": 2.5676, + "theoretical_loss": 3.326592037343442, + "tokens_seen": 2932473856 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011337373737373737, + "loss": 2.801, + "theoretical_loss": 3.326586294899787, + "tokens_seen": 2932539392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011335353535353536, + "loss": 2.7573, + "theoretical_loss": 3.326580552620394, + "tokens_seen": 2932604928 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011333333333333333, + "loss": 2.7452, + "theoretical_loss": 3.3265748105052544, + "tokens_seen": 2932670464 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011331313131313131, + "loss": 2.409, + "theoretical_loss": 3.32656906855436, + "tokens_seen": 2932736000 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001132929292929293, + "loss": 2.3301, + "theoretical_loss": 3.326563326767702, + "tokens_seen": 2932801536 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011327272727272728, + "loss": 2.5418, + "theoretical_loss": 3.326557585145273, + "tokens_seen": 2932867072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011325252525252526, + "loss": 2.4264, + "theoretical_loss": 3.326551843687063, + "tokens_seen": 2932932608 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1608006, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.732264995574951, + "objective/train/theoretical_loss": 3.326546102393065, + "objective/train/tokens_used": 1291857376, + "theoretical_loss": 3.326546102393065, + "tokens_seen": 2932998144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011323232323232323, + "loss": 2.5268, + "theoretical_loss": 3.326546102393065, + "tokens_seen": 2932998144 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001132121212121212, + "loss": 2.53, + "theoretical_loss": 3.3265403612632705, + "tokens_seen": 2933063680 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011319191919191919, + "loss": 2.3276, + "theoretical_loss": 3.3265346202976707, + "tokens_seen": 2933129216 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011317171717171717, + "loss": 2.511, + "theoretical_loss": 3.3265288794962573, + "tokens_seen": 2933194752 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011315151515151516, + "loss": 2.4608, + "theoretical_loss": 3.326523138859022, + "tokens_seen": 2933260288 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011313131313131314, + "loss": 2.538, + "theoretical_loss": 3.3265173983859566, + "tokens_seen": 2933325824 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011311111111111111, + "loss": 2.5589, + "theoretical_loss": 3.3265116580770524, + "tokens_seen": 2933391360 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001130909090909091, + "loss": 2.5977, + "theoretical_loss": 3.3265059179323013, + "tokens_seen": 2933456896 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011307070707070708, + "loss": 2.6608, + "theoretical_loss": 3.3265001779516954, + "tokens_seen": 2933522432 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011305050505050506, + "loss": 2.7689, + "theoretical_loss": 3.3264944381352253, + "tokens_seen": 2933587968 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011303030303030303, + "loss": 2.4797, + "theoretical_loss": 3.326488698482883, + "tokens_seen": 2933653504 + }, + { + "epoch": 0.78, + "learning_rate": 0.000113010101010101, + "loss": 2.4523, + "theoretical_loss": 3.3264829589946605, + "tokens_seen": 2933719040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011298989898989899, + "loss": 2.5599, + "theoretical_loss": 3.3264772196705494, + "tokens_seen": 2933784576 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011296969696969697, + "loss": 2.7592, + "theoretical_loss": 3.3264714805105413, + "tokens_seen": 2933850112 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011294949494949496, + "loss": 2.6112, + "theoretical_loss": 3.3264657415146273, + "tokens_seen": 2933915648 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011292929292929294, + "loss": 2.4418, + "theoretical_loss": 3.3264600026827997, + "tokens_seen": 2933981184 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011290909090909091, + "loss": 2.5688, + "theoretical_loss": 3.32645426401505, + "tokens_seen": 2934046720 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001128888888888889, + "loss": 2.5185, + "theoretical_loss": 3.3264485255113696, + "tokens_seen": 2934112256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011286868686868686, + "loss": 2.6356, + "theoretical_loss": 3.3264427871717506, + "tokens_seen": 2934177792 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011284848484848485, + "loss": 2.4495, + "theoretical_loss": 3.326437048996184, + "tokens_seen": 2934243328 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011282828282828283, + "loss": 2.7521, + "theoretical_loss": 3.326431310984662, + "tokens_seen": 2934308864 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001128080808080808, + "loss": 2.6194, + "theoretical_loss": 3.326425573137176, + "tokens_seen": 2934374400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011278787878787879, + "loss": 2.4862, + "theoretical_loss": 3.3264198354537173, + "tokens_seen": 2934439936 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011276767676767677, + "loss": 2.5968, + "theoretical_loss": 3.3264140979342787, + "tokens_seen": 2934505472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011274747474747475, + "loss": 2.5451, + "theoretical_loss": 3.3264083605788506, + "tokens_seen": 2934571008 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1609289, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.37048602104187, + "objective/train/theoretical_loss": 3.326402623387425, + "objective/train/tokens_used": 1293495776, + "theoretical_loss": 3.326402623387425, + "tokens_seen": 2934636544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011272727272727274, + "loss": 2.5539, + "theoretical_loss": 3.326402623387425, + "tokens_seen": 2934636544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011270707070707072, + "loss": 2.5655, + "theoretical_loss": 3.326396886359994, + "tokens_seen": 2934702080 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011268686868686868, + "loss": 2.5888, + "theoretical_loss": 3.3263911494965486, + "tokens_seen": 2934767616 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011266666666666666, + "loss": 2.5285, + "theoretical_loss": 3.326385412797081, + "tokens_seen": 2934833152 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011264646464646465, + "loss": 2.5375, + "theoretical_loss": 3.326379676261583, + "tokens_seen": 2934898688 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011262626262626263, + "loss": 2.6415, + "theoretical_loss": 3.3263739398900456, + "tokens_seen": 2934964224 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011260606060606062, + "loss": 2.5207, + "theoretical_loss": 3.3263682036824607, + "tokens_seen": 2935029760 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011258585858585859, + "loss": 2.7268, + "theoretical_loss": 3.3263624676388197, + "tokens_seen": 2935095296 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011256565656565657, + "loss": 2.5792, + "theoretical_loss": 3.326356731759115, + "tokens_seen": 2935160832 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011254545454545455, + "loss": 2.4507, + "theoretical_loss": 3.3263509960433377, + "tokens_seen": 2935226368 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011252525252525254, + "loss": 2.636, + "theoretical_loss": 3.3263452604914794, + "tokens_seen": 2935291904 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011250505050505051, + "loss": 2.6615, + "theoretical_loss": 3.326339525103532, + "tokens_seen": 2935357440 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011248484848484848, + "loss": 2.4014, + "theoretical_loss": 3.326333789879487, + "tokens_seen": 2935422976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011246464646464646, + "loss": 2.5954, + "theoretical_loss": 3.3263280548193364, + "tokens_seen": 2935488512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011244444444444445, + "loss": 2.5842, + "theoretical_loss": 3.3263223199230714, + "tokens_seen": 2935554048 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011242424242424243, + "loss": 2.5586, + "theoretical_loss": 3.3263165851906837, + "tokens_seen": 2935619584 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011240404040404041, + "loss": 2.6714, + "theoretical_loss": 3.326310850622165, + "tokens_seen": 2935685120 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011238383838383838, + "loss": 2.6391, + "theoretical_loss": 3.3263051162175077, + "tokens_seen": 2935750656 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011236363636363637, + "loss": 2.6144, + "theoretical_loss": 3.3262993819767024, + "tokens_seen": 2935816192 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011234343434343434, + "loss": 2.7439, + "theoretical_loss": 3.326293647899741, + "tokens_seen": 2935881728 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011232323232323232, + "loss": 2.5445, + "theoretical_loss": 3.3262879139866155, + "tokens_seen": 2935947264 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001123030303030303, + "loss": 2.5282, + "theoretical_loss": 3.3262821802373175, + "tokens_seen": 2936012800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011228282828282829, + "loss": 2.7136, + "theoretical_loss": 3.3262764466518386, + "tokens_seen": 2936078336 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011226262626262626, + "loss": 2.524, + "theoretical_loss": 3.32627071323017, + "tokens_seen": 2936143872 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011224242424242424, + "loss": 2.5809, + "theoretical_loss": 3.3262649799723043, + "tokens_seen": 2936209408 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1609706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9183590412139893, + "objective/train/theoretical_loss": 3.326259246878233, + "objective/train/tokens_used": 1295134176, + "theoretical_loss": 3.326259246878233, + "tokens_seen": 2936274944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011222222222222223, + "loss": 2.5671, + "theoretical_loss": 3.326259246878233, + "tokens_seen": 2936274944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011220202020202021, + "loss": 2.526, + "theoretical_loss": 3.3262535139479468, + "tokens_seen": 2936340480 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001121818181818182, + "loss": 2.4673, + "theoretical_loss": 3.326247781181438, + "tokens_seen": 2936406016 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011216161616161615, + "loss": 2.4796, + "theoretical_loss": 3.326242048578698, + "tokens_seen": 2936471552 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011214141414141414, + "loss": 2.5022, + "theoretical_loss": 3.3262363161397195, + "tokens_seen": 2936537088 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011212121212121212, + "loss": 2.6066, + "theoretical_loss": 3.326230583864493, + "tokens_seen": 2936602624 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001121010101010101, + "loss": 2.4463, + "theoretical_loss": 3.3262248517530106, + "tokens_seen": 2936668160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011208080808080809, + "loss": 2.5293, + "theoretical_loss": 3.3262191198052635, + "tokens_seen": 2936733696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011206060606060606, + "loss": 2.5913, + "theoretical_loss": 3.3262133880212446, + "tokens_seen": 2936799232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011204040404040404, + "loss": 2.6632, + "theoretical_loss": 3.3262076564009444, + "tokens_seen": 2936864768 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011202020202020203, + "loss": 2.4133, + "theoretical_loss": 3.3262019249443546, + "tokens_seen": 2936930304 + }, + { + "epoch": 0.78, + "learning_rate": 0.000112, + "loss": 2.3691, + "theoretical_loss": 3.3261961936514677, + "tokens_seen": 2936995840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011197979797979798, + "loss": 2.4551, + "theoretical_loss": 3.3261904625222742, + "tokens_seen": 2937061376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011195959595959595, + "loss": 2.6147, + "theoretical_loss": 3.326184731556767, + "tokens_seen": 2937126912 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011193939393939394, + "loss": 2.6337, + "theoretical_loss": 3.326179000754937, + "tokens_seen": 2937192448 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011191919191919192, + "loss": 2.548, + "theoretical_loss": 3.3261732701167768, + "tokens_seen": 2937257984 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001118989898989899, + "loss": 2.7662, + "theoretical_loss": 3.3261675396422765, + "tokens_seen": 2937323520 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011187878787878789, + "loss": 2.4415, + "theoretical_loss": 3.326161809331429, + "tokens_seen": 2937389056 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011185858585858586, + "loss": 2.7199, + "theoretical_loss": 3.3261560791842255, + "tokens_seen": 2937454592 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011183838383838384, + "loss": 2.7703, + "theoretical_loss": 3.326150349200658, + "tokens_seen": 2937520128 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011181818181818181, + "loss": 2.8096, + "theoretical_loss": 3.3261446193807176, + "tokens_seen": 2937585664 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001117979797979798, + "loss": 2.2722, + "theoretical_loss": 3.3261388897243966, + "tokens_seen": 2937651200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011177777777777778, + "loss": 2.4869, + "theoretical_loss": 3.3261331602316866, + "tokens_seen": 2937716736 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011175757575757577, + "loss": 2.6614, + "theoretical_loss": 3.326127430902579, + "tokens_seen": 2937782272 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011173737373737374, + "loss": 2.7138, + "theoretical_loss": 3.3261217017370655, + "tokens_seen": 2937847808 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1610633, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4321837425231934, + "objective/train/theoretical_loss": 3.3261159727351375, + "objective/train/tokens_used": 1296772576, + "theoretical_loss": 3.3261159727351375, + "tokens_seen": 2937913344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011171717171717172, + "loss": 2.5338, + "theoretical_loss": 3.3261159727351375, + "tokens_seen": 2937913344 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001116969696969697, + "loss": 2.4712, + "theoretical_loss": 3.326110243896788, + "tokens_seen": 2937978880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011167676767676769, + "loss": 2.5505, + "theoretical_loss": 3.326104515222007, + "tokens_seen": 2938044416 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011165656565656566, + "loss": 2.6028, + "theoretical_loss": 3.326098786710787, + "tokens_seen": 2938109952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011163636363636363, + "loss": 2.7057, + "theoretical_loss": 3.32609305836312, + "tokens_seen": 2938175488 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011161616161616161, + "loss": 2.6309, + "theoretical_loss": 3.3260873301789964, + "tokens_seen": 2938241024 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001115959595959596, + "loss": 2.5039, + "theoretical_loss": 3.3260816021584096, + "tokens_seen": 2938306560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011157575757575758, + "loss": 2.5899, + "theoretical_loss": 3.32607587430135, + "tokens_seen": 2938372096 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011155555555555556, + "loss": 2.6186, + "theoretical_loss": 3.3260701466078095, + "tokens_seen": 2938437632 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011153535353535353, + "loss": 2.5333, + "theoretical_loss": 3.3260644190777806, + "tokens_seen": 2938503168 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011151515151515152, + "loss": 2.8804, + "theoretical_loss": 3.3260586917112542, + "tokens_seen": 2938568704 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001114949494949495, + "loss": 2.4777, + "theoretical_loss": 3.326052964508222, + "tokens_seen": 2938634240 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011147474747474747, + "loss": 2.31, + "theoretical_loss": 3.326047237468676, + "tokens_seen": 2938699776 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011145454545454546, + "loss": 2.5971, + "theoretical_loss": 3.3260415105926078, + "tokens_seen": 2938765312 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011143434343434343, + "loss": 2.4956, + "theoretical_loss": 3.326035783880009, + "tokens_seen": 2938830848 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011141414141414141, + "loss": 2.4588, + "theoretical_loss": 3.3260300573308714, + "tokens_seen": 2938896384 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001113939393939394, + "loss": 2.5158, + "theoretical_loss": 3.3260243309451862, + "tokens_seen": 2938961920 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011137373737373738, + "loss": 2.3541, + "theoretical_loss": 3.326018604722946, + "tokens_seen": 2939027456 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011135353535353536, + "loss": 2.7503, + "theoretical_loss": 3.326012878664142, + "tokens_seen": 2939092992 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011133333333333335, + "loss": 2.4821, + "theoretical_loss": 3.3260071527687654, + "tokens_seen": 2939158528 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001113131313131313, + "loss": 2.6369, + "theoretical_loss": 3.326001427036809, + "tokens_seen": 2939224064 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011129292929292929, + "loss": 2.2282, + "theoretical_loss": 3.3259957014682633, + "tokens_seen": 2939289600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011127272727272727, + "loss": 2.7342, + "theoretical_loss": 3.325989976063121, + "tokens_seen": 2939355136 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011125252525252526, + "loss": 2.3762, + "theoretical_loss": 3.3259842508213735, + "tokens_seen": 2939420672 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011123232323232324, + "loss": 2.5331, + "theoretical_loss": 3.325978525743012, + "tokens_seen": 2939486208 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1611825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4790115356445312, + "objective/train/theoretical_loss": 3.3259728008280285, + "objective/train/tokens_used": 1298410976, + "theoretical_loss": 3.3259728008280285, + "tokens_seen": 2939551744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011121212121212121, + "loss": 2.4969, + "theoretical_loss": 3.3259728008280285, + "tokens_seen": 2939551744 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001111919191919192, + "loss": 2.7655, + "theoretical_loss": 3.325967076076415, + "tokens_seen": 2939617280 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011117171717171718, + "loss": 2.427, + "theoretical_loss": 3.3259613514881625, + "tokens_seen": 2939682816 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011115151515151516, + "loss": 2.5228, + "theoretical_loss": 3.3259556270632635, + "tokens_seen": 2939748352 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011113131313131313, + "loss": 2.6434, + "theoretical_loss": 3.3259499028017094, + "tokens_seen": 2939813888 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001111111111111111, + "loss": 2.5845, + "theoretical_loss": 3.325944178703492, + "tokens_seen": 2939879424 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011109090909090909, + "loss": 2.6955, + "theoretical_loss": 3.325938454768602, + "tokens_seen": 2939944960 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011107070707070707, + "loss": 2.7222, + "theoretical_loss": 3.3259327309970326, + "tokens_seen": 2940010496 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011105050505050505, + "loss": 2.5762, + "theoretical_loss": 3.325927007388775, + "tokens_seen": 2940076032 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011103030303030304, + "loss": 2.6349, + "theoretical_loss": 3.3259212839438206, + "tokens_seen": 2940141568 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011101010101010101, + "loss": 2.4277, + "theoretical_loss": 3.325915560662161, + "tokens_seen": 2940207104 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011098989898989899, + "loss": 2.7432, + "theoretical_loss": 3.3259098375437883, + "tokens_seen": 2940272640 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011096969696969698, + "loss": 2.6131, + "theoretical_loss": 3.3259041145886936, + "tokens_seen": 2940338176 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011094949494949495, + "loss": 2.806, + "theoretical_loss": 3.3258983917968696, + "tokens_seen": 2940403712 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011092929292929293, + "loss": 2.6925, + "theoretical_loss": 3.3258926691683075, + "tokens_seen": 2940469248 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011090909090909092, + "loss": 2.5413, + "theoretical_loss": 3.3258869467029983, + "tokens_seen": 2940534784 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011088888888888889, + "loss": 2.7037, + "theoretical_loss": 3.325881224400935, + "tokens_seen": 2940600320 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011086868686868687, + "loss": 2.5241, + "theoretical_loss": 3.3258755022621083, + "tokens_seen": 2940665856 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011084848484848485, + "loss": 2.5935, + "theoretical_loss": 3.3258697802865105, + "tokens_seen": 2940731392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011082828282828284, + "loss": 2.6394, + "theoretical_loss": 3.325864058474133, + "tokens_seen": 2940796928 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011080808080808082, + "loss": 2.576, + "theoretical_loss": 3.3258583368249672, + "tokens_seen": 2940862464 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011078787878787878, + "loss": 2.4306, + "theoretical_loss": 3.325852615339006, + "tokens_seen": 2940928000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011076767676767676, + "loss": 2.6534, + "theoretical_loss": 3.325846894016239, + "tokens_seen": 2940993536 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011074747474747475, + "loss": 2.607, + "theoretical_loss": 3.32584117285666, + "tokens_seen": 2941059072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011072727272727273, + "loss": 2.3431, + "theoretical_loss": 3.32583545186026, + "tokens_seen": 2941124608 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1612404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.758617877960205, + "objective/train/theoretical_loss": 3.3258297310270306, + "objective/train/tokens_used": 1300049376, + "theoretical_loss": 3.3258297310270306, + "tokens_seen": 2941190144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011070707070707071, + "loss": 2.678, + "theoretical_loss": 3.3258297310270306, + "tokens_seen": 2941190144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011068686868686868, + "loss": 2.5589, + "theoretical_loss": 3.3258240103569636, + "tokens_seen": 2941255680 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011066666666666667, + "loss": 2.4059, + "theoretical_loss": 3.3258182898500506, + "tokens_seen": 2941321216 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011064646464646465, + "loss": 2.6307, + "theoretical_loss": 3.325812569506283, + "tokens_seen": 2941386752 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011062626262626264, + "loss": 2.5359, + "theoretical_loss": 3.325806849325653, + "tokens_seen": 2941452288 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011060606060606061, + "loss": 2.6308, + "theoretical_loss": 3.3258011293081524, + "tokens_seen": 2941517824 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011058585858585858, + "loss": 2.5774, + "theoretical_loss": 3.3257954094537725, + "tokens_seen": 2941583360 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011056565656565656, + "loss": 2.5642, + "theoretical_loss": 3.325789689762505, + "tokens_seen": 2941648896 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011054545454545455, + "loss": 2.6757, + "theoretical_loss": 3.325783970234342, + "tokens_seen": 2941714432 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011052525252525253, + "loss": 2.7848, + "theoretical_loss": 3.3257782508692753, + "tokens_seen": 2941779968 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011050505050505051, + "loss": 2.7476, + "theoretical_loss": 3.325772531667296, + "tokens_seen": 2941845504 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001104848484848485, + "loss": 2.7523, + "theoretical_loss": 3.3257668126283964, + "tokens_seen": 2941911040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011046464646464647, + "loss": 2.369, + "theoretical_loss": 3.325761093752568, + "tokens_seen": 2941976576 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011044444444444444, + "loss": 2.6327, + "theoretical_loss": 3.3257553750398023, + "tokens_seen": 2942042112 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011042424242424242, + "loss": 2.5991, + "theoretical_loss": 3.3257496564900912, + "tokens_seen": 2942107648 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001104040404040404, + "loss": 2.53, + "theoretical_loss": 3.3257439381034266, + "tokens_seen": 2942173184 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011038383838383839, + "loss": 2.6983, + "theoretical_loss": 3.3257382198798, + "tokens_seen": 2942238720 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011036363636363636, + "loss": 2.5285, + "theoretical_loss": 3.3257325018192034, + "tokens_seen": 2942304256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011034343434343434, + "loss": 2.4434, + "theoretical_loss": 3.325726783921628, + "tokens_seen": 2942369792 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011032323232323233, + "loss": 2.532, + "theoretical_loss": 3.3257210661870658, + "tokens_seen": 2942435328 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011030303030303031, + "loss": 2.8869, + "theoretical_loss": 3.3257153486155087, + "tokens_seen": 2942500864 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001102828282828283, + "loss": 2.4905, + "theoretical_loss": 3.3257096312069483, + "tokens_seen": 2942566400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011026262626262625, + "loss": 2.5919, + "theoretical_loss": 3.325703913961376, + "tokens_seen": 2942631936 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011024242424242424, + "loss": 2.4003, + "theoretical_loss": 3.325698196878784, + "tokens_seen": 2942697472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011022222222222222, + "loss": 2.658, + "theoretical_loss": 3.325692479959164, + "tokens_seen": 2942763008 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1613400, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0002992153167725, + "objective/train/theoretical_loss": 3.3256867632025076, + "objective/train/tokens_used": 1301687776, + "theoretical_loss": 3.3256867632025076, + "tokens_seen": 2942828544 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001102020202020202, + "loss": 2.729, + "theoretical_loss": 3.3256867632025076, + "tokens_seen": 2942828544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011018181818181819, + "loss": 2.6348, + "theoretical_loss": 3.325681046608806, + "tokens_seen": 2942894080 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011016161616161616, + "loss": 2.3756, + "theoretical_loss": 3.325675330178052, + "tokens_seen": 2942959616 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011014141414141414, + "loss": 2.4326, + "theoretical_loss": 3.3256696139102364, + "tokens_seen": 2943025152 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011012121212121213, + "loss": 2.6285, + "theoretical_loss": 3.3256638978053514, + "tokens_seen": 2943090688 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011010101010101011, + "loss": 2.6596, + "theoretical_loss": 3.3256581818633886, + "tokens_seen": 2943156224 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011008080808080808, + "loss": 2.5413, + "theoretical_loss": 3.3256524660843394, + "tokens_seen": 2943221760 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011006060606060607, + "loss": 2.8548, + "theoretical_loss": 3.325646750468196, + "tokens_seen": 2943287296 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011004040404040404, + "loss": 2.7512, + "theoretical_loss": 3.32564103501495, + "tokens_seen": 2943352832 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011002020202020202, + "loss": 2.6005, + "theoretical_loss": 3.3256353197245936, + "tokens_seen": 2943418368 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011, + "loss": 2.7225, + "theoretical_loss": 3.3256296045971174, + "tokens_seen": 2943483904 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010997979797979799, + "loss": 2.5975, + "theoretical_loss": 3.325623889632514, + "tokens_seen": 2943549440 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010995959595959597, + "loss": 2.6122, + "theoretical_loss": 3.325618174830775, + "tokens_seen": 2943614976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010993939393939394, + "loss": 2.518, + "theoretical_loss": 3.325612460191892, + "tokens_seen": 2943680512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010991919191919191, + "loss": 2.5377, + "theoretical_loss": 3.3256067457158567, + "tokens_seen": 2943746048 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001098989898989899, + "loss": 2.4338, + "theoretical_loss": 3.325601031402661, + "tokens_seen": 2943811584 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010987878787878788, + "loss": 2.6336, + "theoretical_loss": 3.3255953172522963, + "tokens_seen": 2943877120 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010985858585858586, + "loss": 2.7016, + "theoretical_loss": 3.325589603264755, + "tokens_seen": 2943942656 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010983838383838383, + "loss": 2.6037, + "theoretical_loss": 3.3255838894400283, + "tokens_seen": 2944008192 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010981818181818182, + "loss": 2.4089, + "theoretical_loss": 3.3255781757781078, + "tokens_seen": 2944073728 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001097979797979798, + "loss": 2.6908, + "theoretical_loss": 3.325572462278986, + "tokens_seen": 2944139264 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010977777777777779, + "loss": 2.7255, + "theoretical_loss": 3.3255667489426535, + "tokens_seen": 2944204800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010975757575757577, + "loss": 2.58, + "theoretical_loss": 3.325561035769103, + "tokens_seen": 2944270336 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010973737373737373, + "loss": 2.6426, + "theoretical_loss": 3.3255553227583254, + "tokens_seen": 2944335872 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010971717171717171, + "loss": 2.6954, + "theoretical_loss": 3.325549609910314, + "tokens_seen": 2944401408 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1613826, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.518526792526245, + "objective/train/theoretical_loss": 3.3255438972250584, + "objective/train/tokens_used": 1303326176, + "theoretical_loss": 3.3255438972250584, + "tokens_seen": 2944466944 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001096969696969697, + "loss": 2.6183, + "theoretical_loss": 3.3255438972250584, + "tokens_seen": 2944466944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010967676767676768, + "loss": 2.5964, + "theoretical_loss": 3.325538184702552, + "tokens_seen": 2944532480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010965656565656566, + "loss": 2.6953, + "theoretical_loss": 3.325532472342786, + "tokens_seen": 2944598016 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010963636363636365, + "loss": 2.2839, + "theoretical_loss": 3.3255267601457517, + "tokens_seen": 2944663552 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010961616161616162, + "loss": 2.4044, + "theoretical_loss": 3.3255210481114417, + "tokens_seen": 2944729088 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001095959595959596, + "loss": 2.5569, + "theoretical_loss": 3.3255153362398473, + "tokens_seen": 2944794624 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010957575757575757, + "loss": 2.4601, + "theoretical_loss": 3.3255096245309597, + "tokens_seen": 2944860160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010955555555555556, + "loss": 2.7598, + "theoretical_loss": 3.3255039129847717, + "tokens_seen": 2944925696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010953535353535354, + "loss": 2.2931, + "theoretical_loss": 3.3254982016012744, + "tokens_seen": 2944991232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010951515151515151, + "loss": 2.5856, + "theoretical_loss": 3.3254924903804595, + "tokens_seen": 2945056768 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001094949494949495, + "loss": 2.6282, + "theoretical_loss": 3.3254867793223193, + "tokens_seen": 2945122304 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010947474747474748, + "loss": 2.642, + "theoretical_loss": 3.3254810684268445, + "tokens_seen": 2945187840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010945454545454546, + "loss": 2.659, + "theoretical_loss": 3.325475357694028, + "tokens_seen": 2945253376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010943434343434345, + "loss": 2.2815, + "theoretical_loss": 3.325469647123861, + "tokens_seen": 2945318912 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010941414141414142, + "loss": 2.8497, + "theoretical_loss": 3.3254639367163357, + "tokens_seen": 2945384448 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010939393939393939, + "loss": 2.6946, + "theoretical_loss": 3.3254582264714427, + "tokens_seen": 2945449984 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010937373737373737, + "loss": 2.7545, + "theoretical_loss": 3.325452516389175, + "tokens_seen": 2945515520 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010935353535353536, + "loss": 2.5832, + "theoretical_loss": 3.3254468064695235, + "tokens_seen": 2945581056 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010933333333333334, + "loss": 2.7608, + "theoretical_loss": 3.325441096712481, + "tokens_seen": 2945646592 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010931313131313131, + "loss": 2.6582, + "theoretical_loss": 3.3254353871180378, + "tokens_seen": 2945712128 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001092929292929293, + "loss": 2.5069, + "theoretical_loss": 3.325429677686187, + "tokens_seen": 2945777664 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010927272727272728, + "loss": 2.6522, + "theoretical_loss": 3.3254239684169193, + "tokens_seen": 2945843200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010925252525252526, + "loss": 2.6628, + "theoretical_loss": 3.325418259310227, + "tokens_seen": 2945908736 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010923232323232323, + "loss": 2.6827, + "theoretical_loss": 3.3254125503661016, + "tokens_seen": 2945974272 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010921212121212122, + "loss": 2.4938, + "theoretical_loss": 3.3254068415845355, + "tokens_seen": 2946039808 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1615304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8399295806884766, + "objective/train/theoretical_loss": 3.3254011329655198, + "objective/train/tokens_used": 1304964576, + "theoretical_loss": 3.3254011329655198, + "tokens_seen": 2946105344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010919191919191919, + "loss": 2.8637, + "theoretical_loss": 3.3254011329655198, + "tokens_seen": 2946105344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010917171717171717, + "loss": 2.667, + "theoretical_loss": 3.3253954245090465, + "tokens_seen": 2946170880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010915151515151515, + "loss": 2.5635, + "theoretical_loss": 3.325389716215107, + "tokens_seen": 2946236416 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010913131313131314, + "loss": 2.5937, + "theoretical_loss": 3.325384008083694, + "tokens_seen": 2946301952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010911111111111112, + "loss": 2.7726, + "theoretical_loss": 3.3253783001147976, + "tokens_seen": 2946367488 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010909090909090909, + "loss": 2.6751, + "theoretical_loss": 3.3253725923084114, + "tokens_seen": 2946433024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010907070707070708, + "loss": 2.6317, + "theoretical_loss": 3.3253668846645263, + "tokens_seen": 2946498560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010905050505050505, + "loss": 2.6213, + "theoretical_loss": 3.3253611771831335, + "tokens_seen": 2946564096 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010903030303030303, + "loss": 2.4585, + "theoretical_loss": 3.325355469864226, + "tokens_seen": 2946629632 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010901010101010101, + "loss": 2.5077, + "theoretical_loss": 3.3253497627077944, + "tokens_seen": 2946695168 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010898989898989899, + "loss": 2.6593, + "theoretical_loss": 3.325344055713831, + "tokens_seen": 2946760704 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010896969696969697, + "loss": 2.7443, + "theoretical_loss": 3.325338348882328, + "tokens_seen": 2946826240 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010894949494949495, + "loss": 2.6881, + "theoretical_loss": 3.3253326422132763, + "tokens_seen": 2946891776 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010892929292929294, + "loss": 2.3787, + "theoretical_loss": 3.325326935706668, + "tokens_seen": 2946957312 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010890909090909092, + "loss": 2.3944, + "theoretical_loss": 3.325321229362495, + "tokens_seen": 2947022848 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010888888888888888, + "loss": 2.5759, + "theoretical_loss": 3.325315523180749, + "tokens_seen": 2947088384 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010886868686868686, + "loss": 2.6787, + "theoretical_loss": 3.325309817161422, + "tokens_seen": 2947153920 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010884848484848485, + "loss": 2.4837, + "theoretical_loss": 3.325304111304505, + "tokens_seen": 2947219456 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010882828282828283, + "loss": 2.4463, + "theoretical_loss": 3.325298405609991, + "tokens_seen": 2947284992 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010880808080808081, + "loss": 2.3673, + "theoretical_loss": 3.3252927000778705, + "tokens_seen": 2947350528 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001087878787878788, + "loss": 2.5868, + "theoretical_loss": 3.325286994708136, + "tokens_seen": 2947416064 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010876767676767677, + "loss": 2.5394, + "theoretical_loss": 3.3252812895007793, + "tokens_seen": 2947481600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010874747474747475, + "loss": 2.5199, + "theoretical_loss": 3.3252755844557917, + "tokens_seen": 2947547136 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010872727272727274, + "loss": 2.4673, + "theoretical_loss": 3.325269879573165, + "tokens_seen": 2947612672 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001087070707070707, + "loss": 2.4566, + "theoretical_loss": 3.3252641748528915, + "tokens_seen": 2947678208 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1616471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.794506311416626, + "objective/train/theoretical_loss": 3.325258470294963, + "objective/train/tokens_used": 1306602976, + "theoretical_loss": 3.325258470294963, + "tokens_seen": 2947743744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010868686868686869, + "loss": 2.598, + "theoretical_loss": 3.325258470294963, + "tokens_seen": 2947743744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010866666666666666, + "loss": 2.613, + "theoretical_loss": 3.32525276589937, + "tokens_seen": 2947809280 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010864646464646464, + "loss": 2.4786, + "theoretical_loss": 3.3252470616661056, + "tokens_seen": 2947874816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010862626262626263, + "loss": 2.7583, + "theoretical_loss": 3.3252413575951616, + "tokens_seen": 2947940352 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010860606060606061, + "loss": 2.5134, + "theoretical_loss": 3.3252356536865286, + "tokens_seen": 2948005888 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001085858585858586, + "loss": 2.6787, + "theoretical_loss": 3.3252299499401996, + "tokens_seen": 2948071424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010856565656565657, + "loss": 2.6067, + "theoretical_loss": 3.325224246356166, + "tokens_seen": 2948136960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010854545454545455, + "loss": 2.653, + "theoretical_loss": 3.325218542934419, + "tokens_seen": 2948202496 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010852525252525252, + "loss": 2.5313, + "theoretical_loss": 3.3252128396749514, + "tokens_seen": 2948268032 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001085050505050505, + "loss": 2.7618, + "theoretical_loss": 3.3252071365777542, + "tokens_seen": 2948333568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010848484848484849, + "loss": 2.5056, + "theoretical_loss": 3.3252014336428193, + "tokens_seen": 2948399104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010846464646464646, + "loss": 2.5414, + "theoretical_loss": 3.3251957308701385, + "tokens_seen": 2948464640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010844444444444444, + "loss": 2.5955, + "theoretical_loss": 3.325190028259704, + "tokens_seen": 2948530176 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010842424242424243, + "loss": 2.4789, + "theoretical_loss": 3.325184325811507, + "tokens_seen": 2948595712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010840404040404041, + "loss": 2.6843, + "theoretical_loss": 3.325178623525539, + "tokens_seen": 2948661248 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001083838383838384, + "loss": 2.3767, + "theoretical_loss": 3.325172921401793, + "tokens_seen": 2948726784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010836363636363637, + "loss": 2.5728, + "theoretical_loss": 3.3251672194402597, + "tokens_seen": 2948792320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010834343434343434, + "loss": 2.5631, + "theoretical_loss": 3.3251615176409315, + "tokens_seen": 2948857856 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010832323232323232, + "loss": 2.5711, + "theoretical_loss": 3.3251558160037997, + "tokens_seen": 2948923392 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001083030303030303, + "loss": 2.6212, + "theoretical_loss": 3.3251501145288564, + "tokens_seen": 2948988928 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010828282828282829, + "loss": 2.6286, + "theoretical_loss": 3.325144413216093, + "tokens_seen": 2949054464 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010826262626262627, + "loss": 2.7655, + "theoretical_loss": 3.3251387120655025, + "tokens_seen": 2949120000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010824242424242424, + "loss": 2.8169, + "theoretical_loss": 3.325133011077075, + "tokens_seen": 2949185536 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010822222222222223, + "loss": 2.5712, + "theoretical_loss": 3.325127310250803, + "tokens_seen": 2949251072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010820202020202021, + "loss": 2.6203, + "theoretical_loss": 3.3251216095866782, + "tokens_seen": 2949316608 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1616749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1516165733337402, + "objective/train/theoretical_loss": 3.325115909084693, + "objective/train/tokens_used": 1308241376, + "theoretical_loss": 3.325115909084693, + "tokens_seen": 2949382144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010818181818181818, + "loss": 2.5991, + "theoretical_loss": 3.325115909084693, + "tokens_seen": 2949382144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010816161616161616, + "loss": 2.6945, + "theoretical_loss": 3.3251102087448388, + "tokens_seen": 2949447680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010814141414141414, + "loss": 2.5981, + "theoretical_loss": 3.3251045085671067, + "tokens_seen": 2949513216 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010812121212121212, + "loss": 2.6191, + "theoretical_loss": 3.3250988085514894, + "tokens_seen": 2949578752 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001081010101010101, + "loss": 2.5087, + "theoretical_loss": 3.3250931086979785, + "tokens_seen": 2949644288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010808080808080809, + "loss": 2.5923, + "theoretical_loss": 3.3250874090065654, + "tokens_seen": 2949709824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010806060606060607, + "loss": 2.6183, + "theoretical_loss": 3.3250817094772422, + "tokens_seen": 2949775360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010804040404040404, + "loss": 2.7086, + "theoretical_loss": 3.3250760101100005, + "tokens_seen": 2949840896 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010802020202020201, + "loss": 2.6298, + "theoretical_loss": 3.3250703109048323, + "tokens_seen": 2949906432 + }, + { + "epoch": 0.79, + "learning_rate": 0.000108, + "loss": 2.3853, + "theoretical_loss": 3.325064611861729, + "tokens_seen": 2949971968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010797979797979798, + "loss": 2.5412, + "theoretical_loss": 3.3250589129806833, + "tokens_seen": 2950037504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010795959595959596, + "loss": 2.5326, + "theoretical_loss": 3.325053214261686, + "tokens_seen": 2950103040 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010793939393939395, + "loss": 2.6814, + "theoretical_loss": 3.3250475157047292, + "tokens_seen": 2950168576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010791919191919192, + "loss": 2.526, + "theoretical_loss": 3.3250418173098053, + "tokens_seen": 2950234112 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001078989898989899, + "loss": 2.8406, + "theoretical_loss": 3.325036119076905, + "tokens_seen": 2950299648 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010787878787878789, + "loss": 2.6819, + "theoretical_loss": 3.325030421006021, + "tokens_seen": 2950365184 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010785858585858587, + "loss": 2.7354, + "theoretical_loss": 3.3250247230971444, + "tokens_seen": 2950430720 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010783838383838384, + "loss": 2.5749, + "theoretical_loss": 3.325019025350268, + "tokens_seen": 2950496256 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010781818181818181, + "loss": 2.5647, + "theoretical_loss": 3.325013327765382, + "tokens_seen": 2950561792 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001077979797979798, + "loss": 2.5118, + "theoretical_loss": 3.32500763034248, + "tokens_seen": 2950627328 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010777777777777778, + "loss": 2.4043, + "theoretical_loss": 3.3250019330815523, + "tokens_seen": 2950692864 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010775757575757576, + "loss": 2.4916, + "theoretical_loss": 3.3249962359825918, + "tokens_seen": 2950758400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010773737373737375, + "loss": 2.6121, + "theoretical_loss": 3.3249905390455896, + "tokens_seen": 2950823936 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010771717171717172, + "loss": 2.1771, + "theoretical_loss": 3.3249848422705375, + "tokens_seen": 2950889472 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001076969696969697, + "loss": 2.8045, + "theoretical_loss": 3.324979145657428, + "tokens_seen": 2950955008 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1618033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.540438413619995, + "objective/train/theoretical_loss": 3.324973449206252, + "objective/train/tokens_used": 1309879776, + "theoretical_loss": 3.324973449206252, + "tokens_seen": 2951020544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010767676767676769, + "loss": 2.6239, + "theoretical_loss": 3.324973449206252, + "tokens_seen": 2951020544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010765656565656566, + "loss": 2.4172, + "theoretical_loss": 3.324967752917002, + "tokens_seen": 2951086080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010763636363636364, + "loss": 2.5796, + "theoretical_loss": 3.3249620567896696, + "tokens_seen": 2951151616 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010761616161616161, + "loss": 2.4873, + "theoretical_loss": 3.3249563608242467, + "tokens_seen": 2951217152 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001075959595959596, + "loss": 2.5326, + "theoretical_loss": 3.3249506650207246, + "tokens_seen": 2951282688 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010757575757575758, + "loss": 2.6194, + "theoretical_loss": 3.3249449693790956, + "tokens_seen": 2951348224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010755555555555556, + "loss": 2.5648, + "theoretical_loss": 3.3249392738993513, + "tokens_seen": 2951413760 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010753535353535355, + "loss": 2.6425, + "theoretical_loss": 3.3249335785814833, + "tokens_seen": 2951479296 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010751515151515153, + "loss": 2.6687, + "theoretical_loss": 3.324927883425484, + "tokens_seen": 2951544832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010749494949494949, + "loss": 2.5706, + "theoretical_loss": 3.3249221884313447, + "tokens_seen": 2951610368 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010747474747474747, + "loss": 2.2787, + "theoretical_loss": 3.3249164935990576, + "tokens_seen": 2951675904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010745454545454545, + "loss": 2.6493, + "theoretical_loss": 3.324910798928614, + "tokens_seen": 2951741440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010743434343434344, + "loss": 2.6236, + "theoretical_loss": 3.324905104420006, + "tokens_seen": 2951806976 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010741414141414142, + "loss": 2.6284, + "theoretical_loss": 3.3248994100732254, + "tokens_seen": 2951872512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010739393939393939, + "loss": 2.7336, + "theoretical_loss": 3.3248937158882645, + "tokens_seen": 2951938048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010737373737373738, + "loss": 2.5389, + "theoretical_loss": 3.324888021865114, + "tokens_seen": 2952003584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010735353535353536, + "loss": 2.7596, + "theoretical_loss": 3.3248823280037665, + "tokens_seen": 2952069120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010733333333333334, + "loss": 2.9191, + "theoretical_loss": 3.3248766343042133, + "tokens_seen": 2952134656 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010731313131313132, + "loss": 2.5895, + "theoretical_loss": 3.324870940766447, + "tokens_seen": 2952200192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010729292929292929, + "loss": 2.439, + "theoretical_loss": 3.3248652473904587, + "tokens_seen": 2952265728 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010727272727272727, + "loss": 2.3604, + "theoretical_loss": 3.324859554176241, + "tokens_seen": 2952331264 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010725252525252525, + "loss": 2.5904, + "theoretical_loss": 3.3248538611237843, + "tokens_seen": 2952396800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010723232323232324, + "loss": 2.5589, + "theoretical_loss": 3.324848168233082, + "tokens_seen": 2952462336 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010721212121212122, + "loss": 2.6628, + "theoretical_loss": 3.324842475504125, + "tokens_seen": 2952527872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010719191919191919, + "loss": 2.4778, + "theoretical_loss": 3.324836782936905, + "tokens_seen": 2952593408 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1618610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3691909313201904, + "objective/train/theoretical_loss": 3.3248310905314145, + "objective/train/tokens_used": 1311518176, + "theoretical_loss": 3.3248310905314145, + "tokens_seen": 2952658944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010717171717171718, + "loss": 2.4928, + "theoretical_loss": 3.3248310905314145, + "tokens_seen": 2952658944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010715151515151515, + "loss": 2.3124, + "theoretical_loss": 3.324825398287645, + "tokens_seen": 2952724480 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010713131313131313, + "loss": 2.6518, + "theoretical_loss": 3.324819706205588, + "tokens_seen": 2952790016 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010711111111111111, + "loss": 2.4696, + "theoretical_loss": 3.3248140142852356, + "tokens_seen": 2952855552 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001070909090909091, + "loss": 2.7384, + "theoretical_loss": 3.3248083225265797, + "tokens_seen": 2952921088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010707070707070707, + "loss": 2.5929, + "theoretical_loss": 3.3248026309296117, + "tokens_seen": 2952986624 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010705050505050505, + "loss": 2.4477, + "theoretical_loss": 3.324796939494324, + "tokens_seen": 2953052160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010703030303030304, + "loss": 2.5462, + "theoretical_loss": 3.3247912482207083, + "tokens_seen": 2953117696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010701010101010102, + "loss": 2.634, + "theoretical_loss": 3.3247855571087563, + "tokens_seen": 2953183232 + }, + { + "epoch": 0.79, + "learning_rate": 0.000106989898989899, + "loss": 2.414, + "theoretical_loss": 3.3247798661584596, + "tokens_seen": 2953248768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010696969696969696, + "loss": 2.6698, + "theoretical_loss": 3.32477417536981, + "tokens_seen": 2953314304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010694949494949495, + "loss": 2.4008, + "theoretical_loss": 3.3247684847428, + "tokens_seen": 2953379840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010692929292929293, + "loss": 2.7128, + "theoretical_loss": 3.3247627942774205, + "tokens_seen": 2953445376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010690909090909091, + "loss": 2.5687, + "theoretical_loss": 3.3247571039736643, + "tokens_seen": 2953510912 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001068888888888889, + "loss": 2.5782, + "theoretical_loss": 3.3247514138315224, + "tokens_seen": 2953576448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010686868686868687, + "loss": 2.77, + "theoretical_loss": 3.3247457238509868, + "tokens_seen": 2953641984 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010684848484848485, + "loss": 2.6817, + "theoretical_loss": 3.3247400340320494, + "tokens_seen": 2953707520 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010682828282828284, + "loss": 2.5042, + "theoretical_loss": 3.3247343443747024, + "tokens_seen": 2953773056 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010680808080808082, + "loss": 2.6227, + "theoretical_loss": 3.324728654878937, + "tokens_seen": 2953838592 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010678787878787879, + "loss": 2.5374, + "theoretical_loss": 3.324722965544746, + "tokens_seen": 2953904128 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010676767676767676, + "loss": 2.6405, + "theoretical_loss": 3.32471727637212, + "tokens_seen": 2953969664 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010674747474747474, + "loss": 2.4261, + "theoretical_loss": 3.3247115873610515, + "tokens_seen": 2954035200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010672727272727273, + "loss": 2.6452, + "theoretical_loss": 3.324705898511532, + "tokens_seen": 2954100736 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010670707070707071, + "loss": 2.551, + "theoretical_loss": 3.324700209823554, + "tokens_seen": 2954166272 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001066868686868687, + "loss": 2.4004, + "theoretical_loss": 3.3246945212971086, + "tokens_seen": 2954231808 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1619927, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4913690090179443, + "objective/train/theoretical_loss": 3.3246888329321878, + "objective/train/tokens_used": 1313156576, + "theoretical_loss": 3.3246888329321878, + "tokens_seen": 2954297344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010666666666666668, + "loss": 2.3608, + "theoretical_loss": 3.3246888329321878, + "tokens_seen": 2954297344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010664646464646465, + "loss": 2.3978, + "theoretical_loss": 3.3246831447287835, + "tokens_seen": 2954362880 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010662626262626262, + "loss": 2.3883, + "theoretical_loss": 3.3246774566868877, + "tokens_seen": 2954428416 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001066060606060606, + "loss": 2.5867, + "theoretical_loss": 3.324671768806492, + "tokens_seen": 2954493952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010658585858585859, + "loss": 2.5603, + "theoretical_loss": 3.3246660810875883, + "tokens_seen": 2954559488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010656565656565657, + "loss": 2.5406, + "theoretical_loss": 3.3246603935301686, + "tokens_seen": 2954625024 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010654545454545454, + "loss": 2.6231, + "theoretical_loss": 3.3246547061342246, + "tokens_seen": 2954690560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010652525252525253, + "loss": 2.3915, + "theoretical_loss": 3.3246490188997484, + "tokens_seen": 2954756096 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010650505050505051, + "loss": 2.5871, + "theoretical_loss": 3.324643331826731, + "tokens_seen": 2954821632 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001064848484848485, + "loss": 2.2428, + "theoretical_loss": 3.324637644915165, + "tokens_seen": 2954887168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010646464646464648, + "loss": 2.3654, + "theoretical_loss": 3.324631958165042, + "tokens_seen": 2954952704 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010644444444444444, + "loss": 2.5839, + "theoretical_loss": 3.324626271576354, + "tokens_seen": 2955018240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010642424242424242, + "loss": 2.3345, + "theoretical_loss": 3.3246205851490926, + "tokens_seen": 2955083776 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001064040404040404, + "loss": 2.6119, + "theoretical_loss": 3.32461489888325, + "tokens_seen": 2955149312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010638383838383839, + "loss": 2.6239, + "theoretical_loss": 3.3246092127788174, + "tokens_seen": 2955214848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010636363636363637, + "loss": 2.3651, + "theoretical_loss": 3.324603526835787, + "tokens_seen": 2955280384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010634343434343434, + "loss": 2.2928, + "theoretical_loss": 3.324597841054151, + "tokens_seen": 2955345920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010632323232323233, + "loss": 2.3381, + "theoretical_loss": 3.324592155433901, + "tokens_seen": 2955411456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010630303030303031, + "loss": 2.5536, + "theoretical_loss": 3.3245864699750287, + "tokens_seen": 2955476992 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010628282828282828, + "loss": 2.4378, + "theoretical_loss": 3.3245807846775257, + "tokens_seen": 2955542528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010626262626262626, + "loss": 2.5099, + "theoretical_loss": 3.3245750995413843, + "tokens_seen": 2955608064 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010624242424242425, + "loss": 2.3829, + "theoretical_loss": 3.3245694145665956, + "tokens_seen": 2955673600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010622222222222222, + "loss": 2.5726, + "theoretical_loss": 3.3245637297531525, + "tokens_seen": 2955739136 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001062020202020202, + "loss": 2.3272, + "theoretical_loss": 3.3245580451010466, + "tokens_seen": 2955804672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010618181818181819, + "loss": 2.6045, + "theoretical_loss": 3.3245523606102694, + "tokens_seen": 2955870208 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1620587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.624238967895508, + "objective/train/theoretical_loss": 3.3245466762808125, + "objective/train/tokens_used": 1314794976, + "theoretical_loss": 3.3245466762808125, + "tokens_seen": 2955935744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010616161616161617, + "loss": 2.1955, + "theoretical_loss": 3.3245466762808125, + "tokens_seen": 2955935744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010614141414141415, + "loss": 2.4206, + "theoretical_loss": 3.3245409921126683, + "tokens_seen": 2956001280 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010612121212121212, + "loss": 2.6707, + "theoretical_loss": 3.3245353081058284, + "tokens_seen": 2956066816 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001061010101010101, + "loss": 2.3215, + "theoretical_loss": 3.3245296242602844, + "tokens_seen": 2956132352 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010608080808080808, + "loss": 2.3914, + "theoretical_loss": 3.3245239405760287, + "tokens_seen": 2956197888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010606060606060606, + "loss": 2.4059, + "theoretical_loss": 3.324518257053053, + "tokens_seen": 2956263424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010604040404040405, + "loss": 2.5986, + "theoretical_loss": 3.324512573691349, + "tokens_seen": 2956328960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010602020202020202, + "loss": 2.6937, + "theoretical_loss": 3.3245068904909085, + "tokens_seen": 2956394496 + }, + { + "epoch": 0.79, + "learning_rate": 0.000106, + "loss": 2.6134, + "theoretical_loss": 3.3245012074517235, + "tokens_seen": 2956460032 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010597979797979799, + "loss": 2.66, + "theoretical_loss": 3.324495524573786, + "tokens_seen": 2956525568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010595959595959597, + "loss": 2.5629, + "theoretical_loss": 3.324489841857087, + "tokens_seen": 2956591104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010593939393939394, + "loss": 2.4315, + "theoretical_loss": 3.324484159301619, + "tokens_seen": 2956656640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010591919191919191, + "loss": 2.5471, + "theoretical_loss": 3.324478476907374, + "tokens_seen": 2956722176 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001058989898989899, + "loss": 2.5047, + "theoretical_loss": 3.324472794674344, + "tokens_seen": 2956787712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010587878787878788, + "loss": 2.5472, + "theoretical_loss": 3.3244671126025205, + "tokens_seen": 2956853248 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010585858585858586, + "loss": 2.439, + "theoretical_loss": 3.324461430691895, + "tokens_seen": 2956918784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010583838383838385, + "loss": 2.5288, + "theoretical_loss": 3.32445574894246, + "tokens_seen": 2956984320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010581818181818183, + "loss": 2.4758, + "theoretical_loss": 3.3244500673542072, + "tokens_seen": 2957049856 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001057979797979798, + "loss": 2.4169, + "theoretical_loss": 3.324444385927128, + "tokens_seen": 2957115392 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010577777777777778, + "loss": 2.3895, + "theoretical_loss": 3.324438704661215, + "tokens_seen": 2957180928 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010575757575757575, + "loss": 2.4942, + "theoretical_loss": 3.3244330235564594, + "tokens_seen": 2957246464 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010573737373737374, + "loss": 2.2842, + "theoretical_loss": 3.324427342612853, + "tokens_seen": 2957312000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010571717171717172, + "loss": 2.3584, + "theoretical_loss": 3.324421661830389, + "tokens_seen": 2957377536 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010569696969696969, + "loss": 2.5966, + "theoretical_loss": 3.3244159812090572, + "tokens_seen": 2957443072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010567676767676768, + "loss": 2.5507, + "theoretical_loss": 3.324410300748851, + "tokens_seen": 2957508608 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1621841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1909501552581787, + "objective/train/theoretical_loss": 3.3244046204497617, + "objective/train/tokens_used": 1316433376, + "theoretical_loss": 3.3244046204497617, + "tokens_seen": 2957574144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010565656565656566, + "loss": 2.4851, + "theoretical_loss": 3.3244046204497617, + "tokens_seen": 2957574144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010563636363636365, + "loss": 2.5167, + "theoretical_loss": 3.324398940311781, + "tokens_seen": 2957639680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010561616161616163, + "loss": 2.4686, + "theoretical_loss": 3.324393260334901, + "tokens_seen": 2957705216 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010559595959595959, + "loss": 2.3187, + "theoretical_loss": 3.324387580519114, + "tokens_seen": 2957770752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010557575757575757, + "loss": 2.6202, + "theoretical_loss": 3.324381900864411, + "tokens_seen": 2957836288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010555555555555555, + "loss": 2.6112, + "theoretical_loss": 3.3243762213707844, + "tokens_seen": 2957901824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010553535353535354, + "loss": 2.402, + "theoretical_loss": 3.324370542038226, + "tokens_seen": 2957967360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010551515151515152, + "loss": 2.3216, + "theoretical_loss": 3.324364862866727, + "tokens_seen": 2958032896 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010549494949494949, + "loss": 2.4769, + "theoretical_loss": 3.3243591838562807, + "tokens_seen": 2958098432 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010547474747474748, + "loss": 2.5486, + "theoretical_loss": 3.3243535050068775, + "tokens_seen": 2958163968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010545454545454546, + "loss": 2.4215, + "theoretical_loss": 3.32434782631851, + "tokens_seen": 2958229504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010543434343434344, + "loss": 2.4427, + "theoretical_loss": 3.32434214779117, + "tokens_seen": 2958295040 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010541414141414141, + "loss": 2.3774, + "theoretical_loss": 3.3243364694248494, + "tokens_seen": 2958360576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010539393939393938, + "loss": 2.3702, + "theoretical_loss": 3.32433079121954, + "tokens_seen": 2958426112 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010537373737373737, + "loss": 2.3168, + "theoretical_loss": 3.3243251131752336, + "tokens_seen": 2958491648 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010535353535353535, + "loss": 2.4422, + "theoretical_loss": 3.324319435291922, + "tokens_seen": 2958557184 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010533333333333334, + "loss": 2.5293, + "theoretical_loss": 3.3243137575695973, + "tokens_seen": 2958622720 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010531313131313132, + "loss": 2.284, + "theoretical_loss": 3.3243080800082514, + "tokens_seen": 2958688256 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001052929292929293, + "loss": 2.6009, + "theoretical_loss": 3.3243024026078762, + "tokens_seen": 2958753792 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010527272727272728, + "loss": 2.6215, + "theoretical_loss": 3.324296725368463, + "tokens_seen": 2958819328 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010525252525252526, + "loss": 2.4684, + "theoretical_loss": 3.324291048290004, + "tokens_seen": 2958884864 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010523232323232323, + "loss": 2.7368, + "theoretical_loss": 3.3242853713724916, + "tokens_seen": 2958950400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010521212121212121, + "loss": 2.4528, + "theoretical_loss": 3.3242796946159165, + "tokens_seen": 2959015936 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001051919191919192, + "loss": 2.367, + "theoretical_loss": 3.324274018020272, + "tokens_seen": 2959081472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010517171717171717, + "loss": 2.6569, + "theoretical_loss": 3.3242683415855487, + "tokens_seen": 2959147008 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1622322, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.434272050857544, + "objective/train/theoretical_loss": 3.3242626653117395, + "objective/train/tokens_used": 1318071776, + "theoretical_loss": 3.3242626653117395, + "tokens_seen": 2959212544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010515151515151515, + "loss": 2.3888, + "theoretical_loss": 3.3242626653117395, + "tokens_seen": 2959212544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010513131313131314, + "loss": 2.3452, + "theoretical_loss": 3.324256989198836, + "tokens_seen": 2959278080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010511111111111112, + "loss": 2.4652, + "theoretical_loss": 3.324251313246829, + "tokens_seen": 2959343616 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001050909090909091, + "loss": 2.7122, + "theoretical_loss": 3.324245637455712, + "tokens_seen": 2959409152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010507070707070706, + "loss": 2.5425, + "theoretical_loss": 3.3242399618254757, + "tokens_seen": 2959474688 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010505050505050504, + "loss": 2.6049, + "theoretical_loss": 3.3242342863561127, + "tokens_seen": 2959540224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010503030303030303, + "loss": 2.5624, + "theoretical_loss": 3.324228611047615, + "tokens_seen": 2959605760 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010501010101010101, + "loss": 2.3073, + "theoretical_loss": 3.3242229358999733, + "tokens_seen": 2959671296 + }, + { + "epoch": 0.79, + "learning_rate": 0.000104989898989899, + "loss": 2.5305, + "theoretical_loss": 3.324217260913181, + "tokens_seen": 2959736832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010496969696969697, + "loss": 2.6132, + "theoretical_loss": 3.3242115860872286, + "tokens_seen": 2959802368 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010494949494949495, + "loss": 2.5674, + "theoretical_loss": 3.3242059114221085, + "tokens_seen": 2959867904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010492929292929293, + "loss": 2.3998, + "theoretical_loss": 3.324200236917813, + "tokens_seen": 2959933440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010490909090909092, + "loss": 2.4725, + "theoretical_loss": 3.3241945625743337, + "tokens_seen": 2959998976 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010488888888888889, + "loss": 2.4784, + "theoretical_loss": 3.3241888883916624, + "tokens_seen": 2960064512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010486868686868687, + "loss": 2.6038, + "theoretical_loss": 3.324183214369791, + "tokens_seen": 2960130048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010484848484848484, + "loss": 2.459, + "theoretical_loss": 3.3241775405087117, + "tokens_seen": 2960195584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010482828282828283, + "loss": 2.3374, + "theoretical_loss": 3.324171866808416, + "tokens_seen": 2960261120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010480808080808081, + "loss": 2.3654, + "theoretical_loss": 3.324166193268896, + "tokens_seen": 2960326656 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001047878787878788, + "loss": 2.5011, + "theoretical_loss": 3.324160519890143, + "tokens_seen": 2960392192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010476767676767678, + "loss": 2.3735, + "theoretical_loss": 3.3241548466721498, + "tokens_seen": 2960457728 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010474747474747475, + "loss": 2.4604, + "theoretical_loss": 3.3241491736149076, + "tokens_seen": 2960523264 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010472727272727272, + "loss": 2.6346, + "theoretical_loss": 3.3241435007184084, + "tokens_seen": 2960588800 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001047070707070707, + "loss": 2.4402, + "theoretical_loss": 3.3241378279826446, + "tokens_seen": 2960654336 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010468686868686869, + "loss": 2.605, + "theoretical_loss": 3.3241321554076078, + "tokens_seen": 2960719872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010466666666666667, + "loss": 2.6159, + "theoretical_loss": 3.3241264829932895, + "tokens_seen": 2960785408 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1623244, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6110126972198486, + "objective/train/theoretical_loss": 3.324120810739682, + "objective/train/tokens_used": 1319710176, + "theoretical_loss": 3.324120810739682, + "tokens_seen": 2960850944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010464646464646464, + "loss": 2.5337, + "theoretical_loss": 3.324120810739682, + "tokens_seen": 2960850944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010462626262626263, + "loss": 2.6342, + "theoretical_loss": 3.3241151386467767, + "tokens_seen": 2960916480 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010460606060606061, + "loss": 2.5077, + "theoretical_loss": 3.3241094667145665, + "tokens_seen": 2960982016 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001045858585858586, + "loss": 2.4245, + "theoretical_loss": 3.324103794943042, + "tokens_seen": 2961047552 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010456565656565658, + "loss": 2.5697, + "theoretical_loss": 3.3240981233321962, + "tokens_seen": 2961113088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010454545454545454, + "loss": 2.5805, + "theoretical_loss": 3.3240924518820205, + "tokens_seen": 2961178624 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010452525252525252, + "loss": 2.6296, + "theoretical_loss": 3.324086780592507, + "tokens_seen": 2961244160 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001045050505050505, + "loss": 2.5592, + "theoretical_loss": 3.324081109463647, + "tokens_seen": 2961309696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010448484848484849, + "loss": 2.1178, + "theoretical_loss": 3.324075438495433, + "tokens_seen": 2961375232 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010446464646464647, + "loss": 2.4289, + "theoretical_loss": 3.324069767687857, + "tokens_seen": 2961440768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010444444444444445, + "loss": 2.4596, + "theoretical_loss": 3.3240640970409103, + "tokens_seen": 2961506304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010442424242424243, + "loss": 2.4909, + "theoretical_loss": 3.324058426554585, + "tokens_seen": 2961571840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010440404040404041, + "loss": 2.4652, + "theoretical_loss": 3.3240527562288733, + "tokens_seen": 2961637376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010438383838383839, + "loss": 2.5048, + "theoretical_loss": 3.324047086063767, + "tokens_seen": 2961702912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010436363636363636, + "loss": 2.6128, + "theoretical_loss": 3.3240414160592575, + "tokens_seen": 2961768448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010434343434343435, + "loss": 2.371, + "theoretical_loss": 3.3240357462153374, + "tokens_seen": 2961833984 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010432323232323232, + "loss": 2.5723, + "theoretical_loss": 3.324030076531998, + "tokens_seen": 2961899520 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001043030303030303, + "loss": 2.5163, + "theoretical_loss": 3.324024407009232, + "tokens_seen": 2961965056 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010428282828282829, + "loss": 2.1553, + "theoretical_loss": 3.3240187376470307, + "tokens_seen": 2962030592 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010426262626262627, + "loss": 2.403, + "theoretical_loss": 3.324013068445386, + "tokens_seen": 2962096128 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010424242424242425, + "loss": 2.4736, + "theoretical_loss": 3.3240073994042896, + "tokens_seen": 2962161664 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010422222222222222, + "loss": 2.5284, + "theoretical_loss": 3.324001730523734, + "tokens_seen": 2962227200 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001042020202020202, + "loss": 2.4208, + "theoretical_loss": 3.3239960618037108, + "tokens_seen": 2962292736 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010418181818181818, + "loss": 2.2946, + "theoretical_loss": 3.3239903932442116, + "tokens_seen": 2962358272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010416161616161616, + "loss": 2.4546, + "theoretical_loss": 3.323984724845229, + "tokens_seen": 2962423808 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1624479, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.264127254486084, + "objective/train/theoretical_loss": 3.3239790566067544, + "objective/train/tokens_used": 1321348576, + "theoretical_loss": 3.3239790566067544, + "tokens_seen": 2962489344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010414141414141415, + "loss": 2.3267, + "theoretical_loss": 3.3239790566067544, + "tokens_seen": 2962489344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010412121212121212, + "loss": 2.6755, + "theoretical_loss": 3.3239733885287794, + "tokens_seen": 2962554880 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001041010101010101, + "loss": 2.515, + "theoretical_loss": 3.323967720611297, + "tokens_seen": 2962620416 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010408080808080808, + "loss": 2.6878, + "theoretical_loss": 3.323962052854298, + "tokens_seen": 2962685952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010406060606060607, + "loss": 2.3999, + "theoretical_loss": 3.323956385257775, + "tokens_seen": 2962751488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010404040404040405, + "loss": 2.6721, + "theoretical_loss": 3.3239507178217194, + "tokens_seen": 2962817024 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010402020202020202, + "loss": 2.4076, + "theoretical_loss": 3.3239450505461234, + "tokens_seen": 2962882560 + }, + { + "epoch": 0.79, + "learning_rate": 0.000104, + "loss": 2.7202, + "theoretical_loss": 3.3239393834309787, + "tokens_seen": 2962948096 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010397979797979798, + "loss": 2.4769, + "theoretical_loss": 3.3239337164762777, + "tokens_seen": 2963013632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010395959595959596, + "loss": 2.3982, + "theoretical_loss": 3.323928049682012, + "tokens_seen": 2963079168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010393939393939395, + "loss": 2.6043, + "theoretical_loss": 3.323922383048173, + "tokens_seen": 2963144704 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010391919191919193, + "loss": 2.4343, + "theoretical_loss": 3.323916716574753, + "tokens_seen": 2963210240 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001038989898989899, + "loss": 2.683, + "theoretical_loss": 3.3239110502617444, + "tokens_seen": 2963275776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010387878787878788, + "loss": 2.3344, + "theoretical_loss": 3.323905384109139, + "tokens_seen": 2963341312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010385858585858585, + "loss": 2.5157, + "theoretical_loss": 3.3238997181169276, + "tokens_seen": 2963406848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010383838383838384, + "loss": 2.7316, + "theoretical_loss": 3.3238940522851035, + "tokens_seen": 2963472384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010381818181818182, + "loss": 2.5605, + "theoretical_loss": 3.323888386613658, + "tokens_seen": 2963537920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010379797979797979, + "loss": 2.2599, + "theoretical_loss": 3.323882721102583, + "tokens_seen": 2963603456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010377777777777778, + "loss": 2.4524, + "theoretical_loss": 3.3238770557518706, + "tokens_seen": 2963668992 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010375757575757576, + "loss": 2.3724, + "theoretical_loss": 3.3238713905615125, + "tokens_seen": 2963734528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010373737373737374, + "loss": 2.4722, + "theoretical_loss": 3.3238657255315007, + "tokens_seen": 2963800064 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010371717171717173, + "loss": 2.4297, + "theoretical_loss": 3.323860060661827, + "tokens_seen": 2963865600 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001036969696969697, + "loss": 2.2663, + "theoretical_loss": 3.3238543959524836, + "tokens_seen": 2963931136 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010367676767676767, + "loss": 2.5315, + "theoretical_loss": 3.323848731403462, + "tokens_seen": 2963996672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010365656565656565, + "loss": 2.5912, + "theoretical_loss": 3.3238430670147547, + "tokens_seen": 2964062208 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1624832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1099302768707275, + "objective/train/theoretical_loss": 3.3238374027863533, + "objective/train/tokens_used": 1322986976, + "theoretical_loss": 3.3238374027863533, + "tokens_seen": 2964127744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010363636363636364, + "loss": 2.2317, + "theoretical_loss": 3.3238374027863533, + "tokens_seen": 2964127744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010361616161616162, + "loss": 2.2615, + "theoretical_loss": 3.3238317387182494, + "tokens_seen": 2964193280 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001035959595959596, + "loss": 2.7159, + "theoretical_loss": 3.3238260748104356, + "tokens_seen": 2964258816 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010357575757575758, + "loss": 2.6553, + "theoretical_loss": 3.3238204110629033, + "tokens_seen": 2964324352 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010355555555555556, + "loss": 2.5324, + "theoretical_loss": 3.3238147474756445, + "tokens_seen": 2964389888 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010353535353535354, + "loss": 2.6607, + "theoretical_loss": 3.3238090840486514, + "tokens_seen": 2964455424 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010351515151515151, + "loss": 2.4357, + "theoretical_loss": 3.3238034207819154, + "tokens_seen": 2964520960 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001034949494949495, + "loss": 2.6308, + "theoretical_loss": 3.323797757675429, + "tokens_seen": 2964586496 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010347474747474747, + "loss": 2.5054, + "theoretical_loss": 3.3237920947291837, + "tokens_seen": 2964652032 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010345454545454545, + "loss": 2.6337, + "theoretical_loss": 3.3237864319431716, + "tokens_seen": 2964717568 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010343434343434344, + "loss": 2.3644, + "theoretical_loss": 3.3237807693173846, + "tokens_seen": 2964783104 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010341414141414142, + "loss": 2.6203, + "theoretical_loss": 3.323775106851815, + "tokens_seen": 2964848640 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001033939393939394, + "loss": 2.4556, + "theoretical_loss": 3.323769444546454, + "tokens_seen": 2964914176 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010337373737373737, + "loss": 2.5176, + "theoretical_loss": 3.3237637824012936, + "tokens_seen": 2964979712 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010335353535353536, + "loss": 2.4087, + "theoretical_loss": 3.3237581204163265, + "tokens_seen": 2965045248 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010333333333333333, + "loss": 2.3925, + "theoretical_loss": 3.323752458591544, + "tokens_seen": 2965110784 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010331313131313131, + "loss": 2.4198, + "theoretical_loss": 3.323746796926938, + "tokens_seen": 2965176320 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001032929292929293, + "loss": 2.581, + "theoretical_loss": 3.323741135422501, + "tokens_seen": 2965241856 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010327272727272727, + "loss": 2.6013, + "theoretical_loss": 3.3237354740782243, + "tokens_seen": 2965307392 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010325252525252525, + "loss": 2.5642, + "theoretical_loss": 3.3237298128940997, + "tokens_seen": 2965372928 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010323232323232324, + "loss": 2.6024, + "theoretical_loss": 3.32372415187012, + "tokens_seen": 2965438464 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010321212121212122, + "loss": 2.5792, + "theoretical_loss": 3.3237184910062765, + "tokens_seen": 2965504000 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001031919191919192, + "loss": 2.5008, + "theoretical_loss": 3.3237128303025614, + "tokens_seen": 2965569536 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010317171717171719, + "loss": 2.5079, + "theoretical_loss": 3.323707169758966, + "tokens_seen": 2965635072 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010315151515151514, + "loss": 2.3708, + "theoretical_loss": 3.323701509375483, + "tokens_seen": 2965700608 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1626023, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.672511577606201, + "objective/train/theoretical_loss": 3.323695849152104, + "objective/train/tokens_used": 1324625376, + "theoretical_loss": 3.323695849152104, + "tokens_seen": 2965766144 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010313131313131313, + "loss": 2.5631, + "theoretical_loss": 3.323695849152104, + "tokens_seen": 2965766144 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010311111111111111, + "loss": 2.5906, + "theoretical_loss": 3.323690189088821, + "tokens_seen": 2965831680 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001030909090909091, + "loss": 2.5668, + "theoretical_loss": 3.323684529185626, + "tokens_seen": 2965897216 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010307070707070708, + "loss": 2.5538, + "theoretical_loss": 3.323678869442511, + "tokens_seen": 2965962752 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010305050505050505, + "loss": 2.2811, + "theoretical_loss": 3.3236732098594675, + "tokens_seen": 2966028288 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010303030303030303, + "loss": 2.4367, + "theoretical_loss": 3.323667550436488, + "tokens_seen": 2966093824 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010301010101010102, + "loss": 2.6801, + "theoretical_loss": 3.323661891173564, + "tokens_seen": 2966159360 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010298989898989899, + "loss": 2.6832, + "theoretical_loss": 3.3236562320706873, + "tokens_seen": 2966224896 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010296969696969697, + "loss": 2.4707, + "theoretical_loss": 3.3236505731278503, + "tokens_seen": 2966290432 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010294949494949494, + "loss": 2.6052, + "theoretical_loss": 3.323644914345045, + "tokens_seen": 2966355968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010292929292929293, + "loss": 2.3966, + "theoretical_loss": 3.323639255722263, + "tokens_seen": 2966421504 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010290909090909091, + "loss": 2.4385, + "theoretical_loss": 3.323633597259496, + "tokens_seen": 2966487040 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001028888888888889, + "loss": 2.719, + "theoretical_loss": 3.3236279389567365, + "tokens_seen": 2966552576 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010286868686868688, + "loss": 2.3644, + "theoretical_loss": 3.3236222808139764, + "tokens_seen": 2966618112 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010284848484848485, + "loss": 2.6043, + "theoretical_loss": 3.3236166228312074, + "tokens_seen": 2966683648 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010282828282828283, + "loss": 2.5614, + "theoretical_loss": 3.3236109650084216, + "tokens_seen": 2966749184 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001028080808080808, + "loss": 2.5532, + "theoretical_loss": 3.323605307345611, + "tokens_seen": 2966814720 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010278787878787879, + "loss": 2.6673, + "theoretical_loss": 3.323599649842767, + "tokens_seen": 2966880256 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010276767676767677, + "loss": 2.6003, + "theoretical_loss": 3.323593992499882, + "tokens_seen": 2966945792 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010274747474747476, + "loss": 2.2786, + "theoretical_loss": 3.323588335316948, + "tokens_seen": 2967011328 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010272727272727273, + "loss": 2.4837, + "theoretical_loss": 3.3235826782939566, + "tokens_seen": 2967076864 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010270707070707071, + "loss": 2.2114, + "theoretical_loss": 3.3235770214309, + "tokens_seen": 2967142400 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001026868686868687, + "loss": 2.565, + "theoretical_loss": 3.3235713647277705, + "tokens_seen": 2967207936 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010266666666666668, + "loss": 2.7195, + "theoretical_loss": 3.3235657081845593, + "tokens_seen": 2967273472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010264646464646465, + "loss": 2.6002, + "theoretical_loss": 3.323560051801259, + "tokens_seen": 2967339008 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1626612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8011512756347656, + "objective/train/theoretical_loss": 3.3235543955778613, + "objective/train/tokens_used": 1326263776, + "theoretical_loss": 3.3235543955778613, + "tokens_seen": 2967404544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010262626262626262, + "loss": 2.7578, + "theoretical_loss": 3.3235543955778613, + "tokens_seen": 2967404544 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001026060606060606, + "loss": 2.2567, + "theoretical_loss": 3.323548739514358, + "tokens_seen": 2967470080 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010258585858585859, + "loss": 2.5688, + "theoretical_loss": 3.323543083610741, + "tokens_seen": 2967535616 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010256565656565657, + "loss": 2.4983, + "theoretical_loss": 3.3235374278670027, + "tokens_seen": 2967601152 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010254545454545455, + "loss": 2.4021, + "theoretical_loss": 3.3235317722831343, + "tokens_seen": 2967666688 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010252525252525252, + "loss": 2.4752, + "theoretical_loss": 3.323526116859129, + "tokens_seen": 2967732224 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010250505050505051, + "loss": 2.5998, + "theoretical_loss": 3.323520461594977, + "tokens_seen": 2967797760 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010248484848484849, + "loss": 2.5755, + "theoretical_loss": 3.323514806490672, + "tokens_seen": 2967863296 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010246464646464646, + "loss": 2.5622, + "theoretical_loss": 3.323509151546205, + "tokens_seen": 2967928832 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010244444444444445, + "loss": 2.696, + "theoretical_loss": 3.3235034967615675, + "tokens_seen": 2967994368 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010242424242424242, + "loss": 2.6037, + "theoretical_loss": 3.323497842136753, + "tokens_seen": 2968059904 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001024040404040404, + "loss": 2.585, + "theoretical_loss": 3.323492187671752, + "tokens_seen": 2968125440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010238383838383839, + "loss": 2.6302, + "theoretical_loss": 3.323486533366557, + "tokens_seen": 2968190976 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010236363636363637, + "loss": 2.5971, + "theoretical_loss": 3.32348087922116, + "tokens_seen": 2968256512 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010234343434343435, + "loss": 2.5166, + "theoretical_loss": 3.3234752252355526, + "tokens_seen": 2968322048 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010232323232323234, + "loss": 2.5497, + "theoretical_loss": 3.3234695714097278, + "tokens_seen": 2968387584 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001023030303030303, + "loss": 2.5387, + "theoretical_loss": 3.323463917743676, + "tokens_seen": 2968453120 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010228282828282828, + "loss": 2.3017, + "theoretical_loss": 3.32345826423739, + "tokens_seen": 2968518656 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010226262626262626, + "loss": 2.641, + "theoretical_loss": 3.3234526108908624, + "tokens_seen": 2968584192 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010224242424242425, + "loss": 2.3994, + "theoretical_loss": 3.323446957704084, + "tokens_seen": 2968649728 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010222222222222223, + "loss": 2.5104, + "theoretical_loss": 3.3234413046770475, + "tokens_seen": 2968715264 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001022020202020202, + "loss": 2.6517, + "theoretical_loss": 3.3234356518097443, + "tokens_seen": 2968780800 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010218181818181818, + "loss": 2.7413, + "theoretical_loss": 3.323429999102167, + "tokens_seen": 2968846336 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010216161616161617, + "loss": 2.4412, + "theoretical_loss": 3.323424346554307, + "tokens_seen": 2968911872 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010214141414141415, + "loss": 2.4175, + "theoretical_loss": 3.3234186941661568, + "tokens_seen": 2968977408 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1627805, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.652923822402954, + "objective/train/theoretical_loss": 3.3234130419377075, + "objective/train/tokens_used": 1327902176, + "theoretical_loss": 3.3234130419377075, + "tokens_seen": 2969042944 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010212121212121212, + "loss": 2.5858, + "theoretical_loss": 3.3234130419377075, + "tokens_seen": 2969042944 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010210101010101009, + "loss": 2.5745, + "theoretical_loss": 3.323407389868952, + "tokens_seen": 2969108480 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010208080808080808, + "loss": 2.5786, + "theoretical_loss": 3.3234017379598817, + "tokens_seen": 2969174016 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010206060606060606, + "loss": 2.4503, + "theoretical_loss": 3.3233960862104888, + "tokens_seen": 2969239552 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010204040404040404, + "loss": 2.3736, + "theoretical_loss": 3.323390434620765, + "tokens_seen": 2969305088 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010202020202020203, + "loss": 2.3875, + "theoretical_loss": 3.323384783190703, + "tokens_seen": 2969370624 + }, + { + "epoch": 0.8, + "learning_rate": 0.000102, + "loss": 2.6837, + "theoretical_loss": 3.3233791319202934, + "tokens_seen": 2969436160 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010197979797979798, + "loss": 2.6875, + "theoretical_loss": 3.3233734808095297, + "tokens_seen": 2969501696 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010195959595959597, + "loss": 2.2677, + "theoretical_loss": 3.323367829858403, + "tokens_seen": 2969567232 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010193939393939394, + "loss": 2.4074, + "theoretical_loss": 3.3233621790669052, + "tokens_seen": 2969632768 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010191919191919192, + "loss": 2.3802, + "theoretical_loss": 3.323356528435029, + "tokens_seen": 2969698304 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001018989898989899, + "loss": 2.5734, + "theoretical_loss": 3.3233508779627656, + "tokens_seen": 2969763840 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010187878787878788, + "loss": 2.5026, + "theoretical_loss": 3.323345227650107, + "tokens_seen": 2969829376 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010185858585858586, + "loss": 2.6336, + "theoretical_loss": 3.3233395774970456, + "tokens_seen": 2969894912 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010183838383838384, + "loss": 2.6269, + "theoretical_loss": 3.3233339275035734, + "tokens_seen": 2969960448 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010181818181818183, + "loss": 2.4779, + "theoretical_loss": 3.323328277669682, + "tokens_seen": 2970025984 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010179797979797981, + "loss": 2.4965, + "theoretical_loss": 3.3233226279953634, + "tokens_seen": 2970091520 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010177777777777777, + "loss": 2.7358, + "theoretical_loss": 3.3233169784806096, + "tokens_seen": 2970157056 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010175757575757575, + "loss": 2.4341, + "theoretical_loss": 3.3233113291254126, + "tokens_seen": 2970222592 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010173737373737374, + "loss": 2.5575, + "theoretical_loss": 3.323305679929765, + "tokens_seen": 2970288128 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010171717171717172, + "loss": 2.4274, + "theoretical_loss": 3.3233000308936576, + "tokens_seen": 2970353664 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001016969696969697, + "loss": 2.5485, + "theoretical_loss": 3.3232943820170835, + "tokens_seen": 2970419200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010167676767676767, + "loss": 2.495, + "theoretical_loss": 3.323288733300034, + "tokens_seen": 2970484736 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010165656565656566, + "loss": 2.2588, + "theoretical_loss": 3.323283084742501, + "tokens_seen": 2970550272 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010163636363636364, + "loss": 2.6041, + "theoretical_loss": 3.323277436344477, + "tokens_seen": 2970615808 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1628362, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5661630630493164, + "objective/train/theoretical_loss": 3.3232717881059535, + "objective/train/tokens_used": 1329540576, + "theoretical_loss": 3.3232717881059535, + "tokens_seen": 2970681344 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010161616161616163, + "loss": 2.5095, + "theoretical_loss": 3.3232717881059535, + "tokens_seen": 2970681344 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001015959595959596, + "loss": 2.5209, + "theoretical_loss": 3.3232661400269228, + "tokens_seen": 2970746880 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010157575757575757, + "loss": 2.4259, + "theoretical_loss": 3.3232604921073765, + "tokens_seen": 2970812416 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010155555555555555, + "loss": 2.6697, + "theoretical_loss": 3.323254844347307, + "tokens_seen": 2970877952 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010153535353535354, + "loss": 2.4666, + "theoretical_loss": 3.323249196746706, + "tokens_seen": 2970943488 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010151515151515152, + "loss": 2.5946, + "theoretical_loss": 3.323243549305566, + "tokens_seen": 2971009024 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001014949494949495, + "loss": 2.5381, + "theoretical_loss": 3.323237902023878, + "tokens_seen": 2971074560 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010147474747474749, + "loss": 2.4383, + "theoretical_loss": 3.323232254901635, + "tokens_seen": 2971140096 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010145454545454546, + "loss": 2.4008, + "theoretical_loss": 3.323226607938828, + "tokens_seen": 2971205632 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010143434343434343, + "loss": 2.4742, + "theoretical_loss": 3.3232209611354504, + "tokens_seen": 2971271168 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010141414141414141, + "loss": 2.5057, + "theoretical_loss": 3.3232153144914927, + "tokens_seen": 2971336704 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001013939393939394, + "loss": 2.2964, + "theoretical_loss": 3.3232096680069474, + "tokens_seen": 2971402240 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010137373737373738, + "loss": 2.4692, + "theoretical_loss": 3.3232040216818066, + "tokens_seen": 2971467776 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010135353535353535, + "loss": 2.4429, + "theoretical_loss": 3.3231983755160623, + "tokens_seen": 2971533312 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010133333333333333, + "loss": 2.257, + "theoretical_loss": 3.3231927295097066, + "tokens_seen": 2971598848 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010131313131313132, + "loss": 2.5742, + "theoretical_loss": 3.323187083662731, + "tokens_seen": 2971664384 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001012929292929293, + "loss": 2.514, + "theoretical_loss": 3.323181437975128, + "tokens_seen": 2971729920 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010127272727272729, + "loss": 2.4644, + "theoretical_loss": 3.3231757924468894, + "tokens_seen": 2971795456 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010125252525252524, + "loss": 2.5649, + "theoretical_loss": 3.3231701470780073, + "tokens_seen": 2971860992 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010123232323232323, + "loss": 2.3359, + "theoretical_loss": 3.3231645018684732, + "tokens_seen": 2971926528 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010121212121212121, + "loss": 2.532, + "theoretical_loss": 3.3231588568182797, + "tokens_seen": 2971992064 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001011919191919192, + "loss": 2.5608, + "theoretical_loss": 3.3231532119274183, + "tokens_seen": 2972057600 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010117171717171718, + "loss": 2.5313, + "theoretical_loss": 3.3231475671958814, + "tokens_seen": 2972123136 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010115151515151515, + "loss": 2.5096, + "theoretical_loss": 3.323141922623661, + "tokens_seen": 2972188672 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010113131313131313, + "loss": 2.4996, + "theoretical_loss": 3.323136278210749, + "tokens_seen": 2972254208 + }, + { + "debugging/Self-BLEU-5": 0.3606301032207952, + "debugging/distinct-1-grams": 0.7379428181758975, + "debugging/distinct-2-grams": 0.954427516927517, + "debugging/entropy-1-grams": 4.8487704715345625, + "debugging/entropy-2-grams": 5.495200377538124, + "debugging/length": 654.0, + "debugging/num_segments": 3, + "epoch": 0.8, + "objective/train/docs_used": 1629567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8025996685028076, + "objective/train/theoretical_loss": 3.323130633957137, + "objective/train/tokens_used": 1331178976, + "theoretical_loss": 3.323130633957137, + "tokens_seen": 2972319744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010111111111111112, + "loss": 2.6328, + "theoretical_loss": 3.323130633957137, + "tokens_seen": 2972319744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010109090909090909, + "loss": 2.4821, + "theoretical_loss": 3.3231249898628175, + "tokens_seen": 2972385280 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010107070707070707, + "loss": 2.5996, + "theoretical_loss": 3.3231193459277817, + "tokens_seen": 2972450816 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010105050505050506, + "loss": 2.6157, + "theoretical_loss": 3.323113702152023, + "tokens_seen": 2972516352 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010103030303030303, + "loss": 2.6564, + "theoretical_loss": 3.323108058535532, + "tokens_seen": 2972581888 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010101010101010101, + "loss": 2.4422, + "theoretical_loss": 3.323102415078301, + "tokens_seen": 2972647424 + }, + { + "epoch": 0.8, + "learning_rate": 0.000100989898989899, + "loss": 2.7427, + "theoretical_loss": 3.323096771780323, + "tokens_seen": 2972712960 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010096969696969698, + "loss": 2.5979, + "theoretical_loss": 3.323091128641589, + "tokens_seen": 2972778496 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010094949494949496, + "loss": 2.3605, + "theoretical_loss": 3.3230854856620913, + "tokens_seen": 2972844032 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010092929292929293, + "loss": 2.3803, + "theoretical_loss": 3.3230798428418216, + "tokens_seen": 2972909568 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001009090909090909, + "loss": 2.32, + "theoretical_loss": 3.3230742001807725, + "tokens_seen": 2972975104 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010088888888888889, + "loss": 2.7087, + "theoretical_loss": 3.323068557678935, + "tokens_seen": 2973040640 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010086868686868687, + "loss": 2.4285, + "theoretical_loss": 3.3230629153363025, + "tokens_seen": 2973106176 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010084848484848485, + "loss": 2.5873, + "theoretical_loss": 3.323057273152866, + "tokens_seen": 2973171712 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010082828282828283, + "loss": 2.4956, + "theoretical_loss": 3.323051631128618, + "tokens_seen": 2973237248 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010080808080808081, + "loss": 2.3774, + "theoretical_loss": 3.3230459892635498, + "tokens_seen": 2973302784 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010078787878787879, + "loss": 2.7293, + "theoretical_loss": 3.323040347557654, + "tokens_seen": 2973368320 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010076767676767678, + "loss": 2.5439, + "theoretical_loss": 3.3230347060109224, + "tokens_seen": 2973433856 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010074747474747476, + "loss": 2.4594, + "theoretical_loss": 3.3230290646233467, + "tokens_seen": 2973499392 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010072727272727272, + "loss": 2.2493, + "theoretical_loss": 3.3230234233949196, + "tokens_seen": 2973564928 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001007070707070707, + "loss": 2.6673, + "theoretical_loss": 3.323017782325633, + "tokens_seen": 2973630464 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010068686868686869, + "loss": 2.5226, + "theoretical_loss": 3.3230121414154783, + "tokens_seen": 2973696000 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010066666666666667, + "loss": 2.5968, + "theoretical_loss": 3.323006500664448, + "tokens_seen": 2973761536 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010064646464646465, + "loss": 2.4569, + "theoretical_loss": 3.323000860072534, + "tokens_seen": 2973827072 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010062626262626264, + "loss": 2.5074, + "theoretical_loss": 3.322995219639728, + "tokens_seen": 2973892608 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1630141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5066049098968506, + "objective/train/theoretical_loss": 3.3229895793660225, + "objective/train/tokens_used": 1332817376, + "theoretical_loss": 3.3229895793660225, + "tokens_seen": 2973958144 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010060606060606061, + "loss": 2.7624, + "theoretical_loss": 3.3229895793660225, + "tokens_seen": 2973958144 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010058585858585859, + "loss": 2.4993, + "theoretical_loss": 3.3229839392514093, + "tokens_seen": 2974023680 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010056565656565656, + "loss": 2.6361, + "theoretical_loss": 3.3229782992958805, + "tokens_seen": 2974089216 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010054545454545455, + "loss": 2.5112, + "theoretical_loss": 3.3229726594994276, + "tokens_seen": 2974154752 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010052525252525253, + "loss": 2.4302, + "theoretical_loss": 3.3229670198620433, + "tokens_seen": 2974220288 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001005050505050505, + "loss": 2.6349, + "theoretical_loss": 3.3229613803837195, + "tokens_seen": 2974285824 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010048484848484848, + "loss": 2.4098, + "theoretical_loss": 3.3229557410644475, + "tokens_seen": 2974351360 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010046464646464647, + "loss": 2.5599, + "theoretical_loss": 3.32295010190422, + "tokens_seen": 2974416896 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010044444444444445, + "loss": 2.6251, + "theoretical_loss": 3.322944462903029, + "tokens_seen": 2974482432 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010042424242424244, + "loss": 2.3968, + "theoretical_loss": 3.3229388240608664, + "tokens_seen": 2974547968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010040404040404041, + "loss": 2.676, + "theoretical_loss": 3.3229331853777238, + "tokens_seen": 2974613504 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010038383838383838, + "loss": 2.4412, + "theoretical_loss": 3.322927546853594, + "tokens_seen": 2974679040 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010036363636363636, + "loss": 2.524, + "theoretical_loss": 3.3229219084884685, + "tokens_seen": 2974744576 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010034343434343435, + "loss": 2.5821, + "theoretical_loss": 3.322916270282339, + "tokens_seen": 2974810112 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010032323232323233, + "loss": 2.4873, + "theoretical_loss": 3.3229106322351982, + "tokens_seen": 2974875648 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001003030303030303, + "loss": 2.4768, + "theoretical_loss": 3.322904994347038, + "tokens_seen": 2974941184 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010028282828282828, + "loss": 2.6017, + "theoretical_loss": 3.32289935661785, + "tokens_seen": 2975006720 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010026262626262627, + "loss": 2.4328, + "theoretical_loss": 3.3228937190476264, + "tokens_seen": 2975072256 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010024242424242425, + "loss": 2.6474, + "theoretical_loss": 3.3228880816363593, + "tokens_seen": 2975137792 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010022222222222222, + "loss": 2.5821, + "theoretical_loss": 3.322882444384041, + "tokens_seen": 2975203328 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001002020202020202, + "loss": 2.4681, + "theoretical_loss": 3.3228768072906627, + "tokens_seen": 2975268864 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010018181818181818, + "loss": 2.4377, + "theoretical_loss": 3.3228711703562173, + "tokens_seen": 2975334400 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010016161616161616, + "loss": 2.4956, + "theoretical_loss": 3.3228655335806967, + "tokens_seen": 2975399936 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010014141414141414, + "loss": 2.6342, + "theoretical_loss": 3.322859896964092, + "tokens_seen": 2975465472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010012121212121213, + "loss": 2.4517, + "theoretical_loss": 3.3228542605063964, + "tokens_seen": 2975531008 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1631426, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8006060123443604, + "objective/train/theoretical_loss": 3.322848624207601, + "objective/train/tokens_used": 1334455776, + "theoretical_loss": 3.322848624207601, + "tokens_seen": 2975596544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010010101010101011, + "loss": 2.5544, + "theoretical_loss": 3.322848624207601, + "tokens_seen": 2975596544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010008080808080808, + "loss": 2.4466, + "theoretical_loss": 3.322842988067699, + "tokens_seen": 2975662080 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010006060606060607, + "loss": 2.4037, + "theoretical_loss": 3.322837352086681, + "tokens_seen": 2975727616 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010004040404040404, + "loss": 2.5448, + "theoretical_loss": 3.3228317162645395, + "tokens_seen": 2975793152 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010002020202020202, + "loss": 2.6184, + "theoretical_loss": 3.322826080601267, + "tokens_seen": 2975858688 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001, + "loss": 2.4889, + "theoretical_loss": 3.3228204450968555, + "tokens_seen": 2975924224 + }, + { + "epoch": 0.8, + "learning_rate": 9.997979797979798e-05, + "loss": 2.6864, + "theoretical_loss": 3.3228148097512964, + "tokens_seen": 2975989760 + }, + { + "epoch": 0.8, + "learning_rate": 9.995959595959596e-05, + "loss": 2.4242, + "theoretical_loss": 3.322809174564582, + "tokens_seen": 2976055296 + }, + { + "epoch": 0.8, + "learning_rate": 9.993939393939394e-05, + "loss": 2.5347, + "theoretical_loss": 3.3228035395367046, + "tokens_seen": 2976120832 + }, + { + "epoch": 0.8, + "learning_rate": 9.991919191919193e-05, + "loss": 2.6564, + "theoretical_loss": 3.322797904667656, + "tokens_seen": 2976186368 + }, + { + "epoch": 0.8, + "learning_rate": 9.989898989898991e-05, + "loss": 2.5423, + "theoretical_loss": 3.3227922699574277, + "tokens_seen": 2976251904 + }, + { + "epoch": 0.8, + "learning_rate": 9.987878787878787e-05, + "loss": 2.4526, + "theoretical_loss": 3.322786635406013, + "tokens_seen": 2976317440 + }, + { + "epoch": 0.8, + "learning_rate": 9.985858585858585e-05, + "loss": 2.5039, + "theoretical_loss": 3.3227810010134027, + "tokens_seen": 2976382976 + }, + { + "epoch": 0.8, + "learning_rate": 9.983838383838384e-05, + "loss": 2.3558, + "theoretical_loss": 3.3227753667795894, + "tokens_seen": 2976448512 + }, + { + "epoch": 0.8, + "learning_rate": 9.981818181818182e-05, + "loss": 2.5249, + "theoretical_loss": 3.322769732704565, + "tokens_seen": 2976514048 + }, + { + "epoch": 0.8, + "learning_rate": 9.97979797979798e-05, + "loss": 2.2414, + "theoretical_loss": 3.322764098788322, + "tokens_seen": 2976579584 + }, + { + "epoch": 0.8, + "learning_rate": 9.977777777777779e-05, + "loss": 2.4867, + "theoretical_loss": 3.3227584650308515, + "tokens_seen": 2976645120 + }, + { + "epoch": 0.8, + "learning_rate": 9.975757575757576e-05, + "loss": 2.4597, + "theoretical_loss": 3.3227528314321466, + "tokens_seen": 2976710656 + }, + { + "epoch": 0.8, + "learning_rate": 9.973737373737374e-05, + "loss": 2.4007, + "theoretical_loss": 3.3227471979921983, + "tokens_seen": 2976776192 + }, + { + "epoch": 0.8, + "learning_rate": 9.971717171717173e-05, + "loss": 2.5208, + "theoretical_loss": 3.3227415647109995, + "tokens_seen": 2976841728 + }, + { + "epoch": 0.8, + "learning_rate": 9.96969696969697e-05, + "loss": 2.67, + "theoretical_loss": 3.322735931588541, + "tokens_seen": 2976907264 + }, + { + "epoch": 0.8, + "learning_rate": 9.967676767676768e-05, + "loss": 2.66, + "theoretical_loss": 3.3227302986248164, + "tokens_seen": 2976972800 + }, + { + "epoch": 0.8, + "learning_rate": 9.965656565656565e-05, + "loss": 2.3385, + "theoretical_loss": 3.322724665819817, + "tokens_seen": 2977038336 + }, + { + "epoch": 0.8, + "learning_rate": 9.963636363636363e-05, + "loss": 2.6792, + "theoretical_loss": 3.3227190331735343, + "tokens_seen": 2977103872 + }, + { + "epoch": 0.8, + "learning_rate": 9.961616161616162e-05, + "loss": 2.4259, + "theoretical_loss": 3.3227134006859615, + "tokens_seen": 2977169408 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1632651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7429685592651367, + "objective/train/theoretical_loss": 3.32270776835709, + "objective/train/tokens_used": 1336094176, + "theoretical_loss": 3.32270776835709, + "tokens_seen": 2977234944 + }, + { + "epoch": 0.8, + "learning_rate": 9.95959595959596e-05, + "loss": 2.5818, + "theoretical_loss": 3.32270776835709, + "tokens_seen": 2977234944 + }, + { + "epoch": 0.8, + "learning_rate": 9.957575757575759e-05, + "loss": 2.4438, + "theoretical_loss": 3.322702136186911, + "tokens_seen": 2977300480 + }, + { + "epoch": 0.8, + "learning_rate": 9.955555555555556e-05, + "loss": 2.56, + "theoretical_loss": 3.3226965041754184, + "tokens_seen": 2977366016 + }, + { + "epoch": 0.8, + "learning_rate": 9.953535353535354e-05, + "loss": 2.3429, + "theoretical_loss": 3.3226908723226023, + "tokens_seen": 2977431552 + }, + { + "epoch": 0.8, + "learning_rate": 9.951515151515151e-05, + "loss": 2.4863, + "theoretical_loss": 3.322685240628456, + "tokens_seen": 2977497088 + }, + { + "epoch": 0.8, + "learning_rate": 9.94949494949495e-05, + "loss": 2.3997, + "theoretical_loss": 3.3226796090929716, + "tokens_seen": 2977562624 + }, + { + "epoch": 0.8, + "learning_rate": 9.947474747474748e-05, + "loss": 2.4956, + "theoretical_loss": 3.32267397771614, + "tokens_seen": 2977628160 + }, + { + "epoch": 0.8, + "learning_rate": 9.945454545454545e-05, + "loss": 2.5567, + "theoretical_loss": 3.3226683464979545, + "tokens_seen": 2977693696 + }, + { + "epoch": 0.8, + "learning_rate": 9.943434343434343e-05, + "loss": 2.6441, + "theoretical_loss": 3.3226627154384065, + "tokens_seen": 2977759232 + }, + { + "epoch": 0.8, + "learning_rate": 9.941414141414142e-05, + "loss": 2.6207, + "theoretical_loss": 3.322657084537488, + "tokens_seen": 2977824768 + }, + { + "epoch": 0.8, + "learning_rate": 9.93939393939394e-05, + "loss": 2.6676, + "theoretical_loss": 3.3226514537951912, + "tokens_seen": 2977890304 + }, + { + "epoch": 0.8, + "learning_rate": 9.937373737373739e-05, + "loss": 2.6982, + "theoretical_loss": 3.3226458232115084, + "tokens_seen": 2977955840 + }, + { + "epoch": 0.8, + "learning_rate": 9.935353535353534e-05, + "loss": 2.5821, + "theoretical_loss": 3.3226401927864306, + "tokens_seen": 2978021376 + }, + { + "epoch": 0.8, + "learning_rate": 9.933333333333333e-05, + "loss": 2.3624, + "theoretical_loss": 3.3226345625199514, + "tokens_seen": 2978086912 + }, + { + "epoch": 0.8, + "learning_rate": 9.931313131313131e-05, + "loss": 2.4214, + "theoretical_loss": 3.3226289324120617, + "tokens_seen": 2978152448 + }, + { + "epoch": 0.8, + "learning_rate": 9.92929292929293e-05, + "loss": 2.5239, + "theoretical_loss": 3.3226233024627536, + "tokens_seen": 2978217984 + }, + { + "epoch": 0.8, + "learning_rate": 9.927272727272728e-05, + "loss": 2.3103, + "theoretical_loss": 3.32261767267202, + "tokens_seen": 2978283520 + }, + { + "epoch": 0.8, + "learning_rate": 9.925252525252526e-05, + "loss": 2.4685, + "theoretical_loss": 3.322612043039852, + "tokens_seen": 2978349056 + }, + { + "epoch": 0.8, + "learning_rate": 9.923232323232323e-05, + "loss": 2.6564, + "theoretical_loss": 3.322606413566242, + "tokens_seen": 2978414592 + }, + { + "epoch": 0.8, + "learning_rate": 9.921212121212122e-05, + "loss": 2.541, + "theoretical_loss": 3.3226007842511827, + "tokens_seen": 2978480128 + }, + { + "epoch": 0.8, + "learning_rate": 9.91919191919192e-05, + "loss": 2.4527, + "theoretical_loss": 3.322595155094665, + "tokens_seen": 2978545664 + }, + { + "epoch": 0.8, + "learning_rate": 9.917171717171717e-05, + "loss": 2.5931, + "theoretical_loss": 3.322589526096681, + "tokens_seen": 2978611200 + }, + { + "epoch": 0.8, + "learning_rate": 9.915151515151515e-05, + "loss": 2.4904, + "theoretical_loss": 3.322583897257224, + "tokens_seen": 2978676736 + }, + { + "epoch": 0.8, + "learning_rate": 9.913131313131313e-05, + "loss": 2.3442, + "theoretical_loss": 3.322578268576285, + "tokens_seen": 2978742272 + }, + { + "epoch": 0.8, + "learning_rate": 9.911111111111111e-05, + "loss": 2.3608, + "theoretical_loss": 3.322572640053856, + "tokens_seen": 2978807808 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1632994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5539956092834473, + "objective/train/theoretical_loss": 3.3225670116899293, + "objective/train/tokens_used": 1337732576, + "theoretical_loss": 3.3225670116899293, + "tokens_seen": 2978873344 + }, + { + "epoch": 0.8, + "learning_rate": 9.90909090909091e-05, + "loss": 2.6336, + "theoretical_loss": 3.3225670116899293, + "tokens_seen": 2978873344 + }, + { + "epoch": 0.8, + "learning_rate": 9.907070707070708e-05, + "loss": 2.4718, + "theoretical_loss": 3.3225613834844974, + "tokens_seen": 2978938880 + }, + { + "epoch": 0.8, + "learning_rate": 9.905050505050506e-05, + "loss": 2.6972, + "theoretical_loss": 3.322555755437552, + "tokens_seen": 2979004416 + }, + { + "epoch": 0.8, + "learning_rate": 9.903030303030303e-05, + "loss": 2.3916, + "theoretical_loss": 3.3225501275490847, + "tokens_seen": 2979069952 + }, + { + "epoch": 0.8, + "learning_rate": 9.9010101010101e-05, + "loss": 2.5834, + "theoretical_loss": 3.322544499819088, + "tokens_seen": 2979135488 + }, + { + "epoch": 0.8, + "learning_rate": 9.898989898989899e-05, + "loss": 2.5321, + "theoretical_loss": 3.3225388722475544, + "tokens_seen": 2979201024 + }, + { + "epoch": 0.8, + "learning_rate": 9.896969696969697e-05, + "loss": 2.5734, + "theoretical_loss": 3.3225332448344753, + "tokens_seen": 2979266560 + }, + { + "epoch": 0.8, + "learning_rate": 9.894949494949495e-05, + "loss": 2.0626, + "theoretical_loss": 3.3225276175798424, + "tokens_seen": 2979332096 + }, + { + "epoch": 0.8, + "learning_rate": 9.892929292929292e-05, + "loss": 2.6728, + "theoretical_loss": 3.322521990483649, + "tokens_seen": 2979397632 + }, + { + "epoch": 0.8, + "learning_rate": 9.890909090909091e-05, + "loss": 2.5797, + "theoretical_loss": 3.3225163635458856, + "tokens_seen": 2979463168 + }, + { + "epoch": 0.8, + "learning_rate": 9.888888888888889e-05, + "loss": 2.4496, + "theoretical_loss": 3.322510736766546, + "tokens_seen": 2979528704 + }, + { + "epoch": 0.8, + "learning_rate": 9.886868686868688e-05, + "loss": 2.5563, + "theoretical_loss": 3.32250511014562, + "tokens_seen": 2979594240 + }, + { + "epoch": 0.8, + "learning_rate": 9.884848484848486e-05, + "loss": 2.6315, + "theoretical_loss": 3.322499483683102, + "tokens_seen": 2979659776 + }, + { + "epoch": 0.8, + "learning_rate": 9.882828282828283e-05, + "loss": 2.5052, + "theoretical_loss": 3.322493857378983, + "tokens_seen": 2979725312 + }, + { + "epoch": 0.8, + "learning_rate": 9.88080808080808e-05, + "loss": 2.5054, + "theoretical_loss": 3.3224882312332547, + "tokens_seen": 2979790848 + }, + { + "epoch": 0.8, + "learning_rate": 9.878787878787878e-05, + "loss": 2.55, + "theoretical_loss": 3.32248260524591, + "tokens_seen": 2979856384 + }, + { + "epoch": 0.8, + "learning_rate": 9.876767676767677e-05, + "loss": 2.495, + "theoretical_loss": 3.3224769794169404, + "tokens_seen": 2979921920 + }, + { + "epoch": 0.8, + "learning_rate": 9.874747474747475e-05, + "loss": 2.493, + "theoretical_loss": 3.3224713537463377, + "tokens_seen": 2979987456 + }, + { + "epoch": 0.8, + "learning_rate": 9.872727272727274e-05, + "loss": 2.3048, + "theoretical_loss": 3.3224657282340946, + "tokens_seen": 2980052992 + }, + { + "epoch": 0.8, + "learning_rate": 9.870707070707071e-05, + "loss": 2.6678, + "theoretical_loss": 3.322460102880203, + "tokens_seen": 2980118528 + }, + { + "epoch": 0.8, + "learning_rate": 9.868686868686869e-05, + "loss": 2.3736, + "theoretical_loss": 3.3224544776846545, + "tokens_seen": 2980184064 + }, + { + "epoch": 0.8, + "learning_rate": 9.866666666666668e-05, + "loss": 2.5966, + "theoretical_loss": 3.322448852647442, + "tokens_seen": 2980249600 + }, + { + "epoch": 0.8, + "learning_rate": 9.864646464646465e-05, + "loss": 2.4235, + "theoretical_loss": 3.3224432277685567, + "tokens_seen": 2980315136 + }, + { + "epoch": 0.8, + "learning_rate": 9.862626262626263e-05, + "loss": 2.5723, + "theoretical_loss": 3.322437603047991, + "tokens_seen": 2980380672 + }, + { + "epoch": 0.8, + "learning_rate": 9.86060606060606e-05, + "loss": 2.6906, + "theoretical_loss": 3.3224319784857372, + "tokens_seen": 2980446208 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1634203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.450411796569824, + "objective/train/theoretical_loss": 3.322426354081787, + "objective/train/tokens_used": 1339370976, + "theoretical_loss": 3.322426354081787, + "tokens_seen": 2980511744 + }, + { + "epoch": 0.81, + "learning_rate": 9.858585858585858e-05, + "loss": 2.5142, + "theoretical_loss": 3.322426354081787, + "tokens_seen": 2980511744 + }, + { + "epoch": 0.81, + "learning_rate": 9.856565656565657e-05, + "loss": 2.6244, + "theoretical_loss": 3.322420729836133, + "tokens_seen": 2980577280 + }, + { + "epoch": 0.81, + "learning_rate": 9.854545454545455e-05, + "loss": 2.3715, + "theoretical_loss": 3.3224151057487665, + "tokens_seen": 2980642816 + }, + { + "epoch": 0.81, + "learning_rate": 9.852525252525254e-05, + "loss": 2.589, + "theoretical_loss": 3.3224094818196797, + "tokens_seen": 2980708352 + }, + { + "epoch": 0.81, + "learning_rate": 9.85050505050505e-05, + "loss": 2.5575, + "theoretical_loss": 3.3224038580488653, + "tokens_seen": 2980773888 + }, + { + "epoch": 0.81, + "learning_rate": 9.848484848484848e-05, + "loss": 2.4535, + "theoretical_loss": 3.322398234436315, + "tokens_seen": 2980839424 + }, + { + "epoch": 0.81, + "learning_rate": 9.846464646464646e-05, + "loss": 2.5662, + "theoretical_loss": 3.3223926109820208, + "tokens_seen": 2980904960 + }, + { + "epoch": 0.81, + "learning_rate": 9.844444444444444e-05, + "loss": 2.2262, + "theoretical_loss": 3.3223869876859746, + "tokens_seen": 2980970496 + }, + { + "epoch": 0.81, + "learning_rate": 9.842424242424243e-05, + "loss": 2.4309, + "theoretical_loss": 3.3223813645481695, + "tokens_seen": 2981036032 + }, + { + "epoch": 0.81, + "learning_rate": 9.840404040404041e-05, + "loss": 2.4779, + "theoretical_loss": 3.322375741568596, + "tokens_seen": 2981101568 + }, + { + "epoch": 0.81, + "learning_rate": 9.838383838383838e-05, + "loss": 2.609, + "theoretical_loss": 3.322370118747247, + "tokens_seen": 2981167104 + }, + { + "epoch": 0.81, + "learning_rate": 9.836363636363637e-05, + "loss": 2.4902, + "theoretical_loss": 3.3223644960841145, + "tokens_seen": 2981232640 + }, + { + "epoch": 0.81, + "learning_rate": 9.834343434343435e-05, + "loss": 2.4719, + "theoretical_loss": 3.3223588735791907, + "tokens_seen": 2981298176 + }, + { + "epoch": 0.81, + "learning_rate": 9.832323232323233e-05, + "loss": 2.667, + "theoretical_loss": 3.3223532512324674, + "tokens_seen": 2981363712 + }, + { + "epoch": 0.81, + "learning_rate": 9.83030303030303e-05, + "loss": 2.448, + "theoretical_loss": 3.3223476290439367, + "tokens_seen": 2981429248 + }, + { + "epoch": 0.81, + "learning_rate": 9.828282828282828e-05, + "loss": 2.3739, + "theoretical_loss": 3.3223420070135914, + "tokens_seen": 2981494784 + }, + { + "epoch": 0.81, + "learning_rate": 9.826262626262626e-05, + "loss": 2.336, + "theoretical_loss": 3.3223363851414223, + "tokens_seen": 2981560320 + }, + { + "epoch": 0.81, + "learning_rate": 9.824242424242424e-05, + "loss": 2.6361, + "theoretical_loss": 3.322330763427422, + "tokens_seen": 2981625856 + }, + { + "epoch": 0.81, + "learning_rate": 9.822222222222223e-05, + "loss": 2.7764, + "theoretical_loss": 3.322325141871583, + "tokens_seen": 2981691392 + }, + { + "epoch": 0.81, + "learning_rate": 9.820202020202021e-05, + "loss": 2.5865, + "theoretical_loss": 3.3223195204738976, + "tokens_seen": 2981756928 + }, + { + "epoch": 0.81, + "learning_rate": 9.818181818181818e-05, + "loss": 2.5694, + "theoretical_loss": 3.3223138992343566, + "tokens_seen": 2981822464 + }, + { + "epoch": 0.81, + "learning_rate": 9.816161616161617e-05, + "loss": 2.5128, + "theoretical_loss": 3.322308278152953, + "tokens_seen": 2981888000 + }, + { + "epoch": 0.81, + "learning_rate": 9.814141414141414e-05, + "loss": 2.5934, + "theoretical_loss": 3.322302657229679, + "tokens_seen": 2981953536 + }, + { + "epoch": 0.81, + "learning_rate": 9.812121212121212e-05, + "loss": 2.4779, + "theoretical_loss": 3.322297036464526, + "tokens_seen": 2982019072 + }, + { + "epoch": 0.81, + "learning_rate": 9.81010101010101e-05, + "loss": 2.4815, + "theoretical_loss": 3.3222914158574866, + "tokens_seen": 2982084608 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1635397, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.435359001159668, + "objective/train/theoretical_loss": 3.3222857954085527, + "objective/train/tokens_used": 1341009376, + "theoretical_loss": 3.3222857954085527, + "tokens_seen": 2982150144 + }, + { + "epoch": 0.81, + "learning_rate": 9.808080808080807e-05, + "loss": 2.4558, + "theoretical_loss": 3.3222857954085527, + "tokens_seen": 2982150144 + }, + { + "epoch": 0.81, + "learning_rate": 9.806060606060606e-05, + "loss": 2.4866, + "theoretical_loss": 3.3222801751177164, + "tokens_seen": 2982215680 + }, + { + "epoch": 0.81, + "learning_rate": 9.804040404040404e-05, + "loss": 2.6313, + "theoretical_loss": 3.32227455498497, + "tokens_seen": 2982281216 + }, + { + "epoch": 0.81, + "learning_rate": 9.802020202020203e-05, + "loss": 2.4424, + "theoretical_loss": 3.322268935010305, + "tokens_seen": 2982346752 + }, + { + "epoch": 0.81, + "learning_rate": 9.800000000000001e-05, + "loss": 2.5752, + "theoretical_loss": 3.322263315193714, + "tokens_seen": 2982412288 + }, + { + "epoch": 0.81, + "learning_rate": 9.7979797979798e-05, + "loss": 2.7183, + "theoretical_loss": 3.322257695535189, + "tokens_seen": 2982477824 + }, + { + "epoch": 0.81, + "learning_rate": 9.795959595959595e-05, + "loss": 2.5715, + "theoretical_loss": 3.322252076034722, + "tokens_seen": 2982543360 + }, + { + "epoch": 0.81, + "learning_rate": 9.793939393939394e-05, + "loss": 2.6073, + "theoretical_loss": 3.322246456692305, + "tokens_seen": 2982608896 + }, + { + "epoch": 0.81, + "learning_rate": 9.791919191919192e-05, + "loss": 2.5129, + "theoretical_loss": 3.32224083750793, + "tokens_seen": 2982674432 + }, + { + "epoch": 0.81, + "learning_rate": 9.78989898989899e-05, + "loss": 2.5679, + "theoretical_loss": 3.3222352184815893, + "tokens_seen": 2982739968 + }, + { + "epoch": 0.81, + "learning_rate": 9.787878787878789e-05, + "loss": 2.604, + "theoretical_loss": 3.3222295996132756, + "tokens_seen": 2982805504 + }, + { + "epoch": 0.81, + "learning_rate": 9.785858585858586e-05, + "loss": 2.3175, + "theoretical_loss": 3.3222239809029794, + "tokens_seen": 2982871040 + }, + { + "epoch": 0.81, + "learning_rate": 9.783838383838384e-05, + "loss": 2.3842, + "theoretical_loss": 3.3222183623506942, + "tokens_seen": 2982936576 + }, + { + "epoch": 0.81, + "learning_rate": 9.781818181818183e-05, + "loss": 2.4877, + "theoretical_loss": 3.3222127439564115, + "tokens_seen": 2983002112 + }, + { + "epoch": 0.81, + "learning_rate": 9.77979797979798e-05, + "loss": 2.6444, + "theoretical_loss": 3.3222071257201233, + "tokens_seen": 2983067648 + }, + { + "epoch": 0.81, + "learning_rate": 9.777777777777778e-05, + "loss": 2.4417, + "theoretical_loss": 3.322201507641822, + "tokens_seen": 2983133184 + }, + { + "epoch": 0.81, + "learning_rate": 9.775757575757575e-05, + "loss": 2.6145, + "theoretical_loss": 3.3221958897215, + "tokens_seen": 2983198720 + }, + { + "epoch": 0.81, + "learning_rate": 9.773737373737373e-05, + "loss": 2.6227, + "theoretical_loss": 3.3221902719591485, + "tokens_seen": 2983264256 + }, + { + "epoch": 0.81, + "learning_rate": 9.771717171717172e-05, + "loss": 2.6379, + "theoretical_loss": 3.3221846543547597, + "tokens_seen": 2983329792 + }, + { + "epoch": 0.81, + "learning_rate": 9.76969696969697e-05, + "loss": 2.5952, + "theoretical_loss": 3.3221790369083264, + "tokens_seen": 2983395328 + }, + { + "epoch": 0.81, + "learning_rate": 9.767676767676769e-05, + "loss": 2.4386, + "theoretical_loss": 3.3221734196198405, + "tokens_seen": 2983460864 + }, + { + "epoch": 0.81, + "learning_rate": 9.765656565656566e-05, + "loss": 2.586, + "theoretical_loss": 3.322167802489293, + "tokens_seen": 2983526400 + }, + { + "epoch": 0.81, + "learning_rate": 9.763636363636364e-05, + "loss": 2.4066, + "theoretical_loss": 3.322162185516678, + "tokens_seen": 2983591936 + }, + { + "epoch": 0.81, + "learning_rate": 9.761616161616161e-05, + "loss": 2.7248, + "theoretical_loss": 3.3221565687019856, + "tokens_seen": 2983657472 + }, + { + "epoch": 0.81, + "learning_rate": 9.75959595959596e-05, + "loss": 2.4775, + "theoretical_loss": 3.3221509520452095, + "tokens_seen": 2983723008 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1635965, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.528303384780884, + "objective/train/theoretical_loss": 3.322145335546341, + "objective/train/tokens_used": 1342647776, + "theoretical_loss": 3.322145335546341, + "tokens_seen": 2983788544 + }, + { + "epoch": 0.81, + "learning_rate": 9.757575757575758e-05, + "loss": 2.5285, + "theoretical_loss": 3.322145335546341, + "tokens_seen": 2983788544 + }, + { + "epoch": 0.81, + "learning_rate": 9.755555555555556e-05, + "loss": 2.5471, + "theoretical_loss": 3.3221397192053717, + "tokens_seen": 2983854080 + }, + { + "epoch": 0.81, + "learning_rate": 9.753535353535353e-05, + "loss": 2.6329, + "theoretical_loss": 3.3221341030222944, + "tokens_seen": 2983919616 + }, + { + "epoch": 0.81, + "learning_rate": 9.751515151515152e-05, + "loss": 2.4356, + "theoretical_loss": 3.322128486997101, + "tokens_seen": 2983985152 + }, + { + "epoch": 0.81, + "learning_rate": 9.74949494949495e-05, + "loss": 2.3538, + "theoretical_loss": 3.3221228711297837, + "tokens_seen": 2984050688 + }, + { + "epoch": 0.81, + "learning_rate": 9.747474747474748e-05, + "loss": 2.426, + "theoretical_loss": 3.3221172554203346, + "tokens_seen": 2984116224 + }, + { + "epoch": 0.81, + "learning_rate": 9.745454545454547e-05, + "loss": 2.6191, + "theoretical_loss": 3.3221116398687456, + "tokens_seen": 2984181760 + }, + { + "epoch": 0.81, + "learning_rate": 9.743434343434343e-05, + "loss": 2.5898, + "theoretical_loss": 3.322106024475009, + "tokens_seen": 2984247296 + }, + { + "epoch": 0.81, + "learning_rate": 9.741414141414141e-05, + "loss": 2.5121, + "theoretical_loss": 3.3221004092391166, + "tokens_seen": 2984312832 + }, + { + "epoch": 0.81, + "learning_rate": 9.73939393939394e-05, + "loss": 2.7036, + "theoretical_loss": 3.3220947941610612, + "tokens_seen": 2984378368 + }, + { + "epoch": 0.81, + "learning_rate": 9.737373737373738e-05, + "loss": 2.8146, + "theoretical_loss": 3.322089179240834, + "tokens_seen": 2984443904 + }, + { + "epoch": 0.81, + "learning_rate": 9.735353535353536e-05, + "loss": 2.675, + "theoretical_loss": 3.3220835644784277, + "tokens_seen": 2984509440 + }, + { + "epoch": 0.81, + "learning_rate": 9.733333333333333e-05, + "loss": 2.522, + "theoretical_loss": 3.322077949873834, + "tokens_seen": 2984574976 + }, + { + "epoch": 0.81, + "learning_rate": 9.731313131313132e-05, + "loss": 2.5444, + "theoretical_loss": 3.322072335427045, + "tokens_seen": 2984640512 + }, + { + "epoch": 0.81, + "learning_rate": 9.72929292929293e-05, + "loss": 2.5529, + "theoretical_loss": 3.3220667211380537, + "tokens_seen": 2984706048 + }, + { + "epoch": 0.81, + "learning_rate": 9.727272727272727e-05, + "loss": 2.514, + "theoretical_loss": 3.322061107006851, + "tokens_seen": 2984771584 + }, + { + "epoch": 0.81, + "learning_rate": 9.725252525252525e-05, + "loss": 2.3326, + "theoretical_loss": 3.3220554930334294, + "tokens_seen": 2984837120 + }, + { + "epoch": 0.81, + "learning_rate": 9.723232323232322e-05, + "loss": 2.5039, + "theoretical_loss": 3.322049879217781, + "tokens_seen": 2984902656 + }, + { + "epoch": 0.81, + "learning_rate": 9.721212121212121e-05, + "loss": 2.4145, + "theoretical_loss": 3.3220442655598985, + "tokens_seen": 2984968192 + }, + { + "epoch": 0.81, + "learning_rate": 9.719191919191919e-05, + "loss": 2.6796, + "theoretical_loss": 3.3220386520597733, + "tokens_seen": 2985033728 + }, + { + "epoch": 0.81, + "learning_rate": 9.717171717171718e-05, + "loss": 2.684, + "theoretical_loss": 3.322033038717398, + "tokens_seen": 2985099264 + }, + { + "epoch": 0.81, + "learning_rate": 9.715151515151516e-05, + "loss": 2.5216, + "theoretical_loss": 3.322027425532764, + "tokens_seen": 2985164800 + }, + { + "epoch": 0.81, + "learning_rate": 9.713131313131314e-05, + "loss": 2.3106, + "theoretical_loss": 3.322021812505864, + "tokens_seen": 2985230336 + }, + { + "epoch": 0.81, + "learning_rate": 9.711111111111111e-05, + "loss": 2.483, + "theoretical_loss": 3.32201619963669, + "tokens_seen": 2985295872 + }, + { + "epoch": 0.81, + "learning_rate": 9.709090909090909e-05, + "loss": 2.4527, + "theoretical_loss": 3.3220105869252334, + "tokens_seen": 2985361408 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1636997, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.791591167449951, + "objective/train/theoretical_loss": 3.3220049743714877, + "objective/train/tokens_used": 1344286176, + "theoretical_loss": 3.3220049743714877, + "tokens_seen": 2985426944 + }, + { + "epoch": 0.81, + "learning_rate": 9.707070707070707e-05, + "loss": 2.7103, + "theoretical_loss": 3.3220049743714877, + "tokens_seen": 2985426944 + }, + { + "epoch": 0.81, + "learning_rate": 9.705050505050505e-05, + "loss": 2.5692, + "theoretical_loss": 3.3219993619754438, + "tokens_seen": 2985492480 + }, + { + "epoch": 0.81, + "learning_rate": 9.703030303030304e-05, + "loss": 2.572, + "theoretical_loss": 3.3219937497370946, + "tokens_seen": 2985558016 + }, + { + "epoch": 0.81, + "learning_rate": 9.701010101010101e-05, + "loss": 2.3956, + "theoretical_loss": 3.3219881376564317, + "tokens_seen": 2985623552 + }, + { + "epoch": 0.81, + "learning_rate": 9.698989898989899e-05, + "loss": 2.2745, + "theoretical_loss": 3.321982525733447, + "tokens_seen": 2985689088 + }, + { + "epoch": 0.81, + "learning_rate": 9.696969696969698e-05, + "loss": 2.6699, + "theoretical_loss": 3.3219769139681334, + "tokens_seen": 2985754624 + }, + { + "epoch": 0.81, + "learning_rate": 9.694949494949496e-05, + "loss": 2.487, + "theoretical_loss": 3.3219713023604824, + "tokens_seen": 2985820160 + }, + { + "epoch": 0.81, + "learning_rate": 9.692929292929293e-05, + "loss": 2.3644, + "theoretical_loss": 3.3219656909104867, + "tokens_seen": 2985885696 + }, + { + "epoch": 0.81, + "learning_rate": 9.69090909090909e-05, + "loss": 2.5816, + "theoretical_loss": 3.3219600796181377, + "tokens_seen": 2985951232 + }, + { + "epoch": 0.81, + "learning_rate": 9.688888888888888e-05, + "loss": 2.3874, + "theoretical_loss": 3.3219544684834275, + "tokens_seen": 2986016768 + }, + { + "epoch": 0.81, + "learning_rate": 9.686868686868687e-05, + "loss": 2.7093, + "theoretical_loss": 3.321948857506349, + "tokens_seen": 2986082304 + }, + { + "epoch": 0.81, + "learning_rate": 9.684848484848485e-05, + "loss": 2.6423, + "theoretical_loss": 3.3219432466868937, + "tokens_seen": 2986147840 + }, + { + "epoch": 0.81, + "learning_rate": 9.682828282828284e-05, + "loss": 2.4645, + "theoretical_loss": 3.3219376360250537, + "tokens_seen": 2986213376 + }, + { + "epoch": 0.81, + "learning_rate": 9.68080808080808e-05, + "loss": 2.6043, + "theoretical_loss": 3.3219320255208213, + "tokens_seen": 2986278912 + }, + { + "epoch": 0.81, + "learning_rate": 9.678787878787879e-05, + "loss": 2.5893, + "theoretical_loss": 3.3219264151741887, + "tokens_seen": 2986344448 + }, + { + "epoch": 0.81, + "learning_rate": 9.676767676767677e-05, + "loss": 2.4694, + "theoretical_loss": 3.321920804985148, + "tokens_seen": 2986409984 + }, + { + "epoch": 0.81, + "learning_rate": 9.674747474747474e-05, + "loss": 2.5175, + "theoretical_loss": 3.321915194953691, + "tokens_seen": 2986475520 + }, + { + "epoch": 0.81, + "learning_rate": 9.672727272727273e-05, + "loss": 2.6119, + "theoretical_loss": 3.32190958507981, + "tokens_seen": 2986541056 + }, + { + "epoch": 0.81, + "learning_rate": 9.670707070707071e-05, + "loss": 2.6008, + "theoretical_loss": 3.3219039753634974, + "tokens_seen": 2986606592 + }, + { + "epoch": 0.81, + "learning_rate": 9.668686868686868e-05, + "loss": 2.5308, + "theoretical_loss": 3.3218983658047447, + "tokens_seen": 2986672128 + }, + { + "epoch": 0.81, + "learning_rate": 9.666666666666667e-05, + "loss": 2.6675, + "theoretical_loss": 3.3218927564035443, + "tokens_seen": 2986737664 + }, + { + "epoch": 0.81, + "learning_rate": 9.664646464646465e-05, + "loss": 2.5935, + "theoretical_loss": 3.321887147159889, + "tokens_seen": 2986803200 + }, + { + "epoch": 0.81, + "learning_rate": 9.662626262626264e-05, + "loss": 2.5711, + "theoretical_loss": 3.32188153807377, + "tokens_seen": 2986868736 + }, + { + "epoch": 0.81, + "learning_rate": 9.660606060606062e-05, + "loss": 2.5261, + "theoretical_loss": 3.3218759291451794, + "tokens_seen": 2986934272 + }, + { + "epoch": 0.81, + "learning_rate": 9.658585858585858e-05, + "loss": 2.7224, + "theoretical_loss": 3.32187032037411, + "tokens_seen": 2986999808 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1637476, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5945565700531006, + "objective/train/theoretical_loss": 3.3218647117605533, + "objective/train/tokens_used": 1345924576, + "theoretical_loss": 3.3218647117605533, + "tokens_seen": 2987065344 + }, + { + "epoch": 0.81, + "learning_rate": 9.656565656565656e-05, + "loss": 2.5157, + "theoretical_loss": 3.3218647117605533, + "tokens_seen": 2987065344 + }, + { + "epoch": 0.81, + "learning_rate": 9.654545454545454e-05, + "loss": 2.7103, + "theoretical_loss": 3.321859103304502, + "tokens_seen": 2987130880 + }, + { + "epoch": 0.81, + "learning_rate": 9.652525252525253e-05, + "loss": 2.3595, + "theoretical_loss": 3.321853495005948, + "tokens_seen": 2987196416 + }, + { + "epoch": 0.81, + "learning_rate": 9.650505050505051e-05, + "loss": 2.8643, + "theoretical_loss": 3.321847886864883, + "tokens_seen": 2987261952 + }, + { + "epoch": 0.81, + "learning_rate": 9.648484848484848e-05, + "loss": 2.6721, + "theoretical_loss": 3.3218422788812996, + "tokens_seen": 2987327488 + }, + { + "epoch": 0.81, + "learning_rate": 9.646464646464647e-05, + "loss": 2.4907, + "theoretical_loss": 3.32183667105519, + "tokens_seen": 2987393024 + }, + { + "epoch": 0.81, + "learning_rate": 9.644444444444445e-05, + "loss": 2.5364, + "theoretical_loss": 3.3218310633865458, + "tokens_seen": 2987458560 + }, + { + "epoch": 0.81, + "learning_rate": 9.642424242424243e-05, + "loss": 2.7214, + "theoretical_loss": 3.3218254558753593, + "tokens_seen": 2987524096 + }, + { + "epoch": 0.81, + "learning_rate": 9.64040404040404e-05, + "loss": 2.4705, + "theoretical_loss": 3.3218198485216233, + "tokens_seen": 2987589632 + }, + { + "epoch": 0.81, + "learning_rate": 9.638383838383837e-05, + "loss": 2.5054, + "theoretical_loss": 3.321814241325329, + "tokens_seen": 2987655168 + }, + { + "epoch": 0.81, + "learning_rate": 9.636363636363636e-05, + "loss": 2.7286, + "theoretical_loss": 3.3218086342864686, + "tokens_seen": 2987720704 + }, + { + "epoch": 0.81, + "learning_rate": 9.634343434343434e-05, + "loss": 2.5379, + "theoretical_loss": 3.3218030274050347, + "tokens_seen": 2987786240 + }, + { + "epoch": 0.81, + "learning_rate": 9.632323232323233e-05, + "loss": 2.4708, + "theoretical_loss": 3.32179742068102, + "tokens_seen": 2987851776 + }, + { + "epoch": 0.81, + "learning_rate": 9.630303030303031e-05, + "loss": 2.611, + "theoretical_loss": 3.321791814114415, + "tokens_seen": 2987917312 + }, + { + "epoch": 0.81, + "learning_rate": 9.62828282828283e-05, + "loss": 2.4506, + "theoretical_loss": 3.3217862077052134, + "tokens_seen": 2987982848 + }, + { + "epoch": 0.81, + "learning_rate": 9.626262626262627e-05, + "loss": 2.4102, + "theoretical_loss": 3.3217806014534057, + "tokens_seen": 2988048384 + }, + { + "epoch": 0.81, + "learning_rate": 9.624242424242425e-05, + "loss": 2.673, + "theoretical_loss": 3.321774995358986, + "tokens_seen": 2988113920 + }, + { + "epoch": 0.81, + "learning_rate": 9.622222222222222e-05, + "loss": 2.4879, + "theoretical_loss": 3.3217693894219447, + "tokens_seen": 2988179456 + }, + { + "epoch": 0.81, + "learning_rate": 9.62020202020202e-05, + "loss": 2.3391, + "theoretical_loss": 3.321763783642275, + "tokens_seen": 2988244992 + }, + { + "epoch": 0.81, + "learning_rate": 9.618181818181819e-05, + "loss": 2.4129, + "theoretical_loss": 3.3217581780199685, + "tokens_seen": 2988310528 + }, + { + "epoch": 0.81, + "learning_rate": 9.616161616161616e-05, + "loss": 2.4118, + "theoretical_loss": 3.3217525725550177, + "tokens_seen": 2988376064 + }, + { + "epoch": 0.81, + "learning_rate": 9.614141414141414e-05, + "loss": 2.8217, + "theoretical_loss": 3.3217469672474143, + "tokens_seen": 2988441600 + }, + { + "epoch": 0.81, + "learning_rate": 9.612121212121213e-05, + "loss": 2.3171, + "theoretical_loss": 3.3217413620971508, + "tokens_seen": 2988507136 + }, + { + "epoch": 0.81, + "learning_rate": 9.610101010101011e-05, + "loss": 2.477, + "theoretical_loss": 3.3217357571042188, + "tokens_seen": 2988572672 + }, + { + "epoch": 0.81, + "learning_rate": 9.60808080808081e-05, + "loss": 2.6915, + "theoretical_loss": 3.321730152268611, + "tokens_seen": 2988638208 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1638767, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.644864797592163, + "objective/train/theoretical_loss": 3.32172454759032, + "objective/train/tokens_used": 1347562976, + "theoretical_loss": 3.32172454759032, + "tokens_seen": 2988703744 + }, + { + "epoch": 0.81, + "learning_rate": 9.606060606060605e-05, + "loss": 2.5477, + "theoretical_loss": 3.32172454759032, + "tokens_seen": 2988703744 + }, + { + "epoch": 0.81, + "learning_rate": 9.604040404040403e-05, + "loss": 2.6584, + "theoretical_loss": 3.3217189430693366, + "tokens_seen": 2988769280 + }, + { + "epoch": 0.81, + "learning_rate": 9.602020202020202e-05, + "loss": 2.657, + "theoretical_loss": 3.3217133387056537, + "tokens_seen": 2988834816 + }, + { + "epoch": 0.81, + "learning_rate": 9.6e-05, + "loss": 2.4287, + "theoretical_loss": 3.3217077344992636, + "tokens_seen": 2988900352 + }, + { + "epoch": 0.81, + "learning_rate": 9.597979797979799e-05, + "loss": 2.6714, + "theoretical_loss": 3.3217021304501584, + "tokens_seen": 2988965888 + }, + { + "epoch": 0.81, + "learning_rate": 9.595959595959596e-05, + "loss": 2.6827, + "theoretical_loss": 3.3216965265583296, + "tokens_seen": 2989031424 + }, + { + "epoch": 0.81, + "learning_rate": 9.593939393939394e-05, + "loss": 2.5494, + "theoretical_loss": 3.3216909228237697, + "tokens_seen": 2989096960 + }, + { + "epoch": 0.81, + "learning_rate": 9.591919191919192e-05, + "loss": 2.5002, + "theoretical_loss": 3.3216853192464715, + "tokens_seen": 2989162496 + }, + { + "epoch": 0.81, + "learning_rate": 9.589898989898991e-05, + "loss": 2.3638, + "theoretical_loss": 3.321679715826426, + "tokens_seen": 2989228032 + }, + { + "epoch": 0.81, + "learning_rate": 9.587878787878788e-05, + "loss": 2.6297, + "theoretical_loss": 3.321674112563626, + "tokens_seen": 2989293568 + }, + { + "epoch": 0.81, + "learning_rate": 9.585858585858586e-05, + "loss": 2.5227, + "theoretical_loss": 3.3216685094580636, + "tokens_seen": 2989359104 + }, + { + "epoch": 0.81, + "learning_rate": 9.583838383838383e-05, + "loss": 2.525, + "theoretical_loss": 3.321662906509731, + "tokens_seen": 2989424640 + }, + { + "epoch": 0.81, + "learning_rate": 9.581818181818182e-05, + "loss": 2.4028, + "theoretical_loss": 3.32165730371862, + "tokens_seen": 2989490176 + }, + { + "epoch": 0.81, + "learning_rate": 9.57979797979798e-05, + "loss": 2.5176, + "theoretical_loss": 3.3216517010847233, + "tokens_seen": 2989555712 + }, + { + "epoch": 0.81, + "learning_rate": 9.577777777777779e-05, + "loss": 2.5796, + "theoretical_loss": 3.3216460986080323, + "tokens_seen": 2989621248 + }, + { + "epoch": 0.81, + "learning_rate": 9.575757575757577e-05, + "loss": 2.3615, + "theoretical_loss": 3.3216404962885395, + "tokens_seen": 2989686784 + }, + { + "epoch": 0.81, + "learning_rate": 9.573737373737374e-05, + "loss": 2.4772, + "theoretical_loss": 3.3216348941262375, + "tokens_seen": 2989752320 + }, + { + "epoch": 0.81, + "learning_rate": 9.571717171717171e-05, + "loss": 2.5868, + "theoretical_loss": 3.321629292121118, + "tokens_seen": 2989817856 + }, + { + "epoch": 0.81, + "learning_rate": 9.56969696969697e-05, + "loss": 2.5029, + "theoretical_loss": 3.321623690273173, + "tokens_seen": 2989883392 + }, + { + "epoch": 0.81, + "learning_rate": 9.567676767676768e-05, + "loss": 2.5281, + "theoretical_loss": 3.3216180885823947, + "tokens_seen": 2989948928 + }, + { + "epoch": 0.81, + "learning_rate": 9.565656565656566e-05, + "loss": 2.4924, + "theoretical_loss": 3.3216124870487755, + "tokens_seen": 2990014464 + }, + { + "epoch": 0.81, + "learning_rate": 9.563636363636363e-05, + "loss": 2.721, + "theoretical_loss": 3.3216068856723076, + "tokens_seen": 2990080000 + }, + { + "epoch": 0.81, + "learning_rate": 9.561616161616162e-05, + "loss": 2.5547, + "theoretical_loss": 3.3216012844529827, + "tokens_seen": 2990145536 + }, + { + "epoch": 0.81, + "learning_rate": 9.55959595959596e-05, + "loss": 2.3002, + "theoretical_loss": 3.3215956833907936, + "tokens_seen": 2990211072 + }, + { + "epoch": 0.81, + "learning_rate": 9.557575757575758e-05, + "loss": 2.6108, + "theoretical_loss": 3.3215900824857316, + "tokens_seen": 2990276608 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1639487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.895439624786377, + "objective/train/theoretical_loss": 3.3215844817377898, + "objective/train/tokens_used": 1349201376, + "theoretical_loss": 3.3215844817377898, + "tokens_seen": 2990342144 + }, + { + "epoch": 0.81, + "learning_rate": 9.555555555555557e-05, + "loss": 2.6741, + "theoretical_loss": 3.3215844817377898, + "tokens_seen": 2990342144 + }, + { + "epoch": 0.81, + "learning_rate": 9.553535353535353e-05, + "loss": 2.436, + "theoretical_loss": 3.321578881146959, + "tokens_seen": 2990407680 + }, + { + "epoch": 0.81, + "learning_rate": 9.551515151515151e-05, + "loss": 2.6994, + "theoretical_loss": 3.321573280713233, + "tokens_seen": 2990473216 + }, + { + "epoch": 0.81, + "learning_rate": 9.549494949494949e-05, + "loss": 2.5041, + "theoretical_loss": 3.321567680436603, + "tokens_seen": 2990538752 + }, + { + "epoch": 0.81, + "learning_rate": 9.547474747474748e-05, + "loss": 2.3656, + "theoretical_loss": 3.321562080317061, + "tokens_seen": 2990604288 + }, + { + "epoch": 0.81, + "learning_rate": 9.545454545454546e-05, + "loss": 2.4136, + "theoretical_loss": 3.3215564803546, + "tokens_seen": 2990669824 + }, + { + "epoch": 0.81, + "learning_rate": 9.543434343434344e-05, + "loss": 2.7626, + "theoretical_loss": 3.321550880549211, + "tokens_seen": 2990735360 + }, + { + "epoch": 0.81, + "learning_rate": 9.541414141414142e-05, + "loss": 2.6042, + "theoretical_loss": 3.321545280900887, + "tokens_seen": 2990800896 + }, + { + "epoch": 0.81, + "learning_rate": 9.53939393939394e-05, + "loss": 2.6236, + "theoretical_loss": 3.32153968140962, + "tokens_seen": 2990866432 + }, + { + "epoch": 0.81, + "learning_rate": 9.537373737373737e-05, + "loss": 2.4899, + "theoretical_loss": 3.3215340820754022, + "tokens_seen": 2990931968 + }, + { + "epoch": 0.81, + "learning_rate": 9.535353535353535e-05, + "loss": 2.4347, + "theoretical_loss": 3.321528482898225, + "tokens_seen": 2990997504 + }, + { + "epoch": 0.81, + "learning_rate": 9.533333333333334e-05, + "loss": 2.4313, + "theoretical_loss": 3.3215228838780817, + "tokens_seen": 2991063040 + }, + { + "epoch": 0.81, + "learning_rate": 9.531313131313131e-05, + "loss": 2.6008, + "theoretical_loss": 3.3215172850149637, + "tokens_seen": 2991128576 + }, + { + "epoch": 0.81, + "learning_rate": 9.529292929292929e-05, + "loss": 2.7434, + "theoretical_loss": 3.3215116863088636, + "tokens_seen": 2991194112 + }, + { + "epoch": 0.81, + "learning_rate": 9.527272727272728e-05, + "loss": 2.4785, + "theoretical_loss": 3.3215060877597735, + "tokens_seen": 2991259648 + }, + { + "epoch": 0.81, + "learning_rate": 9.525252525252526e-05, + "loss": 2.504, + "theoretical_loss": 3.3215004893676854, + "tokens_seen": 2991325184 + }, + { + "epoch": 0.81, + "learning_rate": 9.523232323232324e-05, + "loss": 2.5967, + "theoretical_loss": 3.3214948911325908, + "tokens_seen": 2991390720 + }, + { + "epoch": 0.81, + "learning_rate": 9.521212121212121e-05, + "loss": 2.5221, + "theoretical_loss": 3.321489293054483, + "tokens_seen": 2991456256 + }, + { + "epoch": 0.81, + "learning_rate": 9.519191919191918e-05, + "loss": 2.5149, + "theoretical_loss": 3.3214836951333537, + "tokens_seen": 2991521792 + }, + { + "epoch": 0.81, + "learning_rate": 9.517171717171717e-05, + "loss": 2.6058, + "theoretical_loss": 3.321478097369195, + "tokens_seen": 2991587328 + }, + { + "epoch": 0.81, + "learning_rate": 9.515151515151515e-05, + "loss": 2.5665, + "theoretical_loss": 3.321472499761999, + "tokens_seen": 2991652864 + }, + { + "epoch": 0.81, + "learning_rate": 9.513131313131314e-05, + "loss": 2.4444, + "theoretical_loss": 3.321466902311758, + "tokens_seen": 2991718400 + }, + { + "epoch": 0.81, + "learning_rate": 9.511111111111111e-05, + "loss": 2.4364, + "theoretical_loss": 3.321461305018464, + "tokens_seen": 2991783936 + }, + { + "epoch": 0.81, + "learning_rate": 9.509090909090909e-05, + "loss": 2.5517, + "theoretical_loss": 3.3214557078821096, + "tokens_seen": 2991849472 + }, + { + "epoch": 0.81, + "learning_rate": 9.507070707070707e-05, + "loss": 2.4857, + "theoretical_loss": 3.3214501109026866, + "tokens_seen": 2991915008 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1640731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4644265174865723, + "objective/train/theoretical_loss": 3.321444514080187, + "objective/train/tokens_used": 1350839776, + "theoretical_loss": 3.321444514080187, + "tokens_seen": 2991980544 + }, + { + "epoch": 0.81, + "learning_rate": 9.505050505050506e-05, + "loss": 2.3224, + "theoretical_loss": 3.321444514080187, + "tokens_seen": 2991980544 + }, + { + "epoch": 0.81, + "learning_rate": 9.503030303030304e-05, + "loss": 2.4088, + "theoretical_loss": 3.321438917414603, + "tokens_seen": 2992046080 + }, + { + "epoch": 0.81, + "learning_rate": 9.501010101010101e-05, + "loss": 2.5782, + "theoretical_loss": 3.321433320905927, + "tokens_seen": 2992111616 + }, + { + "epoch": 0.81, + "learning_rate": 9.498989898989898e-05, + "loss": 2.5021, + "theoretical_loss": 3.3214277245541513, + "tokens_seen": 2992177152 + }, + { + "epoch": 0.81, + "learning_rate": 9.496969696969697e-05, + "loss": 2.4824, + "theoretical_loss": 3.3214221283592678, + "tokens_seen": 2992242688 + }, + { + "epoch": 0.81, + "learning_rate": 9.494949494949495e-05, + "loss": 2.6367, + "theoretical_loss": 3.321416532321269, + "tokens_seen": 2992308224 + }, + { + "epoch": 0.81, + "learning_rate": 9.492929292929294e-05, + "loss": 2.6117, + "theoretical_loss": 3.321410936440146, + "tokens_seen": 2992373760 + }, + { + "epoch": 0.81, + "learning_rate": 9.490909090909092e-05, + "loss": 2.6393, + "theoretical_loss": 3.3214053407158923, + "tokens_seen": 2992439296 + }, + { + "epoch": 0.81, + "learning_rate": 9.488888888888889e-05, + "loss": 2.4657, + "theoretical_loss": 3.3213997451485, + "tokens_seen": 2992504832 + }, + { + "epoch": 0.81, + "learning_rate": 9.486868686868687e-05, + "loss": 2.6736, + "theoretical_loss": 3.32139414973796, + "tokens_seen": 2992570368 + }, + { + "epoch": 0.81, + "learning_rate": 9.484848484848484e-05, + "loss": 2.3112, + "theoretical_loss": 3.3213885544842654, + "tokens_seen": 2992635904 + }, + { + "epoch": 0.81, + "learning_rate": 9.482828282828283e-05, + "loss": 2.5029, + "theoretical_loss": 3.3213829593874085, + "tokens_seen": 2992701440 + }, + { + "epoch": 0.81, + "learning_rate": 9.480808080808081e-05, + "loss": 2.4191, + "theoretical_loss": 3.321377364447381, + "tokens_seen": 2992766976 + }, + { + "epoch": 0.81, + "learning_rate": 9.478787878787878e-05, + "loss": 2.4189, + "theoretical_loss": 3.321371769664175, + "tokens_seen": 2992832512 + }, + { + "epoch": 0.81, + "learning_rate": 9.476767676767677e-05, + "loss": 2.4057, + "theoretical_loss": 3.3213661750377836, + "tokens_seen": 2992898048 + }, + { + "epoch": 0.81, + "learning_rate": 9.474747474747475e-05, + "loss": 2.6243, + "theoretical_loss": 3.321360580568198, + "tokens_seen": 2992963584 + }, + { + "epoch": 0.81, + "learning_rate": 9.472727272727273e-05, + "loss": 2.7507, + "theoretical_loss": 3.3213549862554106, + "tokens_seen": 2993029120 + }, + { + "epoch": 0.81, + "learning_rate": 9.470707070707072e-05, + "loss": 2.4647, + "theoretical_loss": 3.321349392099414, + "tokens_seen": 2993094656 + }, + { + "epoch": 0.81, + "learning_rate": 9.468686868686869e-05, + "loss": 2.5203, + "theoretical_loss": 3.3213437981001994, + "tokens_seen": 2993160192 + }, + { + "epoch": 0.81, + "learning_rate": 9.466666666666666e-05, + "loss": 2.4117, + "theoretical_loss": 3.32133820425776, + "tokens_seen": 2993225728 + }, + { + "epoch": 0.81, + "learning_rate": 9.464646464646464e-05, + "loss": 2.5705, + "theoretical_loss": 3.3213326105720875, + "tokens_seen": 2993291264 + }, + { + "epoch": 0.81, + "learning_rate": 9.462626262626263e-05, + "loss": 2.3789, + "theoretical_loss": 3.321327017043174, + "tokens_seen": 2993356800 + }, + { + "epoch": 0.81, + "learning_rate": 9.460606060606061e-05, + "loss": 2.3137, + "theoretical_loss": 3.3213214236710122, + "tokens_seen": 2993422336 + }, + { + "epoch": 0.81, + "learning_rate": 9.45858585858586e-05, + "loss": 2.5086, + "theoretical_loss": 3.321315830455594, + "tokens_seen": 2993487872 + }, + { + "epoch": 0.81, + "learning_rate": 9.456565656565657e-05, + "loss": 2.6434, + "theoretical_loss": 3.321310237396911, + "tokens_seen": 2993553408 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1641796, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6809685230255127, + "objective/train/theoretical_loss": 3.321304644494956, + "objective/train/tokens_used": 1352478176, + "theoretical_loss": 3.321304644494956, + "tokens_seen": 2993618944 + }, + { + "epoch": 0.81, + "learning_rate": 9.454545454545455e-05, + "loss": 2.5036, + "theoretical_loss": 3.321304644494956, + "tokens_seen": 2993618944 + }, + { + "epoch": 0.81, + "learning_rate": 9.452525252525253e-05, + "loss": 2.3476, + "theoretical_loss": 3.3212990517497207, + "tokens_seen": 2993684480 + }, + { + "epoch": 0.81, + "learning_rate": 9.45050505050505e-05, + "loss": 2.3924, + "theoretical_loss": 3.3212934591611982, + "tokens_seen": 2993750016 + }, + { + "epoch": 0.81, + "learning_rate": 9.448484848484849e-05, + "loss": 2.3414, + "theoretical_loss": 3.3212878667293797, + "tokens_seen": 2993815552 + }, + { + "epoch": 0.81, + "learning_rate": 9.446464646464646e-05, + "loss": 2.4175, + "theoretical_loss": 3.321282274454258, + "tokens_seen": 2993881088 + }, + { + "epoch": 0.81, + "learning_rate": 9.444444444444444e-05, + "loss": 2.4656, + "theoretical_loss": 3.321276682335825, + "tokens_seen": 2993946624 + }, + { + "epoch": 0.81, + "learning_rate": 9.442424242424243e-05, + "loss": 2.4226, + "theoretical_loss": 3.321271090374073, + "tokens_seen": 2994012160 + }, + { + "epoch": 0.81, + "learning_rate": 9.440404040404041e-05, + "loss": 2.5873, + "theoretical_loss": 3.3212654985689936, + "tokens_seen": 2994077696 + }, + { + "epoch": 0.81, + "learning_rate": 9.43838383838384e-05, + "loss": 2.5045, + "theoretical_loss": 3.32125990692058, + "tokens_seen": 2994143232 + }, + { + "epoch": 0.81, + "learning_rate": 9.436363636363636e-05, + "loss": 2.7089, + "theoretical_loss": 3.3212543154288237, + "tokens_seen": 2994208768 + }, + { + "epoch": 0.81, + "learning_rate": 9.434343434343435e-05, + "loss": 2.6085, + "theoretical_loss": 3.321248724093717, + "tokens_seen": 2994274304 + }, + { + "epoch": 0.81, + "learning_rate": 9.432323232323232e-05, + "loss": 2.3246, + "theoretical_loss": 3.3212431329152525, + "tokens_seen": 2994339840 + }, + { + "epoch": 0.81, + "learning_rate": 9.43030303030303e-05, + "loss": 2.405, + "theoretical_loss": 3.3212375418934217, + "tokens_seen": 2994405376 + }, + { + "epoch": 0.81, + "learning_rate": 9.428282828282829e-05, + "loss": 2.5878, + "theoretical_loss": 3.321231951028217, + "tokens_seen": 2994470912 + }, + { + "epoch": 0.81, + "learning_rate": 9.426262626262626e-05, + "loss": 2.6055, + "theoretical_loss": 3.321226360319631, + "tokens_seen": 2994536448 + }, + { + "epoch": 0.81, + "learning_rate": 9.424242424242424e-05, + "loss": 2.8294, + "theoretical_loss": 3.3212207697676552, + "tokens_seen": 2994601984 + }, + { + "epoch": 0.81, + "learning_rate": 9.422222222222223e-05, + "loss": 2.4154, + "theoretical_loss": 3.3212151793722824, + "tokens_seen": 2994667520 + }, + { + "epoch": 0.81, + "learning_rate": 9.420202020202021e-05, + "loss": 2.5477, + "theoretical_loss": 3.3212095891335043, + "tokens_seen": 2994733056 + }, + { + "epoch": 0.81, + "learning_rate": 9.418181818181819e-05, + "loss": 2.3192, + "theoretical_loss": 3.321203999051314, + "tokens_seen": 2994798592 + }, + { + "epoch": 0.81, + "learning_rate": 9.416161616161618e-05, + "loss": 2.4364, + "theoretical_loss": 3.321198409125702, + "tokens_seen": 2994864128 + }, + { + "epoch": 0.81, + "learning_rate": 9.414141414141413e-05, + "loss": 2.6058, + "theoretical_loss": 3.321192819356662, + "tokens_seen": 2994929664 + }, + { + "epoch": 0.81, + "learning_rate": 9.412121212121212e-05, + "loss": 2.394, + "theoretical_loss": 3.321187229744186, + "tokens_seen": 2994995200 + }, + { + "epoch": 0.81, + "learning_rate": 9.41010101010101e-05, + "loss": 2.3382, + "theoretical_loss": 3.3211816402882652, + "tokens_seen": 2995060736 + }, + { + "epoch": 0.81, + "learning_rate": 9.408080808080809e-05, + "loss": 2.5193, + "theoretical_loss": 3.321176050988893, + "tokens_seen": 2995126272 + }, + { + "epoch": 0.81, + "learning_rate": 9.406060606060607e-05, + "loss": 2.4542, + "theoretical_loss": 3.3211704618460614, + "tokens_seen": 2995191808 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1642448, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.427232265472412, + "objective/train/theoretical_loss": 3.3211648728597614, + "objective/train/tokens_used": 1354116576, + "theoretical_loss": 3.3211648728597614, + "tokens_seen": 2995257344 + }, + { + "epoch": 0.81, + "learning_rate": 9.404040404040404e-05, + "loss": 2.3962, + "theoretical_loss": 3.3211648728597614, + "tokens_seen": 2995257344 + }, + { + "epoch": 0.81, + "learning_rate": 9.402020202020202e-05, + "loss": 2.4893, + "theoretical_loss": 3.3211592840299864, + "tokens_seen": 2995322880 + }, + { + "epoch": 0.81, + "learning_rate": 9.400000000000001e-05, + "loss": 2.5253, + "theoretical_loss": 3.3211536953567284, + "tokens_seen": 2995388416 + }, + { + "epoch": 0.81, + "learning_rate": 9.397979797979798e-05, + "loss": 2.5763, + "theoretical_loss": 3.321148106839979, + "tokens_seen": 2995453952 + }, + { + "epoch": 0.81, + "learning_rate": 9.395959595959596e-05, + "loss": 2.6585, + "theoretical_loss": 3.321142518479731, + "tokens_seen": 2995519488 + }, + { + "epoch": 0.81, + "learning_rate": 9.393939393939393e-05, + "loss": 2.492, + "theoretical_loss": 3.321136930275977, + "tokens_seen": 2995585024 + }, + { + "epoch": 0.81, + "learning_rate": 9.391919191919192e-05, + "loss": 2.5827, + "theoretical_loss": 3.321131342228708, + "tokens_seen": 2995650560 + }, + { + "epoch": 0.81, + "learning_rate": 9.38989898989899e-05, + "loss": 2.28, + "theoretical_loss": 3.321125754337917, + "tokens_seen": 2995716096 + }, + { + "epoch": 0.81, + "learning_rate": 9.387878787878788e-05, + "loss": 2.5499, + "theoretical_loss": 3.321120166603596, + "tokens_seen": 2995781632 + }, + { + "epoch": 0.81, + "learning_rate": 9.385858585858587e-05, + "loss": 2.5521, + "theoretical_loss": 3.3211145790257373, + "tokens_seen": 2995847168 + }, + { + "epoch": 0.81, + "learning_rate": 9.383838383838384e-05, + "loss": 2.604, + "theoretical_loss": 3.3211089916043326, + "tokens_seen": 2995912704 + }, + { + "epoch": 0.81, + "learning_rate": 9.381818181818182e-05, + "loss": 2.6427, + "theoretical_loss": 3.3211034043393743, + "tokens_seen": 2995978240 + }, + { + "epoch": 0.81, + "learning_rate": 9.37979797979798e-05, + "loss": 2.5917, + "theoretical_loss": 3.3210978172308554, + "tokens_seen": 2996043776 + }, + { + "epoch": 0.81, + "learning_rate": 9.377777777777778e-05, + "loss": 2.363, + "theoretical_loss": 3.3210922302787673, + "tokens_seen": 2996109312 + }, + { + "epoch": 0.81, + "learning_rate": 9.375757575757576e-05, + "loss": 2.6328, + "theoretical_loss": 3.3210866434831026, + "tokens_seen": 2996174848 + }, + { + "epoch": 0.81, + "learning_rate": 9.373737373737375e-05, + "loss": 2.3785, + "theoretical_loss": 3.3210810568438527, + "tokens_seen": 2996240384 + }, + { + "epoch": 0.81, + "learning_rate": 9.371717171717172e-05, + "loss": 2.6895, + "theoretical_loss": 3.3210754703610106, + "tokens_seen": 2996305920 + }, + { + "epoch": 0.81, + "learning_rate": 9.36969696969697e-05, + "loss": 2.4293, + "theoretical_loss": 3.3210698840345687, + "tokens_seen": 2996371456 + }, + { + "epoch": 0.81, + "learning_rate": 9.367676767676768e-05, + "loss": 2.5331, + "theoretical_loss": 3.321064297864518, + "tokens_seen": 2996436992 + }, + { + "epoch": 0.81, + "learning_rate": 9.365656565656567e-05, + "loss": 2.5706, + "theoretical_loss": 3.3210587118508523, + "tokens_seen": 2996502528 + }, + { + "epoch": 0.81, + "learning_rate": 9.363636363636364e-05, + "loss": 2.6263, + "theoretical_loss": 3.3210531259935627, + "tokens_seen": 2996568064 + }, + { + "epoch": 0.81, + "learning_rate": 9.361616161616161e-05, + "loss": 2.6323, + "theoretical_loss": 3.3210475402926414, + "tokens_seen": 2996633600 + }, + { + "epoch": 0.81, + "learning_rate": 9.359595959595959e-05, + "loss": 2.3604, + "theoretical_loss": 3.321041954748081, + "tokens_seen": 2996699136 + }, + { + "epoch": 0.81, + "learning_rate": 9.357575757575758e-05, + "loss": 2.3678, + "theoretical_loss": 3.3210363693598737, + "tokens_seen": 2996764672 + }, + { + "epoch": 0.81, + "learning_rate": 9.355555555555556e-05, + "loss": 2.534, + "theoretical_loss": 3.321030784128012, + "tokens_seen": 2996830208 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1643593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4673447608947754, + "objective/train/theoretical_loss": 3.321025199052487, + "objective/train/tokens_used": 1355754976, + "theoretical_loss": 3.321025199052487, + "tokens_seen": 2996895744 + }, + { + "epoch": 0.81, + "learning_rate": 9.353535353535354e-05, + "loss": 2.4258, + "theoretical_loss": 3.321025199052487, + "tokens_seen": 2996895744 + }, + { + "epoch": 0.82, + "learning_rate": 9.351515151515151e-05, + "loss": 2.46, + "theoretical_loss": 3.321019614133292, + "tokens_seen": 2996961280 + }, + { + "epoch": 0.82, + "learning_rate": 9.34949494949495e-05, + "loss": 2.4526, + "theoretical_loss": 3.321014029370419, + "tokens_seen": 2997026816 + }, + { + "epoch": 0.82, + "learning_rate": 9.347474747474748e-05, + "loss": 2.5625, + "theoretical_loss": 3.3210084447638595, + "tokens_seen": 2997092352 + }, + { + "epoch": 0.82, + "learning_rate": 9.345454545454545e-05, + "loss": 2.5541, + "theoretical_loss": 3.3210028603136066, + "tokens_seen": 2997157888 + }, + { + "epoch": 0.82, + "learning_rate": 9.343434343434344e-05, + "loss": 2.5962, + "theoretical_loss": 3.320997276019652, + "tokens_seen": 2997223424 + }, + { + "epoch": 0.82, + "learning_rate": 9.341414141414141e-05, + "loss": 2.6032, + "theoretical_loss": 3.320991691881988, + "tokens_seen": 2997288960 + }, + { + "epoch": 0.82, + "learning_rate": 9.339393939393939e-05, + "loss": 2.4633, + "theoretical_loss": 3.3209861079006067, + "tokens_seen": 2997354496 + }, + { + "epoch": 0.82, + "learning_rate": 9.337373737373738e-05, + "loss": 2.5132, + "theoretical_loss": 3.320980524075501, + "tokens_seen": 2997420032 + }, + { + "epoch": 0.82, + "learning_rate": 9.335353535353536e-05, + "loss": 2.5201, + "theoretical_loss": 3.320974940406662, + "tokens_seen": 2997485568 + }, + { + "epoch": 0.82, + "learning_rate": 9.333333333333334e-05, + "loss": 2.4253, + "theoretical_loss": 3.3209693568940826, + "tokens_seen": 2997551104 + }, + { + "epoch": 0.82, + "learning_rate": 9.331313131313133e-05, + "loss": 2.6305, + "theoretical_loss": 3.320963773537755, + "tokens_seen": 2997616640 + }, + { + "epoch": 0.82, + "learning_rate": 9.329292929292928e-05, + "loss": 2.4189, + "theoretical_loss": 3.3209581903376715, + "tokens_seen": 2997682176 + }, + { + "epoch": 0.82, + "learning_rate": 9.327272727272727e-05, + "loss": 2.3646, + "theoretical_loss": 3.320952607293824, + "tokens_seen": 2997747712 + }, + { + "epoch": 0.82, + "learning_rate": 9.325252525252525e-05, + "loss": 2.576, + "theoretical_loss": 3.3209470244062045, + "tokens_seen": 2997813248 + }, + { + "epoch": 0.82, + "learning_rate": 9.323232323232324e-05, + "loss": 2.3266, + "theoretical_loss": 3.320941441674806, + "tokens_seen": 2997878784 + }, + { + "epoch": 0.82, + "learning_rate": 9.321212121212122e-05, + "loss": 2.5517, + "theoretical_loss": 3.32093585909962, + "tokens_seen": 2997944320 + }, + { + "epoch": 0.82, + "learning_rate": 9.319191919191919e-05, + "loss": 2.3708, + "theoretical_loss": 3.320930276680639, + "tokens_seen": 2998009856 + }, + { + "epoch": 0.82, + "learning_rate": 9.317171717171717e-05, + "loss": 2.5419, + "theoretical_loss": 3.320924694417855, + "tokens_seen": 2998075392 + }, + { + "epoch": 0.82, + "learning_rate": 9.315151515151516e-05, + "loss": 2.4367, + "theoretical_loss": 3.3209191123112607, + "tokens_seen": 2998140928 + }, + { + "epoch": 0.82, + "learning_rate": 9.313131313131314e-05, + "loss": 2.3948, + "theoretical_loss": 3.3209135303608477, + "tokens_seen": 2998206464 + }, + { + "epoch": 0.82, + "learning_rate": 9.311111111111111e-05, + "loss": 2.5559, + "theoretical_loss": 3.320907948566609, + "tokens_seen": 2998272000 + }, + { + "epoch": 0.82, + "learning_rate": 9.309090909090908e-05, + "loss": 2.4195, + "theoretical_loss": 3.320902366928536, + "tokens_seen": 2998337536 + }, + { + "epoch": 0.82, + "learning_rate": 9.307070707070707e-05, + "loss": 2.5077, + "theoretical_loss": 3.3208967854466214, + "tokens_seen": 2998403072 + }, + { + "epoch": 0.82, + "learning_rate": 9.305050505050505e-05, + "loss": 2.5182, + "theoretical_loss": 3.320891204120857, + "tokens_seen": 2998468608 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1644133, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.513659954071045, + "objective/train/theoretical_loss": 3.3208856229512356, + "objective/train/tokens_used": 1357393376, + "theoretical_loss": 3.3208856229512356, + "tokens_seen": 2998534144 + }, + { + "epoch": 0.82, + "learning_rate": 9.303030303030303e-05, + "loss": 2.4814, + "theoretical_loss": 3.3208856229512356, + "tokens_seen": 2998534144 + }, + { + "epoch": 0.82, + "learning_rate": 9.301010101010102e-05, + "loss": 2.285, + "theoretical_loss": 3.320880041937749, + "tokens_seen": 2998599680 + }, + { + "epoch": 0.82, + "learning_rate": 9.298989898989899e-05, + "loss": 2.5273, + "theoretical_loss": 3.3208744610803898, + "tokens_seen": 2998665216 + }, + { + "epoch": 0.82, + "learning_rate": 9.296969696969697e-05, + "loss": 2.4731, + "theoretical_loss": 3.3208688803791495, + "tokens_seen": 2998730752 + }, + { + "epoch": 0.82, + "learning_rate": 9.294949494949496e-05, + "loss": 2.5174, + "theoretical_loss": 3.320863299834021, + "tokens_seen": 2998796288 + }, + { + "epoch": 0.82, + "learning_rate": 9.292929292929293e-05, + "loss": 2.5767, + "theoretical_loss": 3.3208577194449966, + "tokens_seen": 2998861824 + }, + { + "epoch": 0.82, + "learning_rate": 9.290909090909091e-05, + "loss": 2.3321, + "theoretical_loss": 3.320852139212068, + "tokens_seen": 2998927360 + }, + { + "epoch": 0.82, + "learning_rate": 9.288888888888888e-05, + "loss": 2.4857, + "theoretical_loss": 3.3208465591352274, + "tokens_seen": 2998992896 + }, + { + "epoch": 0.82, + "learning_rate": 9.286868686868687e-05, + "loss": 2.6687, + "theoretical_loss": 3.3208409792144677, + "tokens_seen": 2999058432 + }, + { + "epoch": 0.82, + "learning_rate": 9.284848484848485e-05, + "loss": 2.3643, + "theoretical_loss": 3.3208353994497806, + "tokens_seen": 2999123968 + }, + { + "epoch": 0.82, + "learning_rate": 9.282828282828283e-05, + "loss": 2.4023, + "theoretical_loss": 3.320829819841158, + "tokens_seen": 2999189504 + }, + { + "epoch": 0.82, + "learning_rate": 9.280808080808082e-05, + "loss": 2.5221, + "theoretical_loss": 3.320824240388593, + "tokens_seen": 2999255040 + }, + { + "epoch": 0.82, + "learning_rate": 9.27878787878788e-05, + "loss": 2.4745, + "theoretical_loss": 3.320818661092077, + "tokens_seen": 2999320576 + }, + { + "epoch": 0.82, + "learning_rate": 9.276767676767676e-05, + "loss": 2.435, + "theoretical_loss": 3.320813081951603, + "tokens_seen": 2999386112 + }, + { + "epoch": 0.82, + "learning_rate": 9.274747474747474e-05, + "loss": 2.3566, + "theoretical_loss": 3.3208075029671624, + "tokens_seen": 2999451648 + }, + { + "epoch": 0.82, + "learning_rate": 9.272727272727273e-05, + "loss": 2.4218, + "theoretical_loss": 3.320801924138748, + "tokens_seen": 2999517184 + }, + { + "epoch": 0.82, + "learning_rate": 9.270707070707071e-05, + "loss": 2.5465, + "theoretical_loss": 3.320796345466352, + "tokens_seen": 2999582720 + }, + { + "epoch": 0.82, + "learning_rate": 9.26868686868687e-05, + "loss": 2.5968, + "theoretical_loss": 3.3207907669499663, + "tokens_seen": 2999648256 + }, + { + "epoch": 0.82, + "learning_rate": 9.266666666666666e-05, + "loss": 2.538, + "theoretical_loss": 3.320785188589584, + "tokens_seen": 2999713792 + }, + { + "epoch": 0.82, + "learning_rate": 9.264646464646465e-05, + "loss": 2.665, + "theoretical_loss": 3.320779610385196, + "tokens_seen": 2999779328 + }, + { + "epoch": 0.82, + "learning_rate": 9.262626262626263e-05, + "loss": 2.4289, + "theoretical_loss": 3.3207740323367956, + "tokens_seen": 2999844864 + }, + { + "epoch": 0.82, + "learning_rate": 9.260606060606062e-05, + "loss": 2.46, + "theoretical_loss": 3.320768454444374, + "tokens_seen": 2999910400 + }, + { + "epoch": 0.82, + "learning_rate": 9.258585858585859e-05, + "loss": 2.366, + "theoretical_loss": 3.3207628767079242, + "tokens_seen": 2999975936 + }, + { + "epoch": 0.82, + "learning_rate": 9.256565656565656e-05, + "loss": 2.5349, + "theoretical_loss": 3.3207572991274388, + "tokens_seen": 3000041472 + }, + { + "epoch": 0.82, + "learning_rate": 9.254545454545454e-05, + "loss": 2.3487, + "theoretical_loss": 3.3207517217029094, + "tokens_seen": 3000107008 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1645467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9238919019699097, + "objective/train/theoretical_loss": 3.3207461444343283, + "objective/train/tokens_used": 1359031776, + "theoretical_loss": 3.3207461444343283, + "tokens_seen": 3000172544 + }, + { + "epoch": 0.82, + "learning_rate": 9.252525252525253e-05, + "loss": 2.4001, + "theoretical_loss": 3.3207461444343283, + "tokens_seen": 3000172544 + }, + { + "epoch": 0.82, + "learning_rate": 9.250505050505051e-05, + "loss": 2.4868, + "theoretical_loss": 3.3207405673216877, + "tokens_seen": 3000238080 + }, + { + "epoch": 0.82, + "learning_rate": 9.24848484848485e-05, + "loss": 2.469, + "theoretical_loss": 3.3207349903649797, + "tokens_seen": 3000303616 + }, + { + "epoch": 0.82, + "learning_rate": 9.246464646464646e-05, + "loss": 2.3101, + "theoretical_loss": 3.320729413564197, + "tokens_seen": 3000369152 + }, + { + "epoch": 0.82, + "learning_rate": 9.244444444444445e-05, + "loss": 2.4178, + "theoretical_loss": 3.3207238369193313, + "tokens_seen": 3000434688 + }, + { + "epoch": 0.82, + "learning_rate": 9.242424242424242e-05, + "loss": 2.5205, + "theoretical_loss": 3.3207182604303753, + "tokens_seen": 3000500224 + }, + { + "epoch": 0.82, + "learning_rate": 9.24040404040404e-05, + "loss": 2.3117, + "theoretical_loss": 3.3207126840973213, + "tokens_seen": 3000565760 + }, + { + "epoch": 0.82, + "learning_rate": 9.238383838383839e-05, + "loss": 2.3767, + "theoretical_loss": 3.320707107920161, + "tokens_seen": 3000631296 + }, + { + "epoch": 0.82, + "learning_rate": 9.236363636363637e-05, + "loss": 2.5327, + "theoretical_loss": 3.3207015318988873, + "tokens_seen": 3000696832 + }, + { + "epoch": 0.82, + "learning_rate": 9.234343434343434e-05, + "loss": 2.4754, + "theoretical_loss": 3.3206959560334917, + "tokens_seen": 3000762368 + }, + { + "epoch": 0.82, + "learning_rate": 9.232323232323232e-05, + "loss": 2.5305, + "theoretical_loss": 3.320690380323967, + "tokens_seen": 3000827904 + }, + { + "epoch": 0.82, + "learning_rate": 9.230303030303031e-05, + "loss": 2.5921, + "theoretical_loss": 3.320684804770305, + "tokens_seen": 3000893440 + }, + { + "epoch": 0.82, + "learning_rate": 9.228282828282829e-05, + "loss": 2.6496, + "theoretical_loss": 3.3206792293724985, + "tokens_seen": 3000958976 + }, + { + "epoch": 0.82, + "learning_rate": 9.226262626262628e-05, + "loss": 2.8048, + "theoretical_loss": 3.3206736541305393, + "tokens_seen": 3001024512 + }, + { + "epoch": 0.82, + "learning_rate": 9.224242424242423e-05, + "loss": 2.5274, + "theoretical_loss": 3.3206680790444194, + "tokens_seen": 3001090048 + }, + { + "epoch": 0.82, + "learning_rate": 9.222222222222222e-05, + "loss": 2.3691, + "theoretical_loss": 3.3206625041141318, + "tokens_seen": 3001155584 + }, + { + "epoch": 0.82, + "learning_rate": 9.22020202020202e-05, + "loss": 2.474, + "theoretical_loss": 3.3206569293396684, + "tokens_seen": 3001221120 + }, + { + "epoch": 0.82, + "learning_rate": 9.218181818181819e-05, + "loss": 2.4257, + "theoretical_loss": 3.3206513547210212, + "tokens_seen": 3001286656 + }, + { + "epoch": 0.82, + "learning_rate": 9.216161616161617e-05, + "loss": 2.4736, + "theoretical_loss": 3.320645780258183, + "tokens_seen": 3001352192 + }, + { + "epoch": 0.82, + "learning_rate": 9.214141414141414e-05, + "loss": 2.2585, + "theoretical_loss": 3.320640205951145, + "tokens_seen": 3001417728 + }, + { + "epoch": 0.82, + "learning_rate": 9.212121212121212e-05, + "loss": 2.3869, + "theoretical_loss": 3.3206346317999005, + "tokens_seen": 3001483264 + }, + { + "epoch": 0.82, + "learning_rate": 9.210101010101011e-05, + "loss": 2.4894, + "theoretical_loss": 3.3206290578044415, + "tokens_seen": 3001548800 + }, + { + "epoch": 0.82, + "learning_rate": 9.208080808080808e-05, + "loss": 2.3798, + "theoretical_loss": 3.32062348396476, + "tokens_seen": 3001614336 + }, + { + "epoch": 0.82, + "learning_rate": 9.206060606060606e-05, + "loss": 2.3831, + "theoretical_loss": 3.3206179102808484, + "tokens_seen": 3001679872 + }, + { + "epoch": 0.82, + "learning_rate": 9.204040404040403e-05, + "loss": 2.4612, + "theoretical_loss": 3.3206123367526987, + "tokens_seen": 3001745408 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1646067, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.591918706893921, + "objective/train/theoretical_loss": 3.3206067633803036, + "objective/train/tokens_used": 1360670176, + "theoretical_loss": 3.3206067633803036, + "tokens_seen": 3001810944 + }, + { + "epoch": 0.82, + "learning_rate": 9.202020202020202e-05, + "loss": 2.593, + "theoretical_loss": 3.3206067633803036, + "tokens_seen": 3001810944 + }, + { + "epoch": 0.82, + "learning_rate": 9.2e-05, + "loss": 2.5132, + "theoretical_loss": 3.320601190163655, + "tokens_seen": 3001876480 + }, + { + "epoch": 0.82, + "learning_rate": 9.197979797979798e-05, + "loss": 2.6735, + "theoretical_loss": 3.320595617102745, + "tokens_seen": 3001942016 + }, + { + "epoch": 0.82, + "learning_rate": 9.195959595959597e-05, + "loss": 2.4658, + "theoretical_loss": 3.320590044197566, + "tokens_seen": 3002007552 + }, + { + "epoch": 0.82, + "learning_rate": 9.193939393939395e-05, + "loss": 2.4072, + "theoretical_loss": 3.320584471448111, + "tokens_seen": 3002073088 + }, + { + "epoch": 0.82, + "learning_rate": 9.191919191919192e-05, + "loss": 2.6033, + "theoretical_loss": 3.320578898854371, + "tokens_seen": 3002138624 + }, + { + "epoch": 0.82, + "learning_rate": 9.189898989898989e-05, + "loss": 2.5524, + "theoretical_loss": 3.3205733264163393, + "tokens_seen": 3002204160 + }, + { + "epoch": 0.82, + "learning_rate": 9.187878787878788e-05, + "loss": 2.6801, + "theoretical_loss": 3.3205677541340073, + "tokens_seen": 3002269696 + }, + { + "epoch": 0.82, + "learning_rate": 9.185858585858586e-05, + "loss": 2.7446, + "theoretical_loss": 3.320562182007368, + "tokens_seen": 3002335232 + }, + { + "epoch": 0.82, + "learning_rate": 9.183838383838384e-05, + "loss": 2.3572, + "theoretical_loss": 3.3205566100364132, + "tokens_seen": 3002400768 + }, + { + "epoch": 0.82, + "learning_rate": 9.181818181818182e-05, + "loss": 2.5174, + "theoretical_loss": 3.320551038221135, + "tokens_seen": 3002466304 + }, + { + "epoch": 0.82, + "learning_rate": 9.17979797979798e-05, + "loss": 2.4731, + "theoretical_loss": 3.320545466561526, + "tokens_seen": 3002531840 + }, + { + "epoch": 0.82, + "learning_rate": 9.177777777777778e-05, + "loss": 2.4013, + "theoretical_loss": 3.3205398950575784, + "tokens_seen": 3002597376 + }, + { + "epoch": 0.82, + "learning_rate": 9.175757575757577e-05, + "loss": 2.6641, + "theoretical_loss": 3.3205343237092846, + "tokens_seen": 3002662912 + }, + { + "epoch": 0.82, + "learning_rate": 9.173737373737375e-05, + "loss": 2.5352, + "theoretical_loss": 3.320528752516636, + "tokens_seen": 3002728448 + }, + { + "epoch": 0.82, + "learning_rate": 9.171717171717171e-05, + "loss": 2.4333, + "theoretical_loss": 3.320523181479626, + "tokens_seen": 3002793984 + }, + { + "epoch": 0.82, + "learning_rate": 9.169696969696969e-05, + "loss": 2.5558, + "theoretical_loss": 3.3205176105982463, + "tokens_seen": 3002859520 + }, + { + "epoch": 0.82, + "learning_rate": 9.167676767676768e-05, + "loss": 2.4448, + "theoretical_loss": 3.320512039872489, + "tokens_seen": 3002925056 + }, + { + "epoch": 0.82, + "learning_rate": 9.165656565656566e-05, + "loss": 2.3997, + "theoretical_loss": 3.320506469302347, + "tokens_seen": 3002990592 + }, + { + "epoch": 0.82, + "learning_rate": 9.163636363636364e-05, + "loss": 2.4648, + "theoretical_loss": 3.320500898887812, + "tokens_seen": 3003056128 + }, + { + "epoch": 0.82, + "learning_rate": 9.161616161616161e-05, + "loss": 2.3938, + "theoretical_loss": 3.3204953286288763, + "tokens_seen": 3003121664 + }, + { + "epoch": 0.82, + "learning_rate": 9.15959595959596e-05, + "loss": 2.5355, + "theoretical_loss": 3.3204897585255324, + "tokens_seen": 3003187200 + }, + { + "epoch": 0.82, + "learning_rate": 9.157575757575758e-05, + "loss": 2.4006, + "theoretical_loss": 3.3204841885777725, + "tokens_seen": 3003252736 + }, + { + "epoch": 0.82, + "learning_rate": 9.155555555555555e-05, + "loss": 2.4675, + "theoretical_loss": 3.3204786187855886, + "tokens_seen": 3003318272 + }, + { + "epoch": 0.82, + "learning_rate": 9.153535353535354e-05, + "loss": 2.1985, + "theoretical_loss": 3.3204730491489727, + "tokens_seen": 3003383808 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1647443, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.251216173171997, + "objective/train/theoretical_loss": 3.320467479667918, + "objective/train/tokens_used": 1362308576, + "theoretical_loss": 3.320467479667918, + "tokens_seen": 3003449344 + }, + { + "epoch": 0.82, + "learning_rate": 9.151515151515152e-05, + "loss": 2.5003, + "theoretical_loss": 3.320467479667918, + "tokens_seen": 3003449344 + }, + { + "epoch": 0.82, + "learning_rate": 9.149494949494949e-05, + "loss": 2.509, + "theoretical_loss": 3.3204619103424164, + "tokens_seen": 3003514880 + }, + { + "epoch": 0.82, + "learning_rate": 9.147474747474747e-05, + "loss": 2.4573, + "theoretical_loss": 3.3204563411724597, + "tokens_seen": 3003580416 + }, + { + "epoch": 0.82, + "learning_rate": 9.145454545454546e-05, + "loss": 2.5154, + "theoretical_loss": 3.3204507721580403, + "tokens_seen": 3003645952 + }, + { + "epoch": 0.82, + "learning_rate": 9.143434343434344e-05, + "loss": 2.6629, + "theoretical_loss": 3.320445203299151, + "tokens_seen": 3003711488 + }, + { + "epoch": 0.82, + "learning_rate": 9.141414141414143e-05, + "loss": 2.3778, + "theoretical_loss": 3.3204396345957834, + "tokens_seen": 3003777024 + }, + { + "epoch": 0.82, + "learning_rate": 9.13939393939394e-05, + "loss": 2.3765, + "theoretical_loss": 3.3204340660479303, + "tokens_seen": 3003842560 + }, + { + "epoch": 0.82, + "learning_rate": 9.137373737373737e-05, + "loss": 2.4171, + "theoretical_loss": 3.320428497655584, + "tokens_seen": 3003908096 + }, + { + "epoch": 0.82, + "learning_rate": 9.135353535353535e-05, + "loss": 2.3372, + "theoretical_loss": 3.320422929418736, + "tokens_seen": 3003973632 + }, + { + "epoch": 0.82, + "learning_rate": 9.133333333333334e-05, + "loss": 2.5542, + "theoretical_loss": 3.320417361337379, + "tokens_seen": 3004039168 + }, + { + "epoch": 0.82, + "learning_rate": 9.131313131313132e-05, + "loss": 2.4977, + "theoretical_loss": 3.320411793411506, + "tokens_seen": 3004104704 + }, + { + "epoch": 0.82, + "learning_rate": 9.129292929292929e-05, + "loss": 2.4926, + "theoretical_loss": 3.3204062256411078, + "tokens_seen": 3004170240 + }, + { + "epoch": 0.82, + "learning_rate": 9.127272727272727e-05, + "loss": 2.4593, + "theoretical_loss": 3.320400658026178, + "tokens_seen": 3004235776 + }, + { + "epoch": 0.82, + "learning_rate": 9.125252525252526e-05, + "loss": 2.3455, + "theoretical_loss": 3.320395090566708, + "tokens_seen": 3004301312 + }, + { + "epoch": 0.82, + "learning_rate": 9.123232323232324e-05, + "loss": 2.4469, + "theoretical_loss": 3.3203895232626905, + "tokens_seen": 3004366848 + }, + { + "epoch": 0.82, + "learning_rate": 9.121212121212121e-05, + "loss": 2.6561, + "theoretical_loss": 3.3203839561141173, + "tokens_seen": 3004432384 + }, + { + "epoch": 0.82, + "learning_rate": 9.119191919191918e-05, + "loss": 2.5063, + "theoretical_loss": 3.3203783891209815, + "tokens_seen": 3004497920 + }, + { + "epoch": 0.82, + "learning_rate": 9.117171717171717e-05, + "loss": 2.2901, + "theoretical_loss": 3.320372822283275, + "tokens_seen": 3004563456 + }, + { + "epoch": 0.82, + "learning_rate": 9.115151515151515e-05, + "loss": 2.589, + "theoretical_loss": 3.3203672556009898, + "tokens_seen": 3004628992 + }, + { + "epoch": 0.82, + "learning_rate": 9.113131313131313e-05, + "loss": 2.5632, + "theoretical_loss": 3.3203616890741183, + "tokens_seen": 3004694528 + }, + { + "epoch": 0.82, + "learning_rate": 9.111111111111112e-05, + "loss": 2.4651, + "theoretical_loss": 3.320356122702653, + "tokens_seen": 3004760064 + }, + { + "epoch": 0.82, + "learning_rate": 9.10909090909091e-05, + "loss": 2.426, + "theoretical_loss": 3.3203505564865856, + "tokens_seen": 3004825600 + }, + { + "epoch": 0.82, + "learning_rate": 9.107070707070707e-05, + "loss": 2.5257, + "theoretical_loss": 3.320344990425909, + "tokens_seen": 3004891136 + }, + { + "epoch": 0.82, + "learning_rate": 9.105050505050506e-05, + "loss": 2.7284, + "theoretical_loss": 3.3203394245206153, + "tokens_seen": 3004956672 + }, + { + "epoch": 0.82, + "learning_rate": 9.103030303030303e-05, + "loss": 2.4091, + "theoretical_loss": 3.3203338587706965, + "tokens_seen": 3005022208 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1647973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2294132709503174, + "objective/train/theoretical_loss": 3.320328293176145, + "objective/train/tokens_used": 1363946976, + "theoretical_loss": 3.320328293176145, + "tokens_seen": 3005087744 + }, + { + "epoch": 0.82, + "learning_rate": 9.101010101010101e-05, + "loss": 2.3091, + "theoretical_loss": 3.320328293176145, + "tokens_seen": 3005087744 + }, + { + "epoch": 0.82, + "learning_rate": 9.0989898989899e-05, + "loss": 2.3488, + "theoretical_loss": 3.3203227277369534, + "tokens_seen": 3005153280 + }, + { + "epoch": 0.82, + "learning_rate": 9.096969696969697e-05, + "loss": 2.6453, + "theoretical_loss": 3.320317162453114, + "tokens_seen": 3005218816 + }, + { + "epoch": 0.82, + "learning_rate": 9.094949494949495e-05, + "loss": 2.356, + "theoretical_loss": 3.3203115973246184, + "tokens_seen": 3005284352 + }, + { + "epoch": 0.82, + "learning_rate": 9.092929292929293e-05, + "loss": 2.7034, + "theoretical_loss": 3.3203060323514593, + "tokens_seen": 3005349888 + }, + { + "epoch": 0.82, + "learning_rate": 9.090909090909092e-05, + "loss": 2.3826, + "theoretical_loss": 3.320300467533629, + "tokens_seen": 3005415424 + }, + { + "epoch": 0.82, + "learning_rate": 9.08888888888889e-05, + "loss": 2.509, + "theoretical_loss": 3.3202949028711197, + "tokens_seen": 3005480960 + }, + { + "epoch": 0.82, + "learning_rate": 9.086868686868686e-05, + "loss": 2.4096, + "theoretical_loss": 3.320289338363924, + "tokens_seen": 3005546496 + }, + { + "epoch": 0.82, + "learning_rate": 9.084848484848484e-05, + "loss": 2.3328, + "theoretical_loss": 3.3202837740120335, + "tokens_seen": 3005612032 + }, + { + "epoch": 0.82, + "learning_rate": 9.082828282828283e-05, + "loss": 2.1373, + "theoretical_loss": 3.320278209815441, + "tokens_seen": 3005677568 + }, + { + "epoch": 0.82, + "learning_rate": 9.080808080808081e-05, + "loss": 2.5909, + "theoretical_loss": 3.3202726457741387, + "tokens_seen": 3005743104 + }, + { + "epoch": 0.82, + "learning_rate": 9.07878787878788e-05, + "loss": 2.5219, + "theoretical_loss": 3.320267081888119, + "tokens_seen": 3005808640 + }, + { + "epoch": 0.82, + "learning_rate": 9.076767676767676e-05, + "loss": 2.363, + "theoretical_loss": 3.320261518157374, + "tokens_seen": 3005874176 + }, + { + "epoch": 0.82, + "learning_rate": 9.074747474747475e-05, + "loss": 2.5901, + "theoretical_loss": 3.3202559545818957, + "tokens_seen": 3005939712 + }, + { + "epoch": 0.82, + "learning_rate": 9.072727272727273e-05, + "loss": 2.4507, + "theoretical_loss": 3.3202503911616765, + "tokens_seen": 3006005248 + }, + { + "epoch": 0.82, + "learning_rate": 9.070707070707072e-05, + "loss": 2.498, + "theoretical_loss": 3.320244827896709, + "tokens_seen": 3006070784 + }, + { + "epoch": 0.82, + "learning_rate": 9.068686868686869e-05, + "loss": 2.4219, + "theoretical_loss": 3.320239264786986, + "tokens_seen": 3006136320 + }, + { + "epoch": 0.82, + "learning_rate": 9.066666666666667e-05, + "loss": 2.5793, + "theoretical_loss": 3.3202337018324983, + "tokens_seen": 3006201856 + }, + { + "epoch": 0.82, + "learning_rate": 9.064646464646464e-05, + "loss": 2.4463, + "theoretical_loss": 3.3202281390332393, + "tokens_seen": 3006267392 + }, + { + "epoch": 0.82, + "learning_rate": 9.062626262626262e-05, + "loss": 2.5076, + "theoretical_loss": 3.3202225763892015, + "tokens_seen": 3006332928 + }, + { + "epoch": 0.82, + "learning_rate": 9.060606060606061e-05, + "loss": 2.5213, + "theoretical_loss": 3.320217013900376, + "tokens_seen": 3006398464 + }, + { + "epoch": 0.82, + "learning_rate": 9.058585858585859e-05, + "loss": 2.7152, + "theoretical_loss": 3.320211451566756, + "tokens_seen": 3006464000 + }, + { + "epoch": 0.82, + "learning_rate": 9.056565656565658e-05, + "loss": 2.6251, + "theoretical_loss": 3.3202058893883333, + "tokens_seen": 3006529536 + }, + { + "epoch": 0.82, + "learning_rate": 9.054545454545455e-05, + "loss": 2.5632, + "theoretical_loss": 3.320200327365101, + "tokens_seen": 3006595072 + }, + { + "epoch": 0.82, + "learning_rate": 9.052525252525253e-05, + "loss": 2.29, + "theoretical_loss": 3.3201947654970505, + "tokens_seen": 3006660608 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1649106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2692012786865234, + "objective/train/theoretical_loss": 3.3201892037841745, + "objective/train/tokens_used": 1365585376, + "theoretical_loss": 3.3201892037841745, + "tokens_seen": 3006726144 + }, + { + "epoch": 0.82, + "learning_rate": 9.05050505050505e-05, + "loss": 2.4973, + "theoretical_loss": 3.3201892037841745, + "tokens_seen": 3006726144 + }, + { + "epoch": 0.82, + "learning_rate": 9.048484848484849e-05, + "loss": 2.2388, + "theoretical_loss": 3.320183642226465, + "tokens_seen": 3006791680 + }, + { + "epoch": 0.82, + "learning_rate": 9.046464646464647e-05, + "loss": 2.3712, + "theoretical_loss": 3.3201780808239145, + "tokens_seen": 3006857216 + }, + { + "epoch": 0.82, + "learning_rate": 9.044444444444444e-05, + "loss": 2.3286, + "theoretical_loss": 3.3201725195765155, + "tokens_seen": 3006922752 + }, + { + "epoch": 0.82, + "learning_rate": 9.042424242424242e-05, + "loss": 2.273, + "theoretical_loss": 3.32016695848426, + "tokens_seen": 3006988288 + }, + { + "epoch": 0.82, + "learning_rate": 9.040404040404041e-05, + "loss": 2.5644, + "theoretical_loss": 3.3201613975471402, + "tokens_seen": 3007053824 + }, + { + "epoch": 0.82, + "learning_rate": 9.038383838383839e-05, + "loss": 2.4567, + "theoretical_loss": 3.3201558367651485, + "tokens_seen": 3007119360 + }, + { + "epoch": 0.82, + "learning_rate": 9.036363636363638e-05, + "loss": 2.3618, + "theoretical_loss": 3.3201502761382775, + "tokens_seen": 3007184896 + }, + { + "epoch": 0.82, + "learning_rate": 9.034343434343433e-05, + "loss": 2.4384, + "theoretical_loss": 3.3201447156665194, + "tokens_seen": 3007250432 + }, + { + "epoch": 0.82, + "learning_rate": 9.032323232323232e-05, + "loss": 2.5464, + "theoretical_loss": 3.320139155349866, + "tokens_seen": 3007315968 + }, + { + "epoch": 0.82, + "learning_rate": 9.03030303030303e-05, + "loss": 2.3869, + "theoretical_loss": 3.32013359518831, + "tokens_seen": 3007381504 + }, + { + "epoch": 0.82, + "learning_rate": 9.028282828282828e-05, + "loss": 2.5844, + "theoretical_loss": 3.3201280351818436, + "tokens_seen": 3007447040 + }, + { + "epoch": 0.82, + "learning_rate": 9.026262626262627e-05, + "loss": 2.4821, + "theoretical_loss": 3.320122475330459, + "tokens_seen": 3007512576 + }, + { + "epoch": 0.82, + "learning_rate": 9.024242424242425e-05, + "loss": 2.5283, + "theoretical_loss": 3.320116915634149, + "tokens_seen": 3007578112 + }, + { + "epoch": 0.82, + "learning_rate": 9.022222222222222e-05, + "loss": 2.4325, + "theoretical_loss": 3.320111356092905, + "tokens_seen": 3007643648 + }, + { + "epoch": 0.82, + "learning_rate": 9.02020202020202e-05, + "loss": 2.6219, + "theoretical_loss": 3.3201057967067205, + "tokens_seen": 3007709184 + }, + { + "epoch": 0.82, + "learning_rate": 9.018181818181819e-05, + "loss": 2.4652, + "theoretical_loss": 3.320100237475587, + "tokens_seen": 3007774720 + }, + { + "epoch": 0.82, + "learning_rate": 9.016161616161616e-05, + "loss": 2.4742, + "theoretical_loss": 3.3200946783994962, + "tokens_seen": 3007840256 + }, + { + "epoch": 0.82, + "learning_rate": 9.014141414141415e-05, + "loss": 2.3435, + "theoretical_loss": 3.320089119478441, + "tokens_seen": 3007905792 + }, + { + "epoch": 0.82, + "learning_rate": 9.012121212121212e-05, + "loss": 2.2913, + "theoretical_loss": 3.3200835607124146, + "tokens_seen": 3007971328 + }, + { + "epoch": 0.82, + "learning_rate": 9.01010101010101e-05, + "loss": 2.4227, + "theoretical_loss": 3.3200780021014085, + "tokens_seen": 3008036864 + }, + { + "epoch": 0.82, + "learning_rate": 9.008080808080808e-05, + "loss": 2.5685, + "theoretical_loss": 3.3200724436454143, + "tokens_seen": 3008102400 + }, + { + "epoch": 0.82, + "learning_rate": 9.006060606060607e-05, + "loss": 2.578, + "theoretical_loss": 3.320066885344425, + "tokens_seen": 3008167936 + }, + { + "epoch": 0.82, + "learning_rate": 9.004040404040405e-05, + "loss": 2.3715, + "theoretical_loss": 3.3200613271984336, + "tokens_seen": 3008233472 + }, + { + "epoch": 0.82, + "learning_rate": 9.002020202020202e-05, + "loss": 2.4933, + "theoretical_loss": 3.320055769207431, + "tokens_seen": 3008299008 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1649879, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.466209888458252, + "objective/train/theoretical_loss": 3.3200502113714108, + "objective/train/tokens_used": 1367223776, + "theoretical_loss": 3.3200502113714108, + "tokens_seen": 3008364544 + }, + { + "epoch": 0.82, + "learning_rate": 8.999999999999999e-05, + "loss": 2.3348, + "theoretical_loss": 3.3200502113714108, + "tokens_seen": 3008364544 + }, + { + "epoch": 0.82, + "learning_rate": 8.997979797979798e-05, + "loss": 2.5505, + "theoretical_loss": 3.3200446536903643, + "tokens_seen": 3008430080 + }, + { + "epoch": 0.82, + "learning_rate": 8.995959595959596e-05, + "loss": 2.6145, + "theoretical_loss": 3.3200390961642845, + "tokens_seen": 3008495616 + }, + { + "epoch": 0.82, + "learning_rate": 8.993939393939394e-05, + "loss": 2.5454, + "theoretical_loss": 3.320033538793163, + "tokens_seen": 3008561152 + }, + { + "epoch": 0.82, + "learning_rate": 8.991919191919191e-05, + "loss": 2.4013, + "theoretical_loss": 3.3200279815769926, + "tokens_seen": 3008626688 + }, + { + "epoch": 0.82, + "learning_rate": 8.98989898989899e-05, + "loss": 2.4217, + "theoretical_loss": 3.320022424515766, + "tokens_seen": 3008692224 + }, + { + "epoch": 0.82, + "learning_rate": 8.987878787878788e-05, + "loss": 2.6247, + "theoretical_loss": 3.3200168676094743, + "tokens_seen": 3008757760 + }, + { + "epoch": 0.82, + "learning_rate": 8.985858585858587e-05, + "loss": 2.3474, + "theoretical_loss": 3.320011310858111, + "tokens_seen": 3008823296 + }, + { + "epoch": 0.82, + "learning_rate": 8.983838383838385e-05, + "loss": 2.4393, + "theoretical_loss": 3.320005754261668, + "tokens_seen": 3008888832 + }, + { + "epoch": 0.82, + "learning_rate": 8.981818181818182e-05, + "loss": 2.4629, + "theoretical_loss": 3.320000197820137, + "tokens_seen": 3008954368 + }, + { + "epoch": 0.82, + "learning_rate": 8.979797979797979e-05, + "loss": 2.6364, + "theoretical_loss": 3.319994641533511, + "tokens_seen": 3009019904 + }, + { + "epoch": 0.82, + "learning_rate": 8.977777777777778e-05, + "loss": 2.4095, + "theoretical_loss": 3.3199890854017826, + "tokens_seen": 3009085440 + }, + { + "epoch": 0.82, + "learning_rate": 8.975757575757576e-05, + "loss": 2.4089, + "theoretical_loss": 3.319983529424943, + "tokens_seen": 3009150976 + }, + { + "epoch": 0.82, + "learning_rate": 8.973737373737374e-05, + "loss": 2.38, + "theoretical_loss": 3.319977973602986, + "tokens_seen": 3009216512 + }, + { + "epoch": 0.82, + "learning_rate": 8.971717171717173e-05, + "loss": 2.6198, + "theoretical_loss": 3.3199724179359027, + "tokens_seen": 3009282048 + }, + { + "epoch": 0.82, + "learning_rate": 8.96969696969697e-05, + "loss": 2.4144, + "theoretical_loss": 3.3199668624236853, + "tokens_seen": 3009347584 + }, + { + "epoch": 0.82, + "learning_rate": 8.967676767676768e-05, + "loss": 2.3938, + "theoretical_loss": 3.319961307066327, + "tokens_seen": 3009413120 + }, + { + "epoch": 0.82, + "learning_rate": 8.965656565656565e-05, + "loss": 2.6736, + "theoretical_loss": 3.31995575186382, + "tokens_seen": 3009478656 + }, + { + "epoch": 0.82, + "learning_rate": 8.963636363636364e-05, + "loss": 2.4292, + "theoretical_loss": 3.3199501968161558, + "tokens_seen": 3009544192 + }, + { + "epoch": 0.82, + "learning_rate": 8.961616161616162e-05, + "loss": 2.5595, + "theoretical_loss": 3.3199446419233274, + "tokens_seen": 3009609728 + }, + { + "epoch": 0.82, + "learning_rate": 8.959595959595959e-05, + "loss": 2.4893, + "theoretical_loss": 3.319939087185327, + "tokens_seen": 3009675264 + }, + { + "epoch": 0.82, + "learning_rate": 8.957575757575757e-05, + "loss": 2.5559, + "theoretical_loss": 3.319933532602147, + "tokens_seen": 3009740800 + }, + { + "epoch": 0.82, + "learning_rate": 8.955555555555556e-05, + "loss": 2.3606, + "theoretical_loss": 3.3199279781737796, + "tokens_seen": 3009806336 + }, + { + "epoch": 0.82, + "learning_rate": 8.953535353535354e-05, + "loss": 2.6068, + "theoretical_loss": 3.3199224239002167, + "tokens_seen": 3009871872 + }, + { + "epoch": 0.82, + "learning_rate": 8.951515151515153e-05, + "loss": 2.4673, + "theoretical_loss": 3.3199168697814514, + "tokens_seen": 3009937408 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1651225, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.325669527053833, + "objective/train/theoretical_loss": 3.3199113158174756, + "objective/train/tokens_used": 1368862176, + "theoretical_loss": 3.3199113158174756, + "tokens_seen": 3010002944 + }, + { + "epoch": 0.82, + "learning_rate": 8.94949494949495e-05, + "loss": 2.41, + "theoretical_loss": 3.3199113158174756, + "tokens_seen": 3010002944 + }, + { + "epoch": 0.82, + "learning_rate": 8.947474747474747e-05, + "loss": 2.3801, + "theoretical_loss": 3.3199057620082812, + "tokens_seen": 3010068480 + }, + { + "epoch": 0.82, + "learning_rate": 8.945454545454545e-05, + "loss": 2.4696, + "theoretical_loss": 3.3199002083538613, + "tokens_seen": 3010134016 + }, + { + "epoch": 0.82, + "learning_rate": 8.943434343434343e-05, + "loss": 2.2904, + "theoretical_loss": 3.319894654854208, + "tokens_seen": 3010199552 + }, + { + "epoch": 0.82, + "learning_rate": 8.941414141414142e-05, + "loss": 2.5666, + "theoretical_loss": 3.319889101509313, + "tokens_seen": 3010265088 + }, + { + "epoch": 0.82, + "learning_rate": 8.93939393939394e-05, + "loss": 2.3801, + "theoretical_loss": 3.3198835483191695, + "tokens_seen": 3010330624 + }, + { + "epoch": 0.82, + "learning_rate": 8.937373737373737e-05, + "loss": 2.3553, + "theoretical_loss": 3.319877995283769, + "tokens_seen": 3010396160 + }, + { + "epoch": 0.82, + "learning_rate": 8.935353535353536e-05, + "loss": 2.488, + "theoretical_loss": 3.319872442403105, + "tokens_seen": 3010461696 + }, + { + "epoch": 0.82, + "learning_rate": 8.933333333333334e-05, + "loss": 2.5858, + "theoretical_loss": 3.3198668896771686, + "tokens_seen": 3010527232 + }, + { + "epoch": 0.82, + "learning_rate": 8.931313131313132e-05, + "loss": 2.4689, + "theoretical_loss": 3.3198613371059524, + "tokens_seen": 3010592768 + }, + { + "epoch": 0.82, + "learning_rate": 8.92929292929293e-05, + "loss": 2.2521, + "theoretical_loss": 3.319855784689449, + "tokens_seen": 3010658304 + }, + { + "epoch": 0.82, + "learning_rate": 8.927272727272727e-05, + "loss": 2.5345, + "theoretical_loss": 3.319850232427651, + "tokens_seen": 3010723840 + }, + { + "epoch": 0.82, + "learning_rate": 8.925252525252525e-05, + "loss": 2.5216, + "theoretical_loss": 3.31984468032055, + "tokens_seen": 3010789376 + }, + { + "epoch": 0.82, + "learning_rate": 8.923232323232323e-05, + "loss": 2.2822, + "theoretical_loss": 3.3198391283681383, + "tokens_seen": 3010854912 + }, + { + "epoch": 0.82, + "learning_rate": 8.921212121212122e-05, + "loss": 2.439, + "theoretical_loss": 3.3198335765704092, + "tokens_seen": 3010920448 + }, + { + "epoch": 0.82, + "learning_rate": 8.91919191919192e-05, + "loss": 2.6148, + "theoretical_loss": 3.3198280249273546, + "tokens_seen": 3010985984 + }, + { + "epoch": 0.82, + "learning_rate": 8.917171717171717e-05, + "loss": 2.354, + "theoretical_loss": 3.319822473438966, + "tokens_seen": 3011051520 + }, + { + "epoch": 0.82, + "learning_rate": 8.915151515151516e-05, + "loss": 2.294, + "theoretical_loss": 3.319816922105237, + "tokens_seen": 3011117056 + }, + { + "epoch": 0.82, + "learning_rate": 8.913131313131313e-05, + "loss": 2.6274, + "theoretical_loss": 3.3198113709261587, + "tokens_seen": 3011182592 + }, + { + "epoch": 0.82, + "learning_rate": 8.911111111111111e-05, + "loss": 2.34, + "theoretical_loss": 3.319805819901724, + "tokens_seen": 3011248128 + }, + { + "epoch": 0.82, + "learning_rate": 8.90909090909091e-05, + "loss": 2.5116, + "theoretical_loss": 3.3198002690319255, + "tokens_seen": 3011313664 + }, + { + "epoch": 0.82, + "learning_rate": 8.907070707070706e-05, + "loss": 2.3855, + "theoretical_loss": 3.3197947183167553, + "tokens_seen": 3011379200 + }, + { + "epoch": 0.82, + "learning_rate": 8.905050505050505e-05, + "loss": 2.5659, + "theoretical_loss": 3.319789167756206, + "tokens_seen": 3011444736 + }, + { + "epoch": 0.82, + "learning_rate": 8.903030303030303e-05, + "loss": 2.5068, + "theoretical_loss": 3.319783617350269, + "tokens_seen": 3011510272 + }, + { + "epoch": 0.82, + "learning_rate": 8.901010101010102e-05, + "loss": 2.3397, + "theoretical_loss": 3.3197780670989374, + "tokens_seen": 3011575808 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1651868, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4228270053863525, + "objective/train/theoretical_loss": 3.319772517002204, + "objective/train/tokens_used": 1370500576, + "theoretical_loss": 3.319772517002204, + "tokens_seen": 3011641344 + }, + { + "epoch": 0.82, + "learning_rate": 8.8989898989899e-05, + "loss": 2.3949, + "theoretical_loss": 3.319772517002204, + "tokens_seen": 3011641344 + }, + { + "epoch": 0.82, + "learning_rate": 8.896969696969698e-05, + "loss": 2.4053, + "theoretical_loss": 3.31976696706006, + "tokens_seen": 3011706880 + }, + { + "epoch": 0.82, + "learning_rate": 8.894949494949494e-05, + "loss": 2.5134, + "theoretical_loss": 3.319761417272498, + "tokens_seen": 3011772416 + }, + { + "epoch": 0.82, + "learning_rate": 8.892929292929293e-05, + "loss": 2.3324, + "theoretical_loss": 3.319755867639511, + "tokens_seen": 3011837952 + }, + { + "epoch": 0.82, + "learning_rate": 8.890909090909091e-05, + "loss": 2.3563, + "theoretical_loss": 3.319750318161091, + "tokens_seen": 3011903488 + }, + { + "epoch": 0.82, + "learning_rate": 8.888888888888889e-05, + "loss": 2.4773, + "theoretical_loss": 3.3197447688372295, + "tokens_seen": 3011969024 + }, + { + "epoch": 0.82, + "learning_rate": 8.886868686868688e-05, + "loss": 2.4749, + "theoretical_loss": 3.3197392196679205, + "tokens_seen": 3012034560 + }, + { + "epoch": 0.82, + "learning_rate": 8.884848484848485e-05, + "loss": 2.5039, + "theoretical_loss": 3.3197336706531546, + "tokens_seen": 3012100096 + }, + { + "epoch": 0.82, + "learning_rate": 8.882828282828283e-05, + "loss": 2.6036, + "theoretical_loss": 3.3197281217929255, + "tokens_seen": 3012165632 + }, + { + "epoch": 0.82, + "learning_rate": 8.880808080808082e-05, + "loss": 2.3934, + "theoretical_loss": 3.319722573087225, + "tokens_seen": 3012231168 + }, + { + "epoch": 0.82, + "learning_rate": 8.878787878787879e-05, + "loss": 2.5044, + "theoretical_loss": 3.319717024536045, + "tokens_seen": 3012296704 + }, + { + "epoch": 0.82, + "learning_rate": 8.876767676767677e-05, + "loss": 2.4502, + "theoretical_loss": 3.3197114761393784, + "tokens_seen": 3012362240 + }, + { + "epoch": 0.82, + "learning_rate": 8.874747474747474e-05, + "loss": 2.4458, + "theoretical_loss": 3.3197059278972176, + "tokens_seen": 3012427776 + }, + { + "epoch": 0.82, + "learning_rate": 8.872727272727272e-05, + "loss": 2.294, + "theoretical_loss": 3.3197003798095546, + "tokens_seen": 3012493312 + }, + { + "epoch": 0.82, + "learning_rate": 8.870707070707071e-05, + "loss": 2.5854, + "theoretical_loss": 3.3196948318763817, + "tokens_seen": 3012558848 + }, + { + "epoch": 0.82, + "learning_rate": 8.868686868686869e-05, + "loss": 2.4789, + "theoretical_loss": 3.3196892840976915, + "tokens_seen": 3012624384 + }, + { + "epoch": 0.82, + "learning_rate": 8.866666666666668e-05, + "loss": 2.3251, + "theoretical_loss": 3.319683736473476, + "tokens_seen": 3012689920 + }, + { + "epoch": 0.82, + "learning_rate": 8.864646464646465e-05, + "loss": 2.3248, + "theoretical_loss": 3.319678189003728, + "tokens_seen": 3012755456 + }, + { + "epoch": 0.82, + "learning_rate": 8.862626262626263e-05, + "loss": 2.4053, + "theoretical_loss": 3.3196726416884395, + "tokens_seen": 3012820992 + }, + { + "epoch": 0.82, + "learning_rate": 8.86060606060606e-05, + "loss": 2.4879, + "theoretical_loss": 3.319667094527603, + "tokens_seen": 3012886528 + }, + { + "epoch": 0.82, + "learning_rate": 8.858585858585858e-05, + "loss": 2.5869, + "theoretical_loss": 3.3196615475212106, + "tokens_seen": 3012952064 + }, + { + "epoch": 0.82, + "learning_rate": 8.856565656565657e-05, + "loss": 2.3966, + "theoretical_loss": 3.319656000669255, + "tokens_seen": 3013017600 + }, + { + "epoch": 0.82, + "learning_rate": 8.854545454545455e-05, + "loss": 2.4875, + "theoretical_loss": 3.3196504539717284, + "tokens_seen": 3013083136 + }, + { + "epoch": 0.82, + "learning_rate": 8.852525252525252e-05, + "loss": 2.6143, + "theoretical_loss": 3.319644907428623, + "tokens_seen": 3013148672 + }, + { + "epoch": 0.82, + "learning_rate": 8.850505050505051e-05, + "loss": 2.3296, + "theoretical_loss": 3.3196393610399317, + "tokens_seen": 3013214208 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1652881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.490133285522461, + "objective/train/theoretical_loss": 3.3196338148056457, + "objective/train/tokens_used": 1372138976, + "theoretical_loss": 3.3196338148056457, + "tokens_seen": 3013279744 + }, + { + "epoch": 0.82, + "learning_rate": 8.848484848484849e-05, + "loss": 2.3001, + "theoretical_loss": 3.3196338148056457, + "tokens_seen": 3013279744 + }, + { + "epoch": 0.83, + "learning_rate": 8.846464646464648e-05, + "loss": 2.2848, + "theoretical_loss": 3.3196282687257583, + "tokens_seen": 3013345280 + }, + { + "epoch": 0.83, + "learning_rate": 8.844444444444446e-05, + "loss": 2.4237, + "theoretical_loss": 3.3196227228002617, + "tokens_seen": 3013410816 + }, + { + "epoch": 0.83, + "learning_rate": 8.842424242424242e-05, + "loss": 2.4549, + "theoretical_loss": 3.3196171770291483, + "tokens_seen": 3013476352 + }, + { + "epoch": 0.83, + "learning_rate": 8.84040404040404e-05, + "loss": 2.3531, + "theoretical_loss": 3.31961163141241, + "tokens_seen": 3013541888 + }, + { + "epoch": 0.83, + "learning_rate": 8.838383838383838e-05, + "loss": 2.3479, + "theoretical_loss": 3.3196060859500394, + "tokens_seen": 3013607424 + }, + { + "epoch": 0.83, + "learning_rate": 8.836363636363637e-05, + "loss": 2.4355, + "theoretical_loss": 3.319600540642029, + "tokens_seen": 3013672960 + }, + { + "epoch": 0.83, + "learning_rate": 8.834343434343435e-05, + "loss": 2.5351, + "theoretical_loss": 3.319594995488371, + "tokens_seen": 3013738496 + }, + { + "epoch": 0.83, + "learning_rate": 8.832323232323232e-05, + "loss": 2.3865, + "theoretical_loss": 3.3195894504890573, + "tokens_seen": 3013804032 + }, + { + "epoch": 0.83, + "learning_rate": 8.83030303030303e-05, + "loss": 2.782, + "theoretical_loss": 3.3195839056440812, + "tokens_seen": 3013869568 + }, + { + "epoch": 0.83, + "learning_rate": 8.828282828282829e-05, + "loss": 2.2539, + "theoretical_loss": 3.3195783609534346, + "tokens_seen": 3013935104 + }, + { + "epoch": 0.83, + "learning_rate": 8.826262626262626e-05, + "loss": 2.4539, + "theoretical_loss": 3.3195728164171094, + "tokens_seen": 3014000640 + }, + { + "epoch": 0.83, + "learning_rate": 8.824242424242424e-05, + "loss": 2.5259, + "theoretical_loss": 3.3195672720350986, + "tokens_seen": 3014066176 + }, + { + "epoch": 0.83, + "learning_rate": 8.822222222222221e-05, + "loss": 2.3327, + "theoretical_loss": 3.319561727807394, + "tokens_seen": 3014131712 + }, + { + "epoch": 0.83, + "learning_rate": 8.82020202020202e-05, + "loss": 2.5884, + "theoretical_loss": 3.319556183733989, + "tokens_seen": 3014197248 + }, + { + "epoch": 0.83, + "learning_rate": 8.818181818181818e-05, + "loss": 2.3731, + "theoretical_loss": 3.3195506398148744, + "tokens_seen": 3014262784 + }, + { + "epoch": 0.83, + "learning_rate": 8.816161616161617e-05, + "loss": 2.3613, + "theoretical_loss": 3.3195450960500437, + "tokens_seen": 3014328320 + }, + { + "epoch": 0.83, + "learning_rate": 8.814141414141415e-05, + "loss": 2.5168, + "theoretical_loss": 3.319539552439489, + "tokens_seen": 3014393856 + }, + { + "epoch": 0.83, + "learning_rate": 8.812121212121213e-05, + "loss": 2.297, + "theoretical_loss": 3.3195340089832026, + "tokens_seen": 3014459392 + }, + { + "epoch": 0.83, + "learning_rate": 8.81010101010101e-05, + "loss": 2.5237, + "theoretical_loss": 3.3195284656811763, + "tokens_seen": 3014524928 + }, + { + "epoch": 0.83, + "learning_rate": 8.808080808080808e-05, + "loss": 2.568, + "theoretical_loss": 3.3195229225334035, + "tokens_seen": 3014590464 + }, + { + "epoch": 0.83, + "learning_rate": 8.806060606060606e-05, + "loss": 2.4569, + "theoretical_loss": 3.319517379539876, + "tokens_seen": 3014656000 + }, + { + "epoch": 0.83, + "learning_rate": 8.804040404040404e-05, + "loss": 2.2409, + "theoretical_loss": 3.3195118367005856, + "tokens_seen": 3014721536 + }, + { + "epoch": 0.83, + "learning_rate": 8.802020202020203e-05, + "loss": 2.3662, + "theoretical_loss": 3.3195062940155258, + "tokens_seen": 3014787072 + }, + { + "epoch": 0.83, + "learning_rate": 8.8e-05, + "loss": 2.5758, + "theoretical_loss": 3.3195007514846884, + "tokens_seen": 3014852608 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1653310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.427609920501709, + "objective/train/theoretical_loss": 3.3194952091080654, + "objective/train/tokens_used": 1373777376, + "theoretical_loss": 3.3194952091080654, + "tokens_seen": 3014918144 + }, + { + "epoch": 0.83, + "learning_rate": 8.797979797979798e-05, + "loss": 2.3898, + "theoretical_loss": 3.3194952091080654, + "tokens_seen": 3014918144 + }, + { + "epoch": 0.83, + "learning_rate": 8.795959595959597e-05, + "loss": 2.3826, + "theoretical_loss": 3.3194896668856497, + "tokens_seen": 3014983680 + }, + { + "epoch": 0.83, + "learning_rate": 8.793939393939395e-05, + "loss": 2.4972, + "theoretical_loss": 3.3194841248174334, + "tokens_seen": 3015049216 + }, + { + "epoch": 0.83, + "learning_rate": 8.791919191919192e-05, + "loss": 2.4198, + "theoretical_loss": 3.319478582903409, + "tokens_seen": 3015114752 + }, + { + "epoch": 0.83, + "learning_rate": 8.789898989898989e-05, + "loss": 2.5367, + "theoretical_loss": 3.3194730411435684, + "tokens_seen": 3015180288 + }, + { + "epoch": 0.83, + "learning_rate": 8.787878787878787e-05, + "loss": 2.2519, + "theoretical_loss": 3.3194674995379048, + "tokens_seen": 3015245824 + }, + { + "epoch": 0.83, + "learning_rate": 8.785858585858586e-05, + "loss": 2.29, + "theoretical_loss": 3.3194619580864098, + "tokens_seen": 3015311360 + }, + { + "epoch": 0.83, + "learning_rate": 8.783838383838384e-05, + "loss": 2.4601, + "theoretical_loss": 3.3194564167890763, + "tokens_seen": 3015376896 + }, + { + "epoch": 0.83, + "learning_rate": 8.781818181818183e-05, + "loss": 2.4991, + "theoretical_loss": 3.3194508756458965, + "tokens_seen": 3015442432 + }, + { + "epoch": 0.83, + "learning_rate": 8.77979797979798e-05, + "loss": 2.2787, + "theoretical_loss": 3.319445334656862, + "tokens_seen": 3015507968 + }, + { + "epoch": 0.83, + "learning_rate": 8.777777777777778e-05, + "loss": 2.6063, + "theoretical_loss": 3.319439793821967, + "tokens_seen": 3015573504 + }, + { + "epoch": 0.83, + "learning_rate": 8.775757575757576e-05, + "loss": 2.6197, + "theoretical_loss": 3.3194342531412016, + "tokens_seen": 3015639040 + }, + { + "epoch": 0.83, + "learning_rate": 8.773737373737373e-05, + "loss": 2.5613, + "theoretical_loss": 3.3194287126145596, + "tokens_seen": 3015704576 + }, + { + "epoch": 0.83, + "learning_rate": 8.771717171717172e-05, + "loss": 2.4235, + "theoretical_loss": 3.319423172242033, + "tokens_seen": 3015770112 + }, + { + "epoch": 0.83, + "learning_rate": 8.76969696969697e-05, + "loss": 2.4856, + "theoretical_loss": 3.3194176320236144, + "tokens_seen": 3015835648 + }, + { + "epoch": 0.83, + "learning_rate": 8.767676767676767e-05, + "loss": 2.4678, + "theoretical_loss": 3.319412091959296, + "tokens_seen": 3015901184 + }, + { + "epoch": 0.83, + "learning_rate": 8.765656565656566e-05, + "loss": 2.4605, + "theoretical_loss": 3.31940655204907, + "tokens_seen": 3015966720 + }, + { + "epoch": 0.83, + "learning_rate": 8.763636363636364e-05, + "loss": 2.4836, + "theoretical_loss": 3.319401012292929, + "tokens_seen": 3016032256 + }, + { + "epoch": 0.83, + "learning_rate": 8.761616161616163e-05, + "loss": 2.6518, + "theoretical_loss": 3.319395472690865, + "tokens_seen": 3016097792 + }, + { + "epoch": 0.83, + "learning_rate": 8.759595959595961e-05, + "loss": 2.426, + "theoretical_loss": 3.3193899332428707, + "tokens_seen": 3016163328 + }, + { + "epoch": 0.83, + "learning_rate": 8.757575757575757e-05, + "loss": 2.4637, + "theoretical_loss": 3.3193843939489382, + "tokens_seen": 3016228864 + }, + { + "epoch": 0.83, + "learning_rate": 8.755555555555555e-05, + "loss": 2.6203, + "theoretical_loss": 3.3193788548090604, + "tokens_seen": 3016294400 + }, + { + "epoch": 0.83, + "learning_rate": 8.753535353535353e-05, + "loss": 2.649, + "theoretical_loss": 3.319373315823229, + "tokens_seen": 3016359936 + }, + { + "epoch": 0.83, + "learning_rate": 8.751515151515152e-05, + "loss": 2.4175, + "theoretical_loss": 3.319367776991437, + "tokens_seen": 3016425472 + }, + { + "epoch": 0.83, + "learning_rate": 8.74949494949495e-05, + "loss": 2.7281, + "theoretical_loss": 3.3193622383136763, + "tokens_seen": 3016491008 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1654610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4548187255859375, + "objective/train/theoretical_loss": 3.3193566997899397, + "objective/train/tokens_used": 1375415776, + "theoretical_loss": 3.3193566997899397, + "tokens_seen": 3016556544 + }, + { + "epoch": 0.83, + "learning_rate": 8.747474747474747e-05, + "loss": 2.3618, + "theoretical_loss": 3.3193566997899397, + "tokens_seen": 3016556544 + }, + { + "epoch": 0.83, + "learning_rate": 8.745454545454546e-05, + "loss": 2.55, + "theoretical_loss": 3.3193511614202187, + "tokens_seen": 3016622080 + }, + { + "epoch": 0.83, + "learning_rate": 8.743434343434344e-05, + "loss": 2.3869, + "theoretical_loss": 3.319345623204507, + "tokens_seen": 3016687616 + }, + { + "epoch": 0.83, + "learning_rate": 8.741414141414142e-05, + "loss": 2.4419, + "theoretical_loss": 3.319340085142796, + "tokens_seen": 3016753152 + }, + { + "epoch": 0.83, + "learning_rate": 8.73939393939394e-05, + "loss": 2.6702, + "theoretical_loss": 3.319334547235078, + "tokens_seen": 3016818688 + }, + { + "epoch": 0.83, + "learning_rate": 8.737373737373736e-05, + "loss": 2.3728, + "theoretical_loss": 3.319329009481346, + "tokens_seen": 3016884224 + }, + { + "epoch": 0.83, + "learning_rate": 8.735353535353535e-05, + "loss": 2.4164, + "theoretical_loss": 3.319323471881592, + "tokens_seen": 3016949760 + }, + { + "epoch": 0.83, + "learning_rate": 8.733333333333333e-05, + "loss": 2.5133, + "theoretical_loss": 3.3193179344358086, + "tokens_seen": 3017015296 + }, + { + "epoch": 0.83, + "learning_rate": 8.731313131313132e-05, + "loss": 2.5721, + "theoretical_loss": 3.3193123971439875, + "tokens_seen": 3017080832 + }, + { + "epoch": 0.83, + "learning_rate": 8.72929292929293e-05, + "loss": 2.6428, + "theoretical_loss": 3.319306860006122, + "tokens_seen": 3017146368 + }, + { + "epoch": 0.83, + "learning_rate": 8.727272727272728e-05, + "loss": 2.4393, + "theoretical_loss": 3.319301323022204, + "tokens_seen": 3017211904 + }, + { + "epoch": 0.83, + "learning_rate": 8.725252525252526e-05, + "loss": 2.5009, + "theoretical_loss": 3.319295786192226, + "tokens_seen": 3017277440 + }, + { + "epoch": 0.83, + "learning_rate": 8.723232323232324e-05, + "loss": 2.5459, + "theoretical_loss": 3.3192902495161802, + "tokens_seen": 3017342976 + }, + { + "epoch": 0.83, + "learning_rate": 8.721212121212121e-05, + "loss": 2.5739, + "theoretical_loss": 3.319284712994059, + "tokens_seen": 3017408512 + }, + { + "epoch": 0.83, + "learning_rate": 8.71919191919192e-05, + "loss": 2.6264, + "theoretical_loss": 3.319279176625855, + "tokens_seen": 3017474048 + }, + { + "epoch": 0.83, + "learning_rate": 8.717171717171718e-05, + "loss": 2.3934, + "theoretical_loss": 3.3192736404115606, + "tokens_seen": 3017539584 + }, + { + "epoch": 0.83, + "learning_rate": 8.715151515151515e-05, + "loss": 2.5696, + "theoretical_loss": 3.319268104351168, + "tokens_seen": 3017605120 + }, + { + "epoch": 0.83, + "learning_rate": 8.713131313131313e-05, + "loss": 2.4748, + "theoretical_loss": 3.3192625684446693, + "tokens_seen": 3017670656 + }, + { + "epoch": 0.83, + "learning_rate": 8.711111111111112e-05, + "loss": 2.4787, + "theoretical_loss": 3.3192570326920574, + "tokens_seen": 3017736192 + }, + { + "epoch": 0.83, + "learning_rate": 8.70909090909091e-05, + "loss": 2.4315, + "theoretical_loss": 3.3192514970933242, + "tokens_seen": 3017801728 + }, + { + "epoch": 0.83, + "learning_rate": 8.707070707070708e-05, + "loss": 2.5148, + "theoretical_loss": 3.3192459616484626, + "tokens_seen": 3017867264 + }, + { + "epoch": 0.83, + "learning_rate": 8.705050505050504e-05, + "loss": 2.4388, + "theoretical_loss": 3.319240426357465, + "tokens_seen": 3017932800 + }, + { + "epoch": 0.83, + "learning_rate": 8.703030303030302e-05, + "loss": 2.5074, + "theoretical_loss": 3.319234891220323, + "tokens_seen": 3017998336 + }, + { + "epoch": 0.83, + "learning_rate": 8.701010101010101e-05, + "loss": 2.5012, + "theoretical_loss": 3.31922935623703, + "tokens_seen": 3018063872 + }, + { + "epoch": 0.83, + "learning_rate": 8.698989898989899e-05, + "loss": 2.4872, + "theoretical_loss": 3.3192238214075775, + "tokens_seen": 3018129408 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1655301, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8258864879608154, + "objective/train/theoretical_loss": 3.3192182867319584, + "objective/train/tokens_used": 1377054176, + "theoretical_loss": 3.3192182867319584, + "tokens_seen": 3018194944 + }, + { + "epoch": 0.83, + "learning_rate": 8.696969696969698e-05, + "loss": 2.5668, + "theoretical_loss": 3.3192182867319584, + "tokens_seen": 3018194944 + }, + { + "epoch": 0.83, + "learning_rate": 8.694949494949495e-05, + "loss": 2.6382, + "theoretical_loss": 3.319212752210165, + "tokens_seen": 3018260480 + }, + { + "epoch": 0.83, + "learning_rate": 8.692929292929293e-05, + "loss": 2.6348, + "theoretical_loss": 3.3192072178421896, + "tokens_seen": 3018326016 + }, + { + "epoch": 0.83, + "learning_rate": 8.690909090909091e-05, + "loss": 2.4547, + "theoretical_loss": 3.3192016836280245, + "tokens_seen": 3018391552 + }, + { + "epoch": 0.83, + "learning_rate": 8.68888888888889e-05, + "loss": 2.6279, + "theoretical_loss": 3.319196149567662, + "tokens_seen": 3018457088 + }, + { + "epoch": 0.83, + "learning_rate": 8.686868686868687e-05, + "loss": 2.4594, + "theoretical_loss": 3.3191906156610953, + "tokens_seen": 3018522624 + }, + { + "epoch": 0.83, + "learning_rate": 8.684848484848485e-05, + "loss": 2.8098, + "theoretical_loss": 3.3191850819083157, + "tokens_seen": 3018588160 + }, + { + "epoch": 0.83, + "learning_rate": 8.682828282828282e-05, + "loss": 2.3764, + "theoretical_loss": 3.3191795483093163, + "tokens_seen": 3018653696 + }, + { + "epoch": 0.83, + "learning_rate": 8.680808080808081e-05, + "loss": 2.4448, + "theoretical_loss": 3.319174014864089, + "tokens_seen": 3018719232 + }, + { + "epoch": 0.83, + "learning_rate": 8.678787878787879e-05, + "loss": 2.3655, + "theoretical_loss": 3.3191684815726266, + "tokens_seen": 3018784768 + }, + { + "epoch": 0.83, + "learning_rate": 8.676767676767678e-05, + "loss": 2.7354, + "theoretical_loss": 3.319162948434921, + "tokens_seen": 3018850304 + }, + { + "epoch": 0.83, + "learning_rate": 8.674747474747476e-05, + "loss": 2.3429, + "theoretical_loss": 3.3191574154509658, + "tokens_seen": 3018915840 + }, + { + "epoch": 0.83, + "learning_rate": 8.672727272727273e-05, + "loss": 2.3568, + "theoretical_loss": 3.319151882620752, + "tokens_seen": 3018981376 + }, + { + "epoch": 0.83, + "learning_rate": 8.67070707070707e-05, + "loss": 2.4945, + "theoretical_loss": 3.319146349944272, + "tokens_seen": 3019046912 + }, + { + "epoch": 0.83, + "learning_rate": 8.668686868686868e-05, + "loss": 2.4137, + "theoretical_loss": 3.3191408174215193, + "tokens_seen": 3019112448 + }, + { + "epoch": 0.83, + "learning_rate": 8.666666666666667e-05, + "loss": 2.6186, + "theoretical_loss": 3.319135285052486, + "tokens_seen": 3019177984 + }, + { + "epoch": 0.83, + "learning_rate": 8.664646464646465e-05, + "loss": 2.4917, + "theoretical_loss": 3.3191297528371635, + "tokens_seen": 3019243520 + }, + { + "epoch": 0.83, + "learning_rate": 8.662626262626262e-05, + "loss": 2.4559, + "theoretical_loss": 3.319124220775545, + "tokens_seen": 3019309056 + }, + { + "epoch": 0.83, + "learning_rate": 8.66060606060606e-05, + "loss": 2.4907, + "theoretical_loss": 3.319118688867623, + "tokens_seen": 3019374592 + }, + { + "epoch": 0.83, + "learning_rate": 8.658585858585859e-05, + "loss": 2.5039, + "theoretical_loss": 3.319113157113389, + "tokens_seen": 3019440128 + }, + { + "epoch": 0.83, + "learning_rate": 8.656565656565657e-05, + "loss": 2.4518, + "theoretical_loss": 3.319107625512837, + "tokens_seen": 3019505664 + }, + { + "epoch": 0.83, + "learning_rate": 8.654545454545456e-05, + "loss": 2.4552, + "theoretical_loss": 3.3191020940659577, + "tokens_seen": 3019571200 + }, + { + "epoch": 0.83, + "learning_rate": 8.652525252525252e-05, + "loss": 2.3483, + "theoretical_loss": 3.3190965627727445, + "tokens_seen": 3019636736 + }, + { + "epoch": 0.83, + "learning_rate": 8.65050505050505e-05, + "loss": 2.6056, + "theoretical_loss": 3.3190910316331896, + "tokens_seen": 3019702272 + }, + { + "epoch": 0.83, + "learning_rate": 8.648484848484848e-05, + "loss": 2.2051, + "theoretical_loss": 3.3190855006472857, + "tokens_seen": 3019767808 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1656638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5503478050231934, + "objective/train/theoretical_loss": 3.319079969815024, + "objective/train/tokens_used": 1378692576, + "theoretical_loss": 3.319079969815024, + "tokens_seen": 3019833344 + }, + { + "epoch": 0.83, + "learning_rate": 8.646464646464647e-05, + "loss": 2.4142, + "theoretical_loss": 3.319079969815024, + "tokens_seen": 3019833344 + }, + { + "epoch": 0.83, + "learning_rate": 8.644444444444445e-05, + "loss": 2.3171, + "theoretical_loss": 3.3190744391363984, + "tokens_seen": 3019898880 + }, + { + "epoch": 0.83, + "learning_rate": 8.642424242424242e-05, + "loss": 2.3807, + "theoretical_loss": 3.3190689086114005, + "tokens_seen": 3019964416 + }, + { + "epoch": 0.83, + "learning_rate": 8.64040404040404e-05, + "loss": 2.7218, + "theoretical_loss": 3.3190633782400223, + "tokens_seen": 3020029952 + }, + { + "epoch": 0.83, + "learning_rate": 8.638383838383839e-05, + "loss": 2.4483, + "theoretical_loss": 3.3190578480222577, + "tokens_seen": 3020095488 + }, + { + "epoch": 0.83, + "learning_rate": 8.636363636363636e-05, + "loss": 2.5796, + "theoretical_loss": 3.3190523179580973, + "tokens_seen": 3020161024 + }, + { + "epoch": 0.83, + "learning_rate": 8.634343434343434e-05, + "loss": 2.4652, + "theoretical_loss": 3.3190467880475345, + "tokens_seen": 3020226560 + }, + { + "epoch": 0.83, + "learning_rate": 8.632323232323233e-05, + "loss": 2.8986, + "theoretical_loss": 3.3190412582905617, + "tokens_seen": 3020292096 + }, + { + "epoch": 0.83, + "learning_rate": 8.63030303030303e-05, + "loss": 2.3869, + "theoretical_loss": 3.319035728687171, + "tokens_seen": 3020357632 + }, + { + "epoch": 0.83, + "learning_rate": 8.628282828282828e-05, + "loss": 2.4596, + "theoretical_loss": 3.319030199237355, + "tokens_seen": 3020423168 + }, + { + "epoch": 0.83, + "learning_rate": 8.626262626262627e-05, + "loss": 2.3974, + "theoretical_loss": 3.319024669941106, + "tokens_seen": 3020488704 + }, + { + "epoch": 0.83, + "learning_rate": 8.624242424242425e-05, + "loss": 2.3929, + "theoretical_loss": 3.3190191407984164, + "tokens_seen": 3020554240 + }, + { + "epoch": 0.83, + "learning_rate": 8.622222222222223e-05, + "loss": 2.5243, + "theoretical_loss": 3.319013611809279, + "tokens_seen": 3020619776 + }, + { + "epoch": 0.83, + "learning_rate": 8.62020202020202e-05, + "loss": 2.2296, + "theoretical_loss": 3.3190080829736854, + "tokens_seen": 3020685312 + }, + { + "epoch": 0.83, + "learning_rate": 8.618181818181817e-05, + "loss": 2.6387, + "theoretical_loss": 3.319002554291629, + "tokens_seen": 3020750848 + }, + { + "epoch": 0.83, + "learning_rate": 8.616161616161616e-05, + "loss": 2.3839, + "theoretical_loss": 3.318997025763101, + "tokens_seen": 3020816384 + }, + { + "epoch": 0.83, + "learning_rate": 8.614141414141414e-05, + "loss": 2.5364, + "theoretical_loss": 3.3189914973880947, + "tokens_seen": 3020881920 + }, + { + "epoch": 0.83, + "learning_rate": 8.612121212121213e-05, + "loss": 2.3448, + "theoretical_loss": 3.318985969166602, + "tokens_seen": 3020947456 + }, + { + "epoch": 0.83, + "learning_rate": 8.61010101010101e-05, + "loss": 2.1768, + "theoretical_loss": 3.318980441098616, + "tokens_seen": 3021012992 + }, + { + "epoch": 0.83, + "learning_rate": 8.608080808080808e-05, + "loss": 2.3057, + "theoretical_loss": 3.3189749131841286, + "tokens_seen": 3021078528 + }, + { + "epoch": 0.83, + "learning_rate": 8.606060606060606e-05, + "loss": 2.4457, + "theoretical_loss": 3.3189693854231326, + "tokens_seen": 3021144064 + }, + { + "epoch": 0.83, + "learning_rate": 8.604040404040405e-05, + "loss": 2.4754, + "theoretical_loss": 3.3189638578156195, + "tokens_seen": 3021209600 + }, + { + "epoch": 0.83, + "learning_rate": 8.602020202020203e-05, + "loss": 2.2733, + "theoretical_loss": 3.3189583303615824, + "tokens_seen": 3021275136 + }, + { + "epoch": 0.83, + "learning_rate": 8.599999999999999e-05, + "loss": 2.2137, + "theoretical_loss": 3.3189528030610136, + "tokens_seen": 3021340672 + }, + { + "epoch": 0.83, + "learning_rate": 8.597979797979797e-05, + "loss": 2.372, + "theoretical_loss": 3.318947275913906, + "tokens_seen": 3021406208 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1657764, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7943544387817383, + "objective/train/theoretical_loss": 3.318941748920251, + "objective/train/tokens_used": 1380330976, + "theoretical_loss": 3.318941748920251, + "tokens_seen": 3021471744 + }, + { + "epoch": 0.83, + "learning_rate": 8.595959595959596e-05, + "loss": 2.5209, + "theoretical_loss": 3.318941748920251, + "tokens_seen": 3021471744 + }, + { + "epoch": 0.83, + "learning_rate": 8.593939393939394e-05, + "loss": 2.3356, + "theoretical_loss": 3.318936222080042, + "tokens_seen": 3021537280 + }, + { + "epoch": 0.83, + "learning_rate": 8.591919191919193e-05, + "loss": 2.6387, + "theoretical_loss": 3.318930695393271, + "tokens_seen": 3021602816 + }, + { + "epoch": 0.83, + "learning_rate": 8.589898989898991e-05, + "loss": 2.4358, + "theoretical_loss": 3.3189251688599297, + "tokens_seen": 3021668352 + }, + { + "epoch": 0.83, + "learning_rate": 8.587878787878788e-05, + "loss": 2.4895, + "theoretical_loss": 3.3189196424800116, + "tokens_seen": 3021733888 + }, + { + "epoch": 0.83, + "learning_rate": 8.585858585858586e-05, + "loss": 2.4941, + "theoretical_loss": 3.318914116253509, + "tokens_seen": 3021799424 + }, + { + "epoch": 0.83, + "learning_rate": 8.583838383838383e-05, + "loss": 2.5948, + "theoretical_loss": 3.3189085901804134, + "tokens_seen": 3021864960 + }, + { + "epoch": 0.83, + "learning_rate": 8.581818181818182e-05, + "loss": 2.4123, + "theoretical_loss": 3.3189030642607182, + "tokens_seen": 3021930496 + }, + { + "epoch": 0.83, + "learning_rate": 8.57979797979798e-05, + "loss": 2.3932, + "theoretical_loss": 3.3188975384944155, + "tokens_seen": 3021996032 + }, + { + "epoch": 0.83, + "learning_rate": 8.577777777777777e-05, + "loss": 2.4914, + "theoretical_loss": 3.3188920128814976, + "tokens_seen": 3022061568 + }, + { + "epoch": 0.83, + "learning_rate": 8.575757575757576e-05, + "loss": 2.4409, + "theoretical_loss": 3.318886487421957, + "tokens_seen": 3022127104 + }, + { + "epoch": 0.83, + "learning_rate": 8.573737373737374e-05, + "loss": 2.5333, + "theoretical_loss": 3.3188809621157858, + "tokens_seen": 3022192640 + }, + { + "epoch": 0.83, + "learning_rate": 8.571717171717172e-05, + "loss": 2.3906, + "theoretical_loss": 3.318875436962977, + "tokens_seen": 3022258176 + }, + { + "epoch": 0.83, + "learning_rate": 8.569696969696971e-05, + "loss": 2.5242, + "theoretical_loss": 3.3188699119635228, + "tokens_seen": 3022323712 + }, + { + "epoch": 0.83, + "learning_rate": 8.567676767676768e-05, + "loss": 2.5151, + "theoretical_loss": 3.3188643871174155, + "tokens_seen": 3022389248 + }, + { + "epoch": 0.83, + "learning_rate": 8.565656565656565e-05, + "loss": 2.6633, + "theoretical_loss": 3.3188588624246473, + "tokens_seen": 3022454784 + }, + { + "epoch": 0.83, + "learning_rate": 8.563636363636363e-05, + "loss": 2.4314, + "theoretical_loss": 3.318853337885211, + "tokens_seen": 3022520320 + }, + { + "epoch": 0.83, + "learning_rate": 8.561616161616162e-05, + "loss": 2.4341, + "theoretical_loss": 3.318847813499099, + "tokens_seen": 3022585856 + }, + { + "epoch": 0.83, + "learning_rate": 8.55959595959596e-05, + "loss": 2.4042, + "theoretical_loss": 3.318842289266304, + "tokens_seen": 3022651392 + }, + { + "epoch": 0.83, + "learning_rate": 8.557575757575757e-05, + "loss": 2.3619, + "theoretical_loss": 3.318836765186817, + "tokens_seen": 3022716928 + }, + { + "epoch": 0.83, + "learning_rate": 8.555555555555556e-05, + "loss": 2.358, + "theoretical_loss": 3.3188312412606327, + "tokens_seen": 3022782464 + }, + { + "epoch": 0.83, + "learning_rate": 8.553535353535354e-05, + "loss": 2.4972, + "theoretical_loss": 3.3188257174877416, + "tokens_seen": 3022848000 + }, + { + "epoch": 0.83, + "learning_rate": 8.551515151515152e-05, + "loss": 2.3981, + "theoretical_loss": 3.3188201938681368, + "tokens_seen": 3022913536 + }, + { + "epoch": 0.83, + "learning_rate": 8.54949494949495e-05, + "loss": 2.3559, + "theoretical_loss": 3.3188146704018107, + "tokens_seen": 3022979072 + }, + { + "epoch": 0.83, + "learning_rate": 8.547474747474748e-05, + "loss": 2.2652, + "theoretical_loss": 3.318809147088756, + "tokens_seen": 3023044608 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1658338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.480128049850464, + "objective/train/theoretical_loss": 3.318803623928965, + "objective/train/tokens_used": 1381969376, + "theoretical_loss": 3.318803623928965, + "tokens_seen": 3023110144 + }, + { + "epoch": 0.83, + "learning_rate": 8.545454545454545e-05, + "loss": 2.5001, + "theoretical_loss": 3.318803623928965, + "tokens_seen": 3023110144 + }, + { + "epoch": 0.83, + "learning_rate": 8.543434343434343e-05, + "loss": 2.4875, + "theoretical_loss": 3.3187981009224297, + "tokens_seen": 3023175680 + }, + { + "epoch": 0.83, + "learning_rate": 8.541414141414142e-05, + "loss": 2.4253, + "theoretical_loss": 3.318792578069143, + "tokens_seen": 3023241216 + }, + { + "epoch": 0.83, + "learning_rate": 8.53939393939394e-05, + "loss": 2.4554, + "theoretical_loss": 3.3187870553690972, + "tokens_seen": 3023306752 + }, + { + "epoch": 0.83, + "learning_rate": 8.537373737373738e-05, + "loss": 2.3708, + "theoretical_loss": 3.3187815328222845, + "tokens_seen": 3023372288 + }, + { + "epoch": 0.83, + "learning_rate": 8.535353535353535e-05, + "loss": 2.5044, + "theoretical_loss": 3.3187760104286976, + "tokens_seen": 3023437824 + }, + { + "epoch": 0.83, + "learning_rate": 8.533333333333334e-05, + "loss": 2.3458, + "theoretical_loss": 3.3187704881883286, + "tokens_seen": 3023503360 + }, + { + "epoch": 0.83, + "learning_rate": 8.531313131313131e-05, + "loss": 2.4841, + "theoretical_loss": 3.3187649661011704, + "tokens_seen": 3023568896 + }, + { + "epoch": 0.83, + "learning_rate": 8.529292929292929e-05, + "loss": 2.6006, + "theoretical_loss": 3.3187594441672155, + "tokens_seen": 3023634432 + }, + { + "epoch": 0.83, + "learning_rate": 8.527272727272728e-05, + "loss": 2.4445, + "theoretical_loss": 3.3187539223864557, + "tokens_seen": 3023699968 + }, + { + "epoch": 0.83, + "learning_rate": 8.525252525252525e-05, + "loss": 2.4559, + "theoretical_loss": 3.3187484007588837, + "tokens_seen": 3023765504 + }, + { + "epoch": 0.83, + "learning_rate": 8.523232323232323e-05, + "loss": 2.3944, + "theoretical_loss": 3.318742879284492, + "tokens_seen": 3023831040 + }, + { + "epoch": 0.83, + "learning_rate": 8.521212121212122e-05, + "loss": 2.3808, + "theoretical_loss": 3.3187373579632733, + "tokens_seen": 3023896576 + }, + { + "epoch": 0.83, + "learning_rate": 8.51919191919192e-05, + "loss": 2.557, + "theoretical_loss": 3.3187318367952194, + "tokens_seen": 3023962112 + }, + { + "epoch": 0.83, + "learning_rate": 8.517171717171718e-05, + "loss": 2.4328, + "theoretical_loss": 3.3187263157803235, + "tokens_seen": 3024027648 + }, + { + "epoch": 0.83, + "learning_rate": 8.515151515151514e-05, + "loss": 2.4029, + "theoretical_loss": 3.318720794918577, + "tokens_seen": 3024093184 + }, + { + "epoch": 0.83, + "learning_rate": 8.513131313131312e-05, + "loss": 2.4667, + "theoretical_loss": 3.3187152742099735, + "tokens_seen": 3024158720 + }, + { + "epoch": 0.83, + "learning_rate": 8.511111111111111e-05, + "loss": 2.6392, + "theoretical_loss": 3.3187097536545047, + "tokens_seen": 3024224256 + }, + { + "epoch": 0.83, + "learning_rate": 8.509090909090909e-05, + "loss": 2.2707, + "theoretical_loss": 3.3187042332521632, + "tokens_seen": 3024289792 + }, + { + "epoch": 0.83, + "learning_rate": 8.507070707070708e-05, + "loss": 2.5969, + "theoretical_loss": 3.3186987130029415, + "tokens_seen": 3024355328 + }, + { + "epoch": 0.83, + "learning_rate": 8.505050505050506e-05, + "loss": 2.3807, + "theoretical_loss": 3.318693192906832, + "tokens_seen": 3024420864 + }, + { + "epoch": 0.83, + "learning_rate": 8.503030303030303e-05, + "loss": 2.5197, + "theoretical_loss": 3.3186876729638266, + "tokens_seen": 3024486400 + }, + { + "epoch": 0.83, + "learning_rate": 8.501010101010101e-05, + "loss": 2.5681, + "theoretical_loss": 3.318682153173919, + "tokens_seen": 3024551936 + }, + { + "epoch": 0.83, + "learning_rate": 8.4989898989899e-05, + "loss": 2.5106, + "theoretical_loss": 3.3186766335371005, + "tokens_seen": 3024617472 + }, + { + "epoch": 0.83, + "learning_rate": 8.496969696969697e-05, + "loss": 2.2927, + "theoretical_loss": 3.3186711140533642, + "tokens_seen": 3024683008 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1659421, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3318190574645996, + "objective/train/theoretical_loss": 3.318665594722702, + "objective/train/tokens_used": 1383607776, + "theoretical_loss": 3.318665594722702, + "tokens_seen": 3024748544 + }, + { + "epoch": 0.83, + "learning_rate": 8.494949494949495e-05, + "loss": 2.5393, + "theoretical_loss": 3.318665594722702, + "tokens_seen": 3024748544 + }, + { + "epoch": 0.83, + "learning_rate": 8.492929292929292e-05, + "loss": 2.3833, + "theoretical_loss": 3.3186600755451066, + "tokens_seen": 3024814080 + }, + { + "epoch": 0.83, + "learning_rate": 8.490909090909091e-05, + "loss": 2.6365, + "theoretical_loss": 3.3186545565205705, + "tokens_seen": 3024879616 + }, + { + "epoch": 0.83, + "learning_rate": 8.488888888888889e-05, + "loss": 2.3964, + "theoretical_loss": 3.3186490376490863, + "tokens_seen": 3024945152 + }, + { + "epoch": 0.83, + "learning_rate": 8.486868686868687e-05, + "loss": 2.3904, + "theoretical_loss": 3.318643518930646, + "tokens_seen": 3025010688 + }, + { + "epoch": 0.83, + "learning_rate": 8.484848484848486e-05, + "loss": 2.5111, + "theoretical_loss": 3.318638000365242, + "tokens_seen": 3025076224 + }, + { + "epoch": 0.83, + "learning_rate": 8.482828282828283e-05, + "loss": 2.3631, + "theoretical_loss": 3.3186324819528674, + "tokens_seen": 3025141760 + }, + { + "epoch": 0.83, + "learning_rate": 8.480808080808081e-05, + "loss": 2.4628, + "theoretical_loss": 3.318626963693514, + "tokens_seen": 3025207296 + }, + { + "epoch": 0.83, + "learning_rate": 8.478787878787878e-05, + "loss": 2.2574, + "theoretical_loss": 3.318621445587175, + "tokens_seen": 3025272832 + }, + { + "epoch": 0.83, + "learning_rate": 8.476767676767677e-05, + "loss": 2.336, + "theoretical_loss": 3.3186159276338416, + "tokens_seen": 3025338368 + }, + { + "epoch": 0.83, + "learning_rate": 8.474747474747475e-05, + "loss": 2.6326, + "theoretical_loss": 3.3186104098335076, + "tokens_seen": 3025403904 + }, + { + "epoch": 0.83, + "learning_rate": 8.472727272727272e-05, + "loss": 2.3629, + "theoretical_loss": 3.3186048921861646, + "tokens_seen": 3025469440 + }, + { + "epoch": 0.83, + "learning_rate": 8.47070707070707e-05, + "loss": 2.5703, + "theoretical_loss": 3.318599374691805, + "tokens_seen": 3025534976 + }, + { + "epoch": 0.83, + "learning_rate": 8.468686868686869e-05, + "loss": 2.6074, + "theoretical_loss": 3.3185938573504217, + "tokens_seen": 3025600512 + }, + { + "epoch": 0.83, + "learning_rate": 8.466666666666667e-05, + "loss": 2.5822, + "theoretical_loss": 3.318588340162007, + "tokens_seen": 3025666048 + }, + { + "epoch": 0.83, + "learning_rate": 8.464646464646466e-05, + "loss": 2.4586, + "theoretical_loss": 3.3185828231265533, + "tokens_seen": 3025731584 + }, + { + "epoch": 0.83, + "learning_rate": 8.462626262626263e-05, + "loss": 2.5384, + "theoretical_loss": 3.3185773062440527, + "tokens_seen": 3025797120 + }, + { + "epoch": 0.83, + "learning_rate": 8.46060606060606e-05, + "loss": 2.4094, + "theoretical_loss": 3.3185717895144986, + "tokens_seen": 3025862656 + }, + { + "epoch": 0.83, + "learning_rate": 8.458585858585858e-05, + "loss": 2.5627, + "theoretical_loss": 3.3185662729378826, + "tokens_seen": 3025928192 + }, + { + "epoch": 0.83, + "learning_rate": 8.456565656565657e-05, + "loss": 2.5598, + "theoretical_loss": 3.3185607565141972, + "tokens_seen": 3025993728 + }, + { + "epoch": 0.83, + "learning_rate": 8.454545454545455e-05, + "loss": 2.4138, + "theoretical_loss": 3.318555240243435, + "tokens_seen": 3026059264 + }, + { + "epoch": 0.83, + "learning_rate": 8.452525252525253e-05, + "loss": 2.5147, + "theoretical_loss": 3.318549724125589, + "tokens_seen": 3026124800 + }, + { + "epoch": 0.83, + "learning_rate": 8.45050505050505e-05, + "loss": 2.3953, + "theoretical_loss": 3.3185442081606507, + "tokens_seen": 3026190336 + }, + { + "epoch": 0.83, + "learning_rate": 8.448484848484849e-05, + "loss": 2.3997, + "theoretical_loss": 3.318538692348613, + "tokens_seen": 3026255872 + }, + { + "epoch": 0.83, + "learning_rate": 8.446464646464647e-05, + "loss": 2.5486, + "theoretical_loss": 3.3185331766894683, + "tokens_seen": 3026321408 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1660021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.281871795654297, + "objective/train/theoretical_loss": 3.318527661183209, + "objective/train/tokens_used": 1385246176, + "theoretical_loss": 3.318527661183209, + "tokens_seen": 3026386944 + }, + { + "epoch": 0.83, + "learning_rate": 8.444444444444444e-05, + "loss": 2.4894, + "theoretical_loss": 3.318527661183209, + "tokens_seen": 3026386944 + }, + { + "epoch": 0.83, + "learning_rate": 8.442424242424243e-05, + "loss": 2.5351, + "theoretical_loss": 3.318522145829828, + "tokens_seen": 3026452480 + }, + { + "epoch": 0.83, + "learning_rate": 8.44040404040404e-05, + "loss": 2.3844, + "theoretical_loss": 3.318516630629317, + "tokens_seen": 3026518016 + }, + { + "epoch": 0.83, + "learning_rate": 8.438383838383838e-05, + "loss": 2.3252, + "theoretical_loss": 3.318511115581669, + "tokens_seen": 3026583552 + }, + { + "epoch": 0.83, + "learning_rate": 8.436363636363637e-05, + "loss": 2.299, + "theoretical_loss": 3.3185056006868763, + "tokens_seen": 3026649088 + }, + { + "epoch": 0.83, + "learning_rate": 8.434343434343435e-05, + "loss": 2.3768, + "theoretical_loss": 3.3185000859449314, + "tokens_seen": 3026714624 + }, + { + "epoch": 0.83, + "learning_rate": 8.432323232323233e-05, + "loss": 2.489, + "theoretical_loss": 3.318494571355827, + "tokens_seen": 3026780160 + }, + { + "epoch": 0.83, + "learning_rate": 8.43030303030303e-05, + "loss": 2.4351, + "theoretical_loss": 3.3184890569195544, + "tokens_seen": 3026845696 + }, + { + "epoch": 0.83, + "learning_rate": 8.428282828282827e-05, + "loss": 2.4066, + "theoretical_loss": 3.3184835426361077, + "tokens_seen": 3026911232 + }, + { + "epoch": 0.83, + "learning_rate": 8.426262626262626e-05, + "loss": 2.4867, + "theoretical_loss": 3.3184780285054782, + "tokens_seen": 3026976768 + }, + { + "epoch": 0.83, + "learning_rate": 8.424242424242424e-05, + "loss": 2.4346, + "theoretical_loss": 3.318472514527659, + "tokens_seen": 3027042304 + }, + { + "epoch": 0.83, + "learning_rate": 8.422222222222223e-05, + "loss": 2.3971, + "theoretical_loss": 3.318467000702642, + "tokens_seen": 3027107840 + }, + { + "epoch": 0.83, + "learning_rate": 8.420202020202021e-05, + "loss": 2.3927, + "theoretical_loss": 3.31846148703042, + "tokens_seen": 3027173376 + }, + { + "epoch": 0.83, + "learning_rate": 8.418181818181818e-05, + "loss": 2.2368, + "theoretical_loss": 3.3184559735109853, + "tokens_seen": 3027238912 + }, + { + "epoch": 0.83, + "learning_rate": 8.416161616161616e-05, + "loss": 2.1355, + "theoretical_loss": 3.3184504601443305, + "tokens_seen": 3027304448 + }, + { + "epoch": 0.83, + "learning_rate": 8.414141414141415e-05, + "loss": 2.4009, + "theoretical_loss": 3.3184449469304482, + "tokens_seen": 3027369984 + }, + { + "epoch": 0.83, + "learning_rate": 8.412121212121213e-05, + "loss": 2.5795, + "theoretical_loss": 3.318439433869331, + "tokens_seen": 3027435520 + }, + { + "epoch": 0.83, + "learning_rate": 8.41010101010101e-05, + "loss": 2.4062, + "theoretical_loss": 3.3184339209609703, + "tokens_seen": 3027501056 + }, + { + "epoch": 0.83, + "learning_rate": 8.408080808080807e-05, + "loss": 2.2701, + "theoretical_loss": 3.3184284082053597, + "tokens_seen": 3027566592 + }, + { + "epoch": 0.83, + "learning_rate": 8.406060606060606e-05, + "loss": 2.5331, + "theoretical_loss": 3.318422895602491, + "tokens_seen": 3027632128 + }, + { + "epoch": 0.83, + "learning_rate": 8.404040404040404e-05, + "loss": 2.4471, + "theoretical_loss": 3.318417383152357, + "tokens_seen": 3027697664 + }, + { + "epoch": 0.83, + "learning_rate": 8.402020202020202e-05, + "loss": 2.5148, + "theoretical_loss": 3.31841187085495, + "tokens_seen": 3027763200 + }, + { + "epoch": 0.83, + "learning_rate": 8.400000000000001e-05, + "loss": 2.4284, + "theoretical_loss": 3.3184063587102632, + "tokens_seen": 3027828736 + }, + { + "epoch": 0.83, + "learning_rate": 8.397979797979798e-05, + "loss": 2.3382, + "theoretical_loss": 3.318400846718288, + "tokens_seen": 3027894272 + }, + { + "epoch": 0.83, + "learning_rate": 8.395959595959596e-05, + "loss": 2.3401, + "theoretical_loss": 3.318395334879017, + "tokens_seen": 3027959808 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1661280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7463579177856445, + "objective/train/theoretical_loss": 3.318389823192443, + "objective/train/tokens_used": 1386884576, + "theoretical_loss": 3.318389823192443, + "tokens_seen": 3028025344 + }, + { + "epoch": 0.83, + "learning_rate": 8.393939393939393e-05, + "loss": 2.6893, + "theoretical_loss": 3.318389823192443, + "tokens_seen": 3028025344 + }, + { + "epoch": 0.83, + "learning_rate": 8.391919191919192e-05, + "loss": 2.5968, + "theoretical_loss": 3.3183843116585585, + "tokens_seen": 3028090880 + }, + { + "epoch": 0.83, + "learning_rate": 8.38989898989899e-05, + "loss": 2.4658, + "theoretical_loss": 3.318378800277356, + "tokens_seen": 3028156416 + }, + { + "epoch": 0.83, + "learning_rate": 8.387878787878787e-05, + "loss": 2.5697, + "theoretical_loss": 3.318373289048828, + "tokens_seen": 3028221952 + }, + { + "epoch": 0.83, + "learning_rate": 8.385858585858586e-05, + "loss": 2.3935, + "theoretical_loss": 3.3183677779729663, + "tokens_seen": 3028287488 + }, + { + "epoch": 0.83, + "learning_rate": 8.383838383838384e-05, + "loss": 2.3191, + "theoretical_loss": 3.318362267049764, + "tokens_seen": 3028353024 + }, + { + "epoch": 0.83, + "learning_rate": 8.381818181818182e-05, + "loss": 2.4093, + "theoretical_loss": 3.3183567562792136, + "tokens_seen": 3028418560 + }, + { + "epoch": 0.83, + "learning_rate": 8.379797979797981e-05, + "loss": 2.3235, + "theoretical_loss": 3.318351245661307, + "tokens_seen": 3028484096 + }, + { + "epoch": 0.83, + "learning_rate": 8.377777777777779e-05, + "loss": 2.5221, + "theoretical_loss": 3.3183457351960377, + "tokens_seen": 3028549632 + }, + { + "epoch": 0.83, + "learning_rate": 8.375757575757575e-05, + "loss": 2.502, + "theoretical_loss": 3.3183402248833973, + "tokens_seen": 3028615168 + }, + { + "epoch": 0.83, + "learning_rate": 8.373737373737373e-05, + "loss": 2.2369, + "theoretical_loss": 3.3183347147233784, + "tokens_seen": 3028680704 + }, + { + "epoch": 0.83, + "learning_rate": 8.371717171717172e-05, + "loss": 2.4348, + "theoretical_loss": 3.318329204715974, + "tokens_seen": 3028746240 + }, + { + "epoch": 0.83, + "learning_rate": 8.36969696969697e-05, + "loss": 2.6562, + "theoretical_loss": 3.3183236948611756, + "tokens_seen": 3028811776 + }, + { + "epoch": 0.83, + "learning_rate": 8.367676767676768e-05, + "loss": 2.3726, + "theoretical_loss": 3.3183181851589763, + "tokens_seen": 3028877312 + }, + { + "epoch": 0.83, + "learning_rate": 8.365656565656565e-05, + "loss": 2.5966, + "theoretical_loss": 3.3183126756093686, + "tokens_seen": 3028942848 + }, + { + "epoch": 0.83, + "learning_rate": 8.363636363636364e-05, + "loss": 2.5121, + "theoretical_loss": 3.318307166212345, + "tokens_seen": 3029008384 + }, + { + "epoch": 0.83, + "learning_rate": 8.361616161616162e-05, + "loss": 2.2899, + "theoretical_loss": 3.318301656967898, + "tokens_seen": 3029073920 + }, + { + "epoch": 0.83, + "learning_rate": 8.359595959595961e-05, + "loss": 2.5499, + "theoretical_loss": 3.318296147876019, + "tokens_seen": 3029139456 + }, + { + "epoch": 0.83, + "learning_rate": 8.357575757575758e-05, + "loss": 2.3962, + "theoretical_loss": 3.3182906389367024, + "tokens_seen": 3029204992 + }, + { + "epoch": 0.83, + "learning_rate": 8.355555555555555e-05, + "loss": 2.395, + "theoretical_loss": 3.318285130149939, + "tokens_seen": 3029270528 + }, + { + "epoch": 0.83, + "learning_rate": 8.353535353535353e-05, + "loss": 2.4478, + "theoretical_loss": 3.3182796215157224, + "tokens_seen": 3029336064 + }, + { + "epoch": 0.83, + "learning_rate": 8.351515151515152e-05, + "loss": 2.44, + "theoretical_loss": 3.3182741130340445, + "tokens_seen": 3029401600 + }, + { + "epoch": 0.83, + "learning_rate": 8.34949494949495e-05, + "loss": 2.5831, + "theoretical_loss": 3.318268604704898, + "tokens_seen": 3029467136 + }, + { + "epoch": 0.83, + "learning_rate": 8.347474747474748e-05, + "loss": 2.4023, + "theoretical_loss": 3.318263096528275, + "tokens_seen": 3029532672 + }, + { + "epoch": 0.83, + "learning_rate": 8.345454545454545e-05, + "loss": 2.3491, + "theoretical_loss": 3.318257588504168, + "tokens_seen": 3029598208 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1661975, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.476470708847046, + "objective/train/theoretical_loss": 3.31825208063257, + "objective/train/tokens_used": 1388522976, + "theoretical_loss": 3.31825208063257, + "tokens_seen": 3029663744 + }, + { + "epoch": 0.83, + "learning_rate": 8.343434343434344e-05, + "loss": 2.3498, + "theoretical_loss": 3.31825208063257, + "tokens_seen": 3029663744 + }, + { + "epoch": 0.84, + "learning_rate": 8.341414141414141e-05, + "loss": 2.7182, + "theoretical_loss": 3.318246572913474, + "tokens_seen": 3029729280 + }, + { + "epoch": 0.84, + "learning_rate": 8.339393939393939e-05, + "loss": 2.3674, + "theoretical_loss": 3.3182410653468706, + "tokens_seen": 3029794816 + }, + { + "epoch": 0.84, + "learning_rate": 8.337373737373738e-05, + "loss": 2.3421, + "theoretical_loss": 3.318235557932754, + "tokens_seen": 3029860352 + }, + { + "epoch": 0.84, + "learning_rate": 8.335353535353536e-05, + "loss": 2.44, + "theoretical_loss": 3.3182300506711155, + "tokens_seen": 3029925888 + }, + { + "epoch": 0.84, + "learning_rate": 8.333333333333333e-05, + "loss": 2.6024, + "theoretical_loss": 3.318224543561948, + "tokens_seen": 3029991424 + }, + { + "epoch": 0.84, + "learning_rate": 8.331313131313131e-05, + "loss": 2.5514, + "theoretical_loss": 3.318219036605245, + "tokens_seen": 3030056960 + }, + { + "epoch": 0.84, + "learning_rate": 8.32929292929293e-05, + "loss": 2.3569, + "theoretical_loss": 3.3182135298009974, + "tokens_seen": 3030122496 + }, + { + "epoch": 0.84, + "learning_rate": 8.327272727272728e-05, + "loss": 2.2821, + "theoretical_loss": 3.3182080231491984, + "tokens_seen": 3030188032 + }, + { + "epoch": 0.84, + "learning_rate": 8.325252525252527e-05, + "loss": 2.5974, + "theoretical_loss": 3.3182025166498406, + "tokens_seen": 3030253568 + }, + { + "epoch": 0.84, + "learning_rate": 8.323232323232322e-05, + "loss": 2.2281, + "theoretical_loss": 3.3181970103029164, + "tokens_seen": 3030319104 + }, + { + "epoch": 0.84, + "learning_rate": 8.321212121212121e-05, + "loss": 2.4297, + "theoretical_loss": 3.3181915041084182, + "tokens_seen": 3030384640 + }, + { + "epoch": 0.84, + "learning_rate": 8.319191919191919e-05, + "loss": 2.3592, + "theoretical_loss": 3.3181859980663386, + "tokens_seen": 3030450176 + }, + { + "epoch": 0.84, + "learning_rate": 8.317171717171718e-05, + "loss": 2.4375, + "theoretical_loss": 3.3181804921766695, + "tokens_seen": 3030515712 + }, + { + "epoch": 0.84, + "learning_rate": 8.315151515151516e-05, + "loss": 2.4147, + "theoretical_loss": 3.3181749864394043, + "tokens_seen": 3030581248 + }, + { + "epoch": 0.84, + "learning_rate": 8.313131313131313e-05, + "loss": 2.1803, + "theoretical_loss": 3.318169480854535, + "tokens_seen": 3030646784 + }, + { + "epoch": 0.84, + "learning_rate": 8.311111111111111e-05, + "loss": 2.5168, + "theoretical_loss": 3.318163975422054, + "tokens_seen": 3030712320 + }, + { + "epoch": 0.84, + "learning_rate": 8.30909090909091e-05, + "loss": 2.4282, + "theoretical_loss": 3.318158470141954, + "tokens_seen": 3030777856 + }, + { + "epoch": 0.84, + "learning_rate": 8.307070707070707e-05, + "loss": 2.4458, + "theoretical_loss": 3.318152965014227, + "tokens_seen": 3030843392 + }, + { + "epoch": 0.84, + "learning_rate": 8.305050505050505e-05, + "loss": 2.3859, + "theoretical_loss": 3.3181474600388667, + "tokens_seen": 3030908928 + }, + { + "epoch": 0.84, + "learning_rate": 8.303030303030302e-05, + "loss": 2.7279, + "theoretical_loss": 3.3181419552158644, + "tokens_seen": 3030974464 + }, + { + "epoch": 0.84, + "learning_rate": 8.3010101010101e-05, + "loss": 2.3853, + "theoretical_loss": 3.318136450545213, + "tokens_seen": 3031040000 + }, + { + "epoch": 0.84, + "learning_rate": 8.298989898989899e-05, + "loss": 2.6128, + "theoretical_loss": 3.318130946026905, + "tokens_seen": 3031105536 + }, + { + "epoch": 0.84, + "learning_rate": 8.296969696969697e-05, + "loss": 2.5329, + "theoretical_loss": 3.318125441660933, + "tokens_seen": 3031171072 + }, + { + "epoch": 0.84, + "learning_rate": 8.294949494949496e-05, + "loss": 2.4937, + "theoretical_loss": 3.318119937447289, + "tokens_seen": 3031236608 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1662642, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2876884937286377, + "objective/train/theoretical_loss": 3.318114433385966, + "objective/train/tokens_used": 1390161376, + "theoretical_loss": 3.318114433385966, + "tokens_seen": 3031302144 + }, + { + "epoch": 0.84, + "learning_rate": 8.292929292929294e-05, + "loss": 2.1899, + "theoretical_loss": 3.318114433385966, + "tokens_seen": 3031302144 + }, + { + "epoch": 0.84, + "learning_rate": 8.290909090909091e-05, + "loss": 2.4146, + "theoretical_loss": 3.3181089294769563, + "tokens_seen": 3031367680 + }, + { + "epoch": 0.84, + "learning_rate": 8.288888888888888e-05, + "loss": 2.2206, + "theoretical_loss": 3.3181034257202526, + "tokens_seen": 3031433216 + }, + { + "epoch": 0.84, + "learning_rate": 8.286868686868687e-05, + "loss": 2.2775, + "theoretical_loss": 3.318097922115847, + "tokens_seen": 3031498752 + }, + { + "epoch": 0.84, + "learning_rate": 8.284848484848485e-05, + "loss": 2.5515, + "theoretical_loss": 3.318092418663732, + "tokens_seen": 3031564288 + }, + { + "epoch": 0.84, + "learning_rate": 8.282828282828283e-05, + "loss": 2.3613, + "theoretical_loss": 3.318086915363901, + "tokens_seen": 3031629824 + }, + { + "epoch": 0.84, + "learning_rate": 8.28080808080808e-05, + "loss": 2.4062, + "theoretical_loss": 3.3180814122163453, + "tokens_seen": 3031695360 + }, + { + "epoch": 0.84, + "learning_rate": 8.278787878787879e-05, + "loss": 2.3293, + "theoretical_loss": 3.318075909221058, + "tokens_seen": 3031760896 + }, + { + "epoch": 0.84, + "learning_rate": 8.276767676767677e-05, + "loss": 2.3609, + "theoretical_loss": 3.3180704063780313, + "tokens_seen": 3031826432 + }, + { + "epoch": 0.84, + "learning_rate": 8.274747474747476e-05, + "loss": 2.3799, + "theoretical_loss": 3.3180649036872585, + "tokens_seen": 3031891968 + }, + { + "epoch": 0.84, + "learning_rate": 8.272727272727274e-05, + "loss": 2.3762, + "theoretical_loss": 3.318059401148731, + "tokens_seen": 3031957504 + }, + { + "epoch": 0.84, + "learning_rate": 8.27070707070707e-05, + "loss": 2.4328, + "theoretical_loss": 3.3180538987624417, + "tokens_seen": 3032023040 + }, + { + "epoch": 0.84, + "learning_rate": 8.268686868686868e-05, + "loss": 2.4521, + "theoretical_loss": 3.3180483965283836, + "tokens_seen": 3032088576 + }, + { + "epoch": 0.84, + "learning_rate": 8.266666666666667e-05, + "loss": 2.5348, + "theoretical_loss": 3.3180428944465485, + "tokens_seen": 3032154112 + }, + { + "epoch": 0.84, + "learning_rate": 8.264646464646465e-05, + "loss": 2.369, + "theoretical_loss": 3.318037392516929, + "tokens_seen": 3032219648 + }, + { + "epoch": 0.84, + "learning_rate": 8.262626262626263e-05, + "loss": 2.2584, + "theoretical_loss": 3.318031890739518, + "tokens_seen": 3032285184 + }, + { + "epoch": 0.84, + "learning_rate": 8.26060606060606e-05, + "loss": 2.3345, + "theoretical_loss": 3.318026389114308, + "tokens_seen": 3032350720 + }, + { + "epoch": 0.84, + "learning_rate": 8.258585858585859e-05, + "loss": 2.5421, + "theoretical_loss": 3.318020887641291, + "tokens_seen": 3032416256 + }, + { + "epoch": 0.84, + "learning_rate": 8.256565656565657e-05, + "loss": 2.4439, + "theoretical_loss": 3.3180153863204596, + "tokens_seen": 3032481792 + }, + { + "epoch": 0.84, + "learning_rate": 8.254545454545454e-05, + "loss": 2.4823, + "theoretical_loss": 3.318009885151807, + "tokens_seen": 3032547328 + }, + { + "epoch": 0.84, + "learning_rate": 8.252525252525253e-05, + "loss": 2.3049, + "theoretical_loss": 3.3180043841353246, + "tokens_seen": 3032612864 + }, + { + "epoch": 0.84, + "learning_rate": 8.250505050505051e-05, + "loss": 2.4061, + "theoretical_loss": 3.317998883271006, + "tokens_seen": 3032678400 + }, + { + "epoch": 0.84, + "learning_rate": 8.248484848484848e-05, + "loss": 2.4498, + "theoretical_loss": 3.317993382558843, + "tokens_seen": 3032743936 + }, + { + "epoch": 0.84, + "learning_rate": 8.246464646464646e-05, + "loss": 2.5121, + "theoretical_loss": 3.317987881998828, + "tokens_seen": 3032809472 + }, + { + "epoch": 0.84, + "learning_rate": 8.244444444444445e-05, + "loss": 2.5473, + "theoretical_loss": 3.317982381590954, + "tokens_seen": 3032875008 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1663221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0112342834472656, + "objective/train/theoretical_loss": 3.3179768813352135, + "objective/train/tokens_used": 1391799776, + "theoretical_loss": 3.3179768813352135, + "tokens_seen": 3032940544 + }, + { + "epoch": 0.84, + "learning_rate": 8.242424242424243e-05, + "loss": 2.3132, + "theoretical_loss": 3.3179768813352135, + "tokens_seen": 3032940544 + }, + { + "epoch": 0.84, + "learning_rate": 8.240404040404042e-05, + "loss": 2.2845, + "theoretical_loss": 3.3179713812315983, + "tokens_seen": 3033006080 + }, + { + "epoch": 0.84, + "learning_rate": 8.238383838383839e-05, + "loss": 2.5375, + "theoretical_loss": 3.317965881280102, + "tokens_seen": 3033071616 + }, + { + "epoch": 0.84, + "learning_rate": 8.236363636363636e-05, + "loss": 2.6365, + "theoretical_loss": 3.317960381480716, + "tokens_seen": 3033137152 + }, + { + "epoch": 0.84, + "learning_rate": 8.234343434343434e-05, + "loss": 2.3096, + "theoretical_loss": 3.3179548818334337, + "tokens_seen": 3033202688 + }, + { + "epoch": 0.84, + "learning_rate": 8.232323232323233e-05, + "loss": 2.5492, + "theoretical_loss": 3.317949382338247, + "tokens_seen": 3033268224 + }, + { + "epoch": 0.84, + "learning_rate": 8.230303030303031e-05, + "loss": 2.4018, + "theoretical_loss": 3.3179438829951486, + "tokens_seen": 3033333760 + }, + { + "epoch": 0.84, + "learning_rate": 8.228282828282828e-05, + "loss": 2.5228, + "theoretical_loss": 3.3179383838041314, + "tokens_seen": 3033399296 + }, + { + "epoch": 0.84, + "learning_rate": 8.226262626262626e-05, + "loss": 2.4324, + "theoretical_loss": 3.317932884765187, + "tokens_seen": 3033464832 + }, + { + "epoch": 0.84, + "learning_rate": 8.224242424242425e-05, + "loss": 2.4259, + "theoretical_loss": 3.317927385878309, + "tokens_seen": 3033530368 + }, + { + "epoch": 0.84, + "learning_rate": 8.222222222222223e-05, + "loss": 2.3798, + "theoretical_loss": 3.317921887143489, + "tokens_seen": 3033595904 + }, + { + "epoch": 0.84, + "learning_rate": 8.22020202020202e-05, + "loss": 2.3174, + "theoretical_loss": 3.31791638856072, + "tokens_seen": 3033661440 + }, + { + "epoch": 0.84, + "learning_rate": 8.218181818181817e-05, + "loss": 2.4912, + "theoretical_loss": 3.3179108901299945, + "tokens_seen": 3033726976 + }, + { + "epoch": 0.84, + "learning_rate": 8.216161616161616e-05, + "loss": 2.4127, + "theoretical_loss": 3.317905391851305, + "tokens_seen": 3033792512 + }, + { + "epoch": 0.84, + "learning_rate": 8.214141414141414e-05, + "loss": 2.4977, + "theoretical_loss": 3.317899893724644, + "tokens_seen": 3033858048 + }, + { + "epoch": 0.84, + "learning_rate": 8.212121212121212e-05, + "loss": 2.4532, + "theoretical_loss": 3.3178943957500033, + "tokens_seen": 3033923584 + }, + { + "epoch": 0.84, + "learning_rate": 8.210101010101011e-05, + "loss": 2.2835, + "theoretical_loss": 3.3178888979273764, + "tokens_seen": 3033989120 + }, + { + "epoch": 0.84, + "learning_rate": 8.208080808080809e-05, + "loss": 2.3393, + "theoretical_loss": 3.317883400256756, + "tokens_seen": 3034054656 + }, + { + "epoch": 0.84, + "learning_rate": 8.206060606060606e-05, + "loss": 2.3466, + "theoretical_loss": 3.3178779027381333, + "tokens_seen": 3034120192 + }, + { + "epoch": 0.84, + "learning_rate": 8.204040404040405e-05, + "loss": 2.5463, + "theoretical_loss": 3.3178724053715016, + "tokens_seen": 3034185728 + }, + { + "epoch": 0.84, + "learning_rate": 8.202020202020202e-05, + "loss": 2.1825, + "theoretical_loss": 3.3178669081568537, + "tokens_seen": 3034251264 + }, + { + "epoch": 0.84, + "learning_rate": 8.2e-05, + "loss": 2.4553, + "theoretical_loss": 3.3178614110941815, + "tokens_seen": 3034316800 + }, + { + "epoch": 0.84, + "learning_rate": 8.197979797979798e-05, + "loss": 2.4115, + "theoretical_loss": 3.317855914183478, + "tokens_seen": 3034382336 + }, + { + "epoch": 0.84, + "learning_rate": 8.195959595959596e-05, + "loss": 2.4803, + "theoretical_loss": 3.3178504174247356, + "tokens_seen": 3034447872 + }, + { + "epoch": 0.84, + "learning_rate": 8.193939393939394e-05, + "loss": 2.3649, + "theoretical_loss": 3.317844920817947, + "tokens_seen": 3034513408 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1664335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.192890167236328, + "objective/train/theoretical_loss": 3.317839424363104, + "objective/train/tokens_used": 1393438176, + "theoretical_loss": 3.317839424363104, + "tokens_seen": 3034578944 + }, + { + "epoch": 0.84, + "learning_rate": 8.191919191919192e-05, + "loss": 2.3399, + "theoretical_loss": 3.317839424363104, + "tokens_seen": 3034578944 + }, + { + "epoch": 0.84, + "learning_rate": 8.189898989898991e-05, + "loss": 2.5124, + "theoretical_loss": 3.3178339280602, + "tokens_seen": 3034644480 + }, + { + "epoch": 0.84, + "learning_rate": 8.187878787878789e-05, + "loss": 2.3558, + "theoretical_loss": 3.317828431909227, + "tokens_seen": 3034710016 + }, + { + "epoch": 0.84, + "learning_rate": 8.185858585858585e-05, + "loss": 2.7557, + "theoretical_loss": 3.3178229359101774, + "tokens_seen": 3034775552 + }, + { + "epoch": 0.84, + "learning_rate": 8.183838383838383e-05, + "loss": 2.4085, + "theoretical_loss": 3.3178174400630445, + "tokens_seen": 3034841088 + }, + { + "epoch": 0.84, + "learning_rate": 8.181818181818182e-05, + "loss": 2.4967, + "theoretical_loss": 3.31781194436782, + "tokens_seen": 3034906624 + }, + { + "epoch": 0.84, + "learning_rate": 8.17979797979798e-05, + "loss": 2.4474, + "theoretical_loss": 3.3178064488244967, + "tokens_seen": 3034972160 + }, + { + "epoch": 0.84, + "learning_rate": 8.177777777777778e-05, + "loss": 2.5468, + "theoretical_loss": 3.317800953433067, + "tokens_seen": 3035037696 + }, + { + "epoch": 0.84, + "learning_rate": 8.175757575757575e-05, + "loss": 2.4447, + "theoretical_loss": 3.3177954581935234, + "tokens_seen": 3035103232 + }, + { + "epoch": 0.84, + "learning_rate": 8.173737373737374e-05, + "loss": 2.3701, + "theoretical_loss": 3.317789963105859, + "tokens_seen": 3035168768 + }, + { + "epoch": 0.84, + "learning_rate": 8.171717171717172e-05, + "loss": 2.3208, + "theoretical_loss": 3.317784468170066, + "tokens_seen": 3035234304 + }, + { + "epoch": 0.84, + "learning_rate": 8.16969696969697e-05, + "loss": 2.3555, + "theoretical_loss": 3.3177789733861363, + "tokens_seen": 3035299840 + }, + { + "epoch": 0.84, + "learning_rate": 8.167676767676768e-05, + "loss": 2.4335, + "theoretical_loss": 3.317773478754063, + "tokens_seen": 3035365376 + }, + { + "epoch": 0.84, + "learning_rate": 8.165656565656566e-05, + "loss": 2.3018, + "theoretical_loss": 3.3177679842738383, + "tokens_seen": 3035430912 + }, + { + "epoch": 0.84, + "learning_rate": 8.163636363636363e-05, + "loss": 2.6625, + "theoretical_loss": 3.317762489945456, + "tokens_seen": 3035496448 + }, + { + "epoch": 0.84, + "learning_rate": 8.161616161616161e-05, + "loss": 2.488, + "theoretical_loss": 3.3177569957689066, + "tokens_seen": 3035561984 + }, + { + "epoch": 0.84, + "learning_rate": 8.15959595959596e-05, + "loss": 2.648, + "theoretical_loss": 3.3177515017441843, + "tokens_seen": 3035627520 + }, + { + "epoch": 0.84, + "learning_rate": 8.157575757575758e-05, + "loss": 2.5255, + "theoretical_loss": 3.3177460078712806, + "tokens_seen": 3035693056 + }, + { + "epoch": 0.84, + "learning_rate": 8.155555555555557e-05, + "loss": 2.3067, + "theoretical_loss": 3.3177405141501883, + "tokens_seen": 3035758592 + }, + { + "epoch": 0.84, + "learning_rate": 8.153535353535354e-05, + "loss": 2.478, + "theoretical_loss": 3.3177350205809004, + "tokens_seen": 3035824128 + }, + { + "epoch": 0.84, + "learning_rate": 8.151515151515152e-05, + "loss": 2.5763, + "theoretical_loss": 3.317729527163409, + "tokens_seen": 3035889664 + }, + { + "epoch": 0.84, + "learning_rate": 8.149494949494949e-05, + "loss": 2.485, + "theoretical_loss": 3.317724033897706, + "tokens_seen": 3035955200 + }, + { + "epoch": 0.84, + "learning_rate": 8.147474747474748e-05, + "loss": 2.6331, + "theoretical_loss": 3.3177185407837855, + "tokens_seen": 3036020736 + }, + { + "epoch": 0.84, + "learning_rate": 8.145454545454546e-05, + "loss": 2.6253, + "theoretical_loss": 3.3177130478216386, + "tokens_seen": 3036086272 + }, + { + "epoch": 0.84, + "learning_rate": 8.143434343434343e-05, + "loss": 2.5543, + "theoretical_loss": 3.3177075550112587, + "tokens_seen": 3036151808 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1664850, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.734156370162964, + "objective/train/theoretical_loss": 3.3177020623526374, + "objective/train/tokens_used": 1395076576, + "theoretical_loss": 3.3177020623526374, + "tokens_seen": 3036217344 + }, + { + "epoch": 0.84, + "learning_rate": 8.141414141414141e-05, + "loss": 2.5359, + "theoretical_loss": 3.3177020623526374, + "tokens_seen": 3036217344 + }, + { + "epoch": 0.84, + "learning_rate": 8.13939393939394e-05, + "loss": 2.2238, + "theoretical_loss": 3.3176965698457686, + "tokens_seen": 3036282880 + }, + { + "epoch": 0.84, + "learning_rate": 8.137373737373738e-05, + "loss": 2.6056, + "theoretical_loss": 3.3176910774906436, + "tokens_seen": 3036348416 + }, + { + "epoch": 0.84, + "learning_rate": 8.135353535353537e-05, + "loss": 2.439, + "theoretical_loss": 3.3176855852872555, + "tokens_seen": 3036413952 + }, + { + "epoch": 0.84, + "learning_rate": 8.133333333333332e-05, + "loss": 2.4674, + "theoretical_loss": 3.3176800932355968, + "tokens_seen": 3036479488 + }, + { + "epoch": 0.84, + "learning_rate": 8.13131313131313e-05, + "loss": 2.6378, + "theoretical_loss": 3.3176746013356597, + "tokens_seen": 3036545024 + }, + { + "epoch": 0.84, + "learning_rate": 8.129292929292929e-05, + "loss": 2.3898, + "theoretical_loss": 3.3176691095874373, + "tokens_seen": 3036610560 + }, + { + "epoch": 0.84, + "learning_rate": 8.127272727272727e-05, + "loss": 2.3322, + "theoretical_loss": 3.317663617990922, + "tokens_seen": 3036676096 + }, + { + "epoch": 0.84, + "learning_rate": 8.125252525252526e-05, + "loss": 2.572, + "theoretical_loss": 3.317658126546106, + "tokens_seen": 3036741632 + }, + { + "epoch": 0.84, + "learning_rate": 8.123232323232324e-05, + "loss": 2.5319, + "theoretical_loss": 3.3176526352529816, + "tokens_seen": 3036807168 + }, + { + "epoch": 0.84, + "learning_rate": 8.121212121212121e-05, + "loss": 2.3938, + "theoretical_loss": 3.3176471441115423, + "tokens_seen": 3036872704 + }, + { + "epoch": 0.84, + "learning_rate": 8.11919191919192e-05, + "loss": 2.3213, + "theoretical_loss": 3.31764165312178, + "tokens_seen": 3036938240 + }, + { + "epoch": 0.84, + "learning_rate": 8.117171717171718e-05, + "loss": 2.4057, + "theoretical_loss": 3.317636162283687, + "tokens_seen": 3037003776 + }, + { + "epoch": 0.84, + "learning_rate": 8.115151515151515e-05, + "loss": 2.3849, + "theoretical_loss": 3.3176306715972563, + "tokens_seen": 3037069312 + }, + { + "epoch": 0.84, + "learning_rate": 8.113131313131314e-05, + "loss": 2.6266, + "theoretical_loss": 3.31762518106248, + "tokens_seen": 3037134848 + }, + { + "epoch": 0.84, + "learning_rate": 8.11111111111111e-05, + "loss": 2.347, + "theoretical_loss": 3.3176196906793516, + "tokens_seen": 3037200384 + }, + { + "epoch": 0.84, + "learning_rate": 8.109090909090909e-05, + "loss": 2.4266, + "theoretical_loss": 3.317614200447862, + "tokens_seen": 3037265920 + }, + { + "epoch": 0.84, + "learning_rate": 8.107070707070707e-05, + "loss": 2.7149, + "theoretical_loss": 3.3176087103680056, + "tokens_seen": 3037331456 + }, + { + "epoch": 0.84, + "learning_rate": 8.105050505050506e-05, + "loss": 2.5081, + "theoretical_loss": 3.3176032204397736, + "tokens_seen": 3037396992 + }, + { + "epoch": 0.84, + "learning_rate": 8.103030303030304e-05, + "loss": 2.4132, + "theoretical_loss": 3.3175977306631594, + "tokens_seen": 3037462528 + }, + { + "epoch": 0.84, + "learning_rate": 8.101010101010101e-05, + "loss": 2.3554, + "theoretical_loss": 3.3175922410381546, + "tokens_seen": 3037528064 + }, + { + "epoch": 0.84, + "learning_rate": 8.098989898989898e-05, + "loss": 2.5551, + "theoretical_loss": 3.3175867515647526, + "tokens_seen": 3037593600 + }, + { + "epoch": 0.84, + "learning_rate": 8.096969696969697e-05, + "loss": 2.5189, + "theoretical_loss": 3.3175812622429453, + "tokens_seen": 3037659136 + }, + { + "epoch": 0.84, + "learning_rate": 8.094949494949495e-05, + "loss": 2.4834, + "theoretical_loss": 3.317575773072726, + "tokens_seen": 3037724672 + }, + { + "epoch": 0.84, + "learning_rate": 8.092929292929293e-05, + "loss": 2.5021, + "theoretical_loss": 3.317570284054087, + "tokens_seen": 3037790208 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1665791, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.856503963470459, + "objective/train/theoretical_loss": 3.3175647951870197, + "objective/train/tokens_used": 1396714976, + "theoretical_loss": 3.3175647951870197, + "tokens_seen": 3037855744 + }, + { + "epoch": 0.84, + "learning_rate": 8.09090909090909e-05, + "loss": 2.5827, + "theoretical_loss": 3.3175647951870197, + "tokens_seen": 3037855744 + }, + { + "epoch": 0.84, + "learning_rate": 8.088888888888889e-05, + "loss": 2.5805, + "theoretical_loss": 3.317559306471518, + "tokens_seen": 3037921280 + }, + { + "epoch": 0.84, + "learning_rate": 8.086868686868687e-05, + "loss": 2.7304, + "theoretical_loss": 3.3175538179075743, + "tokens_seen": 3037986816 + }, + { + "epoch": 0.84, + "learning_rate": 8.084848484848486e-05, + "loss": 2.5115, + "theoretical_loss": 3.317548329495181, + "tokens_seen": 3038052352 + }, + { + "epoch": 0.84, + "learning_rate": 8.082828282828284e-05, + "loss": 2.5184, + "theoretical_loss": 3.31754284123433, + "tokens_seen": 3038117888 + }, + { + "epoch": 0.84, + "learning_rate": 8.080808080808081e-05, + "loss": 2.5371, + "theoretical_loss": 3.3175373531250147, + "tokens_seen": 3038183424 + }, + { + "epoch": 0.84, + "learning_rate": 8.078787878787878e-05, + "loss": 2.3106, + "theoretical_loss": 3.3175318651672274, + "tokens_seen": 3038248960 + }, + { + "epoch": 0.84, + "learning_rate": 8.076767676767677e-05, + "loss": 2.4675, + "theoretical_loss": 3.3175263773609602, + "tokens_seen": 3038314496 + }, + { + "epoch": 0.84, + "learning_rate": 8.074747474747475e-05, + "loss": 2.5606, + "theoretical_loss": 3.3175208897062065, + "tokens_seen": 3038380032 + }, + { + "epoch": 0.84, + "learning_rate": 8.072727272727273e-05, + "loss": 2.6571, + "theoretical_loss": 3.3175154022029583, + "tokens_seen": 3038445568 + }, + { + "epoch": 0.84, + "learning_rate": 8.070707070707072e-05, + "loss": 2.4533, + "theoretical_loss": 3.317509914851208, + "tokens_seen": 3038511104 + }, + { + "epoch": 0.84, + "learning_rate": 8.068686868686869e-05, + "loss": 2.5453, + "theoretical_loss": 3.317504427650948, + "tokens_seen": 3038576640 + }, + { + "epoch": 0.84, + "learning_rate": 8.066666666666667e-05, + "loss": 2.5643, + "theoretical_loss": 3.3174989406021718, + "tokens_seen": 3038642176 + }, + { + "epoch": 0.84, + "learning_rate": 8.064646464646464e-05, + "loss": 2.4402, + "theoretical_loss": 3.3174934537048713, + "tokens_seen": 3038707712 + }, + { + "epoch": 0.84, + "learning_rate": 8.062626262626263e-05, + "loss": 2.4771, + "theoretical_loss": 3.317487966959039, + "tokens_seen": 3038773248 + }, + { + "epoch": 0.84, + "learning_rate": 8.060606060606061e-05, + "loss": 2.4054, + "theoretical_loss": 3.317482480364667, + "tokens_seen": 3038838784 + }, + { + "epoch": 0.84, + "learning_rate": 8.058585858585858e-05, + "loss": 2.4618, + "theoretical_loss": 3.3174769939217494, + "tokens_seen": 3038904320 + }, + { + "epoch": 0.84, + "learning_rate": 8.056565656565656e-05, + "loss": 2.2871, + "theoretical_loss": 3.3174715076302776, + "tokens_seen": 3038969856 + }, + { + "epoch": 0.84, + "learning_rate": 8.054545454545455e-05, + "loss": 2.4362, + "theoretical_loss": 3.317466021490244, + "tokens_seen": 3039035392 + }, + { + "epoch": 0.84, + "learning_rate": 8.052525252525253e-05, + "loss": 2.6081, + "theoretical_loss": 3.3174605355016418, + "tokens_seen": 3039100928 + }, + { + "epoch": 0.84, + "learning_rate": 8.050505050505052e-05, + "loss": 2.4435, + "theoretical_loss": 3.3174550496644626, + "tokens_seen": 3039166464 + }, + { + "epoch": 0.84, + "learning_rate": 8.048484848484849e-05, + "loss": 2.5942, + "theoretical_loss": 3.3174495639787005, + "tokens_seen": 3039232000 + }, + { + "epoch": 0.84, + "learning_rate": 8.046464646464646e-05, + "loss": 2.4677, + "theoretical_loss": 3.317444078444346, + "tokens_seen": 3039297536 + }, + { + "epoch": 0.84, + "learning_rate": 8.044444444444444e-05, + "loss": 2.3503, + "theoretical_loss": 3.317438593061394, + "tokens_seen": 3039363072 + }, + { + "epoch": 0.84, + "learning_rate": 8.042424242424242e-05, + "loss": 2.6367, + "theoretical_loss": 3.317433107829835, + "tokens_seen": 3039428608 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1666356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3997597694396973, + "objective/train/theoretical_loss": 3.3174276227496633, + "objective/train/tokens_used": 1398353376, + "theoretical_loss": 3.3174276227496633, + "tokens_seen": 3039494144 + }, + { + "epoch": 0.84, + "learning_rate": 8.040404040404041e-05, + "loss": 2.5777, + "theoretical_loss": 3.3174276227496633, + "tokens_seen": 3039494144 + }, + { + "epoch": 0.84, + "learning_rate": 8.038383838383839e-05, + "loss": 2.4705, + "theoretical_loss": 3.31742213782087, + "tokens_seen": 3039559680 + }, + { + "epoch": 0.84, + "learning_rate": 8.036363636363636e-05, + "loss": 2.7066, + "theoretical_loss": 3.3174166530434483, + "tokens_seen": 3039625216 + }, + { + "epoch": 0.84, + "learning_rate": 8.034343434343435e-05, + "loss": 2.583, + "theoretical_loss": 3.3174111684173906, + "tokens_seen": 3039690752 + }, + { + "epoch": 0.84, + "learning_rate": 8.032323232323233e-05, + "loss": 2.5723, + "theoretical_loss": 3.3174056839426895, + "tokens_seen": 3039756288 + }, + { + "epoch": 0.84, + "learning_rate": 8.030303030303031e-05, + "loss": 2.5414, + "theoretical_loss": 3.317400199619338, + "tokens_seen": 3039821824 + }, + { + "epoch": 0.84, + "learning_rate": 8.028282828282829e-05, + "loss": 2.3973, + "theoretical_loss": 3.3173947154473282, + "tokens_seen": 3039887360 + }, + { + "epoch": 0.84, + "learning_rate": 8.026262626262626e-05, + "loss": 2.4839, + "theoretical_loss": 3.3173892314266524, + "tokens_seen": 3039952896 + }, + { + "epoch": 0.84, + "learning_rate": 8.024242424242424e-05, + "loss": 2.3775, + "theoretical_loss": 3.3173837475573036, + "tokens_seen": 3040018432 + }, + { + "epoch": 0.84, + "learning_rate": 8.022222222222222e-05, + "loss": 2.4843, + "theoretical_loss": 3.3173782638392746, + "tokens_seen": 3040083968 + }, + { + "epoch": 0.84, + "learning_rate": 8.020202020202021e-05, + "loss": 2.5318, + "theoretical_loss": 3.3173727802725574, + "tokens_seen": 3040149504 + }, + { + "epoch": 0.84, + "learning_rate": 8.018181818181819e-05, + "loss": 2.6324, + "theoretical_loss": 3.3173672968571446, + "tokens_seen": 3040215040 + }, + { + "epoch": 0.84, + "learning_rate": 8.016161616161616e-05, + "loss": 2.4466, + "theoretical_loss": 3.3173618135930294, + "tokens_seen": 3040280576 + }, + { + "epoch": 0.84, + "learning_rate": 8.014141414141415e-05, + "loss": 2.561, + "theoretical_loss": 3.3173563304802034, + "tokens_seen": 3040346112 + }, + { + "epoch": 0.84, + "learning_rate": 8.012121212121212e-05, + "loss": 2.5293, + "theoretical_loss": 3.31735084751866, + "tokens_seen": 3040411648 + }, + { + "epoch": 0.84, + "learning_rate": 8.01010101010101e-05, + "loss": 2.5092, + "theoretical_loss": 3.3173453647083915, + "tokens_seen": 3040477184 + }, + { + "epoch": 0.84, + "learning_rate": 8.008080808080808e-05, + "loss": 2.2748, + "theoretical_loss": 3.31733988204939, + "tokens_seen": 3040542720 + }, + { + "epoch": 0.84, + "learning_rate": 8.006060606060605e-05, + "loss": 2.5463, + "theoretical_loss": 3.317334399541649, + "tokens_seen": 3040608256 + }, + { + "epoch": 0.84, + "learning_rate": 8.004040404040404e-05, + "loss": 2.3725, + "theoretical_loss": 3.31732891718516, + "tokens_seen": 3040673792 + }, + { + "epoch": 0.84, + "learning_rate": 8.002020202020202e-05, + "loss": 2.4058, + "theoretical_loss": 3.3173234349799166, + "tokens_seen": 3040739328 + }, + { + "epoch": 0.84, + "learning_rate": 8e-05, + "loss": 2.5191, + "theoretical_loss": 3.3173179529259107, + "tokens_seen": 3040804864 + }, + { + "epoch": 0.84, + "learning_rate": 7.997979797979799e-05, + "loss": 2.3644, + "theoretical_loss": 3.317312471023135, + "tokens_seen": 3040870400 + }, + { + "epoch": 0.84, + "learning_rate": 7.995959595959596e-05, + "loss": 2.6913, + "theoretical_loss": 3.3173069892715823, + "tokens_seen": 3040935936 + }, + { + "epoch": 0.84, + "learning_rate": 7.993939393939393e-05, + "loss": 2.4733, + "theoretical_loss": 3.3173015076712447, + "tokens_seen": 3041001472 + }, + { + "epoch": 0.84, + "learning_rate": 7.991919191919192e-05, + "loss": 2.3592, + "theoretical_loss": 3.3172960262221154, + "tokens_seen": 3041067008 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1667402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.068047285079956, + "objective/train/theoretical_loss": 3.3172905449241865, + "objective/train/tokens_used": 1399991776, + "theoretical_loss": 3.3172905449241865, + "tokens_seen": 3041132544 + }, + { + "epoch": 0.84, + "learning_rate": 7.98989898989899e-05, + "loss": 2.2487, + "theoretical_loss": 3.3172905449241865, + "tokens_seen": 3041132544 + }, + { + "epoch": 0.84, + "learning_rate": 7.987878787878788e-05, + "loss": 2.5306, + "theoretical_loss": 3.317285063777451, + "tokens_seen": 3041198080 + }, + { + "epoch": 0.84, + "learning_rate": 7.985858585858587e-05, + "loss": 2.5634, + "theoretical_loss": 3.3172795827819, + "tokens_seen": 3041263616 + }, + { + "epoch": 0.84, + "learning_rate": 7.983838383838384e-05, + "loss": 2.4842, + "theoretical_loss": 3.3172741019375285, + "tokens_seen": 3041329152 + }, + { + "epoch": 0.84, + "learning_rate": 7.981818181818182e-05, + "loss": 2.7777, + "theoretical_loss": 3.3172686212443274, + "tokens_seen": 3041394688 + }, + { + "epoch": 0.84, + "learning_rate": 7.97979797979798e-05, + "loss": 2.5095, + "theoretical_loss": 3.3172631407022894, + "tokens_seen": 3041460224 + }, + { + "epoch": 0.84, + "learning_rate": 7.977777777777778e-05, + "loss": 2.4266, + "theoretical_loss": 3.317257660311408, + "tokens_seen": 3041525760 + }, + { + "epoch": 0.84, + "learning_rate": 7.975757575757576e-05, + "loss": 2.5601, + "theoretical_loss": 3.3172521800716748, + "tokens_seen": 3041591296 + }, + { + "epoch": 0.84, + "learning_rate": 7.973737373737373e-05, + "loss": 2.2851, + "theoretical_loss": 3.3172466999830825, + "tokens_seen": 3041656832 + }, + { + "epoch": 0.84, + "learning_rate": 7.971717171717171e-05, + "loss": 2.167, + "theoretical_loss": 3.317241220045624, + "tokens_seen": 3041722368 + }, + { + "epoch": 0.84, + "learning_rate": 7.96969696969697e-05, + "loss": 2.6428, + "theoretical_loss": 3.317235740259292, + "tokens_seen": 3041787904 + }, + { + "epoch": 0.84, + "learning_rate": 7.967676767676768e-05, + "loss": 2.5417, + "theoretical_loss": 3.3172302606240787, + "tokens_seen": 3041853440 + }, + { + "epoch": 0.84, + "learning_rate": 7.965656565656567e-05, + "loss": 2.4006, + "theoretical_loss": 3.3172247811399767, + "tokens_seen": 3041918976 + }, + { + "epoch": 0.84, + "learning_rate": 7.963636363636364e-05, + "loss": 2.3963, + "theoretical_loss": 3.3172193018069787, + "tokens_seen": 3041984512 + }, + { + "epoch": 0.84, + "learning_rate": 7.961616161616162e-05, + "loss": 2.4501, + "theoretical_loss": 3.317213822625077, + "tokens_seen": 3042050048 + }, + { + "epoch": 0.84, + "learning_rate": 7.959595959595959e-05, + "loss": 2.5003, + "theoretical_loss": 3.317208343594265, + "tokens_seen": 3042115584 + }, + { + "epoch": 0.84, + "learning_rate": 7.957575757575757e-05, + "loss": 2.4536, + "theoretical_loss": 3.3172028647145346, + "tokens_seen": 3042181120 + }, + { + "epoch": 0.84, + "learning_rate": 7.955555555555556e-05, + "loss": 2.5533, + "theoretical_loss": 3.3171973859858785, + "tokens_seen": 3042246656 + }, + { + "epoch": 0.84, + "learning_rate": 7.953535353535353e-05, + "loss": 2.6894, + "theoretical_loss": 3.317191907408289, + "tokens_seen": 3042312192 + }, + { + "epoch": 0.84, + "learning_rate": 7.951515151515151e-05, + "loss": 2.4077, + "theoretical_loss": 3.3171864289817594, + "tokens_seen": 3042377728 + }, + { + "epoch": 0.84, + "learning_rate": 7.94949494949495e-05, + "loss": 2.6312, + "theoretical_loss": 3.3171809507062817, + "tokens_seen": 3042443264 + }, + { + "epoch": 0.84, + "learning_rate": 7.947474747474748e-05, + "loss": 2.3705, + "theoretical_loss": 3.3171754725818485, + "tokens_seen": 3042508800 + }, + { + "epoch": 0.84, + "learning_rate": 7.945454545454547e-05, + "loss": 2.5707, + "theoretical_loss": 3.3171699946084523, + "tokens_seen": 3042574336 + }, + { + "epoch": 0.84, + "learning_rate": 7.943434343434345e-05, + "loss": 2.2673, + "theoretical_loss": 3.3171645167860864, + "tokens_seen": 3042639872 + }, + { + "epoch": 0.84, + "learning_rate": 7.94141414141414e-05, + "loss": 2.5115, + "theoretical_loss": 3.3171590391147427, + "tokens_seen": 3042705408 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1668521, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5300939083099365, + "objective/train/theoretical_loss": 3.317153561594414, + "objective/train/tokens_used": 1401630176, + "theoretical_loss": 3.317153561594414, + "tokens_seen": 3042770944 + }, + { + "epoch": 0.84, + "learning_rate": 7.939393939393939e-05, + "loss": 2.5195, + "theoretical_loss": 3.317153561594414, + "tokens_seen": 3042770944 + }, + { + "epoch": 0.84, + "learning_rate": 7.937373737373737e-05, + "loss": 2.5791, + "theoretical_loss": 3.3171480842250927, + "tokens_seen": 3042836480 + }, + { + "epoch": 0.84, + "learning_rate": 7.935353535353536e-05, + "loss": 2.5705, + "theoretical_loss": 3.3171426070067715, + "tokens_seen": 3042902016 + }, + { + "epoch": 0.84, + "learning_rate": 7.933333333333334e-05, + "loss": 2.4446, + "theoretical_loss": 3.317137129939443, + "tokens_seen": 3042967552 + }, + { + "epoch": 0.84, + "learning_rate": 7.931313131313131e-05, + "loss": 2.2723, + "theoretical_loss": 3.3171316530231003, + "tokens_seen": 3043033088 + }, + { + "epoch": 0.84, + "learning_rate": 7.92929292929293e-05, + "loss": 2.5936, + "theoretical_loss": 3.3171261762577346, + "tokens_seen": 3043098624 + }, + { + "epoch": 0.84, + "learning_rate": 7.927272727272728e-05, + "loss": 2.5481, + "theoretical_loss": 3.31712069964334, + "tokens_seen": 3043164160 + }, + { + "epoch": 0.84, + "learning_rate": 7.925252525252525e-05, + "loss": 2.4877, + "theoretical_loss": 3.3171152231799086, + "tokens_seen": 3043229696 + }, + { + "epoch": 0.84, + "learning_rate": 7.923232323232323e-05, + "loss": 2.5258, + "theoretical_loss": 3.317109746867432, + "tokens_seen": 3043295232 + }, + { + "epoch": 0.84, + "learning_rate": 7.92121212121212e-05, + "loss": 2.5756, + "theoretical_loss": 3.317104270705905, + "tokens_seen": 3043360768 + }, + { + "epoch": 0.84, + "learning_rate": 7.919191919191919e-05, + "loss": 2.4417, + "theoretical_loss": 3.3170987946953177, + "tokens_seen": 3043426304 + }, + { + "epoch": 0.84, + "learning_rate": 7.917171717171717e-05, + "loss": 2.7586, + "theoretical_loss": 3.3170933188356644, + "tokens_seen": 3043491840 + }, + { + "epoch": 0.84, + "learning_rate": 7.915151515151516e-05, + "loss": 2.3759, + "theoretical_loss": 3.3170878431269366, + "tokens_seen": 3043557376 + }, + { + "epoch": 0.84, + "learning_rate": 7.913131313131314e-05, + "loss": 2.5536, + "theoretical_loss": 3.3170823675691277, + "tokens_seen": 3043622912 + }, + { + "epoch": 0.84, + "learning_rate": 7.911111111111111e-05, + "loss": 2.401, + "theoretical_loss": 3.31707689216223, + "tokens_seen": 3043688448 + }, + { + "epoch": 0.84, + "learning_rate": 7.90909090909091e-05, + "loss": 2.4464, + "theoretical_loss": 3.317071416906236, + "tokens_seen": 3043753984 + }, + { + "epoch": 0.84, + "learning_rate": 7.907070707070707e-05, + "loss": 2.5024, + "theoretical_loss": 3.317065941801139, + "tokens_seen": 3043819520 + }, + { + "epoch": 0.84, + "learning_rate": 7.905050505050505e-05, + "loss": 2.694, + "theoretical_loss": 3.3170604668469297, + "tokens_seen": 3043885056 + }, + { + "epoch": 0.84, + "learning_rate": 7.903030303030303e-05, + "loss": 2.3748, + "theoretical_loss": 3.317054992043603, + "tokens_seen": 3043950592 + }, + { + "epoch": 0.84, + "learning_rate": 7.901010101010102e-05, + "loss": 2.5364, + "theoretical_loss": 3.31704951739115, + "tokens_seen": 3044016128 + }, + { + "epoch": 0.84, + "learning_rate": 7.898989898989899e-05, + "loss": 2.4277, + "theoretical_loss": 3.317044042889564, + "tokens_seen": 3044081664 + }, + { + "epoch": 0.84, + "learning_rate": 7.896969696969697e-05, + "loss": 2.5703, + "theoretical_loss": 3.317038568538837, + "tokens_seen": 3044147200 + }, + { + "epoch": 0.84, + "learning_rate": 7.894949494949496e-05, + "loss": 2.6958, + "theoretical_loss": 3.317033094338962, + "tokens_seen": 3044212736 + }, + { + "epoch": 0.84, + "learning_rate": 7.892929292929294e-05, + "loss": 2.3835, + "theoretical_loss": 3.317027620289932, + "tokens_seen": 3044278272 + }, + { + "epoch": 0.84, + "learning_rate": 7.890909090909091e-05, + "loss": 2.3679, + "theoretical_loss": 3.3170221463917384, + "tokens_seen": 3044343808 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1668971, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6453962326049805, + "objective/train/theoretical_loss": 3.317016672644375, + "objective/train/tokens_used": 1403268576, + "theoretical_loss": 3.317016672644375, + "tokens_seen": 3044409344 + }, + { + "epoch": 0.84, + "learning_rate": 7.888888888888888e-05, + "loss": 2.5223, + "theoretical_loss": 3.317016672644375, + "tokens_seen": 3044409344 + }, + { + "epoch": 0.84, + "learning_rate": 7.886868686868686e-05, + "loss": 2.5619, + "theoretical_loss": 3.3170111990478337, + "tokens_seen": 3044474880 + }, + { + "epoch": 0.84, + "learning_rate": 7.884848484848485e-05, + "loss": 2.5507, + "theoretical_loss": 3.3170057256021077, + "tokens_seen": 3044540416 + }, + { + "epoch": 0.84, + "learning_rate": 7.882828282828283e-05, + "loss": 2.4249, + "theoretical_loss": 3.3170002523071886, + "tokens_seen": 3044605952 + }, + { + "epoch": 0.84, + "learning_rate": 7.880808080808082e-05, + "loss": 2.582, + "theoretical_loss": 3.3169947791630703, + "tokens_seen": 3044671488 + }, + { + "epoch": 0.84, + "learning_rate": 7.878787878787879e-05, + "loss": 2.3939, + "theoretical_loss": 3.316989306169744, + "tokens_seen": 3044737024 + }, + { + "epoch": 0.84, + "learning_rate": 7.876767676767677e-05, + "loss": 2.5737, + "theoretical_loss": 3.3169838333272037, + "tokens_seen": 3044802560 + }, + { + "epoch": 0.84, + "learning_rate": 7.874747474747475e-05, + "loss": 2.6263, + "theoretical_loss": 3.3169783606354413, + "tokens_seen": 3044868096 + }, + { + "epoch": 0.84, + "learning_rate": 7.872727272727273e-05, + "loss": 2.5982, + "theoretical_loss": 3.316972888094449, + "tokens_seen": 3044933632 + }, + { + "epoch": 0.84, + "learning_rate": 7.870707070707071e-05, + "loss": 2.5703, + "theoretical_loss": 3.3169674157042195, + "tokens_seen": 3044999168 + }, + { + "epoch": 0.84, + "learning_rate": 7.868686868686868e-05, + "loss": 2.7806, + "theoretical_loss": 3.3169619434647464, + "tokens_seen": 3045064704 + }, + { + "epoch": 0.84, + "learning_rate": 7.866666666666666e-05, + "loss": 2.3478, + "theoretical_loss": 3.3169564713760216, + "tokens_seen": 3045130240 + }, + { + "epoch": 0.84, + "learning_rate": 7.864646464646465e-05, + "loss": 2.4222, + "theoretical_loss": 3.3169509994380375, + "tokens_seen": 3045195776 + }, + { + "epoch": 0.84, + "learning_rate": 7.862626262626263e-05, + "loss": 2.4743, + "theoretical_loss": 3.3169455276507867, + "tokens_seen": 3045261312 + }, + { + "epoch": 0.84, + "learning_rate": 7.860606060606062e-05, + "loss": 2.4583, + "theoretical_loss": 3.3169400560142623, + "tokens_seen": 3045326848 + }, + { + "epoch": 0.84, + "learning_rate": 7.85858585858586e-05, + "loss": 2.5142, + "theoretical_loss": 3.3169345845284566, + "tokens_seen": 3045392384 + }, + { + "epoch": 0.84, + "learning_rate": 7.856565656565656e-05, + "loss": 2.4799, + "theoretical_loss": 3.3169291131933623, + "tokens_seen": 3045457920 + }, + { + "epoch": 0.84, + "learning_rate": 7.854545454545454e-05, + "loss": 2.28, + "theoretical_loss": 3.316923642008972, + "tokens_seen": 3045523456 + }, + { + "epoch": 0.84, + "learning_rate": 7.852525252525252e-05, + "loss": 2.3386, + "theoretical_loss": 3.316918170975278, + "tokens_seen": 3045588992 + }, + { + "epoch": 0.84, + "learning_rate": 7.850505050505051e-05, + "loss": 2.2142, + "theoretical_loss": 3.3169127000922733, + "tokens_seen": 3045654528 + }, + { + "epoch": 0.84, + "learning_rate": 7.848484848484849e-05, + "loss": 2.4112, + "theoretical_loss": 3.31690722935995, + "tokens_seen": 3045720064 + }, + { + "epoch": 0.84, + "learning_rate": 7.846464646464646e-05, + "loss": 2.5582, + "theoretical_loss": 3.316901758778302, + "tokens_seen": 3045785600 + }, + { + "epoch": 0.84, + "learning_rate": 7.844444444444445e-05, + "loss": 2.5313, + "theoretical_loss": 3.3168962883473205, + "tokens_seen": 3045851136 + }, + { + "epoch": 0.84, + "learning_rate": 7.842424242424243e-05, + "loss": 2.4994, + "theoretical_loss": 3.3168908180669985, + "tokens_seen": 3045916672 + }, + { + "epoch": 0.84, + "learning_rate": 7.840404040404041e-05, + "loss": 2.5592, + "theoretical_loss": 3.316885347937329, + "tokens_seen": 3045982208 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1670023, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7244620323181152, + "objective/train/theoretical_loss": 3.3168798779583035, + "objective/train/tokens_used": 1404906976, + "theoretical_loss": 3.3168798779583035, + "tokens_seen": 3046047744 + }, + { + "epoch": 0.84, + "learning_rate": 7.838383838383838e-05, + "loss": 2.5129, + "theoretical_loss": 3.3168798779583035, + "tokens_seen": 3046047744 + }, + { + "epoch": 0.85, + "learning_rate": 7.836363636363636e-05, + "loss": 2.4534, + "theoretical_loss": 3.316874408129916, + "tokens_seen": 3046113280 + }, + { + "epoch": 0.85, + "learning_rate": 7.834343434343434e-05, + "loss": 2.5395, + "theoretical_loss": 3.3168689384521586, + "tokens_seen": 3046178816 + }, + { + "epoch": 0.85, + "learning_rate": 7.832323232323232e-05, + "loss": 2.4973, + "theoretical_loss": 3.316863468925024, + "tokens_seen": 3046244352 + }, + { + "epoch": 0.85, + "learning_rate": 7.830303030303031e-05, + "loss": 2.445, + "theoretical_loss": 3.316857999548504, + "tokens_seen": 3046309888 + }, + { + "epoch": 0.85, + "learning_rate": 7.828282828282829e-05, + "loss": 2.6821, + "theoretical_loss": 3.3168525303225924, + "tokens_seen": 3046375424 + }, + { + "epoch": 0.85, + "learning_rate": 7.826262626262626e-05, + "loss": 2.3535, + "theoretical_loss": 3.316847061247281, + "tokens_seen": 3046440960 + }, + { + "epoch": 0.85, + "learning_rate": 7.824242424242425e-05, + "loss": 2.5673, + "theoretical_loss": 3.316841592322563, + "tokens_seen": 3046506496 + }, + { + "epoch": 0.85, + "learning_rate": 7.822222222222222e-05, + "loss": 2.6083, + "theoretical_loss": 3.3168361235484305, + "tokens_seen": 3046572032 + }, + { + "epoch": 0.85, + "learning_rate": 7.82020202020202e-05, + "loss": 2.6547, + "theoretical_loss": 3.3168306549248765, + "tokens_seen": 3046637568 + }, + { + "epoch": 0.85, + "learning_rate": 7.818181818181818e-05, + "loss": 2.4653, + "theoretical_loss": 3.3168251864518936, + "tokens_seen": 3046703104 + }, + { + "epoch": 0.85, + "learning_rate": 7.816161616161617e-05, + "loss": 2.5581, + "theoretical_loss": 3.316819718129474, + "tokens_seen": 3046768640 + }, + { + "epoch": 0.85, + "learning_rate": 7.814141414141414e-05, + "loss": 2.7203, + "theoretical_loss": 3.31681424995761, + "tokens_seen": 3046834176 + }, + { + "epoch": 0.85, + "learning_rate": 7.812121212121212e-05, + "loss": 2.6148, + "theoretical_loss": 3.3168087819362957, + "tokens_seen": 3046899712 + }, + { + "epoch": 0.85, + "learning_rate": 7.81010101010101e-05, + "loss": 2.5127, + "theoretical_loss": 3.316803314065522, + "tokens_seen": 3046965248 + }, + { + "epoch": 0.85, + "learning_rate": 7.808080808080809e-05, + "loss": 2.3215, + "theoretical_loss": 3.316797846345283, + "tokens_seen": 3047030784 + }, + { + "epoch": 0.85, + "learning_rate": 7.806060606060607e-05, + "loss": 2.4797, + "theoretical_loss": 3.3167923787755704, + "tokens_seen": 3047096320 + }, + { + "epoch": 0.85, + "learning_rate": 7.804040404040403e-05, + "loss": 2.4877, + "theoretical_loss": 3.316786911356377, + "tokens_seen": 3047161856 + }, + { + "epoch": 0.85, + "learning_rate": 7.802020202020201e-05, + "loss": 2.8184, + "theoretical_loss": 3.3167814440876957, + "tokens_seen": 3047227392 + }, + { + "epoch": 0.85, + "learning_rate": 7.8e-05, + "loss": 2.6513, + "theoretical_loss": 3.316775976969519, + "tokens_seen": 3047292928 + }, + { + "epoch": 0.85, + "learning_rate": 7.797979797979798e-05, + "loss": 2.5296, + "theoretical_loss": 3.316770510001839, + "tokens_seen": 3047358464 + }, + { + "epoch": 0.85, + "learning_rate": 7.795959595959597e-05, + "loss": 2.5245, + "theoretical_loss": 3.316765043184649, + "tokens_seen": 3047424000 + }, + { + "epoch": 0.85, + "learning_rate": 7.793939393939394e-05, + "loss": 2.7678, + "theoretical_loss": 3.316759576517941, + "tokens_seen": 3047489536 + }, + { + "epoch": 0.85, + "learning_rate": 7.791919191919192e-05, + "loss": 2.707, + "theoretical_loss": 3.316754110001708, + "tokens_seen": 3047555072 + }, + { + "epoch": 0.85, + "learning_rate": 7.78989898989899e-05, + "loss": 2.6985, + "theoretical_loss": 3.3167486436359432, + "tokens_seen": 3047620608 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1670610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6586220264434814, + "objective/train/theoretical_loss": 3.3167431774206384, + "objective/train/tokens_used": 1406545376, + "theoretical_loss": 3.3167431774206384, + "tokens_seen": 3047686144 + }, + { + "epoch": 0.85, + "learning_rate": 7.787878787878789e-05, + "loss": 2.6564, + "theoretical_loss": 3.3167431774206384, + "tokens_seen": 3047686144 + }, + { + "epoch": 0.85, + "learning_rate": 7.785858585858586e-05, + "loss": 2.5535, + "theoretical_loss": 3.316737711355786, + "tokens_seen": 3047751680 + }, + { + "epoch": 0.85, + "learning_rate": 7.783838383838383e-05, + "loss": 2.5341, + "theoretical_loss": 3.3167322454413792, + "tokens_seen": 3047817216 + }, + { + "epoch": 0.85, + "learning_rate": 7.781818181818181e-05, + "loss": 2.348, + "theoretical_loss": 3.3167267796774107, + "tokens_seen": 3047882752 + }, + { + "epoch": 0.85, + "learning_rate": 7.77979797979798e-05, + "loss": 2.6365, + "theoretical_loss": 3.316721314063873, + "tokens_seen": 3047948288 + }, + { + "epoch": 0.85, + "learning_rate": 7.777777777777778e-05, + "loss": 2.6191, + "theoretical_loss": 3.316715848600759, + "tokens_seen": 3048013824 + }, + { + "epoch": 0.85, + "learning_rate": 7.775757575757577e-05, + "loss": 2.5836, + "theoretical_loss": 3.3167103832880604, + "tokens_seen": 3048079360 + }, + { + "epoch": 0.85, + "learning_rate": 7.773737373737375e-05, + "loss": 2.5794, + "theoretical_loss": 3.3167049181257706, + "tokens_seen": 3048144896 + }, + { + "epoch": 0.85, + "learning_rate": 7.771717171717172e-05, + "loss": 2.364, + "theoretical_loss": 3.316699453113882, + "tokens_seen": 3048210432 + }, + { + "epoch": 0.85, + "learning_rate": 7.769696969696969e-05, + "loss": 2.524, + "theoretical_loss": 3.316693988252387, + "tokens_seen": 3048275968 + }, + { + "epoch": 0.85, + "learning_rate": 7.767676767676767e-05, + "loss": 2.562, + "theoretical_loss": 3.3166885235412784, + "tokens_seen": 3048341504 + }, + { + "epoch": 0.85, + "learning_rate": 7.765656565656566e-05, + "loss": 2.3627, + "theoretical_loss": 3.3166830589805496, + "tokens_seen": 3048407040 + }, + { + "epoch": 0.85, + "learning_rate": 7.763636363636364e-05, + "loss": 2.5919, + "theoretical_loss": 3.316677594570192, + "tokens_seen": 3048472576 + }, + { + "epoch": 0.85, + "learning_rate": 7.761616161616161e-05, + "loss": 2.5561, + "theoretical_loss": 3.3166721303101987, + "tokens_seen": 3048538112 + }, + { + "epoch": 0.85, + "learning_rate": 7.75959595959596e-05, + "loss": 2.4413, + "theoretical_loss": 3.316666666200563, + "tokens_seen": 3048603648 + }, + { + "epoch": 0.85, + "learning_rate": 7.757575757575758e-05, + "loss": 2.6516, + "theoretical_loss": 3.316661202241276, + "tokens_seen": 3048669184 + }, + { + "epoch": 0.85, + "learning_rate": 7.755555555555556e-05, + "loss": 2.5501, + "theoretical_loss": 3.316655738432332, + "tokens_seen": 3048734720 + }, + { + "epoch": 0.85, + "learning_rate": 7.753535353535355e-05, + "loss": 2.4876, + "theoretical_loss": 3.316650274773723, + "tokens_seen": 3048800256 + }, + { + "epoch": 0.85, + "learning_rate": 7.75151515151515e-05, + "loss": 2.3491, + "theoretical_loss": 3.3166448112654408, + "tokens_seen": 3048865792 + }, + { + "epoch": 0.85, + "learning_rate": 7.749494949494949e-05, + "loss": 2.5187, + "theoretical_loss": 3.316639347907479, + "tokens_seen": 3048931328 + }, + { + "epoch": 0.85, + "learning_rate": 7.747474747474747e-05, + "loss": 2.6707, + "theoretical_loss": 3.3166338846998302, + "tokens_seen": 3048996864 + }, + { + "epoch": 0.85, + "learning_rate": 7.745454545454546e-05, + "loss": 2.3096, + "theoretical_loss": 3.316628421642487, + "tokens_seen": 3049062400 + }, + { + "epoch": 0.85, + "learning_rate": 7.743434343434344e-05, + "loss": 2.5512, + "theoretical_loss": 3.316622958735442, + "tokens_seen": 3049127936 + }, + { + "epoch": 0.85, + "learning_rate": 7.741414141414141e-05, + "loss": 2.4984, + "theoretical_loss": 3.316617495978687, + "tokens_seen": 3049193472 + }, + { + "epoch": 0.85, + "learning_rate": 7.73939393939394e-05, + "loss": 2.346, + "theoretical_loss": 3.3166120333722158, + "tokens_seen": 3049259008 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1671708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.587533712387085, + "objective/train/theoretical_loss": 3.3166065709160204, + "objective/train/tokens_used": 1408183776, + "theoretical_loss": 3.3166065709160204, + "tokens_seen": 3049324544 + }, + { + "epoch": 0.85, + "learning_rate": 7.737373737373738e-05, + "loss": 2.4981, + "theoretical_loss": 3.3166065709160204, + "tokens_seen": 3049324544 + }, + { + "epoch": 0.85, + "learning_rate": 7.735353535353535e-05, + "loss": 2.495, + "theoretical_loss": 3.3166011086100937, + "tokens_seen": 3049390080 + }, + { + "epoch": 0.85, + "learning_rate": 7.733333333333333e-05, + "loss": 2.4028, + "theoretical_loss": 3.316595646454428, + "tokens_seen": 3049455616 + }, + { + "epoch": 0.85, + "learning_rate": 7.731313131313132e-05, + "loss": 2.4829, + "theoretical_loss": 3.3165901844490167, + "tokens_seen": 3049521152 + }, + { + "epoch": 0.85, + "learning_rate": 7.729292929292929e-05, + "loss": 2.5542, + "theoretical_loss": 3.316584722593851, + "tokens_seen": 3049586688 + }, + { + "epoch": 0.85, + "learning_rate": 7.727272727272727e-05, + "loss": 2.5382, + "theoretical_loss": 3.3165792608889255, + "tokens_seen": 3049652224 + }, + { + "epoch": 0.85, + "learning_rate": 7.725252525252526e-05, + "loss": 2.6618, + "theoretical_loss": 3.3165737993342312, + "tokens_seen": 3049717760 + }, + { + "epoch": 0.85, + "learning_rate": 7.723232323232324e-05, + "loss": 2.4222, + "theoretical_loss": 3.3165683379297612, + "tokens_seen": 3049783296 + }, + { + "epoch": 0.85, + "learning_rate": 7.721212121212122e-05, + "loss": 2.4644, + "theoretical_loss": 3.3165628766755084, + "tokens_seen": 3049848832 + }, + { + "epoch": 0.85, + "learning_rate": 7.71919191919192e-05, + "loss": 2.5982, + "theoretical_loss": 3.3165574155714657, + "tokens_seen": 3049914368 + }, + { + "epoch": 0.85, + "learning_rate": 7.717171717171716e-05, + "loss": 2.5176, + "theoretical_loss": 3.316551954617625, + "tokens_seen": 3049979904 + }, + { + "epoch": 0.85, + "learning_rate": 7.715151515151515e-05, + "loss": 2.5312, + "theoretical_loss": 3.3165464938139797, + "tokens_seen": 3050045440 + }, + { + "epoch": 0.85, + "learning_rate": 7.713131313131313e-05, + "loss": 2.4443, + "theoretical_loss": 3.3165410331605214, + "tokens_seen": 3050110976 + }, + { + "epoch": 0.85, + "learning_rate": 7.711111111111112e-05, + "loss": 2.5759, + "theoretical_loss": 3.3165355726572434, + "tokens_seen": 3050176512 + }, + { + "epoch": 0.85, + "learning_rate": 7.709090909090909e-05, + "loss": 2.564, + "theoretical_loss": 3.316530112304139, + "tokens_seen": 3050242048 + }, + { + "epoch": 0.85, + "learning_rate": 7.707070707070707e-05, + "loss": 2.5456, + "theoretical_loss": 3.3165246521011995, + "tokens_seen": 3050307584 + }, + { + "epoch": 0.85, + "learning_rate": 7.705050505050506e-05, + "loss": 2.3244, + "theoretical_loss": 3.3165191920484185, + "tokens_seen": 3050373120 + }, + { + "epoch": 0.85, + "learning_rate": 7.703030303030304e-05, + "loss": 2.4519, + "theoretical_loss": 3.3165137321457885, + "tokens_seen": 3050438656 + }, + { + "epoch": 0.85, + "learning_rate": 7.701010101010102e-05, + "loss": 2.2388, + "theoretical_loss": 3.316508272393302, + "tokens_seen": 3050504192 + }, + { + "epoch": 0.85, + "learning_rate": 7.698989898989898e-05, + "loss": 2.4075, + "theoretical_loss": 3.3165028127909513, + "tokens_seen": 3050569728 + }, + { + "epoch": 0.85, + "learning_rate": 7.696969696969696e-05, + "loss": 2.5223, + "theoretical_loss": 3.3164973533387294, + "tokens_seen": 3050635264 + }, + { + "epoch": 0.85, + "learning_rate": 7.694949494949495e-05, + "loss": 2.3208, + "theoretical_loss": 3.3164918940366293, + "tokens_seen": 3050700800 + }, + { + "epoch": 0.85, + "learning_rate": 7.692929292929293e-05, + "loss": 2.5414, + "theoretical_loss": 3.3164864348846432, + "tokens_seen": 3050766336 + }, + { + "epoch": 0.85, + "learning_rate": 7.690909090909092e-05, + "loss": 2.7604, + "theoretical_loss": 3.3164809758827634, + "tokens_seen": 3050831872 + }, + { + "epoch": 0.85, + "learning_rate": 7.68888888888889e-05, + "loss": 2.5467, + "theoretical_loss": 3.3164755170309834, + "tokens_seen": 3050897408 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1672113, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3430919647216797, + "objective/train/theoretical_loss": 3.3164700583292954, + "objective/train/tokens_used": 1409822176, + "theoretical_loss": 3.3164700583292954, + "tokens_seen": 3050962944 + }, + { + "epoch": 0.85, + "learning_rate": 7.686868686868687e-05, + "loss": 2.4189, + "theoretical_loss": 3.3164700583292954, + "tokens_seen": 3050962944 + }, + { + "epoch": 0.85, + "learning_rate": 7.684848484848485e-05, + "loss": 2.4683, + "theoretical_loss": 3.316464599777692, + "tokens_seen": 3051028480 + }, + { + "epoch": 0.85, + "learning_rate": 7.682828282828282e-05, + "loss": 2.6277, + "theoretical_loss": 3.316459141376166, + "tokens_seen": 3051094016 + }, + { + "epoch": 0.85, + "learning_rate": 7.680808080808081e-05, + "loss": 2.6567, + "theoretical_loss": 3.31645368312471, + "tokens_seen": 3051159552 + }, + { + "epoch": 0.85, + "learning_rate": 7.678787878787879e-05, + "loss": 2.5903, + "theoretical_loss": 3.3164482250233163, + "tokens_seen": 3051225088 + }, + { + "epoch": 0.85, + "learning_rate": 7.676767676767676e-05, + "loss": 2.5382, + "theoretical_loss": 3.3164427670719783, + "tokens_seen": 3051290624 + }, + { + "epoch": 0.85, + "learning_rate": 7.674747474747475e-05, + "loss": 2.515, + "theoretical_loss": 3.316437309270688, + "tokens_seen": 3051356160 + }, + { + "epoch": 0.85, + "learning_rate": 7.672727272727273e-05, + "loss": 2.5517, + "theoretical_loss": 3.316431851619438, + "tokens_seen": 3051421696 + }, + { + "epoch": 0.85, + "learning_rate": 7.670707070707071e-05, + "loss": 2.3782, + "theoretical_loss": 3.316426394118222, + "tokens_seen": 3051487232 + }, + { + "epoch": 0.85, + "learning_rate": 7.66868686868687e-05, + "loss": 2.5107, + "theoretical_loss": 3.316420936767031, + "tokens_seen": 3051552768 + }, + { + "epoch": 0.85, + "learning_rate": 7.666666666666667e-05, + "loss": 2.4767, + "theoretical_loss": 3.316415479565859, + "tokens_seen": 3051618304 + }, + { + "epoch": 0.85, + "learning_rate": 7.664646464646464e-05, + "loss": 2.5188, + "theoretical_loss": 3.3164100225146984, + "tokens_seen": 3051683840 + }, + { + "epoch": 0.85, + "learning_rate": 7.662626262626262e-05, + "loss": 2.6542, + "theoretical_loss": 3.3164045656135417, + "tokens_seen": 3051749376 + }, + { + "epoch": 0.85, + "learning_rate": 7.660606060606061e-05, + "loss": 2.3906, + "theoretical_loss": 3.316399108862381, + "tokens_seen": 3051814912 + }, + { + "epoch": 0.85, + "learning_rate": 7.658585858585859e-05, + "loss": 2.6318, + "theoretical_loss": 3.3163936522612096, + "tokens_seen": 3051880448 + }, + { + "epoch": 0.85, + "learning_rate": 7.656565656565656e-05, + "loss": 2.5951, + "theoretical_loss": 3.3163881958100205, + "tokens_seen": 3051945984 + }, + { + "epoch": 0.85, + "learning_rate": 7.654545454545455e-05, + "loss": 2.5271, + "theoretical_loss": 3.3163827395088052, + "tokens_seen": 3052011520 + }, + { + "epoch": 0.85, + "learning_rate": 7.652525252525253e-05, + "loss": 2.4681, + "theoretical_loss": 3.3163772833575575, + "tokens_seen": 3052077056 + }, + { + "epoch": 0.85, + "learning_rate": 7.650505050505051e-05, + "loss": 2.6533, + "theoretical_loss": 3.3163718273562695, + "tokens_seen": 3052142592 + }, + { + "epoch": 0.85, + "learning_rate": 7.648484848484848e-05, + "loss": 2.4145, + "theoretical_loss": 3.316366371504934, + "tokens_seen": 3052208128 + }, + { + "epoch": 0.85, + "learning_rate": 7.646464646464647e-05, + "loss": 2.5068, + "theoretical_loss": 3.3163609158035436, + "tokens_seen": 3052273664 + }, + { + "epoch": 0.85, + "learning_rate": 7.644444444444444e-05, + "loss": 2.8034, + "theoretical_loss": 3.316355460252091, + "tokens_seen": 3052339200 + }, + { + "epoch": 0.85, + "learning_rate": 7.642424242424242e-05, + "loss": 2.6007, + "theoretical_loss": 3.3163500048505687, + "tokens_seen": 3052404736 + }, + { + "epoch": 0.85, + "learning_rate": 7.64040404040404e-05, + "loss": 2.4587, + "theoretical_loss": 3.3163445495989694, + "tokens_seen": 3052470272 + }, + { + "epoch": 0.85, + "learning_rate": 7.638383838383839e-05, + "loss": 2.5191, + "theoretical_loss": 3.3163390944972857, + "tokens_seen": 3052535808 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1673312, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0533852577209473, + "objective/train/theoretical_loss": 3.316333639545511, + "objective/train/tokens_used": 1411460576, + "theoretical_loss": 3.316333639545511, + "tokens_seen": 3052601344 + }, + { + "epoch": 0.85, + "learning_rate": 7.636363636363637e-05, + "loss": 2.5129, + "theoretical_loss": 3.316333639545511, + "tokens_seen": 3052601344 + }, + { + "epoch": 0.85, + "learning_rate": 7.634343434343434e-05, + "loss": 2.5452, + "theoretical_loss": 3.316328184743637, + "tokens_seen": 3052666880 + }, + { + "epoch": 0.85, + "learning_rate": 7.632323232323233e-05, + "loss": 2.4095, + "theoretical_loss": 3.316322730091657, + "tokens_seen": 3052732416 + }, + { + "epoch": 0.85, + "learning_rate": 7.63030303030303e-05, + "loss": 2.565, + "theoretical_loss": 3.3163172755895634, + "tokens_seen": 3052797952 + }, + { + "epoch": 0.85, + "learning_rate": 7.628282828282828e-05, + "loss": 2.5079, + "theoretical_loss": 3.3163118212373486, + "tokens_seen": 3052863488 + }, + { + "epoch": 0.85, + "learning_rate": 7.626262626262627e-05, + "loss": 2.3899, + "theoretical_loss": 3.3163063670350055, + "tokens_seen": 3052929024 + }, + { + "epoch": 0.85, + "learning_rate": 7.624242424242424e-05, + "loss": 2.6336, + "theoretical_loss": 3.316300912982527, + "tokens_seen": 3052994560 + }, + { + "epoch": 0.85, + "learning_rate": 7.622222222222222e-05, + "loss": 2.6825, + "theoretical_loss": 3.3162954590799054, + "tokens_seen": 3053060096 + }, + { + "epoch": 0.85, + "learning_rate": 7.62020202020202e-05, + "loss": 2.5852, + "theoretical_loss": 3.316290005327134, + "tokens_seen": 3053125632 + }, + { + "epoch": 0.85, + "learning_rate": 7.618181818181819e-05, + "loss": 2.5886, + "theoretical_loss": 3.316284551724204, + "tokens_seen": 3053191168 + }, + { + "epoch": 0.85, + "learning_rate": 7.616161616161617e-05, + "loss": 2.7491, + "theoretical_loss": 3.31627909827111, + "tokens_seen": 3053256704 + }, + { + "epoch": 0.85, + "learning_rate": 7.614141414141413e-05, + "loss": 2.417, + "theoretical_loss": 3.3162736449678434, + "tokens_seen": 3053322240 + }, + { + "epoch": 0.85, + "learning_rate": 7.612121212121211e-05, + "loss": 2.3772, + "theoretical_loss": 3.3162681918143972, + "tokens_seen": 3053387776 + }, + { + "epoch": 0.85, + "learning_rate": 7.61010101010101e-05, + "loss": 2.308, + "theoretical_loss": 3.3162627388107637, + "tokens_seen": 3053453312 + }, + { + "epoch": 0.85, + "learning_rate": 7.608080808080808e-05, + "loss": 2.2644, + "theoretical_loss": 3.3162572859569366, + "tokens_seen": 3053518848 + }, + { + "epoch": 0.85, + "learning_rate": 7.606060606060607e-05, + "loss": 2.647, + "theoretical_loss": 3.316251833252908, + "tokens_seen": 3053584384 + }, + { + "epoch": 0.85, + "learning_rate": 7.604040404040405e-05, + "loss": 2.46, + "theoretical_loss": 3.3162463806986695, + "tokens_seen": 3053649920 + }, + { + "epoch": 0.85, + "learning_rate": 7.602020202020202e-05, + "loss": 2.6154, + "theoretical_loss": 3.3162409282942154, + "tokens_seen": 3053715456 + }, + { + "epoch": 0.85, + "learning_rate": 7.6e-05, + "loss": 2.7005, + "theoretical_loss": 3.316235476039538, + "tokens_seen": 3053780992 + }, + { + "epoch": 0.85, + "learning_rate": 7.597979797979799e-05, + "loss": 2.5842, + "theoretical_loss": 3.316230023934629, + "tokens_seen": 3053846528 + }, + { + "epoch": 0.85, + "learning_rate": 7.595959595959596e-05, + "loss": 2.4351, + "theoretical_loss": 3.316224571979482, + "tokens_seen": 3053912064 + }, + { + "epoch": 0.85, + "learning_rate": 7.593939393939394e-05, + "loss": 2.4979, + "theoretical_loss": 3.3162191201740896, + "tokens_seen": 3053977600 + }, + { + "epoch": 0.85, + "learning_rate": 7.591919191919191e-05, + "loss": 2.6221, + "theoretical_loss": 3.3162136685184445, + "tokens_seen": 3054043136 + }, + { + "epoch": 0.85, + "learning_rate": 7.58989898989899e-05, + "loss": 2.5945, + "theoretical_loss": 3.3162082170125387, + "tokens_seen": 3054108672 + }, + { + "epoch": 0.85, + "learning_rate": 7.587878787878788e-05, + "loss": 2.4199, + "theoretical_loss": 3.3162027656563655, + "tokens_seen": 3054174208 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1674056, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5496859550476074, + "objective/train/theoretical_loss": 3.3161973144499175, + "objective/train/tokens_used": 1413098976, + "theoretical_loss": 3.3161973144499175, + "tokens_seen": 3054239744 + }, + { + "epoch": 0.85, + "learning_rate": 7.585858585858586e-05, + "loss": 2.5195, + "theoretical_loss": 3.3161973144499175, + "tokens_seen": 3054239744 + }, + { + "epoch": 0.85, + "learning_rate": 7.583838383838385e-05, + "loss": 2.6354, + "theoretical_loss": 3.316191863393187, + "tokens_seen": 3054305280 + }, + { + "epoch": 0.85, + "learning_rate": 7.581818181818182e-05, + "loss": 2.4182, + "theoretical_loss": 3.3161864124861675, + "tokens_seen": 3054370816 + }, + { + "epoch": 0.85, + "learning_rate": 7.57979797979798e-05, + "loss": 2.6001, + "theoretical_loss": 3.316180961728851, + "tokens_seen": 3054436352 + }, + { + "epoch": 0.85, + "learning_rate": 7.577777777777777e-05, + "loss": 2.6858, + "theoretical_loss": 3.31617551112123, + "tokens_seen": 3054501888 + }, + { + "epoch": 0.85, + "learning_rate": 7.575757575757576e-05, + "loss": 2.4681, + "theoretical_loss": 3.316170060663298, + "tokens_seen": 3054567424 + }, + { + "epoch": 0.85, + "learning_rate": 7.573737373737374e-05, + "loss": 2.7049, + "theoretical_loss": 3.316164610355047, + "tokens_seen": 3054632960 + }, + { + "epoch": 0.85, + "learning_rate": 7.571717171717171e-05, + "loss": 2.4163, + "theoretical_loss": 3.3161591601964697, + "tokens_seen": 3054698496 + }, + { + "epoch": 0.85, + "learning_rate": 7.56969696969697e-05, + "loss": 2.7114, + "theoretical_loss": 3.316153710187559, + "tokens_seen": 3054764032 + }, + { + "epoch": 0.85, + "learning_rate": 7.567676767676768e-05, + "loss": 2.4922, + "theoretical_loss": 3.3161482603283075, + "tokens_seen": 3054829568 + }, + { + "epoch": 0.85, + "learning_rate": 7.565656565656566e-05, + "loss": 2.4424, + "theoretical_loss": 3.316142810618708, + "tokens_seen": 3054895104 + }, + { + "epoch": 0.85, + "learning_rate": 7.563636363636365e-05, + "loss": 2.5637, + "theoretical_loss": 3.316137361058753, + "tokens_seen": 3054960640 + }, + { + "epoch": 0.85, + "learning_rate": 7.561616161616162e-05, + "loss": 2.7039, + "theoretical_loss": 3.3161319116484353, + "tokens_seen": 3055026176 + }, + { + "epoch": 0.85, + "learning_rate": 7.559595959595959e-05, + "loss": 2.5694, + "theoretical_loss": 3.3161264623877473, + "tokens_seen": 3055091712 + }, + { + "epoch": 0.85, + "learning_rate": 7.557575757575757e-05, + "loss": 2.4579, + "theoretical_loss": 3.3161210132766823, + "tokens_seen": 3055157248 + }, + { + "epoch": 0.85, + "learning_rate": 7.555555555555556e-05, + "loss": 2.671, + "theoretical_loss": 3.3161155643152327, + "tokens_seen": 3055222784 + }, + { + "epoch": 0.85, + "learning_rate": 7.553535353535354e-05, + "loss": 2.5609, + "theoretical_loss": 3.316110115503391, + "tokens_seen": 3055288320 + }, + { + "epoch": 0.85, + "learning_rate": 7.551515151515152e-05, + "loss": 2.6479, + "theoretical_loss": 3.3161046668411496, + "tokens_seen": 3055353856 + }, + { + "epoch": 0.85, + "learning_rate": 7.54949494949495e-05, + "loss": 2.5099, + "theoretical_loss": 3.316099218328502, + "tokens_seen": 3055419392 + }, + { + "epoch": 0.85, + "learning_rate": 7.547474747474748e-05, + "loss": 2.7343, + "theoretical_loss": 3.3160937699654403, + "tokens_seen": 3055484928 + }, + { + "epoch": 0.85, + "learning_rate": 7.545454545454546e-05, + "loss": 2.6282, + "theoretical_loss": 3.3160883217519572, + "tokens_seen": 3055550464 + }, + { + "epoch": 0.85, + "learning_rate": 7.543434343434343e-05, + "loss": 2.4051, + "theoretical_loss": 3.316082873688045, + "tokens_seen": 3055616000 + }, + { + "epoch": 0.85, + "learning_rate": 7.541414141414142e-05, + "loss": 2.4086, + "theoretical_loss": 3.316077425773698, + "tokens_seen": 3055681536 + }, + { + "epoch": 0.85, + "learning_rate": 7.539393939393939e-05, + "loss": 2.4319, + "theoretical_loss": 3.316071978008907, + "tokens_seen": 3055747072 + }, + { + "epoch": 0.85, + "learning_rate": 7.537373737373737e-05, + "loss": 2.6793, + "theoretical_loss": 3.316066530393666, + "tokens_seen": 3055812608 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1674689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7513811588287354, + "objective/train/theoretical_loss": 3.316061082927967, + "objective/train/tokens_used": 1414737376, + "theoretical_loss": 3.316061082927967, + "tokens_seen": 3055878144 + }, + { + "epoch": 0.85, + "learning_rate": 7.535353535353536e-05, + "loss": 2.6479, + "theoretical_loss": 3.316061082927967, + "tokens_seen": 3055878144 + }, + { + "epoch": 0.85, + "learning_rate": 7.533333333333334e-05, + "loss": 2.7067, + "theoretical_loss": 3.3160556356118027, + "tokens_seen": 3055943680 + }, + { + "epoch": 0.85, + "learning_rate": 7.531313131313132e-05, + "loss": 2.5991, + "theoretical_loss": 3.316050188445166, + "tokens_seen": 3056009216 + }, + { + "epoch": 0.85, + "learning_rate": 7.52929292929293e-05, + "loss": 2.4816, + "theoretical_loss": 3.3160447414280494, + "tokens_seen": 3056074752 + }, + { + "epoch": 0.85, + "learning_rate": 7.527272727272726e-05, + "loss": 2.6149, + "theoretical_loss": 3.3160392945604458, + "tokens_seen": 3056140288 + }, + { + "epoch": 0.85, + "learning_rate": 7.525252525252525e-05, + "loss": 2.4937, + "theoretical_loss": 3.316033847842348, + "tokens_seen": 3056205824 + }, + { + "epoch": 0.85, + "learning_rate": 7.523232323232323e-05, + "loss": 2.6852, + "theoretical_loss": 3.316028401273748, + "tokens_seen": 3056271360 + }, + { + "epoch": 0.85, + "learning_rate": 7.521212121212122e-05, + "loss": 2.6019, + "theoretical_loss": 3.3160229548546396, + "tokens_seen": 3056336896 + }, + { + "epoch": 0.85, + "learning_rate": 7.51919191919192e-05, + "loss": 2.3766, + "theoretical_loss": 3.3160175085850145, + "tokens_seen": 3056402432 + }, + { + "epoch": 0.85, + "learning_rate": 7.517171717171717e-05, + "loss": 2.5904, + "theoretical_loss": 3.316012062464866, + "tokens_seen": 3056467968 + }, + { + "epoch": 0.85, + "learning_rate": 7.515151515151515e-05, + "loss": 2.4385, + "theoretical_loss": 3.316006616494186, + "tokens_seen": 3056533504 + }, + { + "epoch": 0.85, + "learning_rate": 7.513131313131314e-05, + "loss": 2.5836, + "theoretical_loss": 3.316001170672968, + "tokens_seen": 3056599040 + }, + { + "epoch": 0.85, + "learning_rate": 7.511111111111112e-05, + "loss": 2.5512, + "theoretical_loss": 3.315995725001205, + "tokens_seen": 3056664576 + }, + { + "epoch": 0.85, + "learning_rate": 7.509090909090909e-05, + "loss": 2.4716, + "theoretical_loss": 3.3159902794788887, + "tokens_seen": 3056730112 + }, + { + "epoch": 0.85, + "learning_rate": 7.507070707070706e-05, + "loss": 2.6883, + "theoretical_loss": 3.3159848341060125, + "tokens_seen": 3056795648 + }, + { + "epoch": 0.85, + "learning_rate": 7.505050505050505e-05, + "loss": 2.6142, + "theoretical_loss": 3.3159793888825684, + "tokens_seen": 3056861184 + }, + { + "epoch": 0.85, + "learning_rate": 7.503030303030303e-05, + "loss": 2.4873, + "theoretical_loss": 3.31597394380855, + "tokens_seen": 3056926720 + }, + { + "epoch": 0.85, + "learning_rate": 7.501010101010102e-05, + "loss": 2.3994, + "theoretical_loss": 3.315968498883949, + "tokens_seen": 3056992256 + }, + { + "epoch": 0.85, + "learning_rate": 7.4989898989899e-05, + "loss": 2.5393, + "theoretical_loss": 3.3159630541087592, + "tokens_seen": 3057057792 + }, + { + "epoch": 0.85, + "learning_rate": 7.496969696969697e-05, + "loss": 2.6678, + "theoretical_loss": 3.3159576094829726, + "tokens_seen": 3057123328 + }, + { + "epoch": 0.85, + "learning_rate": 7.494949494949495e-05, + "loss": 2.6532, + "theoretical_loss": 3.3159521650065815, + "tokens_seen": 3057188864 + }, + { + "epoch": 0.85, + "learning_rate": 7.492929292929292e-05, + "loss": 2.7624, + "theoretical_loss": 3.3159467206795794, + "tokens_seen": 3057254400 + }, + { + "epoch": 0.85, + "learning_rate": 7.490909090909091e-05, + "loss": 2.3264, + "theoretical_loss": 3.3159412765019587, + "tokens_seen": 3057319936 + }, + { + "epoch": 0.85, + "learning_rate": 7.488888888888889e-05, + "loss": 2.3513, + "theoretical_loss": 3.3159358324737123, + "tokens_seen": 3057385472 + }, + { + "epoch": 0.85, + "learning_rate": 7.486868686868686e-05, + "loss": 2.3933, + "theoretical_loss": 3.3159303885948326, + "tokens_seen": 3057451008 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1675857, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.530817747116089, + "objective/train/theoretical_loss": 3.3159249448653125, + "objective/train/tokens_used": 1416375776, + "theoretical_loss": 3.3159249448653125, + "tokens_seen": 3057516544 + }, + { + "epoch": 0.85, + "learning_rate": 7.484848484848485e-05, + "loss": 2.508, + "theoretical_loss": 3.3159249448653125, + "tokens_seen": 3057516544 + }, + { + "epoch": 0.85, + "learning_rate": 7.482828282828283e-05, + "loss": 2.4841, + "theoretical_loss": 3.3159195012851446, + "tokens_seen": 3057582080 + }, + { + "epoch": 0.85, + "learning_rate": 7.480808080808081e-05, + "loss": 2.5826, + "theoretical_loss": 3.3159140578543216, + "tokens_seen": 3057647616 + }, + { + "epoch": 0.85, + "learning_rate": 7.47878787878788e-05, + "loss": 2.5008, + "theoretical_loss": 3.315908614572836, + "tokens_seen": 3057713152 + }, + { + "epoch": 0.85, + "learning_rate": 7.476767676767678e-05, + "loss": 2.5316, + "theoretical_loss": 3.315903171440681, + "tokens_seen": 3057778688 + }, + { + "epoch": 0.85, + "learning_rate": 7.474747474747474e-05, + "loss": 2.6011, + "theoretical_loss": 3.315897728457849, + "tokens_seen": 3057844224 + }, + { + "epoch": 0.85, + "learning_rate": 7.472727272727272e-05, + "loss": 2.4907, + "theoretical_loss": 3.3158922856243325, + "tokens_seen": 3057909760 + }, + { + "epoch": 0.85, + "learning_rate": 7.47070707070707e-05, + "loss": 2.4544, + "theoretical_loss": 3.3158868429401247, + "tokens_seen": 3057975296 + }, + { + "epoch": 0.85, + "learning_rate": 7.468686868686869e-05, + "loss": 2.5346, + "theoretical_loss": 3.3158814004052175, + "tokens_seen": 3058040832 + }, + { + "epoch": 0.85, + "learning_rate": 7.466666666666667e-05, + "loss": 2.5592, + "theoretical_loss": 3.3158759580196047, + "tokens_seen": 3058106368 + }, + { + "epoch": 0.85, + "learning_rate": 7.464646464646464e-05, + "loss": 2.4374, + "theoretical_loss": 3.3158705157832786, + "tokens_seen": 3058171904 + }, + { + "epoch": 0.85, + "learning_rate": 7.462626262626263e-05, + "loss": 2.5685, + "theoretical_loss": 3.315865073696231, + "tokens_seen": 3058237440 + }, + { + "epoch": 0.85, + "learning_rate": 7.460606060606061e-05, + "loss": 2.4606, + "theoretical_loss": 3.315859631758456, + "tokens_seen": 3058302976 + }, + { + "epoch": 0.85, + "learning_rate": 7.45858585858586e-05, + "loss": 2.6667, + "theoretical_loss": 3.3158541899699454, + "tokens_seen": 3058368512 + }, + { + "epoch": 0.85, + "learning_rate": 7.456565656565657e-05, + "loss": 2.4679, + "theoretical_loss": 3.3158487483306924, + "tokens_seen": 3058434048 + }, + { + "epoch": 0.85, + "learning_rate": 7.454545454545454e-05, + "loss": 2.4929, + "theoretical_loss": 3.315843306840689, + "tokens_seen": 3058499584 + }, + { + "epoch": 0.85, + "learning_rate": 7.452525252525252e-05, + "loss": 2.4261, + "theoretical_loss": 3.3158378654999288, + "tokens_seen": 3058565120 + }, + { + "epoch": 0.85, + "learning_rate": 7.45050505050505e-05, + "loss": 2.3159, + "theoretical_loss": 3.315832424308404, + "tokens_seen": 3058630656 + }, + { + "epoch": 0.85, + "learning_rate": 7.448484848484849e-05, + "loss": 2.5007, + "theoretical_loss": 3.315826983266107, + "tokens_seen": 3058696192 + }, + { + "epoch": 0.85, + "learning_rate": 7.446464646464647e-05, + "loss": 2.4335, + "theoretical_loss": 3.3158215423730315, + "tokens_seen": 3058761728 + }, + { + "epoch": 0.85, + "learning_rate": 7.444444444444444e-05, + "loss": 2.5351, + "theoretical_loss": 3.31581610162917, + "tokens_seen": 3058827264 + }, + { + "epoch": 0.85, + "learning_rate": 7.442424242424243e-05, + "loss": 2.3689, + "theoretical_loss": 3.315810661034514, + "tokens_seen": 3058892800 + }, + { + "epoch": 0.85, + "learning_rate": 7.44040404040404e-05, + "loss": 2.5123, + "theoretical_loss": 3.3158052205890574, + "tokens_seen": 3058958336 + }, + { + "epoch": 0.85, + "learning_rate": 7.438383838383838e-05, + "loss": 2.56, + "theoretical_loss": 3.3157997802927923, + "tokens_seen": 3059023872 + }, + { + "epoch": 0.85, + "learning_rate": 7.436363636363637e-05, + "loss": 2.6379, + "theoretical_loss": 3.315794340145712, + "tokens_seen": 3059089408 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1676525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.704979181289673, + "objective/train/theoretical_loss": 3.3157889001478087, + "objective/train/tokens_used": 1418014176, + "theoretical_loss": 3.3157889001478087, + "tokens_seen": 3059154944 + }, + { + "epoch": 0.85, + "learning_rate": 7.434343434343435e-05, + "loss": 2.5941, + "theoretical_loss": 3.3157889001478087, + "tokens_seen": 3059154944 + }, + { + "epoch": 0.85, + "learning_rate": 7.432323232323232e-05, + "loss": 2.7479, + "theoretical_loss": 3.3157834602990754, + "tokens_seen": 3059220480 + }, + { + "epoch": 0.85, + "learning_rate": 7.43030303030303e-05, + "loss": 2.7755, + "theoretical_loss": 3.3157780205995047, + "tokens_seen": 3059286016 + }, + { + "epoch": 0.85, + "learning_rate": 7.428282828282829e-05, + "loss": 2.5381, + "theoretical_loss": 3.3157725810490892, + "tokens_seen": 3059351552 + }, + { + "epoch": 0.85, + "learning_rate": 7.426262626262627e-05, + "loss": 2.366, + "theoretical_loss": 3.3157671416478216, + "tokens_seen": 3059417088 + }, + { + "epoch": 0.85, + "learning_rate": 7.424242424242426e-05, + "loss": 2.753, + "theoretical_loss": 3.3157617023956956, + "tokens_seen": 3059482624 + }, + { + "epoch": 0.85, + "learning_rate": 7.422222222222221e-05, + "loss": 2.4703, + "theoretical_loss": 3.3157562632927022, + "tokens_seen": 3059548160 + }, + { + "epoch": 0.85, + "learning_rate": 7.42020202020202e-05, + "loss": 2.4805, + "theoretical_loss": 3.3157508243388354, + "tokens_seen": 3059613696 + }, + { + "epoch": 0.85, + "learning_rate": 7.418181818181818e-05, + "loss": 2.3341, + "theoretical_loss": 3.315745385534087, + "tokens_seen": 3059679232 + }, + { + "epoch": 0.85, + "learning_rate": 7.416161616161617e-05, + "loss": 2.6743, + "theoretical_loss": 3.315739946878451, + "tokens_seen": 3059744768 + }, + { + "epoch": 0.85, + "learning_rate": 7.414141414141415e-05, + "loss": 2.6222, + "theoretical_loss": 3.3157345083719187, + "tokens_seen": 3059810304 + }, + { + "epoch": 0.85, + "learning_rate": 7.412121212121212e-05, + "loss": 2.5243, + "theoretical_loss": 3.3157290700144837, + "tokens_seen": 3059875840 + }, + { + "epoch": 0.85, + "learning_rate": 7.41010101010101e-05, + "loss": 2.5342, + "theoretical_loss": 3.3157236318061387, + "tokens_seen": 3059941376 + }, + { + "epoch": 0.85, + "learning_rate": 7.408080808080809e-05, + "loss": 2.4809, + "theoretical_loss": 3.315718193746876, + "tokens_seen": 3060006912 + }, + { + "epoch": 0.85, + "learning_rate": 7.406060606060606e-05, + "loss": 2.269, + "theoretical_loss": 3.315712755836689, + "tokens_seen": 3060072448 + }, + { + "epoch": 0.85, + "learning_rate": 7.404040404040404e-05, + "loss": 2.3969, + "theoretical_loss": 3.315707318075569, + "tokens_seen": 3060137984 + }, + { + "epoch": 0.85, + "learning_rate": 7.402020202020201e-05, + "loss": 2.406, + "theoretical_loss": 3.3157018804635103, + "tokens_seen": 3060203520 + }, + { + "epoch": 0.85, + "learning_rate": 7.4e-05, + "loss": 2.4797, + "theoretical_loss": 3.3156964430005047, + "tokens_seen": 3060269056 + }, + { + "epoch": 0.85, + "learning_rate": 7.397979797979798e-05, + "loss": 2.6589, + "theoretical_loss": 3.3156910056865456, + "tokens_seen": 3060334592 + }, + { + "epoch": 0.85, + "learning_rate": 7.395959595959596e-05, + "loss": 2.5948, + "theoretical_loss": 3.315685568521625, + "tokens_seen": 3060400128 + }, + { + "epoch": 0.85, + "learning_rate": 7.393939393939395e-05, + "loss": 2.71, + "theoretical_loss": 3.315680131505736, + "tokens_seen": 3060465664 + }, + { + "epoch": 0.85, + "learning_rate": 7.391919191919193e-05, + "loss": 2.4616, + "theoretical_loss": 3.3156746946388713, + "tokens_seen": 3060531200 + }, + { + "epoch": 0.85, + "learning_rate": 7.38989898989899e-05, + "loss": 2.4498, + "theoretical_loss": 3.315669257921024, + "tokens_seen": 3060596736 + }, + { + "epoch": 0.85, + "learning_rate": 7.387878787878787e-05, + "loss": 2.4733, + "theoretical_loss": 3.315663821352186, + "tokens_seen": 3060662272 + }, + { + "epoch": 0.85, + "learning_rate": 7.385858585858586e-05, + "loss": 2.6823, + "theoretical_loss": 3.3156583849323504, + "tokens_seen": 3060727808 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1677413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6018528938293457, + "objective/train/theoretical_loss": 3.3156529486615103, + "objective/train/tokens_used": 1419652576, + "theoretical_loss": 3.3156529486615103, + "tokens_seen": 3060793344 + }, + { + "epoch": 0.85, + "learning_rate": 7.383838383838384e-05, + "loss": 2.4856, + "theoretical_loss": 3.3156529486615103, + "tokens_seen": 3060793344 + }, + { + "epoch": 0.85, + "learning_rate": 7.381818181818182e-05, + "loss": 2.3699, + "theoretical_loss": 3.315647512539658, + "tokens_seen": 3060858880 + }, + { + "epoch": 0.85, + "learning_rate": 7.37979797979798e-05, + "loss": 2.683, + "theoretical_loss": 3.3156420765667862, + "tokens_seen": 3060924416 + }, + { + "epoch": 0.85, + "learning_rate": 7.377777777777778e-05, + "loss": 2.6268, + "theoretical_loss": 3.3156366407428877, + "tokens_seen": 3060989952 + }, + { + "epoch": 0.85, + "learning_rate": 7.375757575757576e-05, + "loss": 2.7515, + "theoretical_loss": 3.3156312050679553, + "tokens_seen": 3061055488 + }, + { + "epoch": 0.85, + "learning_rate": 7.373737373737375e-05, + "loss": 2.3463, + "theoretical_loss": 3.315625769541982, + "tokens_seen": 3061121024 + }, + { + "epoch": 0.85, + "learning_rate": 7.371717171717173e-05, + "loss": 2.5109, + "theoretical_loss": 3.31562033416496, + "tokens_seen": 3061186560 + }, + { + "epoch": 0.85, + "learning_rate": 7.369696969696969e-05, + "loss": 2.6413, + "theoretical_loss": 3.3156148989368823, + "tokens_seen": 3061252096 + }, + { + "epoch": 0.85, + "learning_rate": 7.367676767676767e-05, + "loss": 2.4309, + "theoretical_loss": 3.315609463857742, + "tokens_seen": 3061317632 + }, + { + "epoch": 0.85, + "learning_rate": 7.365656565656566e-05, + "loss": 2.4604, + "theoretical_loss": 3.315604028927531, + "tokens_seen": 3061383168 + }, + { + "epoch": 0.85, + "learning_rate": 7.363636363636364e-05, + "loss": 2.379, + "theoretical_loss": 3.3155985941462425, + "tokens_seen": 3061448704 + }, + { + "epoch": 0.85, + "learning_rate": 7.361616161616162e-05, + "loss": 2.5011, + "theoretical_loss": 3.3155931595138695, + "tokens_seen": 3061514240 + }, + { + "epoch": 0.85, + "learning_rate": 7.35959595959596e-05, + "loss": 2.6115, + "theoretical_loss": 3.315587725030404, + "tokens_seen": 3061579776 + }, + { + "epoch": 0.85, + "learning_rate": 7.357575757575758e-05, + "loss": 2.6024, + "theoretical_loss": 3.3155822906958394, + "tokens_seen": 3061645312 + }, + { + "epoch": 0.85, + "learning_rate": 7.355555555555556e-05, + "loss": 2.5751, + "theoretical_loss": 3.315576856510168, + "tokens_seen": 3061710848 + }, + { + "epoch": 0.85, + "learning_rate": 7.353535353535353e-05, + "loss": 2.5142, + "theoretical_loss": 3.315571422473383, + "tokens_seen": 3061776384 + }, + { + "epoch": 0.85, + "learning_rate": 7.351515151515152e-05, + "loss": 2.6883, + "theoretical_loss": 3.315565988585477, + "tokens_seen": 3061841920 + }, + { + "epoch": 0.85, + "learning_rate": 7.349494949494949e-05, + "loss": 2.4044, + "theoretical_loss": 3.315560554846442, + "tokens_seen": 3061907456 + }, + { + "epoch": 0.85, + "learning_rate": 7.347474747474747e-05, + "loss": 2.43, + "theoretical_loss": 3.3155551212562724, + "tokens_seen": 3061972992 + }, + { + "epoch": 0.85, + "learning_rate": 7.345454545454545e-05, + "loss": 2.4579, + "theoretical_loss": 3.315549687814959, + "tokens_seen": 3062038528 + }, + { + "epoch": 0.85, + "learning_rate": 7.343434343434344e-05, + "loss": 2.4071, + "theoretical_loss": 3.3155442545224956, + "tokens_seen": 3062104064 + }, + { + "epoch": 0.85, + "learning_rate": 7.341414141414142e-05, + "loss": 2.6182, + "theoretical_loss": 3.315538821378875, + "tokens_seen": 3062169600 + }, + { + "epoch": 0.85, + "learning_rate": 7.33939393939394e-05, + "loss": 2.5191, + "theoretical_loss": 3.315533388384089, + "tokens_seen": 3062235136 + }, + { + "epoch": 0.85, + "learning_rate": 7.337373737373738e-05, + "loss": 2.5664, + "theoretical_loss": 3.3155279555381316, + "tokens_seen": 3062300672 + }, + { + "epoch": 0.85, + "learning_rate": 7.335353535353535e-05, + "loss": 2.6448, + "theoretical_loss": 3.315522522840995, + "tokens_seen": 3062366208 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1677869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.561100721359253, + "objective/train/theoretical_loss": 3.315517090292672, + "objective/train/tokens_used": 1421290976, + "theoretical_loss": 3.315517090292672, + "tokens_seen": 3062431744 + }, + { + "epoch": 0.85, + "learning_rate": 7.333333333333333e-05, + "loss": 2.4887, + "theoretical_loss": 3.315517090292672, + "tokens_seen": 3062431744 + }, + { + "epoch": 0.86, + "learning_rate": 7.331313131313132e-05, + "loss": 2.6233, + "theoretical_loss": 3.3155116578931545, + "tokens_seen": 3062497280 + }, + { + "epoch": 0.86, + "learning_rate": 7.32929292929293e-05, + "loss": 2.4811, + "theoretical_loss": 3.3155062256424364, + "tokens_seen": 3062562816 + }, + { + "epoch": 0.86, + "learning_rate": 7.327272727272727e-05, + "loss": 2.4493, + "theoretical_loss": 3.31550079354051, + "tokens_seen": 3062628352 + }, + { + "epoch": 0.86, + "learning_rate": 7.325252525252525e-05, + "loss": 2.5039, + "theoretical_loss": 3.3154953615873684, + "tokens_seen": 3062693888 + }, + { + "epoch": 0.86, + "learning_rate": 7.323232323232324e-05, + "loss": 2.7014, + "theoretical_loss": 3.315489929783004, + "tokens_seen": 3062759424 + }, + { + "epoch": 0.86, + "learning_rate": 7.321212121212122e-05, + "loss": 2.4636, + "theoretical_loss": 3.3154844981274088, + "tokens_seen": 3062824960 + }, + { + "epoch": 0.86, + "learning_rate": 7.319191919191919e-05, + "loss": 2.3658, + "theoretical_loss": 3.315479066620577, + "tokens_seen": 3062890496 + }, + { + "epoch": 0.86, + "learning_rate": 7.317171717171716e-05, + "loss": 2.4867, + "theoretical_loss": 3.3154736352625003, + "tokens_seen": 3062956032 + }, + { + "epoch": 0.86, + "learning_rate": 7.315151515151515e-05, + "loss": 2.33, + "theoretical_loss": 3.3154682040531718, + "tokens_seen": 3063021568 + }, + { + "epoch": 0.86, + "learning_rate": 7.313131313131313e-05, + "loss": 2.2552, + "theoretical_loss": 3.315462772992584, + "tokens_seen": 3063087104 + }, + { + "epoch": 0.86, + "learning_rate": 7.311111111111111e-05, + "loss": 2.6051, + "theoretical_loss": 3.3154573420807303, + "tokens_seen": 3063152640 + }, + { + "epoch": 0.86, + "learning_rate": 7.30909090909091e-05, + "loss": 2.6304, + "theoretical_loss": 3.3154519113176026, + "tokens_seen": 3063218176 + }, + { + "epoch": 0.86, + "learning_rate": 7.307070707070707e-05, + "loss": 2.6308, + "theoretical_loss": 3.3154464807031943, + "tokens_seen": 3063283712 + }, + { + "epoch": 0.86, + "learning_rate": 7.305050505050505e-05, + "loss": 2.3528, + "theoretical_loss": 3.3154410502374976, + "tokens_seen": 3063349248 + }, + { + "epoch": 0.86, + "learning_rate": 7.303030303030304e-05, + "loss": 2.4231, + "theoretical_loss": 3.315435619920506, + "tokens_seen": 3063414784 + }, + { + "epoch": 0.86, + "learning_rate": 7.301010101010101e-05, + "loss": 2.448, + "theoretical_loss": 3.3154301897522114, + "tokens_seen": 3063480320 + }, + { + "epoch": 0.86, + "learning_rate": 7.298989898989899e-05, + "loss": 2.4825, + "theoretical_loss": 3.315424759732607, + "tokens_seen": 3063545856 + }, + { + "epoch": 0.86, + "learning_rate": 7.296969696969697e-05, + "loss": 2.5486, + "theoretical_loss": 3.315419329861686, + "tokens_seen": 3063611392 + }, + { + "epoch": 0.86, + "learning_rate": 7.294949494949495e-05, + "loss": 2.6955, + "theoretical_loss": 3.31541390013944, + "tokens_seen": 3063676928 + }, + { + "epoch": 0.86, + "learning_rate": 7.292929292929293e-05, + "loss": 2.4778, + "theoretical_loss": 3.3154084705658624, + "tokens_seen": 3063742464 + }, + { + "epoch": 0.86, + "learning_rate": 7.290909090909091e-05, + "loss": 2.3729, + "theoretical_loss": 3.3154030411409465, + "tokens_seen": 3063808000 + }, + { + "epoch": 0.86, + "learning_rate": 7.28888888888889e-05, + "loss": 2.6235, + "theoretical_loss": 3.3153976118646837, + "tokens_seen": 3063873536 + }, + { + "epoch": 0.86, + "learning_rate": 7.286868686868688e-05, + "loss": 2.4906, + "theoretical_loss": 3.3153921827370683, + "tokens_seen": 3063939072 + }, + { + "epoch": 0.86, + "learning_rate": 7.284848484848484e-05, + "loss": 2.5501, + "theoretical_loss": 3.3153867537580917, + "tokens_seen": 3064004608 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1679004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.637289047241211, + "objective/train/theoretical_loss": 3.3153813249277473, + "objective/train/tokens_used": 1422929376, + "theoretical_loss": 3.3153813249277473, + "tokens_seen": 3064070144 + }, + { + "epoch": 0.86, + "learning_rate": 7.282828282828282e-05, + "loss": 2.5943, + "theoretical_loss": 3.3153813249277473, + "tokens_seen": 3064070144 + }, + { + "epoch": 0.86, + "learning_rate": 7.28080808080808e-05, + "loss": 2.3298, + "theoretical_loss": 3.315375896246028, + "tokens_seen": 3064135680 + }, + { + "epoch": 0.86, + "learning_rate": 7.278787878787879e-05, + "loss": 2.6728, + "theoretical_loss": 3.3153704677129263, + "tokens_seen": 3064201216 + }, + { + "epoch": 0.86, + "learning_rate": 7.276767676767677e-05, + "loss": 2.6021, + "theoretical_loss": 3.3153650393284346, + "tokens_seen": 3064266752 + }, + { + "epoch": 0.86, + "learning_rate": 7.274747474747474e-05, + "loss": 2.5967, + "theoretical_loss": 3.3153596110925467, + "tokens_seen": 3064332288 + }, + { + "epoch": 0.86, + "learning_rate": 7.272727272727273e-05, + "loss": 2.5462, + "theoretical_loss": 3.315354183005254, + "tokens_seen": 3064397824 + }, + { + "epoch": 0.86, + "learning_rate": 7.270707070707071e-05, + "loss": 2.6929, + "theoretical_loss": 3.315348755066551, + "tokens_seen": 3064463360 + }, + { + "epoch": 0.86, + "learning_rate": 7.26868686868687e-05, + "loss": 2.5676, + "theoretical_loss": 3.315343327276428, + "tokens_seen": 3064528896 + }, + { + "epoch": 0.86, + "learning_rate": 7.266666666666667e-05, + "loss": 2.593, + "theoretical_loss": 3.31533789963488, + "tokens_seen": 3064594432 + }, + { + "epoch": 0.86, + "learning_rate": 7.264646464646464e-05, + "loss": 2.4781, + "theoretical_loss": 3.315332472141899, + "tokens_seen": 3064659968 + }, + { + "epoch": 0.86, + "learning_rate": 7.262626262626262e-05, + "loss": 2.5205, + "theoretical_loss": 3.3153270447974776, + "tokens_seen": 3064725504 + }, + { + "epoch": 0.86, + "learning_rate": 7.26060606060606e-05, + "loss": 2.5351, + "theoretical_loss": 3.315321617601608, + "tokens_seen": 3064791040 + }, + { + "epoch": 0.86, + "learning_rate": 7.258585858585859e-05, + "loss": 2.4188, + "theoretical_loss": 3.3153161905542845, + "tokens_seen": 3064856576 + }, + { + "epoch": 0.86, + "learning_rate": 7.256565656565657e-05, + "loss": 2.3367, + "theoretical_loss": 3.315310763655498, + "tokens_seen": 3064922112 + }, + { + "epoch": 0.86, + "learning_rate": 7.254545454545456e-05, + "loss": 2.4781, + "theoretical_loss": 3.315305336905243, + "tokens_seen": 3064987648 + }, + { + "epoch": 0.86, + "learning_rate": 7.252525252525253e-05, + "loss": 2.5042, + "theoretical_loss": 3.3152999103035112, + "tokens_seen": 3065053184 + }, + { + "epoch": 0.86, + "learning_rate": 7.25050505050505e-05, + "loss": 2.3571, + "theoretical_loss": 3.3152944838502956, + "tokens_seen": 3065118720 + }, + { + "epoch": 0.86, + "learning_rate": 7.248484848484848e-05, + "loss": 2.4047, + "theoretical_loss": 3.315289057545589, + "tokens_seen": 3065184256 + }, + { + "epoch": 0.86, + "learning_rate": 7.246464646464647e-05, + "loss": 2.3912, + "theoretical_loss": 3.3152836313893843, + "tokens_seen": 3065249792 + }, + { + "epoch": 0.86, + "learning_rate": 7.244444444444445e-05, + "loss": 2.5522, + "theoretical_loss": 3.3152782053816736, + "tokens_seen": 3065315328 + }, + { + "epoch": 0.86, + "learning_rate": 7.242424242424242e-05, + "loss": 2.4418, + "theoretical_loss": 3.3152727795224504, + "tokens_seen": 3065380864 + }, + { + "epoch": 0.86, + "learning_rate": 7.24040404040404e-05, + "loss": 2.7271, + "theoretical_loss": 3.3152673538117075, + "tokens_seen": 3065446400 + }, + { + "epoch": 0.86, + "learning_rate": 7.238383838383839e-05, + "loss": 2.5797, + "theoretical_loss": 3.3152619282494373, + "tokens_seen": 3065511936 + }, + { + "epoch": 0.86, + "learning_rate": 7.236363636363637e-05, + "loss": 2.6474, + "theoretical_loss": 3.3152565028356324, + "tokens_seen": 3065577472 + }, + { + "epoch": 0.86, + "learning_rate": 7.234343434343436e-05, + "loss": 2.5777, + "theoretical_loss": 3.315251077570286, + "tokens_seen": 3065643008 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1679560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1986513137817383, + "objective/train/theoretical_loss": 3.315245652453391, + "objective/train/tokens_used": 1424567776, + "theoretical_loss": 3.315245652453391, + "tokens_seen": 3065708544 + }, + { + "epoch": 0.86, + "learning_rate": 7.232323232323231e-05, + "loss": 2.6195, + "theoretical_loss": 3.315245652453391, + "tokens_seen": 3065708544 + }, + { + "epoch": 0.86, + "learning_rate": 7.23030303030303e-05, + "loss": 2.4445, + "theoretical_loss": 3.3152402274849395, + "tokens_seen": 3065774080 + }, + { + "epoch": 0.86, + "learning_rate": 7.228282828282828e-05, + "loss": 2.431, + "theoretical_loss": 3.3152348026649245, + "tokens_seen": 3065839616 + }, + { + "epoch": 0.86, + "learning_rate": 7.226262626262626e-05, + "loss": 2.5059, + "theoretical_loss": 3.315229377993339, + "tokens_seen": 3065905152 + }, + { + "epoch": 0.86, + "learning_rate": 7.224242424242425e-05, + "loss": 2.3823, + "theoretical_loss": 3.3152239534701753, + "tokens_seen": 3065970688 + }, + { + "epoch": 0.86, + "learning_rate": 7.222222222222222e-05, + "loss": 2.4983, + "theoretical_loss": 3.315218529095427, + "tokens_seen": 3066036224 + }, + { + "epoch": 0.86, + "learning_rate": 7.22020202020202e-05, + "loss": 2.2955, + "theoretical_loss": 3.3152131048690863, + "tokens_seen": 3066101760 + }, + { + "epoch": 0.86, + "learning_rate": 7.218181818181819e-05, + "loss": 2.7147, + "theoretical_loss": 3.315207680791146, + "tokens_seen": 3066167296 + }, + { + "epoch": 0.86, + "learning_rate": 7.216161616161617e-05, + "loss": 2.3754, + "theoretical_loss": 3.315202256861599, + "tokens_seen": 3066232832 + }, + { + "epoch": 0.86, + "learning_rate": 7.214141414141414e-05, + "loss": 2.503, + "theoretical_loss": 3.3151968330804378, + "tokens_seen": 3066298368 + }, + { + "epoch": 0.86, + "learning_rate": 7.212121212121213e-05, + "loss": 2.2979, + "theoretical_loss": 3.3151914094476553, + "tokens_seen": 3066363904 + }, + { + "epoch": 0.86, + "learning_rate": 7.21010101010101e-05, + "loss": 2.3723, + "theoretical_loss": 3.3151859859632444, + "tokens_seen": 3066429440 + }, + { + "epoch": 0.86, + "learning_rate": 7.208080808080808e-05, + "loss": 2.5493, + "theoretical_loss": 3.315180562627198, + "tokens_seen": 3066494976 + }, + { + "epoch": 0.86, + "learning_rate": 7.206060606060606e-05, + "loss": 2.273, + "theoretical_loss": 3.3151751394395084, + "tokens_seen": 3066560512 + }, + { + "epoch": 0.86, + "learning_rate": 7.204040404040405e-05, + "loss": 2.7152, + "theoretical_loss": 3.3151697164001686, + "tokens_seen": 3066626048 + }, + { + "epoch": 0.86, + "learning_rate": 7.202020202020203e-05, + "loss": 2.3913, + "theoretical_loss": 3.3151642935091714, + "tokens_seen": 3066691584 + }, + { + "epoch": 0.86, + "learning_rate": 7.2e-05, + "loss": 2.4964, + "theoretical_loss": 3.31515887076651, + "tokens_seen": 3066757120 + }, + { + "epoch": 0.86, + "learning_rate": 7.197979797979797e-05, + "loss": 2.4172, + "theoretical_loss": 3.3151534481721763, + "tokens_seen": 3066822656 + }, + { + "epoch": 0.86, + "learning_rate": 7.195959595959596e-05, + "loss": 2.5056, + "theoretical_loss": 3.3151480257261636, + "tokens_seen": 3066888192 + }, + { + "epoch": 0.86, + "learning_rate": 7.193939393939394e-05, + "loss": 2.6684, + "theoretical_loss": 3.3151426034284643, + "tokens_seen": 3066953728 + }, + { + "epoch": 0.86, + "learning_rate": 7.191919191919192e-05, + "loss": 2.6618, + "theoretical_loss": 3.315137181279072, + "tokens_seen": 3067019264 + }, + { + "epoch": 0.86, + "learning_rate": 7.18989898989899e-05, + "loss": 2.5025, + "theoretical_loss": 3.3151317592779788, + "tokens_seen": 3067084800 + }, + { + "epoch": 0.86, + "learning_rate": 7.187878787878788e-05, + "loss": 2.5933, + "theoretical_loss": 3.3151263374251774, + "tokens_seen": 3067150336 + }, + { + "epoch": 0.86, + "learning_rate": 7.185858585858586e-05, + "loss": 2.4758, + "theoretical_loss": 3.315120915720661, + "tokens_seen": 3067215872 + }, + { + "epoch": 0.86, + "learning_rate": 7.183838383838385e-05, + "loss": 2.3053, + "theoretical_loss": 3.3151154941644223, + "tokens_seen": 3067281408 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1680696, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7343716621398926, + "objective/train/theoretical_loss": 3.315110072756454, + "objective/train/tokens_used": 1426206176, + "theoretical_loss": 3.315110072756454, + "tokens_seen": 3067346944 + }, + { + "epoch": 0.86, + "learning_rate": 7.181818181818183e-05, + "loss": 2.471, + "theoretical_loss": 3.315110072756454, + "tokens_seen": 3067346944 + }, + { + "epoch": 0.86, + "learning_rate": 7.179797979797979e-05, + "loss": 2.5065, + "theoretical_loss": 3.3151046514967484, + "tokens_seen": 3067412480 + }, + { + "epoch": 0.86, + "learning_rate": 7.177777777777777e-05, + "loss": 2.4153, + "theoretical_loss": 3.315099230385299, + "tokens_seen": 3067478016 + }, + { + "epoch": 0.86, + "learning_rate": 7.175757575757576e-05, + "loss": 2.6389, + "theoretical_loss": 3.3150938094220983, + "tokens_seen": 3067543552 + }, + { + "epoch": 0.86, + "learning_rate": 7.173737373737374e-05, + "loss": 2.5182, + "theoretical_loss": 3.3150883886071387, + "tokens_seen": 3067609088 + }, + { + "epoch": 0.86, + "learning_rate": 7.171717171717172e-05, + "loss": 2.5784, + "theoretical_loss": 3.3150829679404135, + "tokens_seen": 3067674624 + }, + { + "epoch": 0.86, + "learning_rate": 7.169696969696971e-05, + "loss": 2.5037, + "theoretical_loss": 3.3150775474219154, + "tokens_seen": 3067740160 + }, + { + "epoch": 0.86, + "learning_rate": 7.167676767676768e-05, + "loss": 2.5332, + "theoretical_loss": 3.3150721270516375, + "tokens_seen": 3067805696 + }, + { + "epoch": 0.86, + "learning_rate": 7.165656565656566e-05, + "loss": 2.4634, + "theoretical_loss": 3.3150667068295716, + "tokens_seen": 3067871232 + }, + { + "epoch": 0.86, + "learning_rate": 7.163636363636363e-05, + "loss": 2.5861, + "theoretical_loss": 3.315061286755711, + "tokens_seen": 3067936768 + }, + { + "epoch": 0.86, + "learning_rate": 7.161616161616162e-05, + "loss": 2.8809, + "theoretical_loss": 3.3150558668300487, + "tokens_seen": 3068002304 + }, + { + "epoch": 0.86, + "learning_rate": 7.15959595959596e-05, + "loss": 2.2718, + "theoretical_loss": 3.3150504470525775, + "tokens_seen": 3068067840 + }, + { + "epoch": 0.86, + "learning_rate": 7.157575757575757e-05, + "loss": 2.7204, + "theoretical_loss": 3.31504502742329, + "tokens_seen": 3068133376 + }, + { + "epoch": 0.86, + "learning_rate": 7.155555555555555e-05, + "loss": 2.5099, + "theoretical_loss": 3.315039607942179, + "tokens_seen": 3068198912 + }, + { + "epoch": 0.86, + "learning_rate": 7.153535353535354e-05, + "loss": 2.6549, + "theoretical_loss": 3.3150341886092374, + "tokens_seen": 3068264448 + }, + { + "epoch": 0.86, + "learning_rate": 7.151515151515152e-05, + "loss": 2.4028, + "theoretical_loss": 3.3150287694244573, + "tokens_seen": 3068329984 + }, + { + "epoch": 0.86, + "learning_rate": 7.14949494949495e-05, + "loss": 2.4487, + "theoretical_loss": 3.3150233503878326, + "tokens_seen": 3068395520 + }, + { + "epoch": 0.86, + "learning_rate": 7.147474747474748e-05, + "loss": 2.5389, + "theoretical_loss": 3.3150179314993555, + "tokens_seen": 3068461056 + }, + { + "epoch": 0.86, + "learning_rate": 7.145454545454545e-05, + "loss": 2.3627, + "theoretical_loss": 3.3150125127590186, + "tokens_seen": 3068526592 + }, + { + "epoch": 0.86, + "learning_rate": 7.143434343434343e-05, + "loss": 2.413, + "theoretical_loss": 3.315007094166815, + "tokens_seen": 3068592128 + }, + { + "epoch": 0.86, + "learning_rate": 7.141414141414141e-05, + "loss": 2.5882, + "theoretical_loss": 3.3150016757227374, + "tokens_seen": 3068657664 + }, + { + "epoch": 0.86, + "learning_rate": 7.13939393939394e-05, + "loss": 2.5108, + "theoretical_loss": 3.3149962574267784, + "tokens_seen": 3068723200 + }, + { + "epoch": 0.86, + "learning_rate": 7.137373737373737e-05, + "loss": 2.5144, + "theoretical_loss": 3.314990839278931, + "tokens_seen": 3068788736 + }, + { + "epoch": 0.86, + "learning_rate": 7.135353535353535e-05, + "loss": 2.2715, + "theoretical_loss": 3.314985421279188, + "tokens_seen": 3068854272 + }, + { + "epoch": 0.86, + "learning_rate": 7.133333333333334e-05, + "loss": 2.5046, + "theoretical_loss": 3.314980003427542, + "tokens_seen": 3068919808 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1681971, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4237277507781982, + "objective/train/theoretical_loss": 3.3149745857239865, + "objective/train/tokens_used": 1427844576, + "theoretical_loss": 3.3149745857239865, + "tokens_seen": 3068985344 + }, + { + "epoch": 0.86, + "learning_rate": 7.131313131313132e-05, + "loss": 2.2561, + "theoretical_loss": 3.3149745857239865, + "tokens_seen": 3068985344 + }, + { + "epoch": 0.86, + "learning_rate": 7.12929292929293e-05, + "loss": 2.4942, + "theoretical_loss": 3.3149691681685134, + "tokens_seen": 3069050880 + }, + { + "epoch": 0.86, + "learning_rate": 7.127272727272728e-05, + "loss": 2.4708, + "theoretical_loss": 3.3149637507611156, + "tokens_seen": 3069116416 + }, + { + "epoch": 0.86, + "learning_rate": 7.125252525252525e-05, + "loss": 2.4615, + "theoretical_loss": 3.314958333501786, + "tokens_seen": 3069181952 + }, + { + "epoch": 0.86, + "learning_rate": 7.123232323232323e-05, + "loss": 2.3969, + "theoretical_loss": 3.314952916390518, + "tokens_seen": 3069247488 + }, + { + "epoch": 0.86, + "learning_rate": 7.121212121212121e-05, + "loss": 2.5021, + "theoretical_loss": 3.3149474994273036, + "tokens_seen": 3069313024 + }, + { + "epoch": 0.86, + "learning_rate": 7.11919191919192e-05, + "loss": 2.4272, + "theoretical_loss": 3.314942082612136, + "tokens_seen": 3069378560 + }, + { + "epoch": 0.86, + "learning_rate": 7.117171717171718e-05, + "loss": 2.5696, + "theoretical_loss": 3.314936665945008, + "tokens_seen": 3069444096 + }, + { + "epoch": 0.86, + "learning_rate": 7.115151515151515e-05, + "loss": 2.4785, + "theoretical_loss": 3.3149312494259116, + "tokens_seen": 3069509632 + }, + { + "epoch": 0.86, + "learning_rate": 7.113131313131314e-05, + "loss": 2.437, + "theoretical_loss": 3.314925833054841, + "tokens_seen": 3069575168 + }, + { + "epoch": 0.86, + "learning_rate": 7.11111111111111e-05, + "loss": 2.3871, + "theoretical_loss": 3.3149204168317876, + "tokens_seen": 3069640704 + }, + { + "epoch": 0.86, + "learning_rate": 7.109090909090909e-05, + "loss": 2.4084, + "theoretical_loss": 3.3149150007567454, + "tokens_seen": 3069706240 + }, + { + "epoch": 0.86, + "learning_rate": 7.107070707070707e-05, + "loss": 2.3469, + "theoretical_loss": 3.3149095848297065, + "tokens_seen": 3069771776 + }, + { + "epoch": 0.86, + "learning_rate": 7.105050505050504e-05, + "loss": 2.2925, + "theoretical_loss": 3.314904169050664, + "tokens_seen": 3069837312 + }, + { + "epoch": 0.86, + "learning_rate": 7.103030303030303e-05, + "loss": 2.4672, + "theoretical_loss": 3.31489875341961, + "tokens_seen": 3069902848 + }, + { + "epoch": 0.86, + "learning_rate": 7.101010101010101e-05, + "loss": 2.4727, + "theoretical_loss": 3.3148933379365384, + "tokens_seen": 3069968384 + }, + { + "epoch": 0.86, + "learning_rate": 7.0989898989899e-05, + "loss": 2.6438, + "theoretical_loss": 3.314887922601441, + "tokens_seen": 3070033920 + }, + { + "epoch": 0.86, + "learning_rate": 7.096969696969698e-05, + "loss": 2.2957, + "theoretical_loss": 3.3148825074143113, + "tokens_seen": 3070099456 + }, + { + "epoch": 0.86, + "learning_rate": 7.094949494949495e-05, + "loss": 2.5545, + "theoretical_loss": 3.314877092375142, + "tokens_seen": 3070164992 + }, + { + "epoch": 0.86, + "learning_rate": 7.092929292929292e-05, + "loss": 2.388, + "theoretical_loss": 3.314871677483925, + "tokens_seen": 3070230528 + }, + { + "epoch": 0.86, + "learning_rate": 7.09090909090909e-05, + "loss": 2.6586, + "theoretical_loss": 3.3148662627406544, + "tokens_seen": 3070296064 + }, + { + "epoch": 0.86, + "learning_rate": 7.088888888888889e-05, + "loss": 2.6173, + "theoretical_loss": 3.3148608481453223, + "tokens_seen": 3070361600 + }, + { + "epoch": 0.86, + "learning_rate": 7.086868686868687e-05, + "loss": 2.4833, + "theoretical_loss": 3.3148554336979217, + "tokens_seen": 3070427136 + }, + { + "epoch": 0.86, + "learning_rate": 7.084848484848486e-05, + "loss": 2.4637, + "theoretical_loss": 3.3148500193984454, + "tokens_seen": 3070492672 + }, + { + "epoch": 0.86, + "learning_rate": 7.082828282828283e-05, + "loss": 2.7387, + "theoretical_loss": 3.3148446052468863, + "tokens_seen": 3070558208 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1682525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2938342094421387, + "objective/train/theoretical_loss": 3.3148391912432364, + "objective/train/tokens_used": 1429482976, + "theoretical_loss": 3.3148391912432364, + "tokens_seen": 3070623744 + }, + { + "epoch": 0.86, + "learning_rate": 7.080808080808081e-05, + "loss": 2.5108, + "theoretical_loss": 3.3148391912432364, + "tokens_seen": 3070623744 + }, + { + "epoch": 0.86, + "learning_rate": 7.07878787878788e-05, + "loss": 2.3536, + "theoretical_loss": 3.3148337773874896, + "tokens_seen": 3070689280 + }, + { + "epoch": 0.86, + "learning_rate": 7.076767676767677e-05, + "loss": 2.5079, + "theoretical_loss": 3.3148283636796383, + "tokens_seen": 3070754816 + }, + { + "epoch": 0.86, + "learning_rate": 7.074747474747475e-05, + "loss": 2.3429, + "theoretical_loss": 3.314822950119675, + "tokens_seen": 3070820352 + }, + { + "epoch": 0.86, + "learning_rate": 7.072727272727272e-05, + "loss": 2.3397, + "theoretical_loss": 3.3148175367075927, + "tokens_seen": 3070885888 + }, + { + "epoch": 0.86, + "learning_rate": 7.07070707070707e-05, + "loss": 2.3227, + "theoretical_loss": 3.3148121234433843, + "tokens_seen": 3070951424 + }, + { + "epoch": 0.86, + "learning_rate": 7.068686868686869e-05, + "loss": 2.5291, + "theoretical_loss": 3.314806710327043, + "tokens_seen": 3071016960 + }, + { + "epoch": 0.86, + "learning_rate": 7.066666666666667e-05, + "loss": 2.5841, + "theoretical_loss": 3.3148012973585606, + "tokens_seen": 3071082496 + }, + { + "epoch": 0.86, + "learning_rate": 7.064646464646466e-05, + "loss": 2.5939, + "theoretical_loss": 3.3147958845379306, + "tokens_seen": 3071148032 + }, + { + "epoch": 0.86, + "learning_rate": 7.062626262626263e-05, + "loss": 2.5207, + "theoretical_loss": 3.3147904718651455, + "tokens_seen": 3071213568 + }, + { + "epoch": 0.86, + "learning_rate": 7.060606060606061e-05, + "loss": 2.622, + "theoretical_loss": 3.3147850593401986, + "tokens_seen": 3071279104 + }, + { + "epoch": 0.86, + "learning_rate": 7.058585858585858e-05, + "loss": 2.6221, + "theoretical_loss": 3.3147796469630824, + "tokens_seen": 3071344640 + }, + { + "epoch": 0.86, + "learning_rate": 7.056565656565656e-05, + "loss": 2.4228, + "theoretical_loss": 3.3147742347337896, + "tokens_seen": 3071410176 + }, + { + "epoch": 0.86, + "learning_rate": 7.054545454545455e-05, + "loss": 2.5531, + "theoretical_loss": 3.314768822652313, + "tokens_seen": 3071475712 + }, + { + "epoch": 0.86, + "learning_rate": 7.052525252525252e-05, + "loss": 2.5823, + "theoretical_loss": 3.3147634107186454, + "tokens_seen": 3071541248 + }, + { + "epoch": 0.86, + "learning_rate": 7.05050505050505e-05, + "loss": 2.5996, + "theoretical_loss": 3.3147579989327802, + "tokens_seen": 3071606784 + }, + { + "epoch": 0.86, + "learning_rate": 7.048484848484849e-05, + "loss": 2.6913, + "theoretical_loss": 3.3147525872947092, + "tokens_seen": 3071672320 + }, + { + "epoch": 0.86, + "learning_rate": 7.046464646464647e-05, + "loss": 2.5188, + "theoretical_loss": 3.3147471758044262, + "tokens_seen": 3071737856 + }, + { + "epoch": 0.86, + "learning_rate": 7.044444444444446e-05, + "loss": 2.5242, + "theoretical_loss": 3.3147417644619233, + "tokens_seen": 3071803392 + }, + { + "epoch": 0.86, + "learning_rate": 7.042424242424244e-05, + "loss": 2.2879, + "theoretical_loss": 3.3147363532671936, + "tokens_seen": 3071868928 + }, + { + "epoch": 0.86, + "learning_rate": 7.04040404040404e-05, + "loss": 2.2478, + "theoretical_loss": 3.31473094222023, + "tokens_seen": 3071934464 + }, + { + "epoch": 0.86, + "learning_rate": 7.038383838383838e-05, + "loss": 2.2967, + "theoretical_loss": 3.3147255313210247, + "tokens_seen": 3072000000 + }, + { + "epoch": 0.86, + "learning_rate": 7.036363636363636e-05, + "loss": 2.5531, + "theoretical_loss": 3.3147201205695715, + "tokens_seen": 3072065536 + }, + { + "epoch": 0.86, + "learning_rate": 7.034343434343435e-05, + "loss": 2.5786, + "theoretical_loss": 3.3147147099658625, + "tokens_seen": 3072131072 + }, + { + "epoch": 0.86, + "learning_rate": 7.032323232323233e-05, + "loss": 2.4711, + "theoretical_loss": 3.3147092995098912, + "tokens_seen": 3072196608 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1683794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.771482467651367, + "objective/train/theoretical_loss": 3.3147038892016494, + "objective/train/tokens_used": 1431121376, + "theoretical_loss": 3.3147038892016494, + "tokens_seen": 3072262144 + }, + { + "epoch": 0.86, + "learning_rate": 7.03030303030303e-05, + "loss": 2.6553, + "theoretical_loss": 3.3147038892016494, + "tokens_seen": 3072262144 + }, + { + "epoch": 0.86, + "learning_rate": 7.028282828282829e-05, + "loss": 2.3029, + "theoretical_loss": 3.3146984790411307, + "tokens_seen": 3072327680 + }, + { + "epoch": 0.86, + "learning_rate": 7.026262626262627e-05, + "loss": 2.5644, + "theoretical_loss": 3.3146930690283276, + "tokens_seen": 3072393216 + }, + { + "epoch": 0.86, + "learning_rate": 7.024242424242424e-05, + "loss": 2.4272, + "theoretical_loss": 3.314687659163233, + "tokens_seen": 3072458752 + }, + { + "epoch": 0.86, + "learning_rate": 7.022222222222222e-05, + "loss": 2.5813, + "theoretical_loss": 3.3146822494458394, + "tokens_seen": 3072524288 + }, + { + "epoch": 0.86, + "learning_rate": 7.02020202020202e-05, + "loss": 2.7149, + "theoretical_loss": 3.3146768398761406, + "tokens_seen": 3072589824 + }, + { + "epoch": 0.86, + "learning_rate": 7.018181818181818e-05, + "loss": 2.5351, + "theoretical_loss": 3.3146714304541285, + "tokens_seen": 3072655360 + }, + { + "epoch": 0.86, + "learning_rate": 7.016161616161616e-05, + "loss": 2.5701, + "theoretical_loss": 3.3146660211797956, + "tokens_seen": 3072720896 + }, + { + "epoch": 0.86, + "learning_rate": 7.014141414141415e-05, + "loss": 2.4705, + "theoretical_loss": 3.314660612053136, + "tokens_seen": 3072786432 + }, + { + "epoch": 0.86, + "learning_rate": 7.012121212121213e-05, + "loss": 2.4087, + "theoretical_loss": 3.3146552030741416, + "tokens_seen": 3072851968 + }, + { + "epoch": 0.86, + "learning_rate": 7.01010101010101e-05, + "loss": 2.467, + "theoretical_loss": 3.314649794242805, + "tokens_seen": 3072917504 + }, + { + "epoch": 0.86, + "learning_rate": 7.008080808080807e-05, + "loss": 2.3489, + "theoretical_loss": 3.31464438555912, + "tokens_seen": 3072983040 + }, + { + "epoch": 0.86, + "learning_rate": 7.006060606060606e-05, + "loss": 2.324, + "theoretical_loss": 3.314638977023079, + "tokens_seen": 3073048576 + }, + { + "epoch": 0.86, + "learning_rate": 7.004040404040404e-05, + "loss": 2.3471, + "theoretical_loss": 3.314633568634674, + "tokens_seen": 3073114112 + }, + { + "epoch": 0.86, + "learning_rate": 7.002020202020202e-05, + "loss": 2.654, + "theoretical_loss": 3.314628160393899, + "tokens_seen": 3073179648 + }, + { + "epoch": 0.86, + "learning_rate": 7.000000000000001e-05, + "loss": 2.512, + "theoretical_loss": 3.314622752300746, + "tokens_seen": 3073245184 + }, + { + "epoch": 0.86, + "learning_rate": 6.997979797979798e-05, + "loss": 2.5784, + "theoretical_loss": 3.3146173443552085, + "tokens_seen": 3073310720 + }, + { + "epoch": 0.86, + "learning_rate": 6.995959595959596e-05, + "loss": 2.6835, + "theoretical_loss": 3.3146119365572786, + "tokens_seen": 3073376256 + }, + { + "epoch": 0.86, + "learning_rate": 6.993939393939395e-05, + "loss": 2.5007, + "theoretical_loss": 3.3146065289069497, + "tokens_seen": 3073441792 + }, + { + "epoch": 0.86, + "learning_rate": 6.991919191919193e-05, + "loss": 2.6728, + "theoretical_loss": 3.3146011214042144, + "tokens_seen": 3073507328 + }, + { + "epoch": 0.86, + "learning_rate": 6.98989898989899e-05, + "loss": 2.5358, + "theoretical_loss": 3.3145957140490654, + "tokens_seen": 3073572864 + }, + { + "epoch": 0.86, + "learning_rate": 6.987878787878787e-05, + "loss": 2.5468, + "theoretical_loss": 3.314590306841496, + "tokens_seen": 3073638400 + }, + { + "epoch": 0.86, + "learning_rate": 6.985858585858585e-05, + "loss": 2.5819, + "theoretical_loss": 3.3145848997814986, + "tokens_seen": 3073703936 + }, + { + "epoch": 0.86, + "learning_rate": 6.983838383838384e-05, + "loss": 2.5914, + "theoretical_loss": 3.3145794928690657, + "tokens_seen": 3073769472 + }, + { + "epoch": 0.86, + "learning_rate": 6.981818181818182e-05, + "loss": 2.4826, + "theoretical_loss": 3.314574086104191, + "tokens_seen": 3073835008 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1684169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.728853464126587, + "objective/train/theoretical_loss": 3.3145686794868667, + "objective/train/tokens_used": 1432759776, + "theoretical_loss": 3.3145686794868667, + "tokens_seen": 3073900544 + }, + { + "epoch": 0.86, + "learning_rate": 6.97979797979798e-05, + "loss": 2.5076, + "theoretical_loss": 3.3145686794868667, + "tokens_seen": 3073900544 + }, + { + "epoch": 0.86, + "learning_rate": 6.977777777777778e-05, + "loss": 2.4032, + "theoretical_loss": 3.314563273017086, + "tokens_seen": 3073966080 + }, + { + "epoch": 0.86, + "learning_rate": 6.975757575757576e-05, + "loss": 2.4293, + "theoretical_loss": 3.314557866694841, + "tokens_seen": 3074031616 + }, + { + "epoch": 0.86, + "learning_rate": 6.973737373737374e-05, + "loss": 2.5094, + "theoretical_loss": 3.314552460520126, + "tokens_seen": 3074097152 + }, + { + "epoch": 0.86, + "learning_rate": 6.971717171717172e-05, + "loss": 2.4498, + "theoretical_loss": 3.314547054492932, + "tokens_seen": 3074162688 + }, + { + "epoch": 0.86, + "learning_rate": 6.96969696969697e-05, + "loss": 2.6062, + "theoretical_loss": 3.314541648613253, + "tokens_seen": 3074228224 + }, + { + "epoch": 0.86, + "learning_rate": 6.967676767676767e-05, + "loss": 2.6434, + "theoretical_loss": 3.314536242881082, + "tokens_seen": 3074293760 + }, + { + "epoch": 0.86, + "learning_rate": 6.965656565656565e-05, + "loss": 2.673, + "theoretical_loss": 3.314530837296411, + "tokens_seen": 3074359296 + }, + { + "epoch": 0.86, + "learning_rate": 6.963636363636364e-05, + "loss": 2.48, + "theoretical_loss": 3.3145254318592325, + "tokens_seen": 3074424832 + }, + { + "epoch": 0.86, + "learning_rate": 6.961616161616162e-05, + "loss": 2.6968, + "theoretical_loss": 3.314520026569541, + "tokens_seen": 3074490368 + }, + { + "epoch": 0.86, + "learning_rate": 6.95959595959596e-05, + "loss": 2.5274, + "theoretical_loss": 3.3145146214273282, + "tokens_seen": 3074555904 + }, + { + "epoch": 0.86, + "learning_rate": 6.957575757575759e-05, + "loss": 2.4694, + "theoretical_loss": 3.3145092164325867, + "tokens_seen": 3074621440 + }, + { + "epoch": 0.86, + "learning_rate": 6.955555555555555e-05, + "loss": 2.46, + "theoretical_loss": 3.3145038115853103, + "tokens_seen": 3074686976 + }, + { + "epoch": 0.86, + "learning_rate": 6.953535353535353e-05, + "loss": 2.5312, + "theoretical_loss": 3.314498406885491, + "tokens_seen": 3074752512 + }, + { + "epoch": 0.86, + "learning_rate": 6.951515151515151e-05, + "loss": 2.4351, + "theoretical_loss": 3.314493002333122, + "tokens_seen": 3074818048 + }, + { + "epoch": 0.86, + "learning_rate": 6.94949494949495e-05, + "loss": 2.5019, + "theoretical_loss": 3.314487597928196, + "tokens_seen": 3074883584 + }, + { + "epoch": 0.86, + "learning_rate": 6.947474747474748e-05, + "loss": 2.3035, + "theoretical_loss": 3.3144821936707056, + "tokens_seen": 3074949120 + }, + { + "epoch": 0.86, + "learning_rate": 6.945454545454545e-05, + "loss": 2.2268, + "theoretical_loss": 3.3144767895606444, + "tokens_seen": 3075014656 + }, + { + "epoch": 0.86, + "learning_rate": 6.943434343434344e-05, + "loss": 2.3458, + "theoretical_loss": 3.3144713855980044, + "tokens_seen": 3075080192 + }, + { + "epoch": 0.86, + "learning_rate": 6.941414141414142e-05, + "loss": 2.7022, + "theoretical_loss": 3.314465981782779, + "tokens_seen": 3075145728 + }, + { + "epoch": 0.86, + "learning_rate": 6.93939393939394e-05, + "loss": 2.5223, + "theoretical_loss": 3.3144605781149608, + "tokens_seen": 3075211264 + }, + { + "epoch": 0.86, + "learning_rate": 6.937373737373737e-05, + "loss": 2.5745, + "theoretical_loss": 3.3144551745945425, + "tokens_seen": 3075276800 + }, + { + "epoch": 0.86, + "learning_rate": 6.935353535353535e-05, + "loss": 2.652, + "theoretical_loss": 3.314449771221517, + "tokens_seen": 3075342336 + }, + { + "epoch": 0.86, + "learning_rate": 6.933333333333333e-05, + "loss": 2.4986, + "theoretical_loss": 3.3144443679958773, + "tokens_seen": 3075407872 + }, + { + "epoch": 0.86, + "learning_rate": 6.931313131313131e-05, + "loss": 2.6499, + "theoretical_loss": 3.3144389649176165, + "tokens_seen": 3075473408 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1684863, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.217968225479126, + "objective/train/theoretical_loss": 3.3144335619867267, + "objective/train/tokens_used": 1434398176, + "theoretical_loss": 3.3144335619867267, + "tokens_seen": 3075538944 + }, + { + "epoch": 0.86, + "learning_rate": 6.92929292929293e-05, + "loss": 2.3854, + "theoretical_loss": 3.3144335619867267, + "tokens_seen": 3075538944 + }, + { + "epoch": 0.86, + "learning_rate": 6.927272727272728e-05, + "loss": 2.7101, + "theoretical_loss": 3.3144281592032017, + "tokens_seen": 3075604480 + }, + { + "epoch": 0.86, + "learning_rate": 6.925252525252525e-05, + "loss": 2.5463, + "theoretical_loss": 3.3144227565670334, + "tokens_seen": 3075670016 + }, + { + "epoch": 0.86, + "learning_rate": 6.923232323232324e-05, + "loss": 2.482, + "theoretical_loss": 3.3144173540782154, + "tokens_seen": 3075735552 + }, + { + "epoch": 0.86, + "learning_rate": 6.92121212121212e-05, + "loss": 2.6025, + "theoretical_loss": 3.3144119517367394, + "tokens_seen": 3075801088 + }, + { + "epoch": 0.86, + "learning_rate": 6.919191919191919e-05, + "loss": 2.5618, + "theoretical_loss": 3.3144065495426, + "tokens_seen": 3075866624 + }, + { + "epoch": 0.86, + "learning_rate": 6.917171717171717e-05, + "loss": 2.6519, + "theoretical_loss": 3.314401147495788, + "tokens_seen": 3075932160 + }, + { + "epoch": 0.86, + "learning_rate": 6.915151515151516e-05, + "loss": 2.4306, + "theoretical_loss": 3.3143957455962982, + "tokens_seen": 3075997696 + }, + { + "epoch": 0.86, + "learning_rate": 6.913131313131313e-05, + "loss": 2.4758, + "theoretical_loss": 3.3143903438441225, + "tokens_seen": 3076063232 + }, + { + "epoch": 0.86, + "learning_rate": 6.911111111111111e-05, + "loss": 2.6594, + "theoretical_loss": 3.3143849422392533, + "tokens_seen": 3076128768 + }, + { + "epoch": 0.86, + "learning_rate": 6.90909090909091e-05, + "loss": 2.4179, + "theoretical_loss": 3.314379540781684, + "tokens_seen": 3076194304 + }, + { + "epoch": 0.86, + "learning_rate": 6.907070707070708e-05, + "loss": 2.3671, + "theoretical_loss": 3.3143741394714077, + "tokens_seen": 3076259840 + }, + { + "epoch": 0.86, + "learning_rate": 6.905050505050506e-05, + "loss": 2.2892, + "theoretical_loss": 3.314368738308417, + "tokens_seen": 3076325376 + }, + { + "epoch": 0.86, + "learning_rate": 6.903030303030302e-05, + "loss": 2.5604, + "theoretical_loss": 3.314363337292704, + "tokens_seen": 3076390912 + }, + { + "epoch": 0.86, + "learning_rate": 6.9010101010101e-05, + "loss": 2.6301, + "theoretical_loss": 3.314357936424263, + "tokens_seen": 3076456448 + }, + { + "epoch": 0.86, + "learning_rate": 6.898989898989899e-05, + "loss": 2.5049, + "theoretical_loss": 3.314352535703086, + "tokens_seen": 3076521984 + }, + { + "epoch": 0.86, + "learning_rate": 6.896969696969697e-05, + "loss": 2.5559, + "theoretical_loss": 3.3143471351291653, + "tokens_seen": 3076587520 + }, + { + "epoch": 0.86, + "learning_rate": 6.894949494949496e-05, + "loss": 2.5282, + "theoretical_loss": 3.314341734702495, + "tokens_seen": 3076653056 + }, + { + "epoch": 0.86, + "learning_rate": 6.892929292929293e-05, + "loss": 2.4398, + "theoretical_loss": 3.314336334423067, + "tokens_seen": 3076718592 + }, + { + "epoch": 0.86, + "learning_rate": 6.890909090909091e-05, + "loss": 2.6514, + "theoretical_loss": 3.3143309342908744, + "tokens_seen": 3076784128 + }, + { + "epoch": 0.86, + "learning_rate": 6.88888888888889e-05, + "loss": 2.6814, + "theoretical_loss": 3.3143255343059104, + "tokens_seen": 3076849664 + }, + { + "epoch": 0.86, + "learning_rate": 6.886868686868688e-05, + "loss": 2.4764, + "theoretical_loss": 3.314320134468167, + "tokens_seen": 3076915200 + }, + { + "epoch": 0.86, + "learning_rate": 6.884848484848485e-05, + "loss": 2.5414, + "theoretical_loss": 3.3143147347776383, + "tokens_seen": 3076980736 + }, + { + "epoch": 0.86, + "learning_rate": 6.882828282828282e-05, + "loss": 2.3015, + "theoretical_loss": 3.3143093352343165, + "tokens_seen": 3077046272 + }, + { + "epoch": 0.86, + "learning_rate": 6.88080808080808e-05, + "loss": 2.5444, + "theoretical_loss": 3.314303935838194, + "tokens_seen": 3077111808 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1686089, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5774176120758057, + "objective/train/theoretical_loss": 3.314298536589264, + "objective/train/tokens_used": 1436036576, + "theoretical_loss": 3.314298536589264, + "tokens_seen": 3077177344 + }, + { + "epoch": 0.86, + "learning_rate": 6.878787878787879e-05, + "loss": 2.6703, + "theoretical_loss": 3.314298536589264, + "tokens_seen": 3077177344 + }, + { + "epoch": 0.86, + "learning_rate": 6.876767676767677e-05, + "loss": 2.3749, + "theoretical_loss": 3.3142931374875197, + "tokens_seen": 3077242880 + }, + { + "epoch": 0.86, + "learning_rate": 6.874747474747476e-05, + "loss": 2.3534, + "theoretical_loss": 3.3142877385329537, + "tokens_seen": 3077308416 + }, + { + "epoch": 0.86, + "learning_rate": 6.872727272727274e-05, + "loss": 2.533, + "theoretical_loss": 3.3142823397255587, + "tokens_seen": 3077373952 + }, + { + "epoch": 0.86, + "learning_rate": 6.870707070707071e-05, + "loss": 2.2779, + "theoretical_loss": 3.314276941065328, + "tokens_seen": 3077439488 + }, + { + "epoch": 0.86, + "learning_rate": 6.868686868686868e-05, + "loss": 2.4877, + "theoretical_loss": 3.3142715425522535, + "tokens_seen": 3077505024 + }, + { + "epoch": 0.86, + "learning_rate": 6.866666666666666e-05, + "loss": 2.3946, + "theoretical_loss": 3.3142661441863295, + "tokens_seen": 3077570560 + }, + { + "epoch": 0.86, + "learning_rate": 6.864646464646465e-05, + "loss": 2.6864, + "theoretical_loss": 3.3142607459675473, + "tokens_seen": 3077636096 + }, + { + "epoch": 0.86, + "learning_rate": 6.862626262626263e-05, + "loss": 2.4446, + "theoretical_loss": 3.3142553478959007, + "tokens_seen": 3077701632 + }, + { + "epoch": 0.86, + "learning_rate": 6.86060606060606e-05, + "loss": 2.4943, + "theoretical_loss": 3.3142499499713827, + "tokens_seen": 3077767168 + }, + { + "epoch": 0.86, + "learning_rate": 6.858585858585859e-05, + "loss": 2.784, + "theoretical_loss": 3.3142445521939856, + "tokens_seen": 3077832704 + }, + { + "epoch": 0.86, + "learning_rate": 6.856565656565657e-05, + "loss": 2.4953, + "theoretical_loss": 3.3142391545637024, + "tokens_seen": 3077898240 + }, + { + "epoch": 0.86, + "learning_rate": 6.854545454545455e-05, + "loss": 2.78, + "theoretical_loss": 3.314233757080526, + "tokens_seen": 3077963776 + }, + { + "epoch": 0.86, + "learning_rate": 6.852525252525254e-05, + "loss": 2.4989, + "theoretical_loss": 3.3142283597444493, + "tokens_seen": 3078029312 + }, + { + "epoch": 0.86, + "learning_rate": 6.85050505050505e-05, + "loss": 2.563, + "theoretical_loss": 3.3142229625554656, + "tokens_seen": 3078094848 + }, + { + "epoch": 0.86, + "learning_rate": 6.848484848484848e-05, + "loss": 2.5226, + "theoretical_loss": 3.314217565513567, + "tokens_seen": 3078160384 + }, + { + "epoch": 0.86, + "learning_rate": 6.846464646464646e-05, + "loss": 2.5321, + "theoretical_loss": 3.3142121686187465, + "tokens_seen": 3078225920 + }, + { + "epoch": 0.86, + "learning_rate": 6.844444444444445e-05, + "loss": 2.5265, + "theoretical_loss": 3.3142067718709973, + "tokens_seen": 3078291456 + }, + { + "epoch": 0.86, + "learning_rate": 6.842424242424243e-05, + "loss": 2.5172, + "theoretical_loss": 3.3142013752703123, + "tokens_seen": 3078356992 + }, + { + "epoch": 0.86, + "learning_rate": 6.84040404040404e-05, + "loss": 2.6621, + "theoretical_loss": 3.3141959788166835, + "tokens_seen": 3078422528 + }, + { + "epoch": 0.86, + "learning_rate": 6.838383838383839e-05, + "loss": 2.5198, + "theoretical_loss": 3.314190582510105, + "tokens_seen": 3078488064 + }, + { + "epoch": 0.86, + "learning_rate": 6.836363636363637e-05, + "loss": 2.6981, + "theoretical_loss": 3.314185186350569, + "tokens_seen": 3078553600 + }, + { + "epoch": 0.86, + "learning_rate": 6.834343434343434e-05, + "loss": 2.3922, + "theoretical_loss": 3.3141797903380685, + "tokens_seen": 3078619136 + }, + { + "epoch": 0.86, + "learning_rate": 6.832323232323232e-05, + "loss": 2.5377, + "theoretical_loss": 3.3141743944725963, + "tokens_seen": 3078684672 + }, + { + "epoch": 0.86, + "learning_rate": 6.830303030303031e-05, + "loss": 2.2504, + "theoretical_loss": 3.314168998754145, + "tokens_seen": 3078750208 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1686821, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.025615692138672, + "objective/train/theoretical_loss": 3.314163603182708, + "objective/train/tokens_used": 1437674976, + "theoretical_loss": 3.314163603182708, + "tokens_seen": 3078815744 + }, + { + "epoch": 0.86, + "learning_rate": 6.828282828282828e-05, + "loss": 2.5985, + "theoretical_loss": 3.314163603182708, + "tokens_seen": 3078815744 + }, + { + "epoch": 0.87, + "learning_rate": 6.826262626262626e-05, + "loss": 2.4834, + "theoretical_loss": 3.314158207758278, + "tokens_seen": 3078881280 + }, + { + "epoch": 0.87, + "learning_rate": 6.824242424242425e-05, + "loss": 2.6475, + "theoretical_loss": 3.3141528124808475, + "tokens_seen": 3078946816 + }, + { + "epoch": 0.87, + "learning_rate": 6.822222222222223e-05, + "loss": 2.5301, + "theoretical_loss": 3.3141474173504095, + "tokens_seen": 3079012352 + }, + { + "epoch": 0.87, + "learning_rate": 6.820202020202021e-05, + "loss": 2.5404, + "theoretical_loss": 3.3141420223669575, + "tokens_seen": 3079077888 + }, + { + "epoch": 0.87, + "learning_rate": 6.818181818181818e-05, + "loss": 2.3773, + "theoretical_loss": 3.3141366275304835, + "tokens_seen": 3079143424 + }, + { + "epoch": 0.87, + "learning_rate": 6.816161616161615e-05, + "loss": 2.3451, + "theoretical_loss": 3.314131232840981, + "tokens_seen": 3079208960 + }, + { + "epoch": 0.87, + "learning_rate": 6.814141414141414e-05, + "loss": 2.5775, + "theoretical_loss": 3.3141258382984424, + "tokens_seen": 3079274496 + }, + { + "epoch": 0.87, + "learning_rate": 6.812121212121212e-05, + "loss": 2.528, + "theoretical_loss": 3.3141204439028606, + "tokens_seen": 3079340032 + }, + { + "epoch": 0.87, + "learning_rate": 6.81010101010101e-05, + "loss": 2.5733, + "theoretical_loss": 3.3141150496542293, + "tokens_seen": 3079405568 + }, + { + "epoch": 0.87, + "learning_rate": 6.808080808080808e-05, + "loss": 2.3317, + "theoretical_loss": 3.31410965555254, + "tokens_seen": 3079471104 + }, + { + "epoch": 0.87, + "learning_rate": 6.806060606060606e-05, + "loss": 2.7141, + "theoretical_loss": 3.3141042615977865, + "tokens_seen": 3079536640 + }, + { + "epoch": 0.87, + "learning_rate": 6.804040404040405e-05, + "loss": 2.3929, + "theoretical_loss": 3.3140988677899617, + "tokens_seen": 3079602176 + }, + { + "epoch": 0.87, + "learning_rate": 6.802020202020203e-05, + "loss": 2.5179, + "theoretical_loss": 3.314093474129058, + "tokens_seen": 3079667712 + }, + { + "epoch": 0.87, + "learning_rate": 6.800000000000001e-05, + "loss": 2.5104, + "theoretical_loss": 3.3140880806150683, + "tokens_seen": 3079733248 + }, + { + "epoch": 0.87, + "learning_rate": 6.797979797979797e-05, + "loss": 2.6084, + "theoretical_loss": 3.314082687247986, + "tokens_seen": 3079798784 + }, + { + "epoch": 0.87, + "learning_rate": 6.795959595959595e-05, + "loss": 2.6733, + "theoretical_loss": 3.3140772940278036, + "tokens_seen": 3079864320 + }, + { + "epoch": 0.87, + "learning_rate": 6.793939393939394e-05, + "loss": 2.5842, + "theoretical_loss": 3.314071900954514, + "tokens_seen": 3079929856 + }, + { + "epoch": 0.87, + "learning_rate": 6.791919191919192e-05, + "loss": 2.5615, + "theoretical_loss": 3.31406650802811, + "tokens_seen": 3079995392 + }, + { + "epoch": 0.87, + "learning_rate": 6.78989898989899e-05, + "loss": 2.5594, + "theoretical_loss": 3.3140611152485846, + "tokens_seen": 3080060928 + }, + { + "epoch": 0.87, + "learning_rate": 6.787878787878789e-05, + "loss": 2.3524, + "theoretical_loss": 3.314055722615931, + "tokens_seen": 3080126464 + }, + { + "epoch": 0.87, + "learning_rate": 6.785858585858586e-05, + "loss": 2.5168, + "theoretical_loss": 3.314050330130141, + "tokens_seen": 3080192000 + }, + { + "epoch": 0.87, + "learning_rate": 6.783838383838384e-05, + "loss": 2.4839, + "theoretical_loss": 3.3140449377912087, + "tokens_seen": 3080257536 + }, + { + "epoch": 0.87, + "learning_rate": 6.781818181818181e-05, + "loss": 2.5763, + "theoretical_loss": 3.314039545599126, + "tokens_seen": 3080323072 + }, + { + "epoch": 0.87, + "learning_rate": 6.77979797979798e-05, + "loss": 2.3635, + "theoretical_loss": 3.314034153553887, + "tokens_seen": 3080388608 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1688234, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.411372184753418, + "objective/train/theoretical_loss": 3.314028761655483, + "objective/train/tokens_used": 1439313376, + "theoretical_loss": 3.314028761655483, + "tokens_seen": 3080454144 + }, + { + "epoch": 0.87, + "learning_rate": 6.777777777777778e-05, + "loss": 2.6097, + "theoretical_loss": 3.314028761655483, + "tokens_seen": 3080454144 + }, + { + "epoch": 0.87, + "learning_rate": 6.775757575757575e-05, + "loss": 2.6007, + "theoretical_loss": 3.314023369903908, + "tokens_seen": 3080519680 + }, + { + "epoch": 0.87, + "learning_rate": 6.773737373737374e-05, + "loss": 2.6046, + "theoretical_loss": 3.3140179782991552, + "tokens_seen": 3080585216 + }, + { + "epoch": 0.87, + "learning_rate": 6.771717171717172e-05, + "loss": 2.5924, + "theoretical_loss": 3.314012586841216, + "tokens_seen": 3080650752 + }, + { + "epoch": 0.87, + "learning_rate": 6.76969696969697e-05, + "loss": 2.6333, + "theoretical_loss": 3.3140071955300847, + "tokens_seen": 3080716288 + }, + { + "epoch": 0.87, + "learning_rate": 6.767676767676769e-05, + "loss": 2.4344, + "theoretical_loss": 3.3140018043657533, + "tokens_seen": 3080781824 + }, + { + "epoch": 0.87, + "learning_rate": 6.765656565656566e-05, + "loss": 2.4619, + "theoretical_loss": 3.3139964133482147, + "tokens_seen": 3080847360 + }, + { + "epoch": 0.87, + "learning_rate": 6.763636363636363e-05, + "loss": 2.3717, + "theoretical_loss": 3.3139910224774627, + "tokens_seen": 3080912896 + }, + { + "epoch": 0.87, + "learning_rate": 6.761616161616161e-05, + "loss": 2.5814, + "theoretical_loss": 3.3139856317534893, + "tokens_seen": 3080978432 + }, + { + "epoch": 0.87, + "learning_rate": 6.75959595959596e-05, + "loss": 2.4031, + "theoretical_loss": 3.3139802411762873, + "tokens_seen": 3081043968 + }, + { + "epoch": 0.87, + "learning_rate": 6.757575757575758e-05, + "loss": 2.2018, + "theoretical_loss": 3.31397485074585, + "tokens_seen": 3081109504 + }, + { + "epoch": 0.87, + "learning_rate": 6.755555555555555e-05, + "loss": 2.4237, + "theoretical_loss": 3.313969460462171, + "tokens_seen": 3081175040 + }, + { + "epoch": 0.87, + "learning_rate": 6.753535353535354e-05, + "loss": 2.7376, + "theoretical_loss": 3.3139640703252415, + "tokens_seen": 3081240576 + }, + { + "epoch": 0.87, + "learning_rate": 6.751515151515152e-05, + "loss": 2.7729, + "theoretical_loss": 3.3139586803350554, + "tokens_seen": 3081306112 + }, + { + "epoch": 0.87, + "learning_rate": 6.74949494949495e-05, + "loss": 2.501, + "theoretical_loss": 3.3139532904916056, + "tokens_seen": 3081371648 + }, + { + "epoch": 0.87, + "learning_rate": 6.747474747474747e-05, + "loss": 2.6857, + "theoretical_loss": 3.3139479007948847, + "tokens_seen": 3081437184 + }, + { + "epoch": 0.87, + "learning_rate": 6.745454545454546e-05, + "loss": 2.4548, + "theoretical_loss": 3.3139425112448864, + "tokens_seen": 3081502720 + }, + { + "epoch": 0.87, + "learning_rate": 6.743434343434343e-05, + "loss": 2.4605, + "theoretical_loss": 3.313937121841602, + "tokens_seen": 3081568256 + }, + { + "epoch": 0.87, + "learning_rate": 6.741414141414141e-05, + "loss": 2.3036, + "theoretical_loss": 3.3139317325850257, + "tokens_seen": 3081633792 + }, + { + "epoch": 0.87, + "learning_rate": 6.73939393939394e-05, + "loss": 2.4159, + "theoretical_loss": 3.31392634347515, + "tokens_seen": 3081699328 + }, + { + "epoch": 0.87, + "learning_rate": 6.737373737373738e-05, + "loss": 2.523, + "theoretical_loss": 3.3139209545119677, + "tokens_seen": 3081764864 + }, + { + "epoch": 0.87, + "learning_rate": 6.735353535353536e-05, + "loss": 2.427, + "theoretical_loss": 3.3139155656954715, + "tokens_seen": 3081830400 + }, + { + "epoch": 0.87, + "learning_rate": 6.733333333333333e-05, + "loss": 2.6438, + "theoretical_loss": 3.313910177025655, + "tokens_seen": 3081895936 + }, + { + "epoch": 0.87, + "learning_rate": 6.731313131313132e-05, + "loss": 2.3858, + "theoretical_loss": 3.3139047885025104, + "tokens_seen": 3081961472 + }, + { + "epoch": 0.87, + "learning_rate": 6.729292929292929e-05, + "loss": 2.5623, + "theoretical_loss": 3.3138994001260307, + "tokens_seen": 3082027008 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1689141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.407421112060547, + "objective/train/theoretical_loss": 3.313894011896209, + "objective/train/tokens_used": 1440951776, + "theoretical_loss": 3.313894011896209, + "tokens_seen": 3082092544 + }, + { + "epoch": 0.87, + "learning_rate": 6.727272727272727e-05, + "loss": 2.515, + "theoretical_loss": 3.313894011896209, + "tokens_seen": 3082092544 + }, + { + "epoch": 0.87, + "learning_rate": 6.725252525252526e-05, + "loss": 2.521, + "theoretical_loss": 3.313888623813038, + "tokens_seen": 3082158080 + }, + { + "epoch": 0.87, + "learning_rate": 6.723232323232323e-05, + "loss": 2.6327, + "theoretical_loss": 3.313883235876511, + "tokens_seen": 3082223616 + }, + { + "epoch": 0.87, + "learning_rate": 6.721212121212121e-05, + "loss": 2.5518, + "theoretical_loss": 3.3138778480866202, + "tokens_seen": 3082289152 + }, + { + "epoch": 0.87, + "learning_rate": 6.71919191919192e-05, + "loss": 2.7246, + "theoretical_loss": 3.3138724604433594, + "tokens_seen": 3082354688 + }, + { + "epoch": 0.87, + "learning_rate": 6.717171717171718e-05, + "loss": 2.5189, + "theoretical_loss": 3.3138670729467203, + "tokens_seen": 3082420224 + }, + { + "epoch": 0.87, + "learning_rate": 6.715151515151516e-05, + "loss": 2.5662, + "theoretical_loss": 3.313861685596697, + "tokens_seen": 3082485760 + }, + { + "epoch": 0.87, + "learning_rate": 6.713131313131312e-05, + "loss": 2.5714, + "theoretical_loss": 3.3138562983932816, + "tokens_seen": 3082551296 + }, + { + "epoch": 0.87, + "learning_rate": 6.71111111111111e-05, + "loss": 2.3845, + "theoretical_loss": 3.313850911336467, + "tokens_seen": 3082616832 + }, + { + "epoch": 0.87, + "learning_rate": 6.709090909090909e-05, + "loss": 2.4743, + "theoretical_loss": 3.313845524426247, + "tokens_seen": 3082682368 + }, + { + "epoch": 0.87, + "learning_rate": 6.707070707070707e-05, + "loss": 2.4326, + "theoretical_loss": 3.313840137662613, + "tokens_seen": 3082747904 + }, + { + "epoch": 0.87, + "learning_rate": 6.705050505050506e-05, + "loss": 2.7347, + "theoretical_loss": 3.3138347510455595, + "tokens_seen": 3082813440 + }, + { + "epoch": 0.87, + "learning_rate": 6.703030303030303e-05, + "loss": 2.4049, + "theoretical_loss": 3.3138293645750783, + "tokens_seen": 3082878976 + }, + { + "epoch": 0.87, + "learning_rate": 6.701010101010101e-05, + "loss": 2.6754, + "theoretical_loss": 3.3138239782511625, + "tokens_seen": 3082944512 + }, + { + "epoch": 0.87, + "learning_rate": 6.6989898989899e-05, + "loss": 2.4863, + "theoretical_loss": 3.3138185920738055, + "tokens_seen": 3083010048 + }, + { + "epoch": 0.87, + "learning_rate": 6.696969696969698e-05, + "loss": 2.6312, + "theoretical_loss": 3.3138132060429992, + "tokens_seen": 3083075584 + }, + { + "epoch": 0.87, + "learning_rate": 6.694949494949495e-05, + "loss": 2.4143, + "theoretical_loss": 3.3138078201587375, + "tokens_seen": 3083141120 + }, + { + "epoch": 0.87, + "learning_rate": 6.692929292929293e-05, + "loss": 2.5004, + "theoretical_loss": 3.313802434421013, + "tokens_seen": 3083206656 + }, + { + "epoch": 0.87, + "learning_rate": 6.69090909090909e-05, + "loss": 2.72, + "theoretical_loss": 3.313797048829818, + "tokens_seen": 3083272192 + }, + { + "epoch": 0.87, + "learning_rate": 6.688888888888889e-05, + "loss": 2.2505, + "theoretical_loss": 3.313791663385146, + "tokens_seen": 3083337728 + }, + { + "epoch": 0.87, + "learning_rate": 6.686868686868687e-05, + "loss": 2.5115, + "theoretical_loss": 3.3137862780869902, + "tokens_seen": 3083403264 + }, + { + "epoch": 0.87, + "learning_rate": 6.684848484848485e-05, + "loss": 2.5423, + "theoretical_loss": 3.313780892935343, + "tokens_seen": 3083468800 + }, + { + "epoch": 0.87, + "learning_rate": 6.682828282828284e-05, + "loss": 2.7063, + "theoretical_loss": 3.313775507930197, + "tokens_seen": 3083534336 + }, + { + "epoch": 0.87, + "learning_rate": 6.680808080808081e-05, + "loss": 2.3624, + "theoretical_loss": 3.313770123071546, + "tokens_seen": 3083599872 + }, + { + "epoch": 0.87, + "learning_rate": 6.678787878787878e-05, + "loss": 2.4882, + "theoretical_loss": 3.3137647383593825, + "tokens_seen": 3083665408 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1689663, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6487629413604736, + "objective/train/theoretical_loss": 3.313759353793699, + "objective/train/tokens_used": 1442590176, + "theoretical_loss": 3.313759353793699, + "tokens_seen": 3083730944 + }, + { + "epoch": 0.87, + "learning_rate": 6.676767676767676e-05, + "loss": 2.641, + "theoretical_loss": 3.313759353793699, + "tokens_seen": 3083730944 + }, + { + "epoch": 0.87, + "learning_rate": 6.674747474747475e-05, + "loss": 2.498, + "theoretical_loss": 3.313753969374489, + "tokens_seen": 3083796480 + }, + { + "epoch": 0.87, + "learning_rate": 6.672727272727273e-05, + "loss": 2.641, + "theoretical_loss": 3.3137485851017447, + "tokens_seen": 3083862016 + }, + { + "epoch": 0.87, + "learning_rate": 6.67070707070707e-05, + "loss": 2.5322, + "theoretical_loss": 3.3137432009754595, + "tokens_seen": 3083927552 + }, + { + "epoch": 0.87, + "learning_rate": 6.668686868686869e-05, + "loss": 2.7787, + "theoretical_loss": 3.3137378169956264, + "tokens_seen": 3083993088 + }, + { + "epoch": 0.87, + "learning_rate": 6.666666666666667e-05, + "loss": 2.5352, + "theoretical_loss": 3.313732433162238, + "tokens_seen": 3084058624 + }, + { + "epoch": 0.87, + "learning_rate": 6.664646464646465e-05, + "loss": 2.5056, + "theoretical_loss": 3.313727049475287, + "tokens_seen": 3084124160 + }, + { + "epoch": 0.87, + "learning_rate": 6.662626262626264e-05, + "loss": 2.6551, + "theoretical_loss": 3.313721665934767, + "tokens_seen": 3084189696 + }, + { + "epoch": 0.87, + "learning_rate": 6.66060606060606e-05, + "loss": 2.6178, + "theoretical_loss": 3.3137162825406707, + "tokens_seen": 3084255232 + }, + { + "epoch": 0.87, + "learning_rate": 6.658585858585858e-05, + "loss": 2.5363, + "theoretical_loss": 3.3137108992929907, + "tokens_seen": 3084320768 + }, + { + "epoch": 0.87, + "learning_rate": 6.656565656565656e-05, + "loss": 2.5089, + "theoretical_loss": 3.31370551619172, + "tokens_seen": 3084386304 + }, + { + "epoch": 0.87, + "learning_rate": 6.654545454545455e-05, + "loss": 2.5706, + "theoretical_loss": 3.3137001332368516, + "tokens_seen": 3084451840 + }, + { + "epoch": 0.87, + "learning_rate": 6.652525252525253e-05, + "loss": 2.658, + "theoretical_loss": 3.313694750428378, + "tokens_seen": 3084517376 + }, + { + "epoch": 0.87, + "learning_rate": 6.650505050505051e-05, + "loss": 2.7094, + "theoretical_loss": 3.313689367766293, + "tokens_seen": 3084582912 + }, + { + "epoch": 0.87, + "learning_rate": 6.648484848484848e-05, + "loss": 2.4594, + "theoretical_loss": 3.313683985250589, + "tokens_seen": 3084648448 + }, + { + "epoch": 0.87, + "learning_rate": 6.646464646464647e-05, + "loss": 2.59, + "theoretical_loss": 3.3136786028812586, + "tokens_seen": 3084713984 + }, + { + "epoch": 0.87, + "learning_rate": 6.644444444444445e-05, + "loss": 2.5546, + "theoretical_loss": 3.313673220658295, + "tokens_seen": 3084779520 + }, + { + "epoch": 0.87, + "learning_rate": 6.642424242424242e-05, + "loss": 2.6838, + "theoretical_loss": 3.3136678385816913, + "tokens_seen": 3084845056 + }, + { + "epoch": 0.87, + "learning_rate": 6.640404040404041e-05, + "loss": 2.5608, + "theoretical_loss": 3.31366245665144, + "tokens_seen": 3084910592 + }, + { + "epoch": 0.87, + "learning_rate": 6.638383838383838e-05, + "loss": 2.6439, + "theoretical_loss": 3.3136570748675345, + "tokens_seen": 3084976128 + }, + { + "epoch": 0.87, + "learning_rate": 6.636363636363636e-05, + "loss": 2.5299, + "theoretical_loss": 3.3136516932299673, + "tokens_seen": 3085041664 + }, + { + "epoch": 0.87, + "learning_rate": 6.634343434343435e-05, + "loss": 2.6604, + "theoretical_loss": 3.313646311738731, + "tokens_seen": 3085107200 + }, + { + "epoch": 0.87, + "learning_rate": 6.632323232323233e-05, + "loss": 2.6763, + "theoretical_loss": 3.3136409303938197, + "tokens_seen": 3085172736 + }, + { + "epoch": 0.87, + "learning_rate": 6.630303030303031e-05, + "loss": 2.4539, + "theoretical_loss": 3.3136355491952254, + "tokens_seen": 3085238272 + }, + { + "epoch": 0.87, + "learning_rate": 6.628282828282828e-05, + "loss": 2.4789, + "theoretical_loss": 3.313630168142941, + "tokens_seen": 3085303808 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1690915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6402039527893066, + "objective/train/theoretical_loss": 3.31362478723696, + "objective/train/tokens_used": 1444228576, + "theoretical_loss": 3.31362478723696, + "tokens_seen": 3085369344 + }, + { + "epoch": 0.87, + "learning_rate": 6.626262626262625e-05, + "loss": 2.5644, + "theoretical_loss": 3.31362478723696, + "tokens_seen": 3085369344 + }, + { + "epoch": 0.87, + "learning_rate": 6.624242424242424e-05, + "loss": 2.526, + "theoretical_loss": 3.3136194064772746, + "tokens_seen": 3085434880 + }, + { + "epoch": 0.87, + "learning_rate": 6.622222222222222e-05, + "loss": 2.8105, + "theoretical_loss": 3.313614025863878, + "tokens_seen": 3085500416 + }, + { + "epoch": 0.87, + "learning_rate": 6.62020202020202e-05, + "loss": 2.4207, + "theoretical_loss": 3.3136086453967635, + "tokens_seen": 3085565952 + }, + { + "epoch": 0.87, + "learning_rate": 6.618181818181818e-05, + "loss": 2.5468, + "theoretical_loss": 3.3136032650759235, + "tokens_seen": 3085631488 + }, + { + "epoch": 0.87, + "learning_rate": 6.616161616161616e-05, + "loss": 2.4694, + "theoretical_loss": 3.313597884901351, + "tokens_seen": 3085697024 + }, + { + "epoch": 0.87, + "learning_rate": 6.614141414141414e-05, + "loss": 2.5222, + "theoretical_loss": 3.313592504873039, + "tokens_seen": 3085762560 + }, + { + "epoch": 0.87, + "learning_rate": 6.612121212121213e-05, + "loss": 2.4348, + "theoretical_loss": 3.3135871249909803, + "tokens_seen": 3085828096 + }, + { + "epoch": 0.87, + "learning_rate": 6.610101010101011e-05, + "loss": 2.5258, + "theoretical_loss": 3.3135817452551684, + "tokens_seen": 3085893632 + }, + { + "epoch": 0.87, + "learning_rate": 6.608080808080808e-05, + "loss": 2.4604, + "theoretical_loss": 3.3135763656655954, + "tokens_seen": 3085959168 + }, + { + "epoch": 0.87, + "learning_rate": 6.606060606060605e-05, + "loss": 2.316, + "theoretical_loss": 3.3135709862222544, + "tokens_seen": 3086024704 + }, + { + "epoch": 0.87, + "learning_rate": 6.604040404040404e-05, + "loss": 2.5543, + "theoretical_loss": 3.313565606925139, + "tokens_seen": 3086090240 + }, + { + "epoch": 0.87, + "learning_rate": 6.602020202020202e-05, + "loss": 2.6276, + "theoretical_loss": 3.3135602277742415, + "tokens_seen": 3086155776 + }, + { + "epoch": 0.87, + "learning_rate": 6.6e-05, + "loss": 2.4826, + "theoretical_loss": 3.313554848769555, + "tokens_seen": 3086221312 + }, + { + "epoch": 0.87, + "learning_rate": 6.597979797979799e-05, + "loss": 2.5305, + "theoretical_loss": 3.313549469911072, + "tokens_seen": 3086286848 + }, + { + "epoch": 0.87, + "learning_rate": 6.595959595959596e-05, + "loss": 2.4179, + "theoretical_loss": 3.313544091198786, + "tokens_seen": 3086352384 + }, + { + "epoch": 0.87, + "learning_rate": 6.593939393939394e-05, + "loss": 2.4637, + "theoretical_loss": 3.3135387126326896, + "tokens_seen": 3086417920 + }, + { + "epoch": 0.87, + "learning_rate": 6.591919191919191e-05, + "loss": 2.3044, + "theoretical_loss": 3.313533334212776, + "tokens_seen": 3086483456 + }, + { + "epoch": 0.87, + "learning_rate": 6.58989898989899e-05, + "loss": 2.5924, + "theoretical_loss": 3.313527955939038, + "tokens_seen": 3086548992 + }, + { + "epoch": 0.87, + "learning_rate": 6.587878787878788e-05, + "loss": 2.6434, + "theoretical_loss": 3.3135225778114683, + "tokens_seen": 3086614528 + }, + { + "epoch": 0.87, + "learning_rate": 6.585858585858585e-05, + "loss": 2.541, + "theoretical_loss": 3.31351719983006, + "tokens_seen": 3086680064 + }, + { + "epoch": 0.87, + "learning_rate": 6.583838383838384e-05, + "loss": 2.6793, + "theoretical_loss": 3.313511821994806, + "tokens_seen": 3086745600 + }, + { + "epoch": 0.87, + "learning_rate": 6.581818181818182e-05, + "loss": 2.5806, + "theoretical_loss": 3.3135064443056996, + "tokens_seen": 3086811136 + }, + { + "epoch": 0.87, + "learning_rate": 6.57979797979798e-05, + "loss": 2.6686, + "theoretical_loss": 3.313501066762733, + "tokens_seen": 3086876672 + }, + { + "epoch": 0.87, + "learning_rate": 6.577777777777779e-05, + "loss": 2.6411, + "theoretical_loss": 3.3134956893658996, + "tokens_seen": 3086942208 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1691543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4736175537109375, + "objective/train/theoretical_loss": 3.3134903121151926, + "objective/train/tokens_used": 1445866976, + "theoretical_loss": 3.3134903121151926, + "tokens_seen": 3087007744 + }, + { + "epoch": 0.87, + "learning_rate": 6.575757575757576e-05, + "loss": 2.5667, + "theoretical_loss": 3.3134903121151926, + "tokens_seen": 3087007744 + }, + { + "epoch": 0.87, + "learning_rate": 6.573737373737373e-05, + "loss": 2.4526, + "theoretical_loss": 3.313484935010604, + "tokens_seen": 3087073280 + }, + { + "epoch": 0.87, + "learning_rate": 6.571717171717171e-05, + "loss": 2.5832, + "theoretical_loss": 3.3134795580521277, + "tokens_seen": 3087138816 + }, + { + "epoch": 0.87, + "learning_rate": 6.56969696969697e-05, + "loss": 2.4107, + "theoretical_loss": 3.313474181239756, + "tokens_seen": 3087204352 + }, + { + "epoch": 0.87, + "learning_rate": 6.567676767676768e-05, + "loss": 2.3361, + "theoretical_loss": 3.313468804573482, + "tokens_seen": 3087269888 + }, + { + "epoch": 0.87, + "learning_rate": 6.565656565656566e-05, + "loss": 2.5359, + "theoretical_loss": 3.3134634280532986, + "tokens_seen": 3087335424 + }, + { + "epoch": 0.87, + "learning_rate": 6.563636363636364e-05, + "loss": 2.6367, + "theoretical_loss": 3.313458051679199, + "tokens_seen": 3087400960 + }, + { + "epoch": 0.87, + "learning_rate": 6.561616161616162e-05, + "loss": 2.4642, + "theoretical_loss": 3.3134526754511757, + "tokens_seen": 3087466496 + }, + { + "epoch": 0.87, + "learning_rate": 6.55959595959596e-05, + "loss": 2.3436, + "theoretical_loss": 3.3134472993692223, + "tokens_seen": 3087532032 + }, + { + "epoch": 0.87, + "learning_rate": 6.557575757575759e-05, + "loss": 2.581, + "theoretical_loss": 3.3134419234333308, + "tokens_seen": 3087597568 + }, + { + "epoch": 0.87, + "learning_rate": 6.555555555555556e-05, + "loss": 2.8591, + "theoretical_loss": 3.3134365476434953, + "tokens_seen": 3087663104 + }, + { + "epoch": 0.87, + "learning_rate": 6.553535353535353e-05, + "loss": 2.4193, + "theoretical_loss": 3.3134311719997074, + "tokens_seen": 3087728640 + }, + { + "epoch": 0.87, + "learning_rate": 6.551515151515151e-05, + "loss": 2.4571, + "theoretical_loss": 3.313425796501961, + "tokens_seen": 3087794176 + }, + { + "epoch": 0.87, + "learning_rate": 6.54949494949495e-05, + "loss": 2.444, + "theoretical_loss": 3.3134204211502487, + "tokens_seen": 3087859712 + }, + { + "epoch": 0.87, + "learning_rate": 6.547474747474748e-05, + "loss": 2.3409, + "theoretical_loss": 3.3134150459445633, + "tokens_seen": 3087925248 + }, + { + "epoch": 0.87, + "learning_rate": 6.545454545454546e-05, + "loss": 2.6659, + "theoretical_loss": 3.313409670884898, + "tokens_seen": 3087990784 + }, + { + "epoch": 0.87, + "learning_rate": 6.543434343434343e-05, + "loss": 2.677, + "theoretical_loss": 3.313404295971245, + "tokens_seen": 3088056320 + }, + { + "epoch": 0.87, + "learning_rate": 6.541414141414142e-05, + "loss": 2.5101, + "theoretical_loss": 3.313398921203599, + "tokens_seen": 3088121856 + }, + { + "epoch": 0.87, + "learning_rate": 6.539393939393939e-05, + "loss": 2.4755, + "theoretical_loss": 3.313393546581951, + "tokens_seen": 3088187392 + }, + { + "epoch": 0.87, + "learning_rate": 6.537373737373737e-05, + "loss": 2.6106, + "theoretical_loss": 3.313388172106295, + "tokens_seen": 3088252928 + }, + { + "epoch": 0.87, + "learning_rate": 6.535353535353536e-05, + "loss": 2.6586, + "theoretical_loss": 3.3133827977766237, + "tokens_seen": 3088318464 + }, + { + "epoch": 0.87, + "learning_rate": 6.533333333333333e-05, + "loss": 2.6604, + "theoretical_loss": 3.3133774235929296, + "tokens_seen": 3088384000 + }, + { + "epoch": 0.87, + "learning_rate": 6.531313131313131e-05, + "loss": 2.5964, + "theoretical_loss": 3.3133720495552064, + "tokens_seen": 3088449536 + }, + { + "epoch": 0.87, + "learning_rate": 6.52929292929293e-05, + "loss": 2.5139, + "theoretical_loss": 3.313366675663447, + "tokens_seen": 3088515072 + }, + { + "epoch": 0.87, + "learning_rate": 6.527272727272728e-05, + "loss": 2.5924, + "theoretical_loss": 3.3133613019176433, + "tokens_seen": 3088580608 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1692731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5280888080596924, + "objective/train/theoretical_loss": 3.3133559283177894, + "objective/train/tokens_used": 1447505376, + "theoretical_loss": 3.3133559283177894, + "tokens_seen": 3088646144 + }, + { + "epoch": 0.87, + "learning_rate": 6.525252525252526e-05, + "loss": 2.5948, + "theoretical_loss": 3.3133559283177894, + "tokens_seen": 3088646144 + }, + { + "epoch": 0.87, + "learning_rate": 6.523232323232325e-05, + "loss": 2.6022, + "theoretical_loss": 3.3133505548638778, + "tokens_seen": 3088711680 + }, + { + "epoch": 0.87, + "learning_rate": 6.52121212121212e-05, + "loss": 2.4306, + "theoretical_loss": 3.3133451815559014, + "tokens_seen": 3088777216 + }, + { + "epoch": 0.87, + "learning_rate": 6.519191919191919e-05, + "loss": 2.3705, + "theoretical_loss": 3.3133398083938532, + "tokens_seen": 3088842752 + }, + { + "epoch": 0.87, + "learning_rate": 6.517171717171717e-05, + "loss": 2.4179, + "theoretical_loss": 3.313334435377726, + "tokens_seen": 3088908288 + }, + { + "epoch": 0.87, + "learning_rate": 6.515151515151516e-05, + "loss": 2.8405, + "theoretical_loss": 3.313329062507513, + "tokens_seen": 3088973824 + }, + { + "epoch": 0.87, + "learning_rate": 6.513131313131314e-05, + "loss": 2.6337, + "theoretical_loss": 3.3133236897832066, + "tokens_seen": 3089039360 + }, + { + "epoch": 0.87, + "learning_rate": 6.511111111111111e-05, + "loss": 2.5904, + "theoretical_loss": 3.313318317204801, + "tokens_seen": 3089104896 + }, + { + "epoch": 0.87, + "learning_rate": 6.50909090909091e-05, + "loss": 2.5566, + "theoretical_loss": 3.3133129447722873, + "tokens_seen": 3089170432 + }, + { + "epoch": 0.87, + "learning_rate": 6.507070707070708e-05, + "loss": 2.5309, + "theoretical_loss": 3.3133075724856598, + "tokens_seen": 3089235968 + }, + { + "epoch": 0.87, + "learning_rate": 6.505050505050505e-05, + "loss": 2.5668, + "theoretical_loss": 3.313302200344911, + "tokens_seen": 3089301504 + }, + { + "epoch": 0.87, + "learning_rate": 6.503030303030303e-05, + "loss": 2.6193, + "theoretical_loss": 3.313296828350034, + "tokens_seen": 3089367040 + }, + { + "epoch": 0.87, + "learning_rate": 6.5010101010101e-05, + "loss": 2.5323, + "theoretical_loss": 3.313291456501022, + "tokens_seen": 3089432576 + }, + { + "epoch": 0.87, + "learning_rate": 6.498989898989899e-05, + "loss": 2.6576, + "theoretical_loss": 3.313286084797867, + "tokens_seen": 3089498112 + }, + { + "epoch": 0.87, + "learning_rate": 6.496969696969697e-05, + "loss": 2.5268, + "theoretical_loss": 3.3132807132405633, + "tokens_seen": 3089563648 + }, + { + "epoch": 0.87, + "learning_rate": 6.494949494949495e-05, + "loss": 2.4203, + "theoretical_loss": 3.3132753418291023, + "tokens_seen": 3089629184 + }, + { + "epoch": 0.87, + "learning_rate": 6.492929292929294e-05, + "loss": 2.6157, + "theoretical_loss": 3.3132699705634785, + "tokens_seen": 3089694720 + }, + { + "epoch": 0.87, + "learning_rate": 6.490909090909091e-05, + "loss": 2.4834, + "theoretical_loss": 3.3132645994436833, + "tokens_seen": 3089760256 + }, + { + "epoch": 0.87, + "learning_rate": 6.488888888888889e-05, + "loss": 2.7207, + "theoretical_loss": 3.313259228469711, + "tokens_seen": 3089825792 + }, + { + "epoch": 0.87, + "learning_rate": 6.486868686868686e-05, + "loss": 2.746, + "theoretical_loss": 3.313253857641554, + "tokens_seen": 3089891328 + }, + { + "epoch": 0.87, + "learning_rate": 6.484848484848485e-05, + "loss": 2.5251, + "theoretical_loss": 3.3132484869592047, + "tokens_seen": 3089956864 + }, + { + "epoch": 0.87, + "learning_rate": 6.482828282828283e-05, + "loss": 2.4626, + "theoretical_loss": 3.3132431164226572, + "tokens_seen": 3090022400 + }, + { + "epoch": 0.87, + "learning_rate": 6.480808080808081e-05, + "loss": 2.5291, + "theoretical_loss": 3.3132377460319034, + "tokens_seen": 3090087936 + }, + { + "epoch": 0.87, + "learning_rate": 6.478787878787879e-05, + "loss": 2.6051, + "theoretical_loss": 3.313232375786937, + "tokens_seen": 3090153472 + }, + { + "epoch": 0.87, + "learning_rate": 6.476767676767677e-05, + "loss": 2.4957, + "theoretical_loss": 3.31322700568775, + "tokens_seen": 3090219008 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1693616, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6141104698181152, + "objective/train/theoretical_loss": 3.3132216357343367, + "objective/train/tokens_used": 1449143776, + "theoretical_loss": 3.3132216357343367, + "tokens_seen": 3090284544 + }, + { + "epoch": 0.87, + "learning_rate": 6.474747474747475e-05, + "loss": 2.7023, + "theoretical_loss": 3.3132216357343367, + "tokens_seen": 3090284544 + }, + { + "epoch": 0.87, + "learning_rate": 6.472727272727274e-05, + "loss": 2.6418, + "theoretical_loss": 3.3132162659266893, + "tokens_seen": 3090350080 + }, + { + "epoch": 0.87, + "learning_rate": 6.470707070707071e-05, + "loss": 2.385, + "theoretical_loss": 3.313210896264801, + "tokens_seen": 3090415616 + }, + { + "epoch": 0.87, + "learning_rate": 6.468686868686868e-05, + "loss": 2.5623, + "theoretical_loss": 3.3132055267486638, + "tokens_seen": 3090481152 + }, + { + "epoch": 0.87, + "learning_rate": 6.466666666666666e-05, + "loss": 2.6563, + "theoretical_loss": 3.3132001573782714, + "tokens_seen": 3090546688 + }, + { + "epoch": 0.87, + "learning_rate": 6.464646464646465e-05, + "loss": 2.5919, + "theoretical_loss": 3.3131947881536172, + "tokens_seen": 3090612224 + }, + { + "epoch": 0.87, + "learning_rate": 6.462626262626263e-05, + "loss": 2.4475, + "theoretical_loss": 3.3131894190746936, + "tokens_seen": 3090677760 + }, + { + "epoch": 0.87, + "learning_rate": 6.460606060606061e-05, + "loss": 2.4951, + "theoretical_loss": 3.3131840501414938, + "tokens_seen": 3090743296 + }, + { + "epoch": 0.87, + "learning_rate": 6.458585858585858e-05, + "loss": 2.5584, + "theoretical_loss": 3.3131786813540107, + "tokens_seen": 3090808832 + }, + { + "epoch": 0.87, + "learning_rate": 6.456565656565657e-05, + "loss": 2.6404, + "theoretical_loss": 3.313173312712237, + "tokens_seen": 3090874368 + }, + { + "epoch": 0.87, + "learning_rate": 6.454545454545455e-05, + "loss": 2.3983, + "theoretical_loss": 3.3131679442161657, + "tokens_seen": 3090939904 + }, + { + "epoch": 0.87, + "learning_rate": 6.452525252525252e-05, + "loss": 2.4885, + "theoretical_loss": 3.31316257586579, + "tokens_seen": 3091005440 + }, + { + "epoch": 0.87, + "learning_rate": 6.45050505050505e-05, + "loss": 2.7371, + "theoretical_loss": 3.3131572076611024, + "tokens_seen": 3091070976 + }, + { + "epoch": 0.87, + "learning_rate": 6.448484848484848e-05, + "loss": 2.3774, + "theoretical_loss": 3.3131518396020967, + "tokens_seen": 3091136512 + }, + { + "epoch": 0.87, + "learning_rate": 6.446464646464646e-05, + "loss": 2.6751, + "theoretical_loss": 3.313146471688765, + "tokens_seen": 3091202048 + }, + { + "epoch": 0.87, + "learning_rate": 6.444444444444444e-05, + "loss": 2.467, + "theoretical_loss": 3.313141103921101, + "tokens_seen": 3091267584 + }, + { + "epoch": 0.87, + "learning_rate": 6.442424242424243e-05, + "loss": 2.5139, + "theoretical_loss": 3.3131357362990967, + "tokens_seen": 3091333120 + }, + { + "epoch": 0.87, + "learning_rate": 6.440404040404041e-05, + "loss": 2.5286, + "theoretical_loss": 3.3131303688227463, + "tokens_seen": 3091398656 + }, + { + "epoch": 0.87, + "learning_rate": 6.43838383838384e-05, + "loss": 2.6779, + "theoretical_loss": 3.3131250014920415, + "tokens_seen": 3091464192 + }, + { + "epoch": 0.87, + "learning_rate": 6.436363636363635e-05, + "loss": 2.5779, + "theoretical_loss": 3.313119634306976, + "tokens_seen": 3091529728 + }, + { + "epoch": 0.87, + "learning_rate": 6.434343434343434e-05, + "loss": 2.5164, + "theoretical_loss": 3.313114267267543, + "tokens_seen": 3091595264 + }, + { + "epoch": 0.87, + "learning_rate": 6.432323232323232e-05, + "loss": 2.3759, + "theoretical_loss": 3.313108900373735, + "tokens_seen": 3091660800 + }, + { + "epoch": 0.87, + "learning_rate": 6.43030303030303e-05, + "loss": 2.4686, + "theoretical_loss": 3.3131035336255446, + "tokens_seen": 3091726336 + }, + { + "epoch": 0.87, + "learning_rate": 6.428282828282829e-05, + "loss": 2.4237, + "theoretical_loss": 3.3130981670229653, + "tokens_seen": 3091791872 + }, + { + "epoch": 0.87, + "learning_rate": 6.426262626262626e-05, + "loss": 2.5719, + "theoretical_loss": 3.31309280056599, + "tokens_seen": 3091857408 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1694218, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4577786922454834, + "objective/train/theoretical_loss": 3.3130874342546117, + "objective/train/tokens_used": 1450782176, + "theoretical_loss": 3.3130874342546117, + "tokens_seen": 3091922944 + }, + { + "epoch": 0.87, + "learning_rate": 6.424242424242424e-05, + "loss": 2.5723, + "theoretical_loss": 3.3130874342546117, + "tokens_seen": 3091922944 + }, + { + "epoch": 0.87, + "learning_rate": 6.422222222222223e-05, + "loss": 2.6279, + "theoretical_loss": 3.3130820680888236, + "tokens_seen": 3091988480 + }, + { + "epoch": 0.87, + "learning_rate": 6.420202020202021e-05, + "loss": 2.7333, + "theoretical_loss": 3.313076702068618, + "tokens_seen": 3092054016 + }, + { + "epoch": 0.87, + "learning_rate": 6.418181818181818e-05, + "loss": 2.5524, + "theoretical_loss": 3.313071336193988, + "tokens_seen": 3092119552 + }, + { + "epoch": 0.87, + "learning_rate": 6.416161616161615e-05, + "loss": 2.7027, + "theoretical_loss": 3.313065970464927, + "tokens_seen": 3092185088 + }, + { + "epoch": 0.87, + "learning_rate": 6.414141414141414e-05, + "loss": 2.7349, + "theoretical_loss": 3.313060604881428, + "tokens_seen": 3092250624 + }, + { + "epoch": 0.87, + "learning_rate": 6.412121212121212e-05, + "loss": 2.508, + "theoretical_loss": 3.3130552394434836, + "tokens_seen": 3092316160 + }, + { + "epoch": 0.87, + "learning_rate": 6.41010101010101e-05, + "loss": 2.4432, + "theoretical_loss": 3.313049874151087, + "tokens_seen": 3092381696 + }, + { + "epoch": 0.87, + "learning_rate": 6.408080808080809e-05, + "loss": 2.3725, + "theoretical_loss": 3.313044509004231, + "tokens_seen": 3092447232 + }, + { + "epoch": 0.87, + "learning_rate": 6.406060606060606e-05, + "loss": 2.3277, + "theoretical_loss": 3.313039144002908, + "tokens_seen": 3092512768 + }, + { + "epoch": 0.87, + "learning_rate": 6.404040404040404e-05, + "loss": 2.402, + "theoretical_loss": 3.313033779147112, + "tokens_seen": 3092578304 + }, + { + "epoch": 0.87, + "learning_rate": 6.402020202020203e-05, + "loss": 2.3483, + "theoretical_loss": 3.313028414436836, + "tokens_seen": 3092643840 + }, + { + "epoch": 0.87, + "learning_rate": 6.4e-05, + "loss": 2.5002, + "theoretical_loss": 3.313023049872072, + "tokens_seen": 3092709376 + }, + { + "epoch": 0.87, + "learning_rate": 6.397979797979798e-05, + "loss": 2.766, + "theoretical_loss": 3.313017685452814, + "tokens_seen": 3092774912 + }, + { + "epoch": 0.87, + "learning_rate": 6.395959595959597e-05, + "loss": 2.7042, + "theoretical_loss": 3.3130123211790536, + "tokens_seen": 3092840448 + }, + { + "epoch": 0.87, + "learning_rate": 6.393939393939394e-05, + "loss": 2.5653, + "theoretical_loss": 3.3130069570507854, + "tokens_seen": 3092905984 + }, + { + "epoch": 0.87, + "learning_rate": 6.391919191919192e-05, + "loss": 2.4104, + "theoretical_loss": 3.3130015930680012, + "tokens_seen": 3092971520 + }, + { + "epoch": 0.87, + "learning_rate": 6.38989898989899e-05, + "loss": 2.4792, + "theoretical_loss": 3.312996229230695, + "tokens_seen": 3093037056 + }, + { + "epoch": 0.87, + "learning_rate": 6.387878787878789e-05, + "loss": 2.6365, + "theoretical_loss": 3.3129908655388585, + "tokens_seen": 3093102592 + }, + { + "epoch": 0.87, + "learning_rate": 6.385858585858587e-05, + "loss": 2.4534, + "theoretical_loss": 3.312985501992485, + "tokens_seen": 3093168128 + }, + { + "epoch": 0.87, + "learning_rate": 6.383838383838383e-05, + "loss": 2.6678, + "theoretical_loss": 3.3129801385915685, + "tokens_seen": 3093233664 + }, + { + "epoch": 0.87, + "learning_rate": 6.381818181818181e-05, + "loss": 2.4763, + "theoretical_loss": 3.312974775336101, + "tokens_seen": 3093299200 + }, + { + "epoch": 0.87, + "learning_rate": 6.37979797979798e-05, + "loss": 2.6092, + "theoretical_loss": 3.3129694122260758, + "tokens_seen": 3093364736 + }, + { + "epoch": 0.87, + "learning_rate": 6.377777777777778e-05, + "loss": 2.4935, + "theoretical_loss": 3.312964049261486, + "tokens_seen": 3093430272 + }, + { + "epoch": 0.87, + "learning_rate": 6.375757575757576e-05, + "loss": 2.6011, + "theoretical_loss": 3.3129586864423244, + "tokens_seen": 3093495808 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1695039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.889664649963379, + "objective/train/theoretical_loss": 3.3129533237685838, + "objective/train/tokens_used": 1452420576, + "theoretical_loss": 3.3129533237685838, + "tokens_seen": 3093561344 + }, + { + "epoch": 0.87, + "learning_rate": 6.373737373737373e-05, + "loss": 2.5791, + "theoretical_loss": 3.3129533237685838, + "tokens_seen": 3093561344 + }, + { + "epoch": 0.87, + "learning_rate": 6.371717171717172e-05, + "loss": 2.5139, + "theoretical_loss": 3.312947961240257, + "tokens_seen": 3093626880 + }, + { + "epoch": 0.87, + "learning_rate": 6.36969696969697e-05, + "loss": 2.5922, + "theoretical_loss": 3.3129425988573376, + "tokens_seen": 3093692416 + }, + { + "epoch": 0.87, + "learning_rate": 6.367676767676769e-05, + "loss": 2.5449, + "theoretical_loss": 3.3129372366198186, + "tokens_seen": 3093757952 + }, + { + "epoch": 0.87, + "learning_rate": 6.365656565656566e-05, + "loss": 2.5858, + "theoretical_loss": 3.3129318745276923, + "tokens_seen": 3093823488 + }, + { + "epoch": 0.87, + "learning_rate": 6.363636363636363e-05, + "loss": 2.6157, + "theoretical_loss": 3.3129265125809524, + "tokens_seen": 3093889024 + }, + { + "epoch": 0.87, + "learning_rate": 6.361616161616161e-05, + "loss": 2.6561, + "theoretical_loss": 3.3129211507795913, + "tokens_seen": 3093954560 + }, + { + "epoch": 0.87, + "learning_rate": 6.35959595959596e-05, + "loss": 2.5878, + "theoretical_loss": 3.3129157891236023, + "tokens_seen": 3094020096 + }, + { + "epoch": 0.87, + "learning_rate": 6.357575757575758e-05, + "loss": 2.4784, + "theoretical_loss": 3.312910427612978, + "tokens_seen": 3094085632 + }, + { + "epoch": 0.87, + "learning_rate": 6.355555555555556e-05, + "loss": 2.5126, + "theoretical_loss": 3.312905066247712, + "tokens_seen": 3094151168 + }, + { + "epoch": 0.87, + "learning_rate": 6.353535353535355e-05, + "loss": 2.4724, + "theoretical_loss": 3.312899705027797, + "tokens_seen": 3094216704 + }, + { + "epoch": 0.87, + "learning_rate": 6.351515151515152e-05, + "loss": 2.4286, + "theoretical_loss": 3.312894343953226, + "tokens_seen": 3094282240 + }, + { + "epoch": 0.87, + "learning_rate": 6.349494949494949e-05, + "loss": 2.573, + "theoretical_loss": 3.312888983023991, + "tokens_seen": 3094347776 + }, + { + "epoch": 0.87, + "learning_rate": 6.347474747474747e-05, + "loss": 2.4739, + "theoretical_loss": 3.312883622240087, + "tokens_seen": 3094413312 + }, + { + "epoch": 0.87, + "learning_rate": 6.345454545454546e-05, + "loss": 2.5379, + "theoretical_loss": 3.3128782616015053, + "tokens_seen": 3094478848 + }, + { + "epoch": 0.87, + "learning_rate": 6.343434343434344e-05, + "loss": 2.6715, + "theoretical_loss": 3.3128729011082396, + "tokens_seen": 3094544384 + }, + { + "epoch": 0.87, + "learning_rate": 6.341414141414141e-05, + "loss": 2.5329, + "theoretical_loss": 3.312867540760283, + "tokens_seen": 3094609920 + }, + { + "epoch": 0.87, + "learning_rate": 6.33939393939394e-05, + "loss": 2.4274, + "theoretical_loss": 3.312862180557628, + "tokens_seen": 3094675456 + }, + { + "epoch": 0.87, + "learning_rate": 6.337373737373738e-05, + "loss": 2.5625, + "theoretical_loss": 3.3128568205002678, + "tokens_seen": 3094740992 + }, + { + "epoch": 0.87, + "learning_rate": 6.335353535353536e-05, + "loss": 2.65, + "theoretical_loss": 3.3128514605881954, + "tokens_seen": 3094806528 + }, + { + "epoch": 0.87, + "learning_rate": 6.333333333333335e-05, + "loss": 2.5221, + "theoretical_loss": 3.3128461008214036, + "tokens_seen": 3094872064 + }, + { + "epoch": 0.87, + "learning_rate": 6.33131313131313e-05, + "loss": 2.4092, + "theoretical_loss": 3.312840741199886, + "tokens_seen": 3094937600 + }, + { + "epoch": 0.87, + "learning_rate": 6.329292929292929e-05, + "loss": 2.4415, + "theoretical_loss": 3.3128353817236347, + "tokens_seen": 3095003136 + }, + { + "epoch": 0.87, + "learning_rate": 6.327272727272727e-05, + "loss": 2.613, + "theoretical_loss": 3.3128300223926437, + "tokens_seen": 3095068672 + }, + { + "epoch": 0.87, + "learning_rate": 6.325252525252525e-05, + "loss": 2.7501, + "theoretical_loss": 3.312824663206905, + "tokens_seen": 3095134208 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1695481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6298532485961914, + "objective/train/theoretical_loss": 3.3128193041664122, + "objective/train/tokens_used": 1454058976, + "theoretical_loss": 3.3128193041664122, + "tokens_seen": 3095199744 + }, + { + "epoch": 0.88, + "learning_rate": 6.323232323232324e-05, + "loss": 2.68, + "theoretical_loss": 3.3128193041664122, + "tokens_seen": 3095199744 + }, + { + "epoch": 0.88, + "learning_rate": 6.321212121212121e-05, + "loss": 2.5162, + "theoretical_loss": 3.312813945271158, + "tokens_seen": 3095265280 + }, + { + "epoch": 0.88, + "learning_rate": 6.319191919191919e-05, + "loss": 2.4389, + "theoretical_loss": 3.312808586521135, + "tokens_seen": 3095330816 + }, + { + "epoch": 0.88, + "learning_rate": 6.317171717171718e-05, + "loss": 2.7464, + "theoretical_loss": 3.3128032279163375, + "tokens_seen": 3095396352 + }, + { + "epoch": 0.88, + "learning_rate": 6.315151515151516e-05, + "loss": 2.2844, + "theoretical_loss": 3.3127978694567575, + "tokens_seen": 3095461888 + }, + { + "epoch": 0.88, + "learning_rate": 6.313131313131313e-05, + "loss": 2.5484, + "theoretical_loss": 3.312792511142388, + "tokens_seen": 3095527424 + }, + { + "epoch": 0.88, + "learning_rate": 6.311111111111112e-05, + "loss": 2.5507, + "theoretical_loss": 3.3127871529732222, + "tokens_seen": 3095592960 + }, + { + "epoch": 0.88, + "learning_rate": 6.309090909090909e-05, + "loss": 2.6127, + "theoretical_loss": 3.3127817949492533, + "tokens_seen": 3095658496 + }, + { + "epoch": 0.88, + "learning_rate": 6.307070707070707e-05, + "loss": 2.5326, + "theoretical_loss": 3.3127764370704735, + "tokens_seen": 3095724032 + }, + { + "epoch": 0.88, + "learning_rate": 6.305050505050505e-05, + "loss": 2.5303, + "theoretical_loss": 3.3127710793368768, + "tokens_seen": 3095789568 + }, + { + "epoch": 0.88, + "learning_rate": 6.303030303030304e-05, + "loss": 2.5401, + "theoretical_loss": 3.3127657217484554, + "tokens_seen": 3095855104 + }, + { + "epoch": 0.88, + "learning_rate": 6.301010101010102e-05, + "loss": 2.6611, + "theoretical_loss": 3.312760364305203, + "tokens_seen": 3095920640 + }, + { + "epoch": 0.88, + "learning_rate": 6.298989898989899e-05, + "loss": 2.7871, + "theoretical_loss": 3.312755007007112, + "tokens_seen": 3095986176 + }, + { + "epoch": 0.88, + "learning_rate": 6.296969696969696e-05, + "loss": 2.3965, + "theoretical_loss": 3.3127496498541755, + "tokens_seen": 3096051712 + }, + { + "epoch": 0.88, + "learning_rate": 6.294949494949495e-05, + "loss": 2.5377, + "theoretical_loss": 3.3127442928463866, + "tokens_seen": 3096117248 + }, + { + "epoch": 0.88, + "learning_rate": 6.292929292929293e-05, + "loss": 2.6393, + "theoretical_loss": 3.3127389359837385, + "tokens_seen": 3096182784 + }, + { + "epoch": 0.88, + "learning_rate": 6.290909090909091e-05, + "loss": 2.5779, + "theoretical_loss": 3.312733579266224, + "tokens_seen": 3096248320 + }, + { + "epoch": 0.88, + "learning_rate": 6.288888888888888e-05, + "loss": 2.6446, + "theoretical_loss": 3.3127282226938357, + "tokens_seen": 3096313856 + }, + { + "epoch": 0.88, + "learning_rate": 6.286868686868687e-05, + "loss": 2.4689, + "theoretical_loss": 3.3127228662665673, + "tokens_seen": 3096379392 + }, + { + "epoch": 0.88, + "learning_rate": 6.284848484848485e-05, + "loss": 2.503, + "theoretical_loss": 3.3127175099844113, + "tokens_seen": 3096444928 + }, + { + "epoch": 0.88, + "learning_rate": 6.282828282828284e-05, + "loss": 2.7658, + "theoretical_loss": 3.312712153847361, + "tokens_seen": 3096510464 + }, + { + "epoch": 0.88, + "learning_rate": 6.280808080808082e-05, + "loss": 2.5225, + "theoretical_loss": 3.3127067978554092, + "tokens_seen": 3096576000 + }, + { + "epoch": 0.88, + "learning_rate": 6.278787878787878e-05, + "loss": 2.6234, + "theoretical_loss": 3.3127014420085494, + "tokens_seen": 3096641536 + }, + { + "epoch": 0.88, + "learning_rate": 6.276767676767676e-05, + "loss": 2.5125, + "theoretical_loss": 3.3126960863067736, + "tokens_seen": 3096707072 + }, + { + "epoch": 0.88, + "learning_rate": 6.274747474747475e-05, + "loss": 2.4877, + "theoretical_loss": 3.3126907307500755, + "tokens_seen": 3096772608 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1696582, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7248613834381104, + "objective/train/theoretical_loss": 3.3126853753384484, + "objective/train/tokens_used": 1455697376, + "theoretical_loss": 3.3126853753384484, + "tokens_seen": 3096838144 + }, + { + "epoch": 0.88, + "learning_rate": 6.272727272727273e-05, + "loss": 2.5119, + "theoretical_loss": 3.3126853753384484, + "tokens_seen": 3096838144 + }, + { + "epoch": 0.88, + "learning_rate": 6.270707070707071e-05, + "loss": 2.5106, + "theoretical_loss": 3.3126800200718844, + "tokens_seen": 3096903680 + }, + { + "epoch": 0.88, + "learning_rate": 6.26868686868687e-05, + "loss": 2.4791, + "theoretical_loss": 3.312674664950377, + "tokens_seen": 3096969216 + }, + { + "epoch": 0.88, + "learning_rate": 6.266666666666667e-05, + "loss": 2.531, + "theoretical_loss": 3.312669309973919, + "tokens_seen": 3097034752 + }, + { + "epoch": 0.88, + "learning_rate": 6.264646464646465e-05, + "loss": 2.5963, + "theoretical_loss": 3.3126639551425043, + "tokens_seen": 3097100288 + }, + { + "epoch": 0.88, + "learning_rate": 6.262626262626262e-05, + "loss": 2.4545, + "theoretical_loss": 3.3126586004561247, + "tokens_seen": 3097165824 + }, + { + "epoch": 0.88, + "learning_rate": 6.26060606060606e-05, + "loss": 2.5032, + "theoretical_loss": 3.3126532459147735, + "tokens_seen": 3097231360 + }, + { + "epoch": 0.88, + "learning_rate": 6.258585858585859e-05, + "loss": 2.4115, + "theoretical_loss": 3.312647891518444, + "tokens_seen": 3097296896 + }, + { + "epoch": 0.88, + "learning_rate": 6.256565656565656e-05, + "loss": 2.7151, + "theoretical_loss": 3.3126425372671293, + "tokens_seen": 3097362432 + }, + { + "epoch": 0.88, + "learning_rate": 6.254545454545454e-05, + "loss": 2.6109, + "theoretical_loss": 3.312637183160822, + "tokens_seen": 3097427968 + }, + { + "epoch": 0.88, + "learning_rate": 6.252525252525253e-05, + "loss": 2.5439, + "theoretical_loss": 3.3126318291995154, + "tokens_seen": 3097493504 + }, + { + "epoch": 0.88, + "learning_rate": 6.250505050505051e-05, + "loss": 2.4795, + "theoretical_loss": 3.3126264753832024, + "tokens_seen": 3097559040 + }, + { + "epoch": 0.88, + "learning_rate": 6.248484848484848e-05, + "loss": 2.361, + "theoretical_loss": 3.312621121711876, + "tokens_seen": 3097624576 + }, + { + "epoch": 0.88, + "learning_rate": 6.246464646464647e-05, + "loss": 2.7573, + "theoretical_loss": 3.312615768185529, + "tokens_seen": 3097690112 + }, + { + "epoch": 0.88, + "learning_rate": 6.244444444444444e-05, + "loss": 2.4943, + "theoretical_loss": 3.312610414804155, + "tokens_seen": 3097755648 + }, + { + "epoch": 0.88, + "learning_rate": 6.242424242424242e-05, + "loss": 2.4398, + "theoretical_loss": 3.3126050615677465, + "tokens_seen": 3097821184 + }, + { + "epoch": 0.88, + "learning_rate": 6.24040404040404e-05, + "loss": 2.8358, + "theoretical_loss": 3.312599708476297, + "tokens_seen": 3097886720 + }, + { + "epoch": 0.88, + "learning_rate": 6.238383838383839e-05, + "loss": 2.4262, + "theoretical_loss": 3.3125943555297983, + "tokens_seen": 3097952256 + }, + { + "epoch": 0.88, + "learning_rate": 6.236363636363636e-05, + "loss": 2.4278, + "theoretical_loss": 3.3125890027282447, + "tokens_seen": 3098017792 + }, + { + "epoch": 0.88, + "learning_rate": 6.234343434343434e-05, + "loss": 2.6825, + "theoretical_loss": 3.3125836500716286, + "tokens_seen": 3098083328 + }, + { + "epoch": 0.88, + "learning_rate": 6.232323232323233e-05, + "loss": 2.5118, + "theoretical_loss": 3.3125782975599436, + "tokens_seen": 3098148864 + }, + { + "epoch": 0.88, + "learning_rate": 6.230303030303031e-05, + "loss": 2.6161, + "theoretical_loss": 3.312572945193182, + "tokens_seen": 3098214400 + }, + { + "epoch": 0.88, + "learning_rate": 6.228282828282828e-05, + "loss": 2.5364, + "theoretical_loss": 3.312567592971337, + "tokens_seen": 3098279936 + }, + { + "epoch": 0.88, + "learning_rate": 6.226262626262627e-05, + "loss": 2.3895, + "theoretical_loss": 3.312562240894402, + "tokens_seen": 3098345472 + }, + { + "epoch": 0.88, + "learning_rate": 6.224242424242425e-05, + "loss": 2.5503, + "theoretical_loss": 3.3125568889623693, + "tokens_seen": 3098411008 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1697795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.322500705718994, + "objective/train/theoretical_loss": 3.3125515371752323, + "objective/train/tokens_used": 1457335776, + "theoretical_loss": 3.3125515371752323, + "tokens_seen": 3098476544 + }, + { + "epoch": 0.88, + "learning_rate": 6.222222222222222e-05, + "loss": 2.4702, + "theoretical_loss": 3.3125515371752323, + "tokens_seen": 3098476544 + }, + { + "epoch": 0.88, + "learning_rate": 6.22020202020202e-05, + "loss": 2.762, + "theoretical_loss": 3.3125461855329847, + "tokens_seen": 3098542080 + }, + { + "epoch": 0.88, + "learning_rate": 6.218181818181817e-05, + "loss": 2.4064, + "theoretical_loss": 3.3125408340356186, + "tokens_seen": 3098607616 + }, + { + "epoch": 0.88, + "learning_rate": 6.216161616161616e-05, + "loss": 2.3894, + "theoretical_loss": 3.312535482683127, + "tokens_seen": 3098673152 + }, + { + "epoch": 0.88, + "learning_rate": 6.214141414141414e-05, + "loss": 2.5943, + "theoretical_loss": 3.312530131475503, + "tokens_seen": 3098738688 + }, + { + "epoch": 0.88, + "learning_rate": 6.212121212121213e-05, + "loss": 2.6939, + "theoretical_loss": 3.3125247804127405, + "tokens_seen": 3098804224 + }, + { + "epoch": 0.88, + "learning_rate": 6.21010101010101e-05, + "loss": 2.7059, + "theoretical_loss": 3.3125194294948312, + "tokens_seen": 3098869760 + }, + { + "epoch": 0.88, + "learning_rate": 6.208080808080808e-05, + "loss": 2.3517, + "theoretical_loss": 3.312514078721769, + "tokens_seen": 3098935296 + }, + { + "epoch": 0.88, + "learning_rate": 6.206060606060606e-05, + "loss": 2.6556, + "theoretical_loss": 3.3125087280935466, + "tokens_seen": 3099000832 + }, + { + "epoch": 0.88, + "learning_rate": 6.204040404040405e-05, + "loss": 2.6908, + "theoretical_loss": 3.312503377610157, + "tokens_seen": 3099066368 + }, + { + "epoch": 0.88, + "learning_rate": 6.202020202020202e-05, + "loss": 2.4926, + "theoretical_loss": 3.3124980272715936, + "tokens_seen": 3099131904 + }, + { + "epoch": 0.88, + "learning_rate": 6.2e-05, + "loss": 2.5133, + "theoretical_loss": 3.3124926770778487, + "tokens_seen": 3099197440 + }, + { + "epoch": 0.88, + "learning_rate": 6.197979797979799e-05, + "loss": 2.6487, + "theoretical_loss": 3.3124873270289155, + "tokens_seen": 3099262976 + }, + { + "epoch": 0.88, + "learning_rate": 6.195959595959596e-05, + "loss": 2.5897, + "theoretical_loss": 3.3124819771247878, + "tokens_seen": 3099328512 + }, + { + "epoch": 0.88, + "learning_rate": 6.193939393939394e-05, + "loss": 2.5271, + "theoretical_loss": 3.3124766273654576, + "tokens_seen": 3099394048 + }, + { + "epoch": 0.88, + "learning_rate": 6.191919191919192e-05, + "loss": 2.5876, + "theoretical_loss": 3.3124712777509187, + "tokens_seen": 3099459584 + }, + { + "epoch": 0.88, + "learning_rate": 6.18989898989899e-05, + "loss": 2.367, + "theoretical_loss": 3.3124659282811635, + "tokens_seen": 3099525120 + }, + { + "epoch": 0.88, + "learning_rate": 6.187878787878788e-05, + "loss": 2.2925, + "theoretical_loss": 3.312460578956186, + "tokens_seen": 3099590656 + }, + { + "epoch": 0.88, + "learning_rate": 6.185858585858586e-05, + "loss": 2.4821, + "theoretical_loss": 3.3124552297759777, + "tokens_seen": 3099656192 + }, + { + "epoch": 0.88, + "learning_rate": 6.183838383838383e-05, + "loss": 2.6006, + "theoretical_loss": 3.3124498807405325, + "tokens_seen": 3099721728 + }, + { + "epoch": 0.88, + "learning_rate": 6.181818181818182e-05, + "loss": 2.6217, + "theoretical_loss": 3.312444531849844, + "tokens_seen": 3099787264 + }, + { + "epoch": 0.88, + "learning_rate": 6.17979797979798e-05, + "loss": 2.6884, + "theoretical_loss": 3.312439183103904, + "tokens_seen": 3099852800 + }, + { + "epoch": 0.88, + "learning_rate": 6.177777777777779e-05, + "loss": 2.7982, + "theoretical_loss": 3.3124338345027065, + "tokens_seen": 3099918336 + }, + { + "epoch": 0.88, + "learning_rate": 6.175757575757576e-05, + "loss": 2.5943, + "theoretical_loss": 3.312428486046244, + "tokens_seen": 3099983872 + }, + { + "epoch": 0.88, + "learning_rate": 6.173737373737374e-05, + "loss": 2.6628, + "theoretical_loss": 3.3124231377345095, + "tokens_seen": 3100049408 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1698452, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5663528442382812, + "objective/train/theoretical_loss": 3.312417789567496, + "objective/train/tokens_used": 1458974176, + "theoretical_loss": 3.312417789567496, + "tokens_seen": 3100114944 + }, + { + "epoch": 0.88, + "learning_rate": 6.171717171717172e-05, + "loss": 2.5289, + "theoretical_loss": 3.312417789567496, + "tokens_seen": 3100114944 + }, + { + "epoch": 0.88, + "learning_rate": 6.16969696969697e-05, + "loss": 2.4182, + "theoretical_loss": 3.3124124415451974, + "tokens_seen": 3100180480 + }, + { + "epoch": 0.88, + "learning_rate": 6.167676767676768e-05, + "loss": 2.5353, + "theoretical_loss": 3.3124070936676056, + "tokens_seen": 3100246016 + }, + { + "epoch": 0.88, + "learning_rate": 6.165656565656566e-05, + "loss": 2.9345, + "theoretical_loss": 3.3124017459347144, + "tokens_seen": 3100311552 + }, + { + "epoch": 0.88, + "learning_rate": 6.163636363636363e-05, + "loss": 2.572, + "theoretical_loss": 3.312396398346516, + "tokens_seen": 3100377088 + }, + { + "epoch": 0.88, + "learning_rate": 6.161616161616162e-05, + "loss": 2.3813, + "theoretical_loss": 3.3123910509030043, + "tokens_seen": 3100442624 + }, + { + "epoch": 0.88, + "learning_rate": 6.15959595959596e-05, + "loss": 2.4589, + "theoretical_loss": 3.3123857036041717, + "tokens_seen": 3100508160 + }, + { + "epoch": 0.88, + "learning_rate": 6.157575757575757e-05, + "loss": 2.4343, + "theoretical_loss": 3.3123803564500114, + "tokens_seen": 3100573696 + }, + { + "epoch": 0.88, + "learning_rate": 6.155555555555555e-05, + "loss": 2.5572, + "theoretical_loss": 3.3123750094405167, + "tokens_seen": 3100639232 + }, + { + "epoch": 0.88, + "learning_rate": 6.153535353535354e-05, + "loss": 2.6416, + "theoretical_loss": 3.3123696625756804, + "tokens_seen": 3100704768 + }, + { + "epoch": 0.88, + "learning_rate": 6.151515151515152e-05, + "loss": 2.5129, + "theoretical_loss": 3.3123643158554956, + "tokens_seen": 3100770304 + }, + { + "epoch": 0.88, + "learning_rate": 6.14949494949495e-05, + "loss": 2.5191, + "theoretical_loss": 3.312358969279955, + "tokens_seen": 3100835840 + }, + { + "epoch": 0.88, + "learning_rate": 6.147474747474748e-05, + "loss": 2.4498, + "theoretical_loss": 3.312353622849052, + "tokens_seen": 3100901376 + }, + { + "epoch": 0.88, + "learning_rate": 6.145454545454546e-05, + "loss": 2.5606, + "theoretical_loss": 3.31234827656278, + "tokens_seen": 3100966912 + }, + { + "epoch": 0.88, + "learning_rate": 6.143434343434343e-05, + "loss": 2.687, + "theoretical_loss": 3.3123429304211314, + "tokens_seen": 3101032448 + }, + { + "epoch": 0.88, + "learning_rate": 6.141414141414142e-05, + "loss": 2.4729, + "theoretical_loss": 3.312337584424099, + "tokens_seen": 3101097984 + }, + { + "epoch": 0.88, + "learning_rate": 6.13939393939394e-05, + "loss": 2.5053, + "theoretical_loss": 3.3123322385716767, + "tokens_seen": 3101163520 + }, + { + "epoch": 0.88, + "learning_rate": 6.137373737373737e-05, + "loss": 2.1983, + "theoretical_loss": 3.312326892863857, + "tokens_seen": 3101229056 + }, + { + "epoch": 0.88, + "learning_rate": 6.135353535353535e-05, + "loss": 2.5121, + "theoretical_loss": 3.312321547300633, + "tokens_seen": 3101294592 + }, + { + "epoch": 0.88, + "learning_rate": 6.133333333333334e-05, + "loss": 2.509, + "theoretical_loss": 3.3123162018819974, + "tokens_seen": 3101360128 + }, + { + "epoch": 0.88, + "learning_rate": 6.131313131313131e-05, + "loss": 2.6087, + "theoretical_loss": 3.3123108566079438, + "tokens_seen": 3101425664 + }, + { + "epoch": 0.88, + "learning_rate": 6.129292929292929e-05, + "loss": 2.3842, + "theoretical_loss": 3.312305511478465, + "tokens_seen": 3101491200 + }, + { + "epoch": 0.88, + "learning_rate": 6.127272727272728e-05, + "loss": 2.6654, + "theoretical_loss": 3.3123001664935545, + "tokens_seen": 3101556736 + }, + { + "epoch": 0.88, + "learning_rate": 6.125252525252526e-05, + "loss": 2.4942, + "theoretical_loss": 3.312294821653204, + "tokens_seen": 3101622272 + }, + { + "epoch": 0.88, + "learning_rate": 6.123232323232323e-05, + "loss": 2.4403, + "theoretical_loss": 3.3122894769574085, + "tokens_seen": 3101687808 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1699609, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.601219415664673, + "objective/train/theoretical_loss": 3.312284132406159, + "objective/train/tokens_used": 1460612576, + "theoretical_loss": 3.312284132406159, + "tokens_seen": 3101753344 + }, + { + "epoch": 0.88, + "learning_rate": 6.121212121212121e-05, + "loss": 2.6571, + "theoretical_loss": 3.312284132406159, + "tokens_seen": 3101753344 + }, + { + "epoch": 0.88, + "learning_rate": 6.11919191919192e-05, + "loss": 2.7138, + "theoretical_loss": 3.3122787879994497, + "tokens_seen": 3101818880 + }, + { + "epoch": 0.88, + "learning_rate": 6.117171717171717e-05, + "loss": 2.5408, + "theoretical_loss": 3.312273443737274, + "tokens_seen": 3101884416 + }, + { + "epoch": 0.88, + "learning_rate": 6.115151515151515e-05, + "loss": 2.4109, + "theoretical_loss": 3.3122680996196237, + "tokens_seen": 3101949952 + }, + { + "epoch": 0.88, + "learning_rate": 6.113131313131314e-05, + "loss": 2.7438, + "theoretical_loss": 3.3122627556464925, + "tokens_seen": 3102015488 + }, + { + "epoch": 0.88, + "learning_rate": 6.111111111111111e-05, + "loss": 2.3149, + "theoretical_loss": 3.312257411817874, + "tokens_seen": 3102081024 + }, + { + "epoch": 0.88, + "learning_rate": 6.109090909090909e-05, + "loss": 2.5854, + "theoretical_loss": 3.31225206813376, + "tokens_seen": 3102146560 + }, + { + "epoch": 0.88, + "learning_rate": 6.107070707070708e-05, + "loss": 2.5051, + "theoretical_loss": 3.3122467245941447, + "tokens_seen": 3102212096 + }, + { + "epoch": 0.88, + "learning_rate": 6.105050505050505e-05, + "loss": 2.6118, + "theoretical_loss": 3.3122413811990206, + "tokens_seen": 3102277632 + }, + { + "epoch": 0.88, + "learning_rate": 6.1030303030303036e-05, + "loss": 2.4403, + "theoretical_loss": 3.312236037948381, + "tokens_seen": 3102343168 + }, + { + "epoch": 0.88, + "learning_rate": 6.101010101010101e-05, + "loss": 2.6937, + "theoretical_loss": 3.312230694842218, + "tokens_seen": 3102408704 + }, + { + "epoch": 0.88, + "learning_rate": 6.098989898989899e-05, + "loss": 2.5728, + "theoretical_loss": 3.312225351880526, + "tokens_seen": 3102474240 + }, + { + "epoch": 0.88, + "learning_rate": 6.0969696969696975e-05, + "loss": 2.5153, + "theoretical_loss": 3.312220009063297, + "tokens_seen": 3102539776 + }, + { + "epoch": 0.88, + "learning_rate": 6.094949494949495e-05, + "loss": 2.6498, + "theoretical_loss": 3.312214666390525, + "tokens_seen": 3102605312 + }, + { + "epoch": 0.88, + "learning_rate": 6.092929292929293e-05, + "loss": 2.4861, + "theoretical_loss": 3.312209323862202, + "tokens_seen": 3102670848 + }, + { + "epoch": 0.88, + "learning_rate": 6.0909090909090906e-05, + "loss": 2.4796, + "theoretical_loss": 3.312203981478322, + "tokens_seen": 3102736384 + }, + { + "epoch": 0.88, + "learning_rate": 6.088888888888889e-05, + "loss": 2.4995, + "theoretical_loss": 3.3121986392388774, + "tokens_seen": 3102801920 + }, + { + "epoch": 0.88, + "learning_rate": 6.0868686868686874e-05, + "loss": 2.5709, + "theoretical_loss": 3.3121932971438612, + "tokens_seen": 3102867456 + }, + { + "epoch": 0.88, + "learning_rate": 6.0848484848484845e-05, + "loss": 2.7927, + "theoretical_loss": 3.312187955193267, + "tokens_seen": 3102932992 + }, + { + "epoch": 0.88, + "learning_rate": 6.082828282828283e-05, + "loss": 2.6521, + "theoretical_loss": 3.3121826133870873, + "tokens_seen": 3102998528 + }, + { + "epoch": 0.88, + "learning_rate": 6.080808080808081e-05, + "loss": 2.7619, + "theoretical_loss": 3.3121772717253157, + "tokens_seen": 3103064064 + }, + { + "epoch": 0.88, + "learning_rate": 6.078787878787879e-05, + "loss": 2.5576, + "theoretical_loss": 3.3121719302079446, + "tokens_seen": 3103129600 + }, + { + "epoch": 0.88, + "learning_rate": 6.0767676767676774e-05, + "loss": 2.759, + "theoretical_loss": 3.312166588834968, + "tokens_seen": 3103195136 + }, + { + "epoch": 0.88, + "learning_rate": 6.0747474747474744e-05, + "loss": 2.5586, + "theoretical_loss": 3.3121612476063778, + "tokens_seen": 3103260672 + }, + { + "epoch": 0.88, + "learning_rate": 6.072727272727273e-05, + "loss": 2.6071, + "theoretical_loss": 3.3121559065221673, + "tokens_seen": 3103326208 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1699977, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4875283241271973, + "objective/train/theoretical_loss": 3.3121505655823302, + "objective/train/tokens_used": 1462250976, + "theoretical_loss": 3.3121505655823302, + "tokens_seen": 3103391744 + }, + { + "epoch": 0.88, + "learning_rate": 6.070707070707071e-05, + "loss": 2.5343, + "theoretical_loss": 3.3121505655823302, + "tokens_seen": 3103391744 + }, + { + "epoch": 0.88, + "learning_rate": 6.068686868686869e-05, + "loss": 2.5134, + "theoretical_loss": 3.312145224786859, + "tokens_seen": 3103457280 + }, + { + "epoch": 0.88, + "learning_rate": 6.0666666666666666e-05, + "loss": 2.5927, + "theoretical_loss": 3.312139884135747, + "tokens_seen": 3103522816 + }, + { + "epoch": 0.88, + "learning_rate": 6.064646464646465e-05, + "loss": 2.5497, + "theoretical_loss": 3.312134543628987, + "tokens_seen": 3103588352 + }, + { + "epoch": 0.88, + "learning_rate": 6.062626262626263e-05, + "loss": 2.3856, + "theoretical_loss": 3.3121292032665726, + "tokens_seen": 3103653888 + }, + { + "epoch": 0.88, + "learning_rate": 6.060606060606061e-05, + "loss": 2.4927, + "theoretical_loss": 3.3121238630484964, + "tokens_seen": 3103719424 + }, + { + "epoch": 0.88, + "learning_rate": 6.058585858585858e-05, + "loss": 2.5122, + "theoretical_loss": 3.3121185229747514, + "tokens_seen": 3103784960 + }, + { + "epoch": 0.88, + "learning_rate": 6.0565656565656566e-05, + "loss": 2.3729, + "theoretical_loss": 3.3121131830453305, + "tokens_seen": 3103850496 + }, + { + "epoch": 0.88, + "learning_rate": 6.054545454545455e-05, + "loss": 2.4443, + "theoretical_loss": 3.312107843260227, + "tokens_seen": 3103916032 + }, + { + "epoch": 0.88, + "learning_rate": 6.052525252525253e-05, + "loss": 2.577, + "theoretical_loss": 3.3121025036194345, + "tokens_seen": 3103981568 + }, + { + "epoch": 0.88, + "learning_rate": 6.0505050505050504e-05, + "loss": 2.6435, + "theoretical_loss": 3.312097164122945, + "tokens_seen": 3104047104 + }, + { + "epoch": 0.88, + "learning_rate": 6.048484848484848e-05, + "loss": 2.397, + "theoretical_loss": 3.3120918247707527, + "tokens_seen": 3104112640 + }, + { + "epoch": 0.88, + "learning_rate": 6.0464646464646465e-05, + "loss": 2.5688, + "theoretical_loss": 3.3120864855628493, + "tokens_seen": 3104178176 + }, + { + "epoch": 0.88, + "learning_rate": 6.044444444444445e-05, + "loss": 2.5518, + "theoretical_loss": 3.3120811464992292, + "tokens_seen": 3104243712 + }, + { + "epoch": 0.88, + "learning_rate": 6.0424242424242426e-05, + "loss": 2.5791, + "theoretical_loss": 3.3120758075798844, + "tokens_seen": 3104309248 + }, + { + "epoch": 0.88, + "learning_rate": 6.0404040404040404e-05, + "loss": 2.418, + "theoretical_loss": 3.3120704688048086, + "tokens_seen": 3104374784 + }, + { + "epoch": 0.88, + "learning_rate": 6.038383838383839e-05, + "loss": 2.4642, + "theoretical_loss": 3.3120651301739947, + "tokens_seen": 3104440320 + }, + { + "epoch": 0.88, + "learning_rate": 6.0363636363636365e-05, + "loss": 2.5801, + "theoretical_loss": 3.3120597916874357, + "tokens_seen": 3104505856 + }, + { + "epoch": 0.88, + "learning_rate": 6.034343434343435e-05, + "loss": 2.5841, + "theoretical_loss": 3.312054453345125, + "tokens_seen": 3104571392 + }, + { + "epoch": 0.88, + "learning_rate": 6.032323232323232e-05, + "loss": 2.4319, + "theoretical_loss": 3.3120491151470546, + "tokens_seen": 3104636928 + }, + { + "epoch": 0.88, + "learning_rate": 6.03030303030303e-05, + "loss": 2.4052, + "theoretical_loss": 3.3120437770932183, + "tokens_seen": 3104702464 + }, + { + "epoch": 0.88, + "learning_rate": 6.028282828282829e-05, + "loss": 2.6067, + "theoretical_loss": 3.3120384391836097, + "tokens_seen": 3104768000 + }, + { + "epoch": 0.88, + "learning_rate": 6.0262626262626264e-05, + "loss": 2.5073, + "theoretical_loss": 3.312033101418221, + "tokens_seen": 3104833536 + }, + { + "epoch": 0.88, + "learning_rate": 6.024242424242424e-05, + "loss": 2.5479, + "theoretical_loss": 3.3120277637970457, + "tokens_seen": 3104899072 + }, + { + "epoch": 0.88, + "learning_rate": 6.0222222222222225e-05, + "loss": 2.352, + "theoretical_loss": 3.3120224263200764, + "tokens_seen": 3104964608 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1701277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4853217601776123, + "objective/train/theoretical_loss": 3.312017088987307, + "objective/train/tokens_used": 1463889376, + "theoretical_loss": 3.312017088987307, + "tokens_seen": 3105030144 + }, + { + "epoch": 0.88, + "learning_rate": 6.02020202020202e-05, + "loss": 2.5866, + "theoretical_loss": 3.312017088987307, + "tokens_seen": 3105030144 + }, + { + "epoch": 0.88, + "learning_rate": 6.0181818181818187e-05, + "loss": 2.4091, + "theoretical_loss": 3.3120117517987295, + "tokens_seen": 3105095680 + }, + { + "epoch": 0.88, + "learning_rate": 6.016161616161616e-05, + "loss": 2.6715, + "theoretical_loss": 3.3120064147543378, + "tokens_seen": 3105161216 + }, + { + "epoch": 0.88, + "learning_rate": 6.014141414141414e-05, + "loss": 2.646, + "theoretical_loss": 3.312001077854125, + "tokens_seen": 3105226752 + }, + { + "epoch": 0.88, + "learning_rate": 6.0121212121212125e-05, + "loss": 2.3669, + "theoretical_loss": 3.311995741098083, + "tokens_seen": 3105292288 + }, + { + "epoch": 0.88, + "learning_rate": 6.01010101010101e-05, + "loss": 2.6533, + "theoretical_loss": 3.311990404486206, + "tokens_seen": 3105357824 + }, + { + "epoch": 0.88, + "learning_rate": 6.0080808080808086e-05, + "loss": 2.4392, + "theoretical_loss": 3.3119850680184872, + "tokens_seen": 3105423360 + }, + { + "epoch": 0.88, + "learning_rate": 6.0060606060606056e-05, + "loss": 2.5074, + "theoretical_loss": 3.311979731694919, + "tokens_seen": 3105488896 + }, + { + "epoch": 0.88, + "learning_rate": 6.004040404040404e-05, + "loss": 2.555, + "theoretical_loss": 3.3119743955154943, + "tokens_seen": 3105554432 + }, + { + "epoch": 0.88, + "learning_rate": 6.0020202020202024e-05, + "loss": 2.5081, + "theoretical_loss": 3.311969059480207, + "tokens_seen": 3105619968 + }, + { + "epoch": 0.88, + "learning_rate": 6e-05, + "loss": 2.7133, + "theoretical_loss": 3.3119637235890496, + "tokens_seen": 3105685504 + }, + { + "epoch": 0.88, + "learning_rate": 5.997979797979798e-05, + "loss": 2.3401, + "theoretical_loss": 3.311958387842015, + "tokens_seen": 3105751040 + }, + { + "epoch": 0.88, + "learning_rate": 5.995959595959596e-05, + "loss": 2.5552, + "theoretical_loss": 3.311953052239097, + "tokens_seen": 3105816576 + }, + { + "epoch": 0.88, + "learning_rate": 5.993939393939394e-05, + "loss": 2.5798, + "theoretical_loss": 3.311947716780288, + "tokens_seen": 3105882112 + }, + { + "epoch": 0.88, + "learning_rate": 5.9919191919191924e-05, + "loss": 2.7055, + "theoretical_loss": 3.311942381465581, + "tokens_seen": 3105947648 + }, + { + "epoch": 0.88, + "learning_rate": 5.9898989898989894e-05, + "loss": 2.7204, + "theoretical_loss": 3.31193704629497, + "tokens_seen": 3106013184 + }, + { + "epoch": 0.88, + "learning_rate": 5.987878787878788e-05, + "loss": 2.4615, + "theoretical_loss": 3.3119317112684468, + "tokens_seen": 3106078720 + }, + { + "epoch": 0.88, + "learning_rate": 5.985858585858586e-05, + "loss": 2.5158, + "theoretical_loss": 3.311926376386005, + "tokens_seen": 3106144256 + }, + { + "epoch": 0.88, + "learning_rate": 5.983838383838384e-05, + "loss": 2.6355, + "theoretical_loss": 3.311921041647638, + "tokens_seen": 3106209792 + }, + { + "epoch": 0.88, + "learning_rate": 5.981818181818182e-05, + "loss": 2.6673, + "theoretical_loss": 3.311915707053339, + "tokens_seen": 3106275328 + }, + { + "epoch": 0.88, + "learning_rate": 5.97979797979798e-05, + "loss": 2.4921, + "theoretical_loss": 3.3119103726031005, + "tokens_seen": 3106340864 + }, + { + "epoch": 0.88, + "learning_rate": 5.977777777777778e-05, + "loss": 2.6558, + "theoretical_loss": 3.3119050382969153, + "tokens_seen": 3106406400 + }, + { + "epoch": 0.88, + "learning_rate": 5.975757575757576e-05, + "loss": 2.3377, + "theoretical_loss": 3.3118997041347775, + "tokens_seen": 3106471936 + }, + { + "epoch": 0.88, + "learning_rate": 5.973737373737374e-05, + "loss": 2.5013, + "theoretical_loss": 3.311894370116679, + "tokens_seen": 3106537472 + }, + { + "epoch": 0.88, + "learning_rate": 5.9717171717171716e-05, + "loss": 2.5437, + "theoretical_loss": 3.311889036242614, + "tokens_seen": 3106603008 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1702465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5862417221069336, + "objective/train/theoretical_loss": 3.3118837025125747, + "objective/train/tokens_used": 1465527776, + "theoretical_loss": 3.3118837025125747, + "tokens_seen": 3106668544 + }, + { + "epoch": 0.88, + "learning_rate": 5.96969696969697e-05, + "loss": 2.5774, + "theoretical_loss": 3.3118837025125747, + "tokens_seen": 3106668544 + }, + { + "epoch": 0.88, + "learning_rate": 5.967676767676768e-05, + "loss": 2.5998, + "theoretical_loss": 3.3118783689265547, + "tokens_seen": 3106734080 + }, + { + "epoch": 0.88, + "learning_rate": 5.965656565656566e-05, + "loss": 2.7441, + "theoretical_loss": 3.311873035484547, + "tokens_seen": 3106799616 + }, + { + "epoch": 0.88, + "learning_rate": 5.963636363636363e-05, + "loss": 2.5806, + "theoretical_loss": 3.3118677021865444, + "tokens_seen": 3106865152 + }, + { + "epoch": 0.88, + "learning_rate": 5.9616161616161615e-05, + "loss": 2.6239, + "theoretical_loss": 3.31186236903254, + "tokens_seen": 3106930688 + }, + { + "epoch": 0.88, + "learning_rate": 5.95959595959596e-05, + "loss": 2.4948, + "theoretical_loss": 3.3118570360225275, + "tokens_seen": 3106996224 + }, + { + "epoch": 0.88, + "learning_rate": 5.957575757575758e-05, + "loss": 2.4716, + "theoretical_loss": 3.311851703156499, + "tokens_seen": 3107061760 + }, + { + "epoch": 0.88, + "learning_rate": 5.955555555555556e-05, + "loss": 2.6291, + "theoretical_loss": 3.3118463704344485, + "tokens_seen": 3107127296 + }, + { + "epoch": 0.88, + "learning_rate": 5.953535353535354e-05, + "loss": 2.415, + "theoretical_loss": 3.311841037856368, + "tokens_seen": 3107192832 + }, + { + "epoch": 0.88, + "learning_rate": 5.9515151515151515e-05, + "loss": 2.6237, + "theoretical_loss": 3.3118357054222516, + "tokens_seen": 3107258368 + }, + { + "epoch": 0.88, + "learning_rate": 5.94949494949495e-05, + "loss": 2.6566, + "theoretical_loss": 3.311830373132092, + "tokens_seen": 3107323904 + }, + { + "epoch": 0.88, + "learning_rate": 5.9474747474747476e-05, + "loss": 2.4192, + "theoretical_loss": 3.311825040985882, + "tokens_seen": 3107389440 + }, + { + "epoch": 0.88, + "learning_rate": 5.945454545454545e-05, + "loss": 2.45, + "theoretical_loss": 3.3118197089836157, + "tokens_seen": 3107454976 + }, + { + "epoch": 0.88, + "learning_rate": 5.943434343434344e-05, + "loss": 2.4953, + "theoretical_loss": 3.311814377125285, + "tokens_seen": 3107520512 + }, + { + "epoch": 0.88, + "learning_rate": 5.9414141414141414e-05, + "loss": 2.3781, + "theoretical_loss": 3.3118090454108833, + "tokens_seen": 3107586048 + }, + { + "epoch": 0.88, + "learning_rate": 5.93939393939394e-05, + "loss": 2.3685, + "theoretical_loss": 3.311803713840404, + "tokens_seen": 3107651584 + }, + { + "epoch": 0.88, + "learning_rate": 5.9373737373737376e-05, + "loss": 2.4293, + "theoretical_loss": 3.3117983824138397, + "tokens_seen": 3107717120 + }, + { + "epoch": 0.88, + "learning_rate": 5.935353535353535e-05, + "loss": 2.2755, + "theoretical_loss": 3.3117930511311835, + "tokens_seen": 3107782656 + }, + { + "epoch": 0.88, + "learning_rate": 5.933333333333334e-05, + "loss": 2.3488, + "theoretical_loss": 3.311787719992429, + "tokens_seen": 3107848192 + }, + { + "epoch": 0.88, + "learning_rate": 5.9313131313131314e-05, + "loss": 2.5903, + "theoretical_loss": 3.3117823889975693, + "tokens_seen": 3107913728 + }, + { + "epoch": 0.88, + "learning_rate": 5.929292929292929e-05, + "loss": 2.5463, + "theoretical_loss": 3.311777058146597, + "tokens_seen": 3107979264 + }, + { + "epoch": 0.88, + "learning_rate": 5.9272727272727275e-05, + "loss": 2.3332, + "theoretical_loss": 3.3117717274395053, + "tokens_seen": 3108044800 + }, + { + "epoch": 0.88, + "learning_rate": 5.925252525252525e-05, + "loss": 2.5112, + "theoretical_loss": 3.311766396876288, + "tokens_seen": 3108110336 + }, + { + "epoch": 0.88, + "learning_rate": 5.9232323232323236e-05, + "loss": 2.4528, + "theoretical_loss": 3.3117610664569366, + "tokens_seen": 3108175872 + }, + { + "epoch": 0.88, + "learning_rate": 5.921212121212121e-05, + "loss": 2.6739, + "theoretical_loss": 3.3117557361814454, + "tokens_seen": 3108241408 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1702852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.631697416305542, + "objective/train/theoretical_loss": 3.311750406049807, + "objective/train/tokens_used": 1467166176, + "theoretical_loss": 3.311750406049807, + "tokens_seen": 3108306944 + }, + { + "epoch": 0.88, + "learning_rate": 5.919191919191919e-05, + "loss": 2.6014, + "theoretical_loss": 3.311750406049807, + "tokens_seen": 3108306944 + }, + { + "epoch": 0.88, + "learning_rate": 5.9171717171717175e-05, + "loss": 2.4699, + "theoretical_loss": 3.3117450760620155, + "tokens_seen": 3108372480 + }, + { + "epoch": 0.88, + "learning_rate": 5.915151515151515e-05, + "loss": 2.6776, + "theoretical_loss": 3.3117397462180627, + "tokens_seen": 3108438016 + }, + { + "epoch": 0.88, + "learning_rate": 5.9131313131313136e-05, + "loss": 2.5916, + "theoretical_loss": 3.311734416517942, + "tokens_seen": 3108503552 + }, + { + "epoch": 0.88, + "learning_rate": 5.911111111111111e-05, + "loss": 2.4766, + "theoretical_loss": 3.3117290869616465, + "tokens_seen": 3108569088 + }, + { + "epoch": 0.88, + "learning_rate": 5.909090909090909e-05, + "loss": 2.5136, + "theoretical_loss": 3.31172375754917, + "tokens_seen": 3108634624 + }, + { + "epoch": 0.88, + "learning_rate": 5.9070707070707074e-05, + "loss": 2.3885, + "theoretical_loss": 3.3117184282805048, + "tokens_seen": 3108700160 + }, + { + "epoch": 0.88, + "learning_rate": 5.905050505050505e-05, + "loss": 2.513, + "theoretical_loss": 3.311713099155644, + "tokens_seen": 3108765696 + }, + { + "epoch": 0.88, + "learning_rate": 5.903030303030303e-05, + "loss": 2.4896, + "theoretical_loss": 3.311707770174581, + "tokens_seen": 3108831232 + }, + { + "epoch": 0.88, + "learning_rate": 5.901010101010101e-05, + "loss": 2.7205, + "theoretical_loss": 3.311702441337309, + "tokens_seen": 3108896768 + }, + { + "epoch": 0.88, + "learning_rate": 5.898989898989899e-05, + "loss": 2.6017, + "theoretical_loss": 3.3116971126438206, + "tokens_seen": 3108962304 + }, + { + "epoch": 0.88, + "learning_rate": 5.8969696969696973e-05, + "loss": 2.3786, + "theoretical_loss": 3.3116917840941094, + "tokens_seen": 3109027840 + }, + { + "epoch": 0.88, + "learning_rate": 5.894949494949495e-05, + "loss": 2.4606, + "theoretical_loss": 3.311686455688168, + "tokens_seen": 3109093376 + }, + { + "epoch": 0.88, + "learning_rate": 5.892929292929293e-05, + "loss": 2.5305, + "theoretical_loss": 3.3116811274259903, + "tokens_seen": 3109158912 + }, + { + "epoch": 0.88, + "learning_rate": 5.890909090909091e-05, + "loss": 2.3527, + "theoretical_loss": 3.3116757993075683, + "tokens_seen": 3109224448 + }, + { + "epoch": 0.88, + "learning_rate": 5.888888888888889e-05, + "loss": 2.4679, + "theoretical_loss": 3.3116704713328957, + "tokens_seen": 3109289984 + }, + { + "epoch": 0.88, + "learning_rate": 5.886868686868687e-05, + "loss": 2.5124, + "theoretical_loss": 3.3116651435019655, + "tokens_seen": 3109355520 + }, + { + "epoch": 0.88, + "learning_rate": 5.884848484848485e-05, + "loss": 2.3745, + "theoretical_loss": 3.311659815814771, + "tokens_seen": 3109421056 + }, + { + "epoch": 0.88, + "learning_rate": 5.882828282828283e-05, + "loss": 2.3728, + "theoretical_loss": 3.311654488271305, + "tokens_seen": 3109486592 + }, + { + "epoch": 0.88, + "learning_rate": 5.880808080808081e-05, + "loss": 2.5824, + "theoretical_loss": 3.311649160871561, + "tokens_seen": 3109552128 + }, + { + "epoch": 0.88, + "learning_rate": 5.878787878787879e-05, + "loss": 2.4978, + "theoretical_loss": 3.3116438336155314, + "tokens_seen": 3109617664 + }, + { + "epoch": 0.88, + "learning_rate": 5.8767676767676766e-05, + "loss": 2.6183, + "theoretical_loss": 3.31163850650321, + "tokens_seen": 3109683200 + }, + { + "epoch": 0.88, + "learning_rate": 5.874747474747475e-05, + "loss": 2.459, + "theoretical_loss": 3.3116331795345895, + "tokens_seen": 3109748736 + }, + { + "epoch": 0.88, + "learning_rate": 5.872727272727273e-05, + "loss": 2.626, + "theoretical_loss": 3.311627852709663, + "tokens_seen": 3109814272 + }, + { + "epoch": 0.88, + "learning_rate": 5.870707070707071e-05, + "loss": 2.6233, + "theoretical_loss": 3.3116225260284238, + "tokens_seen": 3109879808 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1703988, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0623679161071777, + "objective/train/theoretical_loss": 3.3116171994908647, + "objective/train/tokens_used": 1468804576, + "theoretical_loss": 3.3116171994908647, + "tokens_seen": 3109945344 + }, + { + "epoch": 0.88, + "learning_rate": 5.868686868686869e-05, + "loss": 2.3847, + "theoretical_loss": 3.3116171994908647, + "tokens_seen": 3109945344 + }, + { + "epoch": 0.88, + "learning_rate": 5.8666666666666665e-05, + "loss": 2.6372, + "theoretical_loss": 3.311611873096979, + "tokens_seen": 3110010880 + }, + { + "epoch": 0.88, + "learning_rate": 5.864646464646465e-05, + "loss": 2.4165, + "theoretical_loss": 3.3116065468467597, + "tokens_seen": 3110076416 + }, + { + "epoch": 0.88, + "learning_rate": 5.8626262626262626e-05, + "loss": 2.4378, + "theoretical_loss": 3.3116012207402, + "tokens_seen": 3110141952 + }, + { + "epoch": 0.88, + "learning_rate": 5.860606060606061e-05, + "loss": 2.342, + "theoretical_loss": 3.311595894777293, + "tokens_seen": 3110207488 + }, + { + "epoch": 0.88, + "learning_rate": 5.858585858585859e-05, + "loss": 2.6764, + "theoretical_loss": 3.311590568958032, + "tokens_seen": 3110273024 + }, + { + "epoch": 0.88, + "learning_rate": 5.8565656565656565e-05, + "loss": 2.4949, + "theoretical_loss": 3.3115852432824093, + "tokens_seen": 3110338560 + }, + { + "epoch": 0.88, + "learning_rate": 5.854545454545455e-05, + "loss": 2.5449, + "theoretical_loss": 3.3115799177504193, + "tokens_seen": 3110404096 + }, + { + "epoch": 0.88, + "learning_rate": 5.8525252525252526e-05, + "loss": 2.5519, + "theoretical_loss": 3.311574592362054, + "tokens_seen": 3110469632 + }, + { + "epoch": 0.88, + "learning_rate": 5.85050505050505e-05, + "loss": 2.5924, + "theoretical_loss": 3.3115692671173065, + "tokens_seen": 3110535168 + }, + { + "epoch": 0.88, + "learning_rate": 5.848484848484849e-05, + "loss": 2.6205, + "theoretical_loss": 3.311563942016171, + "tokens_seen": 3110600704 + }, + { + "epoch": 0.88, + "learning_rate": 5.8464646464646464e-05, + "loss": 2.3588, + "theoretical_loss": 3.311558617058639, + "tokens_seen": 3110666240 + }, + { + "epoch": 0.88, + "learning_rate": 5.844444444444445e-05, + "loss": 2.4496, + "theoretical_loss": 3.311553292244705, + "tokens_seen": 3110731776 + }, + { + "epoch": 0.88, + "learning_rate": 5.8424242424242425e-05, + "loss": 2.5144, + "theoretical_loss": 3.3115479675743615, + "tokens_seen": 3110797312 + }, + { + "epoch": 0.88, + "learning_rate": 5.84040404040404e-05, + "loss": 2.3736, + "theoretical_loss": 3.3115426430476016, + "tokens_seen": 3110862848 + }, + { + "epoch": 0.88, + "learning_rate": 5.8383838383838386e-05, + "loss": 2.3605, + "theoretical_loss": 3.311537318664419, + "tokens_seen": 3110928384 + }, + { + "epoch": 0.88, + "learning_rate": 5.8363636363636364e-05, + "loss": 2.2743, + "theoretical_loss": 3.3115319944248056, + "tokens_seen": 3110993920 + }, + { + "epoch": 0.88, + "learning_rate": 5.834343434343435e-05, + "loss": 2.5728, + "theoretical_loss": 3.3115266703287554, + "tokens_seen": 3111059456 + }, + { + "epoch": 0.88, + "learning_rate": 5.8323232323232325e-05, + "loss": 2.46, + "theoretical_loss": 3.311521346376261, + "tokens_seen": 3111124992 + }, + { + "epoch": 0.88, + "learning_rate": 5.83030303030303e-05, + "loss": 2.6491, + "theoretical_loss": 3.311516022567316, + "tokens_seen": 3111190528 + }, + { + "epoch": 0.88, + "learning_rate": 5.8282828282828286e-05, + "loss": 2.6912, + "theoretical_loss": 3.311510698901914, + "tokens_seen": 3111256064 + }, + { + "epoch": 0.88, + "learning_rate": 5.826262626262627e-05, + "loss": 2.4674, + "theoretical_loss": 3.3115053753800465, + "tokens_seen": 3111321600 + }, + { + "epoch": 0.88, + "learning_rate": 5.824242424242424e-05, + "loss": 2.6155, + "theoretical_loss": 3.311500052001708, + "tokens_seen": 3111387136 + }, + { + "epoch": 0.88, + "learning_rate": 5.8222222222222224e-05, + "loss": 2.3785, + "theoretical_loss": 3.311494728766891, + "tokens_seen": 3111452672 + }, + { + "epoch": 0.88, + "learning_rate": 5.82020202020202e-05, + "loss": 2.4621, + "theoretical_loss": 3.311489405675588, + "tokens_seen": 3111518208 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1704703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7959706783294678, + "objective/train/theoretical_loss": 3.311484082727794, + "objective/train/tokens_used": 1470442976, + "theoretical_loss": 3.311484082727794, + "tokens_seen": 3111583744 + }, + { + "epoch": 0.89, + "learning_rate": 5.8181818181818185e-05, + "loss": 2.7426, + "theoretical_loss": 3.311484082727794, + "tokens_seen": 3111583744 + }, + { + "epoch": 0.89, + "learning_rate": 5.816161616161616e-05, + "loss": 2.6539, + "theoretical_loss": 3.3114787599235003, + "tokens_seen": 3111649280 + }, + { + "epoch": 0.89, + "learning_rate": 5.814141414141414e-05, + "loss": 2.4339, + "theoretical_loss": 3.311473437262701, + "tokens_seen": 3111714816 + }, + { + "epoch": 0.89, + "learning_rate": 5.8121212121212124e-05, + "loss": 2.5017, + "theoretical_loss": 3.3114681147453884, + "tokens_seen": 3111780352 + }, + { + "epoch": 0.89, + "learning_rate": 5.81010101010101e-05, + "loss": 2.5124, + "theoretical_loss": 3.3114627923715565, + "tokens_seen": 3111845888 + }, + { + "epoch": 0.89, + "learning_rate": 5.808080808080808e-05, + "loss": 2.3113, + "theoretical_loss": 3.311457470141198, + "tokens_seen": 3111911424 + }, + { + "epoch": 0.89, + "learning_rate": 5.806060606060606e-05, + "loss": 2.5038, + "theoretical_loss": 3.3114521480543058, + "tokens_seen": 3111976960 + }, + { + "epoch": 0.89, + "learning_rate": 5.804040404040404e-05, + "loss": 2.4724, + "theoretical_loss": 3.311446826110873, + "tokens_seen": 3112042496 + }, + { + "epoch": 0.89, + "learning_rate": 5.802020202020202e-05, + "loss": 2.7757, + "theoretical_loss": 3.311441504310893, + "tokens_seen": 3112108032 + }, + { + "epoch": 0.89, + "learning_rate": 5.800000000000001e-05, + "loss": 2.4715, + "theoretical_loss": 3.311436182654359, + "tokens_seen": 3112173568 + }, + { + "epoch": 0.89, + "learning_rate": 5.797979797979798e-05, + "loss": 2.571, + "theoretical_loss": 3.3114308611412637, + "tokens_seen": 3112239104 + }, + { + "epoch": 0.89, + "learning_rate": 5.795959595959596e-05, + "loss": 2.375, + "theoretical_loss": 3.311425539771601, + "tokens_seen": 3112304640 + }, + { + "epoch": 0.89, + "learning_rate": 5.793939393939394e-05, + "loss": 2.3272, + "theoretical_loss": 3.311420218545363, + "tokens_seen": 3112370176 + }, + { + "epoch": 0.89, + "learning_rate": 5.791919191919192e-05, + "loss": 2.5437, + "theoretical_loss": 3.3114148974625435, + "tokens_seen": 3112435712 + }, + { + "epoch": 0.89, + "learning_rate": 5.78989898989899e-05, + "loss": 2.4327, + "theoretical_loss": 3.3114095765231353, + "tokens_seen": 3112501248 + }, + { + "epoch": 0.89, + "learning_rate": 5.787878787878788e-05, + "loss": 2.5862, + "theoretical_loss": 3.3114042557271315, + "tokens_seen": 3112566784 + }, + { + "epoch": 0.89, + "learning_rate": 5.785858585858586e-05, + "loss": 2.6509, + "theoretical_loss": 3.3113989350745254, + "tokens_seen": 3112632320 + }, + { + "epoch": 0.89, + "learning_rate": 5.7838383838383845e-05, + "loss": 2.4821, + "theoretical_loss": 3.3113936145653105, + "tokens_seen": 3112697856 + }, + { + "epoch": 0.89, + "learning_rate": 5.7818181818181815e-05, + "loss": 2.5516, + "theoretical_loss": 3.311388294199479, + "tokens_seen": 3112763392 + }, + { + "epoch": 0.89, + "learning_rate": 5.77979797979798e-05, + "loss": 2.7557, + "theoretical_loss": 3.3113829739770244, + "tokens_seen": 3112828928 + }, + { + "epoch": 0.89, + "learning_rate": 5.7777777777777776e-05, + "loss": 2.4398, + "theoretical_loss": 3.31137765389794, + "tokens_seen": 3112894464 + }, + { + "epoch": 0.89, + "learning_rate": 5.775757575757576e-05, + "loss": 2.5363, + "theoretical_loss": 3.311372333962219, + "tokens_seen": 3112960000 + }, + { + "epoch": 0.89, + "learning_rate": 5.7737373737373744e-05, + "loss": 2.5353, + "theoretical_loss": 3.3113670141698544, + "tokens_seen": 3113025536 + }, + { + "epoch": 0.89, + "learning_rate": 5.7717171717171715e-05, + "loss": 2.4473, + "theoretical_loss": 3.311361694520839, + "tokens_seen": 3113091072 + }, + { + "epoch": 0.89, + "learning_rate": 5.76969696969697e-05, + "loss": 2.6608, + "theoretical_loss": 3.311356375015166, + "tokens_seen": 3113156608 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1705619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7808351516723633, + "objective/train/theoretical_loss": 3.311351055652829, + "objective/train/tokens_used": 1472081376, + "theoretical_loss": 3.311351055652829, + "tokens_seen": 3113222144 + }, + { + "epoch": 0.89, + "learning_rate": 5.7676767676767676e-05, + "loss": 2.6632, + "theoretical_loss": 3.311351055652829, + "tokens_seen": 3113222144 + }, + { + "epoch": 0.89, + "learning_rate": 5.765656565656566e-05, + "loss": 2.3446, + "theoretical_loss": 3.311345736433821, + "tokens_seen": 3113287680 + }, + { + "epoch": 0.89, + "learning_rate": 5.763636363636364e-05, + "loss": 2.4078, + "theoretical_loss": 3.3113404173581342, + "tokens_seen": 3113353216 + }, + { + "epoch": 0.89, + "learning_rate": 5.7616161616161614e-05, + "loss": 2.5743, + "theoretical_loss": 3.3113350984257632, + "tokens_seen": 3113418752 + }, + { + "epoch": 0.89, + "learning_rate": 5.75959595959596e-05, + "loss": 2.3902, + "theoretical_loss": 3.3113297796367, + "tokens_seen": 3113484288 + }, + { + "epoch": 0.89, + "learning_rate": 5.757575757575758e-05, + "loss": 2.5539, + "theoretical_loss": 3.3113244609909382, + "tokens_seen": 3113549824 + }, + { + "epoch": 0.89, + "learning_rate": 5.755555555555555e-05, + "loss": 2.6848, + "theoretical_loss": 3.311319142488471, + "tokens_seen": 3113615360 + }, + { + "epoch": 0.89, + "learning_rate": 5.7535353535353537e-05, + "loss": 2.4893, + "theoretical_loss": 3.311313824129291, + "tokens_seen": 3113680896 + }, + { + "epoch": 0.89, + "learning_rate": 5.7515151515151514e-05, + "loss": 2.6103, + "theoretical_loss": 3.311308505913392, + "tokens_seen": 3113746432 + }, + { + "epoch": 0.89, + "learning_rate": 5.74949494949495e-05, + "loss": 2.5074, + "theoretical_loss": 3.3113031878407666, + "tokens_seen": 3113811968 + }, + { + "epoch": 0.89, + "learning_rate": 5.7474747474747475e-05, + "loss": 2.452, + "theoretical_loss": 3.3112978699114084, + "tokens_seen": 3113877504 + }, + { + "epoch": 0.89, + "learning_rate": 5.745454545454545e-05, + "loss": 2.5468, + "theoretical_loss": 3.31129255212531, + "tokens_seen": 3113943040 + }, + { + "epoch": 0.89, + "learning_rate": 5.7434343434343436e-05, + "loss": 2.5905, + "theoretical_loss": 3.3112872344824646, + "tokens_seen": 3114008576 + }, + { + "epoch": 0.89, + "learning_rate": 5.741414141414142e-05, + "loss": 2.6261, + "theoretical_loss": 3.311281916982866, + "tokens_seen": 3114074112 + }, + { + "epoch": 0.89, + "learning_rate": 5.73939393939394e-05, + "loss": 2.3937, + "theoretical_loss": 3.3112765996265066, + "tokens_seen": 3114139648 + }, + { + "epoch": 0.89, + "learning_rate": 5.7373737373737374e-05, + "loss": 2.4801, + "theoretical_loss": 3.3112712824133794, + "tokens_seen": 3114205184 + }, + { + "epoch": 0.89, + "learning_rate": 5.735353535353535e-05, + "loss": 2.5464, + "theoretical_loss": 3.3112659653434786, + "tokens_seen": 3114270720 + }, + { + "epoch": 0.89, + "learning_rate": 5.7333333333333336e-05, + "loss": 2.4611, + "theoretical_loss": 3.311260648416796, + "tokens_seen": 3114336256 + }, + { + "epoch": 0.89, + "learning_rate": 5.731313131313132e-05, + "loss": 2.4733, + "theoretical_loss": 3.3112553316333253, + "tokens_seen": 3114401792 + }, + { + "epoch": 0.89, + "learning_rate": 5.729292929292929e-05, + "loss": 2.4378, + "theoretical_loss": 3.31125001499306, + "tokens_seen": 3114467328 + }, + { + "epoch": 0.89, + "learning_rate": 5.7272727272727274e-05, + "loss": 2.5558, + "theoretical_loss": 3.311244698495993, + "tokens_seen": 3114532864 + }, + { + "epoch": 0.89, + "learning_rate": 5.725252525252525e-05, + "loss": 2.5575, + "theoretical_loss": 3.3112393821421167, + "tokens_seen": 3114598400 + }, + { + "epoch": 0.89, + "learning_rate": 5.7232323232323235e-05, + "loss": 2.6434, + "theoretical_loss": 3.3112340659314254, + "tokens_seen": 3114663936 + }, + { + "epoch": 0.89, + "learning_rate": 5.721212121212121e-05, + "loss": 2.4846, + "theoretical_loss": 3.3112287498639117, + "tokens_seen": 3114729472 + }, + { + "epoch": 0.89, + "learning_rate": 5.719191919191919e-05, + "loss": 2.5234, + "theoretical_loss": 3.3112234339395687, + "tokens_seen": 3114795008 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1706661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7602922916412354, + "objective/train/theoretical_loss": 3.311218118158389, + "objective/train/tokens_used": 1473719776, + "theoretical_loss": 3.311218118158389, + "tokens_seen": 3114860544 + }, + { + "epoch": 0.89, + "learning_rate": 5.717171717171717e-05, + "loss": 2.4291, + "theoretical_loss": 3.311218118158389, + "tokens_seen": 3114860544 + }, + { + "epoch": 0.89, + "learning_rate": 5.715151515151516e-05, + "loss": 2.6124, + "theoretical_loss": 3.311212802520367, + "tokens_seen": 3114926080 + }, + { + "epoch": 0.89, + "learning_rate": 5.7131313131313134e-05, + "loss": 2.4069, + "theoretical_loss": 3.311207487025495, + "tokens_seen": 3114991616 + }, + { + "epoch": 0.89, + "learning_rate": 5.711111111111111e-05, + "loss": 2.3906, + "theoretical_loss": 3.311202171673766, + "tokens_seen": 3115057152 + }, + { + "epoch": 0.89, + "learning_rate": 5.709090909090909e-05, + "loss": 2.5687, + "theoretical_loss": 3.3111968564651733, + "tokens_seen": 3115122688 + }, + { + "epoch": 0.89, + "learning_rate": 5.707070707070707e-05, + "loss": 2.5649, + "theoretical_loss": 3.3111915413997104, + "tokens_seen": 3115188224 + }, + { + "epoch": 0.89, + "learning_rate": 5.705050505050506e-05, + "loss": 2.5813, + "theoretical_loss": 3.31118622647737, + "tokens_seen": 3115253760 + }, + { + "epoch": 0.89, + "learning_rate": 5.703030303030303e-05, + "loss": 2.3948, + "theoretical_loss": 3.3111809116981457, + "tokens_seen": 3115319296 + }, + { + "epoch": 0.89, + "learning_rate": 5.701010101010101e-05, + "loss": 2.539, + "theoretical_loss": 3.3111755970620296, + "tokens_seen": 3115384832 + }, + { + "epoch": 0.89, + "learning_rate": 5.6989898989898995e-05, + "loss": 2.422, + "theoretical_loss": 3.3111702825690164, + "tokens_seen": 3115450368 + }, + { + "epoch": 0.89, + "learning_rate": 5.696969696969697e-05, + "loss": 2.5662, + "theoretical_loss": 3.3111649682190976, + "tokens_seen": 3115515904 + }, + { + "epoch": 0.89, + "learning_rate": 5.694949494949495e-05, + "loss": 2.6884, + "theoretical_loss": 3.311159654012268, + "tokens_seen": 3115581440 + }, + { + "epoch": 0.89, + "learning_rate": 5.692929292929293e-05, + "loss": 2.5474, + "theoretical_loss": 3.3111543399485193, + "tokens_seen": 3115646976 + }, + { + "epoch": 0.89, + "learning_rate": 5.690909090909091e-05, + "loss": 2.4619, + "theoretical_loss": 3.311149026027845, + "tokens_seen": 3115712512 + }, + { + "epoch": 0.89, + "learning_rate": 5.6888888888888895e-05, + "loss": 2.5182, + "theoretical_loss": 3.3111437122502387, + "tokens_seen": 3115778048 + }, + { + "epoch": 0.89, + "learning_rate": 5.6868686868686865e-05, + "loss": 2.6564, + "theoretical_loss": 3.3111383986156935, + "tokens_seen": 3115843584 + }, + { + "epoch": 0.89, + "learning_rate": 5.684848484848485e-05, + "loss": 2.5293, + "theoretical_loss": 3.3111330851242022, + "tokens_seen": 3115909120 + }, + { + "epoch": 0.89, + "learning_rate": 5.6828282828282826e-05, + "loss": 2.2689, + "theoretical_loss": 3.3111277717757583, + "tokens_seen": 3115974656 + }, + { + "epoch": 0.89, + "learning_rate": 5.680808080808081e-05, + "loss": 2.4963, + "theoretical_loss": 3.3111224585703543, + "tokens_seen": 3116040192 + }, + { + "epoch": 0.89, + "learning_rate": 5.6787878787878794e-05, + "loss": 2.4439, + "theoretical_loss": 3.3111171455079838, + "tokens_seen": 3116105728 + }, + { + "epoch": 0.89, + "learning_rate": 5.6767676767676764e-05, + "loss": 2.5048, + "theoretical_loss": 3.31111183258864, + "tokens_seen": 3116171264 + }, + { + "epoch": 0.89, + "learning_rate": 5.674747474747475e-05, + "loss": 2.5524, + "theoretical_loss": 3.311106519812316, + "tokens_seen": 3116236800 + }, + { + "epoch": 0.89, + "learning_rate": 5.672727272727273e-05, + "loss": 2.4357, + "theoretical_loss": 3.311101207179005, + "tokens_seen": 3116302336 + }, + { + "epoch": 0.89, + "learning_rate": 5.670707070707071e-05, + "loss": 2.4106, + "theoretical_loss": 3.3110958946887, + "tokens_seen": 3116367872 + }, + { + "epoch": 0.89, + "learning_rate": 5.668686868686869e-05, + "loss": 2.562, + "theoretical_loss": 3.311090582341394, + "tokens_seen": 3116433408 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1707241, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.480644464492798, + "objective/train/theoretical_loss": 3.31108527013708, + "objective/train/tokens_used": 1475358176, + "theoretical_loss": 3.31108527013708, + "tokens_seen": 3116498944 + }, + { + "epoch": 0.89, + "learning_rate": 5.6666666666666664e-05, + "loss": 2.6308, + "theoretical_loss": 3.31108527013708, + "tokens_seen": 3116498944 + }, + { + "epoch": 0.89, + "learning_rate": 5.664646464646465e-05, + "loss": 2.5987, + "theoretical_loss": 3.3110799580757515, + "tokens_seen": 3116564480 + }, + { + "epoch": 0.89, + "learning_rate": 5.662626262626263e-05, + "loss": 2.5669, + "theoretical_loss": 3.311074646157402, + "tokens_seen": 3116630016 + }, + { + "epoch": 0.89, + "learning_rate": 5.66060606060606e-05, + "loss": 2.4388, + "theoretical_loss": 3.311069334382024, + "tokens_seen": 3116695552 + }, + { + "epoch": 0.89, + "learning_rate": 5.6585858585858586e-05, + "loss": 2.6134, + "theoretical_loss": 3.311064022749611, + "tokens_seen": 3116761088 + }, + { + "epoch": 0.89, + "learning_rate": 5.656565656565657e-05, + "loss": 2.7051, + "theoretical_loss": 3.3110587112601557, + "tokens_seen": 3116826624 + }, + { + "epoch": 0.89, + "learning_rate": 5.654545454545455e-05, + "loss": 2.6074, + "theoretical_loss": 3.311053399913652, + "tokens_seen": 3116892160 + }, + { + "epoch": 0.89, + "learning_rate": 5.652525252525253e-05, + "loss": 2.2297, + "theoretical_loss": 3.3110480887100926, + "tokens_seen": 3116957696 + }, + { + "epoch": 0.89, + "learning_rate": 5.65050505050505e-05, + "loss": 2.5668, + "theoretical_loss": 3.3110427776494706, + "tokens_seen": 3117023232 + }, + { + "epoch": 0.89, + "learning_rate": 5.6484848484848486e-05, + "loss": 2.3114, + "theoretical_loss": 3.3110374667317792, + "tokens_seen": 3117088768 + }, + { + "epoch": 0.89, + "learning_rate": 5.646464646464647e-05, + "loss": 2.6691, + "theoretical_loss": 3.3110321559570117, + "tokens_seen": 3117154304 + }, + { + "epoch": 0.89, + "learning_rate": 5.644444444444445e-05, + "loss": 2.5942, + "theoretical_loss": 3.311026845325161, + "tokens_seen": 3117219840 + }, + { + "epoch": 0.89, + "learning_rate": 5.6424242424242424e-05, + "loss": 2.3339, + "theoretical_loss": 3.31102153483622, + "tokens_seen": 3117285376 + }, + { + "epoch": 0.89, + "learning_rate": 5.64040404040404e-05, + "loss": 2.5438, + "theoretical_loss": 3.311016224490183, + "tokens_seen": 3117350912 + }, + { + "epoch": 0.89, + "learning_rate": 5.6383838383838385e-05, + "loss": 2.4655, + "theoretical_loss": 3.3110109142870416, + "tokens_seen": 3117416448 + }, + { + "epoch": 0.89, + "learning_rate": 5.636363636363637e-05, + "loss": 2.3874, + "theoretical_loss": 3.31100560422679, + "tokens_seen": 3117481984 + }, + { + "epoch": 0.89, + "learning_rate": 5.634343434343434e-05, + "loss": 2.2287, + "theoretical_loss": 3.311000294309421, + "tokens_seen": 3117547520 + }, + { + "epoch": 0.89, + "learning_rate": 5.6323232323232324e-05, + "loss": 2.6364, + "theoretical_loss": 3.3109949845349282, + "tokens_seen": 3117613056 + }, + { + "epoch": 0.89, + "learning_rate": 5.630303030303031e-05, + "loss": 2.6803, + "theoretical_loss": 3.310989674903304, + "tokens_seen": 3117678592 + }, + { + "epoch": 0.89, + "learning_rate": 5.6282828282828285e-05, + "loss": 2.348, + "theoretical_loss": 3.3109843654145417, + "tokens_seen": 3117744128 + }, + { + "epoch": 0.89, + "learning_rate": 5.626262626262627e-05, + "loss": 2.6195, + "theoretical_loss": 3.310979056068635, + "tokens_seen": 3117809664 + }, + { + "epoch": 0.89, + "learning_rate": 5.624242424242424e-05, + "loss": 2.4176, + "theoretical_loss": 3.310973746865577, + "tokens_seen": 3117875200 + }, + { + "epoch": 0.89, + "learning_rate": 5.622222222222222e-05, + "loss": 2.6039, + "theoretical_loss": 3.3109684378053603, + "tokens_seen": 3117940736 + }, + { + "epoch": 0.89, + "learning_rate": 5.620202020202021e-05, + "loss": 2.492, + "theoretical_loss": 3.310963128887978, + "tokens_seen": 3118006272 + }, + { + "epoch": 0.89, + "learning_rate": 5.6181818181818184e-05, + "loss": 2.5854, + "theoretical_loss": 3.3109578201134244, + "tokens_seen": 3118071808 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1708404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5623645782470703, + "objective/train/theoretical_loss": 3.310952511481691, + "objective/train/tokens_used": 1476996576, + "theoretical_loss": 3.310952511481691, + "tokens_seen": 3118137344 + }, + { + "epoch": 0.89, + "learning_rate": 5.616161616161616e-05, + "loss": 2.3662, + "theoretical_loss": 3.310952511481691, + "tokens_seen": 3118137344 + }, + { + "epoch": 0.89, + "learning_rate": 5.6141414141414145e-05, + "loss": 2.6509, + "theoretical_loss": 3.3109472029927725, + "tokens_seen": 3118202880 + }, + { + "epoch": 0.89, + "learning_rate": 5.612121212121212e-05, + "loss": 2.7009, + "theoretical_loss": 3.310941894646661, + "tokens_seen": 3118268416 + }, + { + "epoch": 0.89, + "learning_rate": 5.6101010101010106e-05, + "loss": 2.5428, + "theoretical_loss": 3.31093658644335, + "tokens_seen": 3118333952 + }, + { + "epoch": 0.89, + "learning_rate": 5.608080808080808e-05, + "loss": 2.6387, + "theoretical_loss": 3.310931278382833, + "tokens_seen": 3118399488 + }, + { + "epoch": 0.89, + "learning_rate": 5.606060606060606e-05, + "loss": 2.6081, + "theoretical_loss": 3.3109259704651026, + "tokens_seen": 3118465024 + }, + { + "epoch": 0.89, + "learning_rate": 5.6040404040404045e-05, + "loss": 2.549, + "theoretical_loss": 3.310920662690152, + "tokens_seen": 3118530560 + }, + { + "epoch": 0.89, + "learning_rate": 5.602020202020202e-05, + "loss": 2.6525, + "theoretical_loss": 3.3109153550579746, + "tokens_seen": 3118596096 + }, + { + "epoch": 0.89, + "learning_rate": 5.6e-05, + "loss": 2.5174, + "theoretical_loss": 3.310910047568564, + "tokens_seen": 3118661632 + }, + { + "epoch": 0.89, + "learning_rate": 5.5979797979797976e-05, + "loss": 2.5809, + "theoretical_loss": 3.3109047402219125, + "tokens_seen": 3118727168 + }, + { + "epoch": 0.89, + "learning_rate": 5.595959595959596e-05, + "loss": 2.608, + "theoretical_loss": 3.3108994330180135, + "tokens_seen": 3118792704 + }, + { + "epoch": 0.89, + "learning_rate": 5.5939393939393944e-05, + "loss": 2.4211, + "theoretical_loss": 3.3108941259568607, + "tokens_seen": 3118858240 + }, + { + "epoch": 0.89, + "learning_rate": 5.591919191919192e-05, + "loss": 2.5102, + "theoretical_loss": 3.3108888190384467, + "tokens_seen": 3118923776 + }, + { + "epoch": 0.89, + "learning_rate": 5.58989898989899e-05, + "loss": 2.7221, + "theoretical_loss": 3.3108835122627647, + "tokens_seen": 3118989312 + }, + { + "epoch": 0.89, + "learning_rate": 5.587878787878788e-05, + "loss": 2.5556, + "theoretical_loss": 3.310878205629808, + "tokens_seen": 3119054848 + }, + { + "epoch": 0.89, + "learning_rate": 5.585858585858586e-05, + "loss": 2.6132, + "theoretical_loss": 3.3108728991395697, + "tokens_seen": 3119120384 + }, + { + "epoch": 0.89, + "learning_rate": 5.5838383838383844e-05, + "loss": 2.4856, + "theoretical_loss": 3.310867592792043, + "tokens_seen": 3119185920 + }, + { + "epoch": 0.89, + "learning_rate": 5.5818181818181814e-05, + "loss": 2.6071, + "theoretical_loss": 3.3108622865872213, + "tokens_seen": 3119251456 + }, + { + "epoch": 0.89, + "learning_rate": 5.57979797979798e-05, + "loss": 2.4906, + "theoretical_loss": 3.3108569805250974, + "tokens_seen": 3119316992 + }, + { + "epoch": 0.89, + "learning_rate": 5.577777777777778e-05, + "loss": 2.2396, + "theoretical_loss": 3.3108516746056647, + "tokens_seen": 3119382528 + }, + { + "epoch": 0.89, + "learning_rate": 5.575757575757576e-05, + "loss": 2.4673, + "theoretical_loss": 3.310846368828916, + "tokens_seen": 3119448064 + }, + { + "epoch": 0.89, + "learning_rate": 5.5737373737373736e-05, + "loss": 2.254, + "theoretical_loss": 3.310841063194845, + "tokens_seen": 3119513600 + }, + { + "epoch": 0.89, + "learning_rate": 5.5717171717171714e-05, + "loss": 2.4005, + "theoretical_loss": 3.3108357577034444, + "tokens_seen": 3119579136 + }, + { + "epoch": 0.89, + "learning_rate": 5.56969696969697e-05, + "loss": 2.421, + "theoretical_loss": 3.3108304523547076, + "tokens_seen": 3119644672 + }, + { + "epoch": 0.89, + "learning_rate": 5.567676767676768e-05, + "loss": 2.4974, + "theoretical_loss": 3.310825147148628, + "tokens_seen": 3119710208 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1709630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9203097820281982, + "objective/train/theoretical_loss": 3.310819842085198, + "objective/train/tokens_used": 1478634976, + "theoretical_loss": 3.310819842085198, + "tokens_seen": 3119775744 + }, + { + "epoch": 0.89, + "learning_rate": 5.565656565656565e-05, + "loss": 2.7455, + "theoretical_loss": 3.310819842085198, + "tokens_seen": 3119775744 + }, + { + "epoch": 0.89, + "learning_rate": 5.5636363636363636e-05, + "loss": 2.4473, + "theoretical_loss": 3.3108145371644113, + "tokens_seen": 3119841280 + }, + { + "epoch": 0.89, + "learning_rate": 5.561616161616162e-05, + "loss": 2.3788, + "theoretical_loss": 3.3108092323862617, + "tokens_seen": 3119906816 + }, + { + "epoch": 0.89, + "learning_rate": 5.55959595959596e-05, + "loss": 2.349, + "theoretical_loss": 3.310803927750741, + "tokens_seen": 3119972352 + }, + { + "epoch": 0.89, + "learning_rate": 5.557575757575758e-05, + "loss": 2.5108, + "theoretical_loss": 3.3107986232578437, + "tokens_seen": 3120037888 + }, + { + "epoch": 0.89, + "learning_rate": 5.555555555555555e-05, + "loss": 2.4122, + "theoretical_loss": 3.3107933189075616, + "tokens_seen": 3120103424 + }, + { + "epoch": 0.89, + "learning_rate": 5.5535353535353535e-05, + "loss": 2.7762, + "theoretical_loss": 3.310788014699889, + "tokens_seen": 3120168960 + }, + { + "epoch": 0.89, + "learning_rate": 5.551515151515152e-05, + "loss": 2.5402, + "theoretical_loss": 3.3107827106348187, + "tokens_seen": 3120234496 + }, + { + "epoch": 0.89, + "learning_rate": 5.5494949494949497e-05, + "loss": 2.4868, + "theoretical_loss": 3.3107774067123437, + "tokens_seen": 3120300032 + }, + { + "epoch": 0.89, + "learning_rate": 5.5474747474747474e-05, + "loss": 2.5715, + "theoretical_loss": 3.3107721029324573, + "tokens_seen": 3120365568 + }, + { + "epoch": 0.89, + "learning_rate": 5.545454545454546e-05, + "loss": 2.5473, + "theoretical_loss": 3.310766799295153, + "tokens_seen": 3120431104 + }, + { + "epoch": 0.89, + "learning_rate": 5.5434343434343435e-05, + "loss": 2.5751, + "theoretical_loss": 3.3107614958004232, + "tokens_seen": 3120496640 + }, + { + "epoch": 0.89, + "learning_rate": 5.541414141414142e-05, + "loss": 2.696, + "theoretical_loss": 3.3107561924482622, + "tokens_seen": 3120562176 + }, + { + "epoch": 0.89, + "learning_rate": 5.539393939393939e-05, + "loss": 2.5909, + "theoretical_loss": 3.310750889238662, + "tokens_seen": 3120627712 + }, + { + "epoch": 0.89, + "learning_rate": 5.537373737373737e-05, + "loss": 2.5694, + "theoretical_loss": 3.3107455861716164, + "tokens_seen": 3120693248 + }, + { + "epoch": 0.89, + "learning_rate": 5.535353535353536e-05, + "loss": 2.3176, + "theoretical_loss": 3.3107402832471187, + "tokens_seen": 3120758784 + }, + { + "epoch": 0.89, + "learning_rate": 5.5333333333333334e-05, + "loss": 2.5623, + "theoretical_loss": 3.3107349804651616, + "tokens_seen": 3120824320 + }, + { + "epoch": 0.89, + "learning_rate": 5.531313131313132e-05, + "loss": 2.3698, + "theoretical_loss": 3.3107296778257385, + "tokens_seen": 3120889856 + }, + { + "epoch": 0.89, + "learning_rate": 5.529292929292929e-05, + "loss": 2.5032, + "theoretical_loss": 3.3107243753288427, + "tokens_seen": 3120955392 + }, + { + "epoch": 0.89, + "learning_rate": 5.527272727272727e-05, + "loss": 2.6077, + "theoretical_loss": 3.310719072974467, + "tokens_seen": 3121020928 + }, + { + "epoch": 0.89, + "learning_rate": 5.525252525252526e-05, + "loss": 2.4741, + "theoretical_loss": 3.310713770762605, + "tokens_seen": 3121086464 + }, + { + "epoch": 0.89, + "learning_rate": 5.5232323232323234e-05, + "loss": 2.7724, + "theoretical_loss": 3.31070846869325, + "tokens_seen": 3121152000 + }, + { + "epoch": 0.89, + "learning_rate": 5.521212121212121e-05, + "loss": 2.7443, + "theoretical_loss": 3.3107031667663946, + "tokens_seen": 3121217536 + }, + { + "epoch": 0.89, + "learning_rate": 5.5191919191919195e-05, + "loss": 2.5582, + "theoretical_loss": 3.310697864982032, + "tokens_seen": 3121283072 + }, + { + "epoch": 0.89, + "learning_rate": 5.517171717171717e-05, + "loss": 2.5742, + "theoretical_loss": 3.3106925633401563, + "tokens_seen": 3121348608 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1710132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.622396945953369, + "objective/train/theoretical_loss": 3.3106872618407595, + "objective/train/tokens_used": 1480273376, + "theoretical_loss": 3.3106872618407595, + "tokens_seen": 3121414144 + }, + { + "epoch": 0.89, + "learning_rate": 5.5151515151515156e-05, + "loss": 2.3752, + "theoretical_loss": 3.3106872618407595, + "tokens_seen": 3121414144 + }, + { + "epoch": 0.89, + "learning_rate": 5.5131313131313127e-05, + "loss": 2.2602, + "theoretical_loss": 3.3106819604838353, + "tokens_seen": 3121479680 + }, + { + "epoch": 0.89, + "learning_rate": 5.511111111111111e-05, + "loss": 2.5267, + "theoretical_loss": 3.3106766592693777, + "tokens_seen": 3121545216 + }, + { + "epoch": 0.89, + "learning_rate": 5.5090909090909094e-05, + "loss": 2.4669, + "theoretical_loss": 3.310671358197378, + "tokens_seen": 3121610752 + }, + { + "epoch": 0.89, + "learning_rate": 5.507070707070707e-05, + "loss": 2.4228, + "theoretical_loss": 3.3106660572678313, + "tokens_seen": 3121676288 + }, + { + "epoch": 0.89, + "learning_rate": 5.5050505050505056e-05, + "loss": 2.5149, + "theoretical_loss": 3.3106607564807295, + "tokens_seen": 3121741824 + }, + { + "epoch": 0.89, + "learning_rate": 5.503030303030303e-05, + "loss": 2.3352, + "theoretical_loss": 3.3106554558360664, + "tokens_seen": 3121807360 + }, + { + "epoch": 0.89, + "learning_rate": 5.501010101010101e-05, + "loss": 2.603, + "theoretical_loss": 3.3106501553338346, + "tokens_seen": 3121872896 + }, + { + "epoch": 0.89, + "learning_rate": 5.4989898989898994e-05, + "loss": 2.4747, + "theoretical_loss": 3.3106448549740284, + "tokens_seen": 3121938432 + }, + { + "epoch": 0.89, + "learning_rate": 5.496969696969697e-05, + "loss": 2.4625, + "theoretical_loss": 3.3106395547566394, + "tokens_seen": 3122003968 + }, + { + "epoch": 0.89, + "learning_rate": 5.494949494949495e-05, + "loss": 2.6623, + "theoretical_loss": 3.3106342546816623, + "tokens_seen": 3122069504 + }, + { + "epoch": 0.89, + "learning_rate": 5.492929292929293e-05, + "loss": 2.3486, + "theoretical_loss": 3.3106289547490895, + "tokens_seen": 3122135040 + }, + { + "epoch": 0.89, + "learning_rate": 5.490909090909091e-05, + "loss": 2.5842, + "theoretical_loss": 3.3106236549589143, + "tokens_seen": 3122200576 + }, + { + "epoch": 0.89, + "learning_rate": 5.4888888888888893e-05, + "loss": 2.6396, + "theoretical_loss": 3.3106183553111297, + "tokens_seen": 3122266112 + }, + { + "epoch": 0.89, + "learning_rate": 5.4868686868686864e-05, + "loss": 2.4955, + "theoretical_loss": 3.3106130558057294, + "tokens_seen": 3122331648 + }, + { + "epoch": 0.89, + "learning_rate": 5.484848484848485e-05, + "loss": 2.3153, + "theoretical_loss": 3.310607756442706, + "tokens_seen": 3122397184 + }, + { + "epoch": 0.89, + "learning_rate": 5.482828282828283e-05, + "loss": 2.5075, + "theoretical_loss": 3.310602457222053, + "tokens_seen": 3122462720 + }, + { + "epoch": 0.89, + "learning_rate": 5.480808080808081e-05, + "loss": 2.5018, + "theoretical_loss": 3.3105971581437634, + "tokens_seen": 3122528256 + }, + { + "epoch": 0.89, + "learning_rate": 5.4787878787878786e-05, + "loss": 2.4604, + "theoretical_loss": 3.310591859207831, + "tokens_seen": 3122593792 + }, + { + "epoch": 0.89, + "learning_rate": 5.476767676767677e-05, + "loss": 2.6068, + "theoretical_loss": 3.310586560414248, + "tokens_seen": 3122659328 + }, + { + "epoch": 0.89, + "learning_rate": 5.474747474747475e-05, + "loss": 2.6024, + "theoretical_loss": 3.3105812617630086, + "tokens_seen": 3122724864 + }, + { + "epoch": 0.89, + "learning_rate": 5.472727272727273e-05, + "loss": 2.2696, + "theoretical_loss": 3.3105759632541054, + "tokens_seen": 3122790400 + }, + { + "epoch": 0.89, + "learning_rate": 5.470707070707071e-05, + "loss": 2.5905, + "theoretical_loss": 3.3105706648875315, + "tokens_seen": 3122855936 + }, + { + "epoch": 0.89, + "learning_rate": 5.4686868686868686e-05, + "loss": 2.3147, + "theoretical_loss": 3.3105653666632806, + "tokens_seen": 3122921472 + }, + { + "epoch": 0.89, + "learning_rate": 5.466666666666667e-05, + "loss": 2.4372, + "theoretical_loss": 3.310560068581345, + "tokens_seen": 3122987008 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1711620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.502155065536499, + "objective/train/theoretical_loss": 3.310554770641719, + "objective/train/tokens_used": 1481911776, + "theoretical_loss": 3.310554770641719, + "tokens_seen": 3123052544 + }, + { + "epoch": 0.89, + "learning_rate": 5.464646464646465e-05, + "loss": 2.5965, + "theoretical_loss": 3.310554770641719, + "tokens_seen": 3123052544 + }, + { + "epoch": 0.89, + "learning_rate": 5.462626262626263e-05, + "loss": 2.5797, + "theoretical_loss": 3.310549472844395, + "tokens_seen": 3123118080 + }, + { + "epoch": 0.89, + "learning_rate": 5.460606060606061e-05, + "loss": 2.5122, + "theoretical_loss": 3.3105441751893667, + "tokens_seen": 3123183616 + }, + { + "epoch": 0.89, + "learning_rate": 5.4585858585858585e-05, + "loss": 2.6127, + "theoretical_loss": 3.3105388776766267, + "tokens_seen": 3123249152 + }, + { + "epoch": 0.89, + "learning_rate": 5.456565656565657e-05, + "loss": 2.449, + "theoretical_loss": 3.310533580306169, + "tokens_seen": 3123314688 + }, + { + "epoch": 0.89, + "learning_rate": 5.4545454545454546e-05, + "loss": 2.55, + "theoretical_loss": 3.3105282830779856, + "tokens_seen": 3123380224 + }, + { + "epoch": 0.89, + "learning_rate": 5.4525252525252523e-05, + "loss": 2.3981, + "theoretical_loss": 3.310522985992071, + "tokens_seen": 3123445760 + }, + { + "epoch": 0.89, + "learning_rate": 5.450505050505051e-05, + "loss": 2.5463, + "theoretical_loss": 3.310517689048418, + "tokens_seen": 3123511296 + }, + { + "epoch": 0.89, + "learning_rate": 5.4484848484848485e-05, + "loss": 2.642, + "theoretical_loss": 3.310512392247019, + "tokens_seen": 3123576832 + }, + { + "epoch": 0.89, + "learning_rate": 5.446464646464647e-05, + "loss": 2.3345, + "theoretical_loss": 3.3105070955878677, + "tokens_seen": 3123642368 + }, + { + "epoch": 0.89, + "learning_rate": 5.444444444444444e-05, + "loss": 2.3782, + "theoretical_loss": 3.310501799070958, + "tokens_seen": 3123707904 + }, + { + "epoch": 0.89, + "learning_rate": 5.442424242424242e-05, + "loss": 2.5189, + "theoretical_loss": 3.3104965026962825, + "tokens_seen": 3123773440 + }, + { + "epoch": 0.89, + "learning_rate": 5.440404040404041e-05, + "loss": 2.4072, + "theoretical_loss": 3.310491206463834, + "tokens_seen": 3123838976 + }, + { + "epoch": 0.89, + "learning_rate": 5.4383838383838384e-05, + "loss": 2.4807, + "theoretical_loss": 3.310485910373606, + "tokens_seen": 3123904512 + }, + { + "epoch": 0.89, + "learning_rate": 5.436363636363637e-05, + "loss": 2.3751, + "theoretical_loss": 3.3104806144255923, + "tokens_seen": 3123970048 + }, + { + "epoch": 0.89, + "learning_rate": 5.4343434343434345e-05, + "loss": 2.4799, + "theoretical_loss": 3.310475318619785, + "tokens_seen": 3124035584 + }, + { + "epoch": 0.89, + "learning_rate": 5.432323232323232e-05, + "loss": 2.5382, + "theoretical_loss": 3.3104700229561783, + "tokens_seen": 3124101120 + }, + { + "epoch": 0.89, + "learning_rate": 5.4303030303030306e-05, + "loss": 2.5655, + "theoretical_loss": 3.310464727434765, + "tokens_seen": 3124166656 + }, + { + "epoch": 0.89, + "learning_rate": 5.4282828282828284e-05, + "loss": 2.6128, + "theoretical_loss": 3.310459432055538, + "tokens_seen": 3124232192 + }, + { + "epoch": 0.89, + "learning_rate": 5.426262626262626e-05, + "loss": 2.5549, + "theoretical_loss": 3.3104541368184908, + "tokens_seen": 3124297728 + }, + { + "epoch": 0.89, + "learning_rate": 5.4242424242424245e-05, + "loss": 2.4359, + "theoretical_loss": 3.3104488417236166, + "tokens_seen": 3124363264 + }, + { + "epoch": 0.89, + "learning_rate": 5.422222222222222e-05, + "loss": 2.558, + "theoretical_loss": 3.310443546770909, + "tokens_seen": 3124428800 + }, + { + "epoch": 0.89, + "learning_rate": 5.4202020202020206e-05, + "loss": 2.5778, + "theoretical_loss": 3.31043825196036, + "tokens_seen": 3124494336 + }, + { + "epoch": 0.89, + "learning_rate": 5.418181818181818e-05, + "loss": 2.2413, + "theoretical_loss": 3.3104329572919644, + "tokens_seen": 3124559872 + }, + { + "epoch": 0.89, + "learning_rate": 5.416161616161616e-05, + "loss": 2.5551, + "theoretical_loss": 3.310427662765714, + "tokens_seen": 3124625408 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1712296, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1693756580352783, + "objective/train/theoretical_loss": 3.310422368381603, + "objective/train/tokens_used": 1483550176, + "theoretical_loss": 3.310422368381603, + "tokens_seen": 3124690944 + }, + { + "epoch": 0.89, + "learning_rate": 5.4141414141414144e-05, + "loss": 2.454, + "theoretical_loss": 3.310422368381603, + "tokens_seen": 3124690944 + }, + { + "epoch": 0.89, + "learning_rate": 5.412121212121212e-05, + "loss": 2.5508, + "theoretical_loss": 3.310417074139624, + "tokens_seen": 3124756480 + }, + { + "epoch": 0.89, + "learning_rate": 5.4101010101010105e-05, + "loss": 2.5891, + "theoretical_loss": 3.3104117800397703, + "tokens_seen": 3124822016 + }, + { + "epoch": 0.89, + "learning_rate": 5.408080808080808e-05, + "loss": 2.3371, + "theoretical_loss": 3.3104064860820355, + "tokens_seen": 3124887552 + }, + { + "epoch": 0.89, + "learning_rate": 5.406060606060606e-05, + "loss": 2.5875, + "theoretical_loss": 3.3104011922664123, + "tokens_seen": 3124953088 + }, + { + "epoch": 0.89, + "learning_rate": 5.4040404040404044e-05, + "loss": 2.436, + "theoretical_loss": 3.310395898592894, + "tokens_seen": 3125018624 + }, + { + "epoch": 0.89, + "learning_rate": 5.402020202020202e-05, + "loss": 2.5147, + "theoretical_loss": 3.3103906050614738, + "tokens_seen": 3125084160 + }, + { + "epoch": 0.89, + "learning_rate": 5.4e-05, + "loss": 2.6758, + "theoretical_loss": 3.310385311672145, + "tokens_seen": 3125149696 + }, + { + "epoch": 0.89, + "learning_rate": 5.397979797979798e-05, + "loss": 2.3758, + "theoretical_loss": 3.3103800184249015, + "tokens_seen": 3125215232 + }, + { + "epoch": 0.89, + "learning_rate": 5.395959595959596e-05, + "loss": 2.5026, + "theoretical_loss": 3.3103747253197353, + "tokens_seen": 3125280768 + }, + { + "epoch": 0.89, + "learning_rate": 5.393939393939394e-05, + "loss": 2.3838, + "theoretical_loss": 3.31036943235664, + "tokens_seen": 3125346304 + }, + { + "epoch": 0.89, + "learning_rate": 5.391919191919192e-05, + "loss": 2.4366, + "theoretical_loss": 3.310364139535609, + "tokens_seen": 3125411840 + }, + { + "epoch": 0.89, + "learning_rate": 5.38989898989899e-05, + "loss": 2.4601, + "theoretical_loss": 3.310358846856636, + "tokens_seen": 3125477376 + }, + { + "epoch": 0.89, + "learning_rate": 5.387878787878788e-05, + "loss": 2.6361, + "theoretical_loss": 3.3103535543197133, + "tokens_seen": 3125542912 + }, + { + "epoch": 0.89, + "learning_rate": 5.385858585858586e-05, + "loss": 2.6607, + "theoretical_loss": 3.3103482619248346, + "tokens_seen": 3125608448 + }, + { + "epoch": 0.89, + "learning_rate": 5.383838383838384e-05, + "loss": 2.6644, + "theoretical_loss": 3.310342969671993, + "tokens_seen": 3125673984 + }, + { + "epoch": 0.89, + "learning_rate": 5.381818181818182e-05, + "loss": 2.4883, + "theoretical_loss": 3.310337677561181, + "tokens_seen": 3125739520 + }, + { + "epoch": 0.89, + "learning_rate": 5.37979797979798e-05, + "loss": 2.4689, + "theoretical_loss": 3.3103323855923934, + "tokens_seen": 3125805056 + }, + { + "epoch": 0.89, + "learning_rate": 5.377777777777778e-05, + "loss": 2.6823, + "theoretical_loss": 3.310327093765622, + "tokens_seen": 3125870592 + }, + { + "epoch": 0.89, + "learning_rate": 5.3757575757575765e-05, + "loss": 2.5791, + "theoretical_loss": 3.3103218020808605, + "tokens_seen": 3125936128 + }, + { + "epoch": 0.89, + "learning_rate": 5.3737373737373735e-05, + "loss": 2.4405, + "theoretical_loss": 3.3103165105381023, + "tokens_seen": 3126001664 + }, + { + "epoch": 0.89, + "learning_rate": 5.371717171717172e-05, + "loss": 2.4747, + "theoretical_loss": 3.3103112191373407, + "tokens_seen": 3126067200 + }, + { + "epoch": 0.89, + "learning_rate": 5.3696969696969696e-05, + "loss": 2.5219, + "theoretical_loss": 3.3103059278785683, + "tokens_seen": 3126132736 + }, + { + "epoch": 0.89, + "learning_rate": 5.367676767676768e-05, + "loss": 2.6093, + "theoretical_loss": 3.3103006367617787, + "tokens_seen": 3126198272 + }, + { + "epoch": 0.89, + "learning_rate": 5.365656565656566e-05, + "loss": 2.4798, + "theoretical_loss": 3.3102953457869653, + "tokens_seen": 3126263808 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1713214, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.179457902908325, + "objective/train/theoretical_loss": 3.310290054954121, + "objective/train/tokens_used": 1485188576, + "theoretical_loss": 3.310290054954121, + "tokens_seen": 3126329344 + }, + { + "epoch": 0.89, + "learning_rate": 5.3636363636363635e-05, + "loss": 2.3429, + "theoretical_loss": 3.310290054954121, + "tokens_seen": 3126329344 + }, + { + "epoch": 0.89, + "learning_rate": 5.361616161616162e-05, + "loss": 2.5034, + "theoretical_loss": 3.310284764263239, + "tokens_seen": 3126394880 + }, + { + "epoch": 0.89, + "learning_rate": 5.3595959595959596e-05, + "loss": 2.4623, + "theoretical_loss": 3.3102794737143126, + "tokens_seen": 3126460416 + }, + { + "epoch": 0.89, + "learning_rate": 5.357575757575757e-05, + "loss": 2.5542, + "theoretical_loss": 3.3102741833073352, + "tokens_seen": 3126525952 + }, + { + "epoch": 0.89, + "learning_rate": 5.355555555555556e-05, + "loss": 2.7197, + "theoretical_loss": 3.3102688930423, + "tokens_seen": 3126591488 + }, + { + "epoch": 0.89, + "learning_rate": 5.3535353535353534e-05, + "loss": 2.6196, + "theoretical_loss": 3.3102636029192, + "tokens_seen": 3126657024 + }, + { + "epoch": 0.89, + "learning_rate": 5.351515151515152e-05, + "loss": 2.5072, + "theoretical_loss": 3.3102583129380285, + "tokens_seen": 3126722560 + }, + { + "epoch": 0.89, + "learning_rate": 5.34949494949495e-05, + "loss": 2.5497, + "theoretical_loss": 3.3102530230987783, + "tokens_seen": 3126788096 + }, + { + "epoch": 0.89, + "learning_rate": 5.347474747474747e-05, + "loss": 2.4407, + "theoretical_loss": 3.3102477334014435, + "tokens_seen": 3126853632 + }, + { + "epoch": 0.89, + "learning_rate": 5.3454545454545457e-05, + "loss": 2.4279, + "theoretical_loss": 3.310242443846017, + "tokens_seen": 3126919168 + }, + { + "epoch": 0.89, + "learning_rate": 5.3434343434343434e-05, + "loss": 2.4822, + "theoretical_loss": 3.3102371544324916, + "tokens_seen": 3126984704 + }, + { + "epoch": 0.89, + "learning_rate": 5.341414141414142e-05, + "loss": 2.5169, + "theoretical_loss": 3.3102318651608607, + "tokens_seen": 3127050240 + }, + { + "epoch": 0.89, + "learning_rate": 5.3393939393939395e-05, + "loss": 2.637, + "theoretical_loss": 3.3102265760311176, + "tokens_seen": 3127115776 + }, + { + "epoch": 0.89, + "learning_rate": 5.337373737373737e-05, + "loss": 2.494, + "theoretical_loss": 3.3102212870432557, + "tokens_seen": 3127181312 + }, + { + "epoch": 0.89, + "learning_rate": 5.3353535353535356e-05, + "loss": 2.5724, + "theoretical_loss": 3.310215998197268, + "tokens_seen": 3127246848 + }, + { + "epoch": 0.89, + "learning_rate": 5.333333333333334e-05, + "loss": 2.3699, + "theoretical_loss": 3.3102107094931474, + "tokens_seen": 3127312384 + }, + { + "epoch": 0.89, + "learning_rate": 5.331313131313131e-05, + "loss": 2.3923, + "theoretical_loss": 3.310205420930888, + "tokens_seen": 3127377920 + }, + { + "epoch": 0.89, + "learning_rate": 5.3292929292929294e-05, + "loss": 2.6086, + "theoretical_loss": 3.3102001325104826, + "tokens_seen": 3127443456 + }, + { + "epoch": 0.89, + "learning_rate": 5.327272727272727e-05, + "loss": 2.401, + "theoretical_loss": 3.310194844231924, + "tokens_seen": 3127508992 + }, + { + "epoch": 0.89, + "learning_rate": 5.3252525252525255e-05, + "loss": 2.2302, + "theoretical_loss": 3.3101895560952057, + "tokens_seen": 3127574528 + }, + { + "epoch": 0.89, + "learning_rate": 5.323232323232324e-05, + "loss": 2.4812, + "theoretical_loss": 3.3101842681003215, + "tokens_seen": 3127640064 + }, + { + "epoch": 0.89, + "learning_rate": 5.321212121212121e-05, + "loss": 2.4541, + "theoretical_loss": 3.3101789802472634, + "tokens_seen": 3127705600 + }, + { + "epoch": 0.89, + "learning_rate": 5.3191919191919194e-05, + "loss": 2.6238, + "theoretical_loss": 3.310173692536026, + "tokens_seen": 3127771136 + }, + { + "epoch": 0.89, + "learning_rate": 5.317171717171717e-05, + "loss": 2.4537, + "theoretical_loss": 3.3101684049666016, + "tokens_seen": 3127836672 + }, + { + "epoch": 0.89, + "learning_rate": 5.3151515151515155e-05, + "loss": 2.4686, + "theoretical_loss": 3.3101631175389836, + "tokens_seen": 3127902208 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1713719, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.412795066833496, + "objective/train/theoretical_loss": 3.310157830253165, + "objective/train/tokens_used": 1486826976, + "theoretical_loss": 3.310157830253165, + "tokens_seen": 3127967744 + }, + { + "epoch": 0.9, + "learning_rate": 5.313131313131313e-05, + "loss": 2.4731, + "theoretical_loss": 3.310157830253165, + "tokens_seen": 3127967744 + }, + { + "epoch": 0.9, + "learning_rate": 5.311111111111111e-05, + "loss": 2.5146, + "theoretical_loss": 3.31015254310914, + "tokens_seen": 3128033280 + }, + { + "epoch": 0.9, + "learning_rate": 5.309090909090909e-05, + "loss": 2.695, + "theoretical_loss": 3.3101472561069007, + "tokens_seen": 3128098816 + }, + { + "epoch": 0.9, + "learning_rate": 5.307070707070708e-05, + "loss": 2.5678, + "theoretical_loss": 3.310141969246441, + "tokens_seen": 3128164352 + }, + { + "epoch": 0.9, + "learning_rate": 5.305050505050505e-05, + "loss": 2.6045, + "theoretical_loss": 3.310136682527754, + "tokens_seen": 3128229888 + }, + { + "epoch": 0.9, + "learning_rate": 5.303030303030303e-05, + "loss": 2.4549, + "theoretical_loss": 3.3101313959508327, + "tokens_seen": 3128295424 + }, + { + "epoch": 0.9, + "learning_rate": 5.301010101010101e-05, + "loss": 2.4012, + "theoretical_loss": 3.3101261095156707, + "tokens_seen": 3128360960 + }, + { + "epoch": 0.9, + "learning_rate": 5.298989898989899e-05, + "loss": 2.5029, + "theoretical_loss": 3.3101208232222605, + "tokens_seen": 3128426496 + }, + { + "epoch": 0.9, + "learning_rate": 5.296969696969697e-05, + "loss": 2.4209, + "theoretical_loss": 3.310115537070596, + "tokens_seen": 3128492032 + }, + { + "epoch": 0.9, + "learning_rate": 5.294949494949495e-05, + "loss": 2.3642, + "theoretical_loss": 3.310110251060671, + "tokens_seen": 3128557568 + }, + { + "epoch": 0.9, + "learning_rate": 5.292929292929293e-05, + "loss": 2.5829, + "theoretical_loss": 3.310104965192477, + "tokens_seen": 3128623104 + }, + { + "epoch": 0.9, + "learning_rate": 5.2909090909090915e-05, + "loss": 2.6321, + "theoretical_loss": 3.3100996794660085, + "tokens_seen": 3128688640 + }, + { + "epoch": 0.9, + "learning_rate": 5.288888888888889e-05, + "loss": 2.6529, + "theoretical_loss": 3.3100943938812586, + "tokens_seen": 3128754176 + }, + { + "epoch": 0.9, + "learning_rate": 5.286868686868687e-05, + "loss": 2.5047, + "theoretical_loss": 3.3100891084382207, + "tokens_seen": 3128819712 + }, + { + "epoch": 0.9, + "learning_rate": 5.2848484848484847e-05, + "loss": 2.399, + "theoretical_loss": 3.310083823136887, + "tokens_seen": 3128885248 + }, + { + "epoch": 0.9, + "learning_rate": 5.282828282828283e-05, + "loss": 2.4684, + "theoretical_loss": 3.310078537977252, + "tokens_seen": 3128950784 + }, + { + "epoch": 0.9, + "learning_rate": 5.2808080808080815e-05, + "loss": 2.4146, + "theoretical_loss": 3.310073252959308, + "tokens_seen": 3129016320 + }, + { + "epoch": 0.9, + "learning_rate": 5.2787878787878785e-05, + "loss": 2.4126, + "theoretical_loss": 3.3100679680830494, + "tokens_seen": 3129081856 + }, + { + "epoch": 0.9, + "learning_rate": 5.276767676767677e-05, + "loss": 2.587, + "theoretical_loss": 3.310062683348468, + "tokens_seen": 3129147392 + }, + { + "epoch": 0.9, + "learning_rate": 5.2747474747474746e-05, + "loss": 2.604, + "theoretical_loss": 3.3100573987555575, + "tokens_seen": 3129212928 + }, + { + "epoch": 0.9, + "learning_rate": 5.272727272727273e-05, + "loss": 2.4922, + "theoretical_loss": 3.3100521143043116, + "tokens_seen": 3129278464 + }, + { + "epoch": 0.9, + "learning_rate": 5.270707070707071e-05, + "loss": 2.659, + "theoretical_loss": 3.3100468299947234, + "tokens_seen": 3129344000 + }, + { + "epoch": 0.9, + "learning_rate": 5.2686868686868684e-05, + "loss": 2.3207, + "theoretical_loss": 3.310041545826786, + "tokens_seen": 3129409536 + }, + { + "epoch": 0.9, + "learning_rate": 5.266666666666667e-05, + "loss": 2.5277, + "theoretical_loss": 3.3100362618004926, + "tokens_seen": 3129475072 + }, + { + "epoch": 0.9, + "learning_rate": 5.264646464646465e-05, + "loss": 2.6204, + "theoretical_loss": 3.3100309779158366, + "tokens_seen": 3129540608 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1714913, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5364954471588135, + "objective/train/theoretical_loss": 3.3100256941728103, + "objective/train/tokens_used": 1488465376, + "theoretical_loss": 3.3100256941728103, + "tokens_seen": 3129606144 + }, + { + "epoch": 0.9, + "learning_rate": 5.262626262626263e-05, + "loss": 2.6241, + "theoretical_loss": 3.3100256941728103, + "tokens_seen": 3129606144 + }, + { + "epoch": 0.9, + "learning_rate": 5.260606060606061e-05, + "loss": 2.529, + "theoretical_loss": 3.3100204105714086, + "tokens_seen": 3129671680 + }, + { + "epoch": 0.9, + "learning_rate": 5.2585858585858584e-05, + "loss": 2.3102, + "theoretical_loss": 3.3100151271116234, + "tokens_seen": 3129737216 + }, + { + "epoch": 0.9, + "learning_rate": 5.256565656565657e-05, + "loss": 2.6465, + "theoretical_loss": 3.310009843793449, + "tokens_seen": 3129802752 + }, + { + "epoch": 0.9, + "learning_rate": 5.254545454545455e-05, + "loss": 2.6128, + "theoretical_loss": 3.3100045606168775, + "tokens_seen": 3129868288 + }, + { + "epoch": 0.9, + "learning_rate": 5.252525252525252e-05, + "loss": 2.366, + "theoretical_loss": 3.309999277581903, + "tokens_seen": 3129933824 + }, + { + "epoch": 0.9, + "learning_rate": 5.2505050505050506e-05, + "loss": 2.5214, + "theoretical_loss": 3.3099939946885186, + "tokens_seen": 3129999360 + }, + { + "epoch": 0.9, + "learning_rate": 5.248484848484848e-05, + "loss": 2.4854, + "theoretical_loss": 3.309988711936717, + "tokens_seen": 3130064896 + }, + { + "epoch": 0.9, + "learning_rate": 5.246464646464647e-05, + "loss": 2.4333, + "theoretical_loss": 3.309983429326492, + "tokens_seen": 3130130432 + }, + { + "epoch": 0.9, + "learning_rate": 5.2444444444444445e-05, + "loss": 2.5563, + "theoretical_loss": 3.3099781468578366, + "tokens_seen": 3130195968 + }, + { + "epoch": 0.9, + "learning_rate": 5.242424242424242e-05, + "loss": 2.2967, + "theoretical_loss": 3.309972864530744, + "tokens_seen": 3130261504 + }, + { + "epoch": 0.9, + "learning_rate": 5.2404040404040406e-05, + "loss": 2.5916, + "theoretical_loss": 3.309967582345208, + "tokens_seen": 3130327040 + }, + { + "epoch": 0.9, + "learning_rate": 5.238383838383839e-05, + "loss": 2.3397, + "theoretical_loss": 3.309962300301221, + "tokens_seen": 3130392576 + }, + { + "epoch": 0.9, + "learning_rate": 5.236363636363636e-05, + "loss": 2.4227, + "theoretical_loss": 3.3099570183987765, + "tokens_seen": 3130458112 + }, + { + "epoch": 0.9, + "learning_rate": 5.2343434343434344e-05, + "loss": 2.515, + "theoretical_loss": 3.3099517366378683, + "tokens_seen": 3130523648 + }, + { + "epoch": 0.9, + "learning_rate": 5.232323232323232e-05, + "loss": 2.4291, + "theoretical_loss": 3.309946455018489, + "tokens_seen": 3130589184 + }, + { + "epoch": 0.9, + "learning_rate": 5.2303030303030305e-05, + "loss": 2.5584, + "theoretical_loss": 3.3099411735406323, + "tokens_seen": 3130654720 + }, + { + "epoch": 0.9, + "learning_rate": 5.228282828282829e-05, + "loss": 2.3128, + "theoretical_loss": 3.309935892204291, + "tokens_seen": 3130720256 + }, + { + "epoch": 0.9, + "learning_rate": 5.226262626262626e-05, + "loss": 2.5589, + "theoretical_loss": 3.3099306110094586, + "tokens_seen": 3130785792 + }, + { + "epoch": 0.9, + "learning_rate": 5.2242424242424243e-05, + "loss": 2.4693, + "theoretical_loss": 3.3099253299561284, + "tokens_seen": 3130851328 + }, + { + "epoch": 0.9, + "learning_rate": 5.222222222222223e-05, + "loss": 2.4384, + "theoretical_loss": 3.3099200490442935, + "tokens_seen": 3130916864 + }, + { + "epoch": 0.9, + "learning_rate": 5.2202020202020205e-05, + "loss": 2.3314, + "theoretical_loss": 3.309914768273947, + "tokens_seen": 3130982400 + }, + { + "epoch": 0.9, + "learning_rate": 5.218181818181818e-05, + "loss": 2.6522, + "theoretical_loss": 3.3099094876450827, + "tokens_seen": 3131047936 + }, + { + "epoch": 0.9, + "learning_rate": 5.216161616161616e-05, + "loss": 2.4337, + "theoretical_loss": 3.309904207157693, + "tokens_seen": 3131113472 + }, + { + "epoch": 0.9, + "learning_rate": 5.214141414141414e-05, + "loss": 2.6077, + "theoretical_loss": 3.3098989268117722, + "tokens_seen": 3131179008 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1715340, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.970646858215332, + "objective/train/theoretical_loss": 3.3098936466073132, + "objective/train/tokens_used": 1490103776, + "theoretical_loss": 3.3098936466073132, + "tokens_seen": 3131244544 + }, + { + "epoch": 0.9, + "learning_rate": 5.212121212121213e-05, + "loss": 2.7176, + "theoretical_loss": 3.3098936466073132, + "tokens_seen": 3131244544 + }, + { + "epoch": 0.9, + "learning_rate": 5.21010101010101e-05, + "loss": 2.5933, + "theoretical_loss": 3.3098883665443086, + "tokens_seen": 3131310080 + }, + { + "epoch": 0.9, + "learning_rate": 5.208080808080808e-05, + "loss": 2.5595, + "theoretical_loss": 3.309883086622752, + "tokens_seen": 3131375616 + }, + { + "epoch": 0.9, + "learning_rate": 5.206060606060606e-05, + "loss": 2.5883, + "theoretical_loss": 3.309877806842637, + "tokens_seen": 3131441152 + }, + { + "epoch": 0.9, + "learning_rate": 5.204040404040404e-05, + "loss": 2.5676, + "theoretical_loss": 3.3098725272039564, + "tokens_seen": 3131506688 + }, + { + "epoch": 0.9, + "learning_rate": 5.2020202020202026e-05, + "loss": 2.3857, + "theoretical_loss": 3.3098672477067037, + "tokens_seen": 3131572224 + }, + { + "epoch": 0.9, + "learning_rate": 5.2e-05, + "loss": 2.5211, + "theoretical_loss": 3.309861968350872, + "tokens_seen": 3131637760 + }, + { + "epoch": 0.9, + "learning_rate": 5.197979797979798e-05, + "loss": 2.2517, + "theoretical_loss": 3.309856689136455, + "tokens_seen": 3131703296 + }, + { + "epoch": 0.9, + "learning_rate": 5.1959595959595965e-05, + "loss": 2.353, + "theoretical_loss": 3.3098514100634455, + "tokens_seen": 3131768832 + }, + { + "epoch": 0.9, + "learning_rate": 5.193939393939394e-05, + "loss": 2.7162, + "theoretical_loss": 3.3098461311318363, + "tokens_seen": 3131834368 + }, + { + "epoch": 0.9, + "learning_rate": 5.191919191919192e-05, + "loss": 2.6389, + "theoretical_loss": 3.309840852341622, + "tokens_seen": 3131899904 + }, + { + "epoch": 0.9, + "learning_rate": 5.1898989898989896e-05, + "loss": 2.5132, + "theoretical_loss": 3.3098355736927947, + "tokens_seen": 3131965440 + }, + { + "epoch": 0.9, + "learning_rate": 5.187878787878788e-05, + "loss": 2.5549, + "theoretical_loss": 3.309830295185348, + "tokens_seen": 3132030976 + }, + { + "epoch": 0.9, + "learning_rate": 5.1858585858585864e-05, + "loss": 2.3926, + "theoretical_loss": 3.3098250168192753, + "tokens_seen": 3132096512 + }, + { + "epoch": 0.9, + "learning_rate": 5.1838383838383835e-05, + "loss": 2.5215, + "theoretical_loss": 3.3098197385945696, + "tokens_seen": 3132162048 + }, + { + "epoch": 0.9, + "learning_rate": 5.181818181818182e-05, + "loss": 2.5081, + "theoretical_loss": 3.3098144605112245, + "tokens_seen": 3132227584 + }, + { + "epoch": 0.9, + "learning_rate": 5.17979797979798e-05, + "loss": 2.6858, + "theoretical_loss": 3.309809182569233, + "tokens_seen": 3132293120 + }, + { + "epoch": 0.9, + "learning_rate": 5.177777777777778e-05, + "loss": 2.4106, + "theoretical_loss": 3.3098039047685885, + "tokens_seen": 3132358656 + }, + { + "epoch": 0.9, + "learning_rate": 5.175757575757576e-05, + "loss": 2.3635, + "theoretical_loss": 3.309798627109284, + "tokens_seen": 3132424192 + }, + { + "epoch": 0.9, + "learning_rate": 5.1737373737373734e-05, + "loss": 2.4203, + "theoretical_loss": 3.3097933495913128, + "tokens_seen": 3132489728 + }, + { + "epoch": 0.9, + "learning_rate": 5.171717171717172e-05, + "loss": 2.3843, + "theoretical_loss": 3.309788072214668, + "tokens_seen": 3132555264 + }, + { + "epoch": 0.9, + "learning_rate": 5.16969696969697e-05, + "loss": 2.5244, + "theoretical_loss": 3.309782794979344, + "tokens_seen": 3132620800 + }, + { + "epoch": 0.9, + "learning_rate": 5.167676767676768e-05, + "loss": 2.5972, + "theoretical_loss": 3.3097775178853324, + "tokens_seen": 3132686336 + }, + { + "epoch": 0.9, + "learning_rate": 5.1656565656565656e-05, + "loss": 2.6212, + "theoretical_loss": 3.309772240932628, + "tokens_seen": 3132751872 + }, + { + "epoch": 0.9, + "learning_rate": 5.1636363636363634e-05, + "loss": 2.5713, + "theoretical_loss": 3.3097669641212226, + "tokens_seen": 3132817408 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1716598, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5470502376556396, + "objective/train/theoretical_loss": 3.30976168745111, + "objective/train/tokens_used": 1491742176, + "theoretical_loss": 3.30976168745111, + "tokens_seen": 3132882944 + }, + { + "epoch": 0.9, + "learning_rate": 5.161616161616162e-05, + "loss": 2.4798, + "theoretical_loss": 3.30976168745111, + "tokens_seen": 3132882944 + }, + { + "epoch": 0.9, + "learning_rate": 5.15959595959596e-05, + "loss": 2.4729, + "theoretical_loss": 3.309756410922285, + "tokens_seen": 3132948480 + }, + { + "epoch": 0.9, + "learning_rate": 5.157575757575757e-05, + "loss": 2.5623, + "theoretical_loss": 3.3097511345347383, + "tokens_seen": 3133014016 + }, + { + "epoch": 0.9, + "learning_rate": 5.1555555555555556e-05, + "loss": 2.6216, + "theoretical_loss": 3.3097458582884647, + "tokens_seen": 3133079552 + }, + { + "epoch": 0.9, + "learning_rate": 5.153535353535354e-05, + "loss": 2.4995, + "theoretical_loss": 3.3097405821834567, + "tokens_seen": 3133145088 + }, + { + "epoch": 0.9, + "learning_rate": 5.151515151515152e-05, + "loss": 2.4496, + "theoretical_loss": 3.3097353062197086, + "tokens_seen": 3133210624 + }, + { + "epoch": 0.9, + "learning_rate": 5.1494949494949494e-05, + "loss": 2.5412, + "theoretical_loss": 3.309730030397213, + "tokens_seen": 3133276160 + }, + { + "epoch": 0.9, + "learning_rate": 5.147474747474747e-05, + "loss": 2.6207, + "theoretical_loss": 3.3097247547159627, + "tokens_seen": 3133341696 + }, + { + "epoch": 0.9, + "learning_rate": 5.1454545454545455e-05, + "loss": 2.494, + "theoretical_loss": 3.309719479175952, + "tokens_seen": 3133407232 + }, + { + "epoch": 0.9, + "learning_rate": 5.143434343434344e-05, + "loss": 2.6045, + "theoretical_loss": 3.3097142037771734, + "tokens_seen": 3133472768 + }, + { + "epoch": 0.9, + "learning_rate": 5.1414141414141416e-05, + "loss": 2.7989, + "theoretical_loss": 3.309708928519621, + "tokens_seen": 3133538304 + }, + { + "epoch": 0.9, + "learning_rate": 5.1393939393939394e-05, + "loss": 2.4449, + "theoretical_loss": 3.3097036534032864, + "tokens_seen": 3133603840 + }, + { + "epoch": 0.9, + "learning_rate": 5.137373737373738e-05, + "loss": 2.2737, + "theoretical_loss": 3.309698378428165, + "tokens_seen": 3133669376 + }, + { + "epoch": 0.9, + "learning_rate": 5.1353535353535355e-05, + "loss": 2.517, + "theoretical_loss": 3.309693103594248, + "tokens_seen": 3133734912 + }, + { + "epoch": 0.9, + "learning_rate": 5.133333333333334e-05, + "loss": 2.5784, + "theoretical_loss": 3.3096878289015303, + "tokens_seen": 3133800448 + }, + { + "epoch": 0.9, + "learning_rate": 5.131313131313131e-05, + "loss": 2.4806, + "theoretical_loss": 3.3096825543500046, + "tokens_seen": 3133865984 + }, + { + "epoch": 0.9, + "learning_rate": 5.129292929292929e-05, + "loss": 2.5138, + "theoretical_loss": 3.3096772799396637, + "tokens_seen": 3133931520 + }, + { + "epoch": 0.9, + "learning_rate": 5.127272727272728e-05, + "loss": 2.3786, + "theoretical_loss": 3.3096720056705013, + "tokens_seen": 3133997056 + }, + { + "epoch": 0.9, + "learning_rate": 5.1252525252525254e-05, + "loss": 2.5306, + "theoretical_loss": 3.309666731542511, + "tokens_seen": 3134062592 + }, + { + "epoch": 0.9, + "learning_rate": 5.123232323232323e-05, + "loss": 2.4192, + "theoretical_loss": 3.309661457555686, + "tokens_seen": 3134128128 + }, + { + "epoch": 0.9, + "learning_rate": 5.121212121212121e-05, + "loss": 2.3612, + "theoretical_loss": 3.3096561837100187, + "tokens_seen": 3134193664 + }, + { + "epoch": 0.9, + "learning_rate": 5.119191919191919e-05, + "loss": 2.5919, + "theoretical_loss": 3.309650910005503, + "tokens_seen": 3134259200 + }, + { + "epoch": 0.9, + "learning_rate": 5.1171717171717177e-05, + "loss": 2.3549, + "theoretical_loss": 3.3096456364421325, + "tokens_seen": 3134324736 + }, + { + "epoch": 0.9, + "learning_rate": 5.115151515151515e-05, + "loss": 2.3419, + "theoretical_loss": 3.3096403630198994, + "tokens_seen": 3134390272 + }, + { + "epoch": 0.9, + "learning_rate": 5.113131313131313e-05, + "loss": 2.76, + "theoretical_loss": 3.3096350897387983, + "tokens_seen": 3134455808 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1717316, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.688441753387451, + "objective/train/theoretical_loss": 3.3096298165988216, + "objective/train/tokens_used": 1493380576, + "theoretical_loss": 3.3096298165988216, + "tokens_seen": 3134521344 + }, + { + "epoch": 0.9, + "learning_rate": 5.1111111111111115e-05, + "loss": 2.6699, + "theoretical_loss": 3.3096298165988216, + "tokens_seen": 3134521344 + }, + { + "epoch": 0.9, + "learning_rate": 5.109090909090909e-05, + "loss": 2.3374, + "theoretical_loss": 3.309624543599963, + "tokens_seen": 3134586880 + }, + { + "epoch": 0.9, + "learning_rate": 5.1070707070707076e-05, + "loss": 2.4117, + "theoretical_loss": 3.3096192707422154, + "tokens_seen": 3134652416 + }, + { + "epoch": 0.9, + "learning_rate": 5.1050505050505046e-05, + "loss": 2.5414, + "theoretical_loss": 3.3096139980255725, + "tokens_seen": 3134717952 + }, + { + "epoch": 0.9, + "learning_rate": 5.103030303030303e-05, + "loss": 2.4691, + "theoretical_loss": 3.309608725450027, + "tokens_seen": 3134783488 + }, + { + "epoch": 0.9, + "learning_rate": 5.1010101010101014e-05, + "loss": 2.6253, + "theoretical_loss": 3.3096034530155727, + "tokens_seen": 3134849024 + }, + { + "epoch": 0.9, + "learning_rate": 5.098989898989899e-05, + "loss": 2.3831, + "theoretical_loss": 3.309598180722203, + "tokens_seen": 3134914560 + }, + { + "epoch": 0.9, + "learning_rate": 5.096969696969697e-05, + "loss": 2.7028, + "theoretical_loss": 3.3095929085699103, + "tokens_seen": 3134980096 + }, + { + "epoch": 0.9, + "learning_rate": 5.094949494949495e-05, + "loss": 2.4241, + "theoretical_loss": 3.3095876365586885, + "tokens_seen": 3135045632 + }, + { + "epoch": 0.9, + "learning_rate": 5.092929292929293e-05, + "loss": 2.5145, + "theoretical_loss": 3.309582364688531, + "tokens_seen": 3135111168 + }, + { + "epoch": 0.9, + "learning_rate": 5.0909090909090914e-05, + "loss": 2.6871, + "theoretical_loss": 3.3095770929594313, + "tokens_seen": 3135176704 + }, + { + "epoch": 0.9, + "learning_rate": 5.0888888888888884e-05, + "loss": 2.5008, + "theoretical_loss": 3.3095718213713816, + "tokens_seen": 3135242240 + }, + { + "epoch": 0.9, + "learning_rate": 5.086868686868687e-05, + "loss": 2.48, + "theoretical_loss": 3.309566549924376, + "tokens_seen": 3135307776 + }, + { + "epoch": 0.9, + "learning_rate": 5.084848484848485e-05, + "loss": 2.7101, + "theoretical_loss": 3.309561278618408, + "tokens_seen": 3135373312 + }, + { + "epoch": 0.9, + "learning_rate": 5.082828282828283e-05, + "loss": 2.6843, + "theoretical_loss": 3.30955600745347, + "tokens_seen": 3135438848 + }, + { + "epoch": 0.9, + "learning_rate": 5.080808080808081e-05, + "loss": 2.5975, + "theoretical_loss": 3.309550736429556, + "tokens_seen": 3135504384 + }, + { + "epoch": 0.9, + "learning_rate": 5.0787878787878784e-05, + "loss": 2.645, + "theoretical_loss": 3.309545465546659, + "tokens_seen": 3135569920 + }, + { + "epoch": 0.9, + "learning_rate": 5.076767676767677e-05, + "loss": 2.4297, + "theoretical_loss": 3.3095401948047725, + "tokens_seen": 3135635456 + }, + { + "epoch": 0.9, + "learning_rate": 5.074747474747475e-05, + "loss": 2.5001, + "theoretical_loss": 3.3095349242038896, + "tokens_seen": 3135700992 + }, + { + "epoch": 0.9, + "learning_rate": 5.072727272727273e-05, + "loss": 2.6638, + "theoretical_loss": 3.3095296537440038, + "tokens_seen": 3135766528 + }, + { + "epoch": 0.9, + "learning_rate": 5.0707070707070706e-05, + "loss": 2.7629, + "theoretical_loss": 3.309524383425108, + "tokens_seen": 3135832064 + }, + { + "epoch": 0.9, + "learning_rate": 5.068686868686869e-05, + "loss": 2.6171, + "theoretical_loss": 3.3095191132471955, + "tokens_seen": 3135897600 + }, + { + "epoch": 0.9, + "learning_rate": 5.066666666666667e-05, + "loss": 2.5431, + "theoretical_loss": 3.3095138432102598, + "tokens_seen": 3135963136 + }, + { + "epoch": 0.9, + "learning_rate": 5.064646464646465e-05, + "loss": 2.3449, + "theoretical_loss": 3.309508573314294, + "tokens_seen": 3136028672 + }, + { + "epoch": 0.9, + "learning_rate": 5.062626262626262e-05, + "loss": 2.6349, + "theoretical_loss": 3.309503303559292, + "tokens_seen": 3136094208 + }, + { + "debugging/Self-BLEU-5": 0.5351812588664681, + "debugging/distinct-1-grams": 0.7698809900384261, + "debugging/distinct-2-grams": 0.9453324138172466, + "debugging/entropy-1-grams": 5.91278720119932, + "debugging/entropy-2-grams": 6.738603264472726, + "debugging/length": 540.6666666666666, + "debugging/num_segments": 12, + "epoch": 0.9, + "objective/train/docs_used": 1718594, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.599855661392212, + "objective/train/theoretical_loss": 3.309498033945246, + "objective/train/tokens_used": 1495018976, + "theoretical_loss": 3.309498033945246, + "tokens_seen": 3136159744 + }, + { + "epoch": 0.9, + "learning_rate": 5.0606060606060606e-05, + "loss": 2.5894, + "theoretical_loss": 3.309498033945246, + "tokens_seen": 3136159744 + }, + { + "epoch": 0.9, + "learning_rate": 5.058585858585859e-05, + "loss": 2.4069, + "theoretical_loss": 3.30949276447215, + "tokens_seen": 3136225280 + }, + { + "epoch": 0.9, + "learning_rate": 5.056565656565657e-05, + "loss": 2.4828, + "theoretical_loss": 3.3094874951399977, + "tokens_seen": 3136290816 + }, + { + "epoch": 0.9, + "learning_rate": 5.0545454545454544e-05, + "loss": 2.1724, + "theoretical_loss": 3.3094822259487815, + "tokens_seen": 3136356352 + }, + { + "epoch": 0.9, + "learning_rate": 5.052525252525253e-05, + "loss": 2.5349, + "theoretical_loss": 3.3094769568984947, + "tokens_seen": 3136421888 + }, + { + "epoch": 0.9, + "learning_rate": 5.0505050505050505e-05, + "loss": 2.5389, + "theoretical_loss": 3.3094716879891317, + "tokens_seen": 3136487424 + }, + { + "epoch": 0.9, + "learning_rate": 5.048484848484849e-05, + "loss": 2.4786, + "theoretical_loss": 3.3094664192206844, + "tokens_seen": 3136552960 + }, + { + "epoch": 0.9, + "learning_rate": 5.0464646464646466e-05, + "loss": 2.7658, + "theoretical_loss": 3.309461150593147, + "tokens_seen": 3136618496 + }, + { + "epoch": 0.9, + "learning_rate": 5.044444444444444e-05, + "loss": 2.5136, + "theoretical_loss": 3.309455882106512, + "tokens_seen": 3136684032 + }, + { + "epoch": 0.9, + "learning_rate": 5.042424242424243e-05, + "loss": 2.5915, + "theoretical_loss": 3.3094506137607738, + "tokens_seen": 3136749568 + }, + { + "epoch": 0.9, + "learning_rate": 5.0404040404040404e-05, + "loss": 2.2775, + "theoretical_loss": 3.309445345555925, + "tokens_seen": 3136815104 + }, + { + "epoch": 0.9, + "learning_rate": 5.038383838383839e-05, + "loss": 2.6138, + "theoretical_loss": 3.309440077491959, + "tokens_seen": 3136880640 + }, + { + "epoch": 0.9, + "learning_rate": 5.036363636363636e-05, + "loss": 2.5571, + "theoretical_loss": 3.3094348095688684, + "tokens_seen": 3136946176 + }, + { + "epoch": 0.9, + "learning_rate": 5.034343434343434e-05, + "loss": 2.0176, + "theoretical_loss": 3.3094295417866477, + "tokens_seen": 3137011712 + }, + { + "epoch": 0.9, + "learning_rate": 5.032323232323233e-05, + "loss": 2.4709, + "theoretical_loss": 3.3094242741452895, + "tokens_seen": 3137077248 + }, + { + "epoch": 0.9, + "learning_rate": 5.0303030303030304e-05, + "loss": 2.6007, + "theoretical_loss": 3.309419006644787, + "tokens_seen": 3137142784 + }, + { + "epoch": 0.9, + "learning_rate": 5.028282828282828e-05, + "loss": 2.6594, + "theoretical_loss": 3.3094137392851337, + "tokens_seen": 3137208320 + }, + { + "epoch": 0.9, + "learning_rate": 5.0262626262626265e-05, + "loss": 2.4839, + "theoretical_loss": 3.3094084720663233, + "tokens_seen": 3137273856 + }, + { + "epoch": 0.9, + "learning_rate": 5.024242424242424e-05, + "loss": 2.4997, + "theoretical_loss": 3.3094032049883486, + "tokens_seen": 3137339392 + }, + { + "epoch": 0.9, + "learning_rate": 5.0222222222222226e-05, + "loss": 2.641, + "theoretical_loss": 3.3093979380512026, + "tokens_seen": 3137404928 + }, + { + "epoch": 0.9, + "learning_rate": 5.0202020202020203e-05, + "loss": 2.401, + "theoretical_loss": 3.309392671254879, + "tokens_seen": 3137470464 + }, + { + "epoch": 0.9, + "learning_rate": 5.018181818181818e-05, + "loss": 2.5622, + "theoretical_loss": 3.309387404599372, + "tokens_seen": 3137536000 + }, + { + "epoch": 0.9, + "learning_rate": 5.0161616161616165e-05, + "loss": 2.5967, + "theoretical_loss": 3.309382138084673, + "tokens_seen": 3137601536 + }, + { + "epoch": 0.9, + "learning_rate": 5.014141414141414e-05, + "loss": 2.4791, + "theoretical_loss": 3.3093768717107763, + "tokens_seen": 3137667072 + }, + { + "epoch": 0.9, + "learning_rate": 5.0121212121212126e-05, + "loss": 2.3914, + "theoretical_loss": 3.3093716054776756, + "tokens_seen": 3137732608 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1719775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0205938816070557, + "objective/train/theoretical_loss": 3.309366339385363, + "objective/train/tokens_used": 1496657376, + "theoretical_loss": 3.309366339385363, + "tokens_seen": 3137798144 + }, + { + "epoch": 0.9, + "learning_rate": 5.01010101010101e-05, + "loss": 2.3732, + "theoretical_loss": 3.309366339385363, + "tokens_seen": 3137798144 + }, + { + "epoch": 0.9, + "learning_rate": 5.008080808080808e-05, + "loss": 2.5349, + "theoretical_loss": 3.3093610734338332, + "tokens_seen": 3137863680 + }, + { + "epoch": 0.9, + "learning_rate": 5.0060606060606064e-05, + "loss": 2.6199, + "theoretical_loss": 3.3093558076230787, + "tokens_seen": 3137929216 + }, + { + "epoch": 0.9, + "learning_rate": 5.004040404040404e-05, + "loss": 2.612, + "theoretical_loss": 3.309350541953093, + "tokens_seen": 3137994752 + }, + { + "epoch": 0.9, + "learning_rate": 5.002020202020202e-05, + "loss": 2.4251, + "theoretical_loss": 3.309345276423869, + "tokens_seen": 3138060288 + }, + { + "epoch": 0.9, + "learning_rate": 5e-05, + "loss": 2.4492, + "theoretical_loss": 3.3093400110354008, + "tokens_seen": 3138125824 + }, + { + "epoch": 0.9, + "learning_rate": 4.997979797979798e-05, + "loss": 2.4638, + "theoretical_loss": 3.3093347457876807, + "tokens_seen": 3138191360 + }, + { + "epoch": 0.9, + "learning_rate": 4.9959595959595964e-05, + "loss": 2.5549, + "theoretical_loss": 3.3093294806807028, + "tokens_seen": 3138256896 + }, + { + "epoch": 0.9, + "learning_rate": 4.9939393939393934e-05, + "loss": 2.5365, + "theoretical_loss": 3.3093242157144602, + "tokens_seen": 3138322432 + }, + { + "epoch": 0.9, + "learning_rate": 4.991919191919192e-05, + "loss": 2.3918, + "theoretical_loss": 3.309318950888946, + "tokens_seen": 3138387968 + }, + { + "epoch": 0.9, + "learning_rate": 4.98989898989899e-05, + "loss": 2.6219, + "theoretical_loss": 3.309313686204154, + "tokens_seen": 3138453504 + }, + { + "epoch": 0.9, + "learning_rate": 4.987878787878788e-05, + "loss": 2.4167, + "theoretical_loss": 3.3093084216600763, + "tokens_seen": 3138519040 + }, + { + "epoch": 0.9, + "learning_rate": 4.985858585858586e-05, + "loss": 2.5313, + "theoretical_loss": 3.309303157256708, + "tokens_seen": 3138584576 + }, + { + "epoch": 0.9, + "learning_rate": 4.983838383838384e-05, + "loss": 2.507, + "theoretical_loss": 3.3092978929940404, + "tokens_seen": 3138650112 + }, + { + "epoch": 0.9, + "learning_rate": 4.981818181818182e-05, + "loss": 2.4198, + "theoretical_loss": 3.3092926288720688, + "tokens_seen": 3138715648 + }, + { + "epoch": 0.9, + "learning_rate": 4.97979797979798e-05, + "loss": 2.2421, + "theoretical_loss": 3.309287364890785, + "tokens_seen": 3138781184 + }, + { + "epoch": 0.9, + "learning_rate": 4.977777777777778e-05, + "loss": 2.4252, + "theoretical_loss": 3.309282101050183, + "tokens_seen": 3138846720 + }, + { + "epoch": 0.9, + "learning_rate": 4.9757575757575756e-05, + "loss": 2.5294, + "theoretical_loss": 3.3092768373502555, + "tokens_seen": 3138912256 + }, + { + "epoch": 0.9, + "learning_rate": 4.973737373737374e-05, + "loss": 2.4322, + "theoretical_loss": 3.309271573790997, + "tokens_seen": 3138977792 + }, + { + "epoch": 0.9, + "learning_rate": 4.971717171717172e-05, + "loss": 2.4483, + "theoretical_loss": 3.3092663103723994, + "tokens_seen": 3139043328 + }, + { + "epoch": 0.9, + "learning_rate": 4.96969696969697e-05, + "loss": 2.4268, + "theoretical_loss": 3.3092610470944566, + "tokens_seen": 3139108864 + }, + { + "epoch": 0.9, + "learning_rate": 4.967676767676767e-05, + "loss": 2.5251, + "theoretical_loss": 3.3092557839571626, + "tokens_seen": 3139174400 + }, + { + "epoch": 0.9, + "learning_rate": 4.9656565656565655e-05, + "loss": 2.271, + "theoretical_loss": 3.3092505209605094, + "tokens_seen": 3139239936 + }, + { + "epoch": 0.9, + "learning_rate": 4.963636363636364e-05, + "loss": 2.6913, + "theoretical_loss": 3.3092452581044913, + "tokens_seen": 3139305472 + }, + { + "epoch": 0.9, + "learning_rate": 4.9616161616161616e-05, + "loss": 2.339, + "theoretical_loss": 3.309239995389101, + "tokens_seen": 3139371008 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1720372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8587496280670166, + "objective/train/theoretical_loss": 3.309234732814333, + "objective/train/tokens_used": 1498295776, + "theoretical_loss": 3.309234732814333, + "tokens_seen": 3139436544 + }, + { + "epoch": 0.9, + "learning_rate": 4.95959595959596e-05, + "loss": 2.5399, + "theoretical_loss": 3.309234732814333, + "tokens_seen": 3139436544 + }, + { + "epoch": 0.9, + "learning_rate": 4.957575757575758e-05, + "loss": 2.4278, + "theoretical_loss": 3.3092294703801786, + "tokens_seen": 3139502080 + }, + { + "epoch": 0.9, + "learning_rate": 4.9555555555555555e-05, + "loss": 2.3393, + "theoretical_loss": 3.3092242080866323, + "tokens_seen": 3139567616 + }, + { + "epoch": 0.9, + "learning_rate": 4.953535353535354e-05, + "loss": 2.5924, + "theoretical_loss": 3.309218945933688, + "tokens_seen": 3139633152 + }, + { + "epoch": 0.9, + "learning_rate": 4.9515151515151516e-05, + "loss": 2.4275, + "theoretical_loss": 3.3092136839213375, + "tokens_seen": 3139698688 + }, + { + "epoch": 0.9, + "learning_rate": 4.949494949494949e-05, + "loss": 2.2654, + "theoretical_loss": 3.309208422049575, + "tokens_seen": 3139764224 + }, + { + "epoch": 0.9, + "learning_rate": 4.947474747474748e-05, + "loss": 2.2858, + "theoretical_loss": 3.3092031603183942, + "tokens_seen": 3139829760 + }, + { + "epoch": 0.9, + "learning_rate": 4.9454545454545454e-05, + "loss": 2.6002, + "theoretical_loss": 3.309197898727788, + "tokens_seen": 3139895296 + }, + { + "epoch": 0.9, + "learning_rate": 4.943434343434344e-05, + "loss": 2.7461, + "theoretical_loss": 3.309192637277749, + "tokens_seen": 3139960832 + }, + { + "epoch": 0.9, + "learning_rate": 4.9414141414141415e-05, + "loss": 2.4524, + "theoretical_loss": 3.3091873759682717, + "tokens_seen": 3140026368 + }, + { + "epoch": 0.9, + "learning_rate": 4.939393939393939e-05, + "loss": 2.5017, + "theoretical_loss": 3.3091821147993485, + "tokens_seen": 3140091904 + }, + { + "epoch": 0.9, + "learning_rate": 4.9373737373737376e-05, + "loss": 2.4882, + "theoretical_loss": 3.3091768537709734, + "tokens_seen": 3140157440 + }, + { + "epoch": 0.9, + "learning_rate": 4.9353535353535354e-05, + "loss": 2.6705, + "theoretical_loss": 3.3091715928831396, + "tokens_seen": 3140222976 + }, + { + "epoch": 0.9, + "learning_rate": 4.933333333333334e-05, + "loss": 2.5789, + "theoretical_loss": 3.3091663321358395, + "tokens_seen": 3140288512 + }, + { + "epoch": 0.9, + "learning_rate": 4.9313131313131315e-05, + "loss": 2.4921, + "theoretical_loss": 3.3091610715290676, + "tokens_seen": 3140354048 + }, + { + "epoch": 0.9, + "learning_rate": 4.929292929292929e-05, + "loss": 2.4943, + "theoretical_loss": 3.3091558110628165, + "tokens_seen": 3140419584 + }, + { + "epoch": 0.9, + "learning_rate": 4.9272727272727276e-05, + "loss": 2.4751, + "theoretical_loss": 3.3091505507370798, + "tokens_seen": 3140485120 + }, + { + "epoch": 0.9, + "learning_rate": 4.925252525252525e-05, + "loss": 2.5114, + "theoretical_loss": 3.309145290551851, + "tokens_seen": 3140550656 + }, + { + "epoch": 0.9, + "learning_rate": 4.923232323232323e-05, + "loss": 2.547, + "theoretical_loss": 3.3091400305071232, + "tokens_seen": 3140616192 + }, + { + "epoch": 0.9, + "learning_rate": 4.9212121212121214e-05, + "loss": 2.6884, + "theoretical_loss": 3.3091347706028893, + "tokens_seen": 3140681728 + }, + { + "epoch": 0.9, + "learning_rate": 4.919191919191919e-05, + "loss": 2.4527, + "theoretical_loss": 3.309129510839143, + "tokens_seen": 3140747264 + }, + { + "epoch": 0.9, + "learning_rate": 4.9171717171717175e-05, + "loss": 2.2926, + "theoretical_loss": 3.309124251215878, + "tokens_seen": 3140812800 + }, + { + "epoch": 0.9, + "learning_rate": 4.915151515151515e-05, + "loss": 2.518, + "theoretical_loss": 3.309118991733087, + "tokens_seen": 3140878336 + }, + { + "epoch": 0.9, + "learning_rate": 4.913131313131313e-05, + "loss": 2.5719, + "theoretical_loss": 3.3091137323907636, + "tokens_seen": 3140943872 + }, + { + "epoch": 0.9, + "learning_rate": 4.9111111111111114e-05, + "loss": 2.5717, + "theoretical_loss": 3.309108473188901, + "tokens_seen": 3141009408 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1721276, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8492307662963867, + "objective/train/theoretical_loss": 3.309103214127493, + "objective/train/tokens_used": 1499934176, + "theoretical_loss": 3.309103214127493, + "tokens_seen": 3141074944 + }, + { + "epoch": 0.9, + "learning_rate": 4.909090909090909e-05, + "loss": 2.6827, + "theoretical_loss": 3.309103214127493, + "tokens_seen": 3141074944 + }, + { + "epoch": 0.9, + "learning_rate": 4.907070707070707e-05, + "loss": 2.543, + "theoretical_loss": 3.309097955206532, + "tokens_seen": 3141140480 + }, + { + "epoch": 0.9, + "learning_rate": 4.905050505050505e-05, + "loss": 2.4554, + "theoretical_loss": 3.3090926964260117, + "tokens_seen": 3141206016 + }, + { + "epoch": 0.9, + "learning_rate": 4.903030303030303e-05, + "loss": 2.6095, + "theoretical_loss": 3.309087437785926, + "tokens_seen": 3141271552 + }, + { + "epoch": 0.9, + "learning_rate": 4.901010101010101e-05, + "loss": 2.3343, + "theoretical_loss": 3.3090821792862677, + "tokens_seen": 3141337088 + }, + { + "epoch": 0.9, + "learning_rate": 4.8989898989899e-05, + "loss": 2.5804, + "theoretical_loss": 3.30907692092703, + "tokens_seen": 3141402624 + }, + { + "epoch": 0.9, + "learning_rate": 4.896969696969697e-05, + "loss": 2.4873, + "theoretical_loss": 3.309071662708207, + "tokens_seen": 3141468160 + }, + { + "epoch": 0.9, + "learning_rate": 4.894949494949495e-05, + "loss": 2.4516, + "theoretical_loss": 3.3090664046297906, + "tokens_seen": 3141533696 + }, + { + "epoch": 0.9, + "learning_rate": 4.892929292929293e-05, + "loss": 2.4883, + "theoretical_loss": 3.3090611466917754, + "tokens_seen": 3141599232 + }, + { + "epoch": 0.9, + "learning_rate": 4.890909090909091e-05, + "loss": 2.6479, + "theoretical_loss": 3.309055888894154, + "tokens_seen": 3141664768 + }, + { + "epoch": 0.9, + "learning_rate": 4.888888888888889e-05, + "loss": 2.4643, + "theoretical_loss": 3.30905063123692, + "tokens_seen": 3141730304 + }, + { + "epoch": 0.9, + "learning_rate": 4.886868686868687e-05, + "loss": 2.5867, + "theoretical_loss": 3.309045373720067, + "tokens_seen": 3141795840 + }, + { + "epoch": 0.9, + "learning_rate": 4.884848484848485e-05, + "loss": 2.5174, + "theoretical_loss": 3.309040116343588, + "tokens_seen": 3141861376 + }, + { + "epoch": 0.9, + "learning_rate": 4.882828282828283e-05, + "loss": 2.3292, + "theoretical_loss": 3.3090348591074763, + "tokens_seen": 3141926912 + }, + { + "epoch": 0.9, + "learning_rate": 4.8808080808080805e-05, + "loss": 2.3899, + "theoretical_loss": 3.3090296020117256, + "tokens_seen": 3141992448 + }, + { + "epoch": 0.9, + "learning_rate": 4.878787878787879e-05, + "loss": 2.571, + "theoretical_loss": 3.3090243450563284, + "tokens_seen": 3142057984 + }, + { + "epoch": 0.9, + "learning_rate": 4.8767676767676767e-05, + "loss": 2.7157, + "theoretical_loss": 3.309019088241279, + "tokens_seen": 3142123520 + }, + { + "epoch": 0.9, + "learning_rate": 4.874747474747475e-05, + "loss": 2.5147, + "theoretical_loss": 3.30901383156657, + "tokens_seen": 3142189056 + }, + { + "epoch": 0.9, + "learning_rate": 4.8727272727272734e-05, + "loss": 2.6067, + "theoretical_loss": 3.309008575032195, + "tokens_seen": 3142254592 + }, + { + "epoch": 0.9, + "learning_rate": 4.8707070707070705e-05, + "loss": 2.7813, + "theoretical_loss": 3.3090033186381476, + "tokens_seen": 3142320128 + }, + { + "epoch": 0.9, + "learning_rate": 4.868686868686869e-05, + "loss": 2.5659, + "theoretical_loss": 3.30899806238442, + "tokens_seen": 3142385664 + }, + { + "epoch": 0.9, + "learning_rate": 4.8666666666666666e-05, + "loss": 2.4175, + "theoretical_loss": 3.3089928062710072, + "tokens_seen": 3142451200 + }, + { + "epoch": 0.9, + "learning_rate": 4.864646464646465e-05, + "loss": 2.6298, + "theoretical_loss": 3.3089875502979016, + "tokens_seen": 3142516736 + }, + { + "epoch": 0.9, + "learning_rate": 4.862626262626263e-05, + "loss": 2.4466, + "theoretical_loss": 3.3089822944650966, + "tokens_seen": 3142582272 + }, + { + "epoch": 0.9, + "learning_rate": 4.8606060606060604e-05, + "loss": 2.6734, + "theoretical_loss": 3.3089770387725856, + "tokens_seen": 3142647808 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1721895, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6944432258605957, + "objective/train/theoretical_loss": 3.308971783220362, + "objective/train/tokens_used": 1501572576, + "theoretical_loss": 3.308971783220362, + "tokens_seen": 3142713344 + }, + { + "epoch": 0.9, + "learning_rate": 4.858585858585859e-05, + "loss": 2.4914, + "theoretical_loss": 3.308971783220362, + "tokens_seen": 3142713344 + }, + { + "epoch": 0.9, + "learning_rate": 4.856565656565657e-05, + "loss": 2.3536, + "theoretical_loss": 3.3089665278084186, + "tokens_seen": 3142778880 + }, + { + "epoch": 0.9, + "learning_rate": 4.854545454545454e-05, + "loss": 2.6213, + "theoretical_loss": 3.308961272536749, + "tokens_seen": 3142844416 + }, + { + "epoch": 0.9, + "learning_rate": 4.852525252525253e-05, + "loss": 2.5149, + "theoretical_loss": 3.3089560174053476, + "tokens_seen": 3142909952 + }, + { + "epoch": 0.9, + "learning_rate": 4.8505050505050504e-05, + "loss": 2.5626, + "theoretical_loss": 3.308950762414206, + "tokens_seen": 3142975488 + }, + { + "epoch": 0.9, + "learning_rate": 4.848484848484849e-05, + "loss": 2.5824, + "theoretical_loss": 3.308945507563319, + "tokens_seen": 3143041024 + }, + { + "epoch": 0.9, + "learning_rate": 4.8464646464646465e-05, + "loss": 2.4857, + "theoretical_loss": 3.3089402528526786, + "tokens_seen": 3143106560 + }, + { + "epoch": 0.9, + "learning_rate": 4.844444444444444e-05, + "loss": 2.4785, + "theoretical_loss": 3.3089349982822793, + "tokens_seen": 3143172096 + }, + { + "epoch": 0.9, + "learning_rate": 4.8424242424242426e-05, + "loss": 2.6559, + "theoretical_loss": 3.3089297438521137, + "tokens_seen": 3143237632 + }, + { + "epoch": 0.9, + "learning_rate": 4.84040404040404e-05, + "loss": 2.3955, + "theoretical_loss": 3.3089244895621754, + "tokens_seen": 3143303168 + }, + { + "epoch": 0.9, + "learning_rate": 4.838383838383839e-05, + "loss": 2.681, + "theoretical_loss": 3.308919235412458, + "tokens_seen": 3143368704 + }, + { + "epoch": 0.9, + "learning_rate": 4.8363636363636364e-05, + "loss": 2.528, + "theoretical_loss": 3.308913981402954, + "tokens_seen": 3143434240 + }, + { + "epoch": 0.9, + "learning_rate": 4.834343434343434e-05, + "loss": 2.5491, + "theoretical_loss": 3.3089087275336575, + "tokens_seen": 3143499776 + }, + { + "epoch": 0.9, + "learning_rate": 4.8323232323232326e-05, + "loss": 2.4465, + "theoretical_loss": 3.3089034738045617, + "tokens_seen": 3143565312 + }, + { + "epoch": 0.9, + "learning_rate": 4.830303030303031e-05, + "loss": 2.3991, + "theoretical_loss": 3.30889822021566, + "tokens_seen": 3143630848 + }, + { + "epoch": 0.9, + "learning_rate": 4.828282828282828e-05, + "loss": 2.4925, + "theoretical_loss": 3.3088929667669458, + "tokens_seen": 3143696384 + }, + { + "epoch": 0.9, + "learning_rate": 4.8262626262626264e-05, + "loss": 2.5359, + "theoretical_loss": 3.308887713458412, + "tokens_seen": 3143761920 + }, + { + "epoch": 0.9, + "learning_rate": 4.824242424242424e-05, + "loss": 2.5774, + "theoretical_loss": 3.308882460290052, + "tokens_seen": 3143827456 + }, + { + "epoch": 0.9, + "learning_rate": 4.8222222222222225e-05, + "loss": 2.6088, + "theoretical_loss": 3.308877207261859, + "tokens_seen": 3143892992 + }, + { + "epoch": 0.9, + "learning_rate": 4.82020202020202e-05, + "loss": 2.4569, + "theoretical_loss": 3.308871954373827, + "tokens_seen": 3143958528 + }, + { + "epoch": 0.9, + "learning_rate": 4.818181818181818e-05, + "loss": 2.5163, + "theoretical_loss": 3.3088667016259494, + "tokens_seen": 3144024064 + }, + { + "epoch": 0.9, + "learning_rate": 4.8161616161616163e-05, + "loss": 2.572, + "theoretical_loss": 3.3088614490182184, + "tokens_seen": 3144089600 + }, + { + "epoch": 0.9, + "learning_rate": 4.814141414141415e-05, + "loss": 2.507, + "theoretical_loss": 3.3088561965506287, + "tokens_seen": 3144155136 + }, + { + "epoch": 0.9, + "learning_rate": 4.8121212121212125e-05, + "loss": 2.5452, + "theoretical_loss": 3.308850944223172, + "tokens_seen": 3144220672 + }, + { + "epoch": 0.9, + "learning_rate": 4.81010101010101e-05, + "loss": 2.3846, + "theoretical_loss": 3.3088456920358436, + "tokens_seen": 3144286208 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1722966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.410130023956299, + "objective/train/theoretical_loss": 3.3088404399886358, + "objective/train/tokens_used": 1503210976, + "theoretical_loss": 3.3088404399886358, + "tokens_seen": 3144351744 + }, + { + "epoch": 0.91, + "learning_rate": 4.808080808080808e-05, + "loss": 2.4863, + "theoretical_loss": 3.3088404399886358, + "tokens_seen": 3144351744 + }, + { + "epoch": 0.91, + "learning_rate": 4.806060606060606e-05, + "loss": 2.4622, + "theoretical_loss": 3.3088351880815416, + "tokens_seen": 3144417280 + }, + { + "epoch": 0.91, + "learning_rate": 4.804040404040405e-05, + "loss": 2.3561, + "theoretical_loss": 3.308829936314555, + "tokens_seen": 3144482816 + }, + { + "epoch": 0.91, + "learning_rate": 4.802020202020202e-05, + "loss": 2.5604, + "theoretical_loss": 3.308824684687669, + "tokens_seen": 3144548352 + }, + { + "epoch": 0.91, + "learning_rate": 4.8e-05, + "loss": 2.348, + "theoretical_loss": 3.3088194332008776, + "tokens_seen": 3144613888 + }, + { + "epoch": 0.91, + "learning_rate": 4.797979797979798e-05, + "loss": 2.6156, + "theoretical_loss": 3.308814181854173, + "tokens_seen": 3144679424 + }, + { + "epoch": 0.91, + "learning_rate": 4.795959595959596e-05, + "loss": 2.3846, + "theoretical_loss": 3.308808930647549, + "tokens_seen": 3144744960 + }, + { + "epoch": 0.91, + "learning_rate": 4.793939393939394e-05, + "loss": 2.5222, + "theoretical_loss": 3.3088036795809996, + "tokens_seen": 3144810496 + }, + { + "epoch": 0.91, + "learning_rate": 4.791919191919192e-05, + "loss": 2.5168, + "theoretical_loss": 3.3087984286545176, + "tokens_seen": 3144876032 + }, + { + "epoch": 0.91, + "learning_rate": 4.78989898989899e-05, + "loss": 2.4877, + "theoretical_loss": 3.308793177868096, + "tokens_seen": 3144941568 + }, + { + "epoch": 0.91, + "learning_rate": 4.7878787878787885e-05, + "loss": 2.6013, + "theoretical_loss": 3.308787927221729, + "tokens_seen": 3145007104 + }, + { + "epoch": 0.91, + "learning_rate": 4.7858585858585855e-05, + "loss": 2.5577, + "theoretical_loss": 3.308782676715409, + "tokens_seen": 3145072640 + }, + { + "epoch": 0.91, + "learning_rate": 4.783838383838384e-05, + "loss": 2.7173, + "theoretical_loss": 3.30877742634913, + "tokens_seen": 3145138176 + }, + { + "epoch": 0.91, + "learning_rate": 4.7818181818181816e-05, + "loss": 2.52, + "theoretical_loss": 3.308772176122885, + "tokens_seen": 3145203712 + }, + { + "epoch": 0.91, + "learning_rate": 4.77979797979798e-05, + "loss": 2.4798, + "theoretical_loss": 3.3087669260366677, + "tokens_seen": 3145269248 + }, + { + "epoch": 0.91, + "learning_rate": 4.7777777777777784e-05, + "loss": 2.2254, + "theoretical_loss": 3.3087616760904712, + "tokens_seen": 3145334784 + }, + { + "epoch": 0.91, + "learning_rate": 4.7757575757575755e-05, + "loss": 2.5628, + "theoretical_loss": 3.3087564262842886, + "tokens_seen": 3145400320 + }, + { + "epoch": 0.91, + "learning_rate": 4.773737373737374e-05, + "loss": 2.4911, + "theoretical_loss": 3.308751176618114, + "tokens_seen": 3145465856 + }, + { + "epoch": 0.91, + "learning_rate": 4.771717171717172e-05, + "loss": 2.5945, + "theoretical_loss": 3.30874592709194, + "tokens_seen": 3145531392 + }, + { + "epoch": 0.91, + "learning_rate": 4.76969696969697e-05, + "loss": 2.4557, + "theoretical_loss": 3.3087406777057606, + "tokens_seen": 3145596928 + }, + { + "epoch": 0.91, + "learning_rate": 4.767676767676768e-05, + "loss": 2.4159, + "theoretical_loss": 3.3087354284595687, + "tokens_seen": 3145662464 + }, + { + "epoch": 0.91, + "learning_rate": 4.7656565656565654e-05, + "loss": 2.4518, + "theoretical_loss": 3.3087301793533577, + "tokens_seen": 3145728000 + }, + { + "epoch": 0.91, + "learning_rate": 4.763636363636364e-05, + "loss": 2.3517, + "theoretical_loss": 3.308724930387121, + "tokens_seen": 3145793536 + }, + { + "epoch": 0.91, + "learning_rate": 4.761616161616162e-05, + "loss": 2.5307, + "theoretical_loss": 3.3087196815608517, + "tokens_seen": 3145859072 + }, + { + "epoch": 0.91, + "learning_rate": 4.759595959595959e-05, + "loss": 2.4236, + "theoretical_loss": 3.3087144328745435, + "tokens_seen": 3145924608 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1724144, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6357920169830322, + "objective/train/theoretical_loss": 3.3087091843281895, + "objective/train/tokens_used": 1504849376, + "theoretical_loss": 3.3087091843281895, + "tokens_seen": 3145990144 + }, + { + "epoch": 0.91, + "learning_rate": 4.7575757575757576e-05, + "loss": 2.5183, + "theoretical_loss": 3.3087091843281895, + "tokens_seen": 3145990144 + }, + { + "epoch": 0.91, + "learning_rate": 4.7555555555555553e-05, + "loss": 2.5566, + "theoretical_loss": 3.3087039359217836, + "tokens_seen": 3146055680 + }, + { + "epoch": 0.91, + "learning_rate": 4.753535353535354e-05, + "loss": 2.6589, + "theoretical_loss": 3.3086986876553186, + "tokens_seen": 3146121216 + }, + { + "epoch": 0.91, + "learning_rate": 4.751515151515152e-05, + "loss": 2.6606, + "theoretical_loss": 3.308693439528788, + "tokens_seen": 3146186752 + }, + { + "epoch": 0.91, + "learning_rate": 4.749494949494949e-05, + "loss": 2.2115, + "theoretical_loss": 3.3086881915421853, + "tokens_seen": 3146252288 + }, + { + "epoch": 0.91, + "learning_rate": 4.7474747474747476e-05, + "loss": 2.6273, + "theoretical_loss": 3.308682943695503, + "tokens_seen": 3146317824 + }, + { + "epoch": 0.91, + "learning_rate": 4.745454545454546e-05, + "loss": 2.4925, + "theoretical_loss": 3.3086776959887363, + "tokens_seen": 3146383360 + }, + { + "epoch": 0.91, + "learning_rate": 4.743434343434344e-05, + "loss": 2.6271, + "theoretical_loss": 3.308672448421877, + "tokens_seen": 3146448896 + }, + { + "epoch": 0.91, + "learning_rate": 4.7414141414141414e-05, + "loss": 2.4854, + "theoretical_loss": 3.3086672009949187, + "tokens_seen": 3146514432 + }, + { + "epoch": 0.91, + "learning_rate": 4.739393939393939e-05, + "loss": 2.5437, + "theoretical_loss": 3.3086619537078548, + "tokens_seen": 3146579968 + }, + { + "epoch": 0.91, + "learning_rate": 4.7373737373737375e-05, + "loss": 2.6661, + "theoretical_loss": 3.3086567065606793, + "tokens_seen": 3146645504 + }, + { + "epoch": 0.91, + "learning_rate": 4.735353535353536e-05, + "loss": 2.5325, + "theoretical_loss": 3.308651459553385, + "tokens_seen": 3146711040 + }, + { + "epoch": 0.91, + "learning_rate": 4.733333333333333e-05, + "loss": 2.4648, + "theoretical_loss": 3.3086462126859653, + "tokens_seen": 3146776576 + }, + { + "epoch": 0.91, + "learning_rate": 4.7313131313131314e-05, + "loss": 2.4399, + "theoretical_loss": 3.3086409659584133, + "tokens_seen": 3146842112 + }, + { + "epoch": 0.91, + "learning_rate": 4.72929292929293e-05, + "loss": 2.414, + "theoretical_loss": 3.3086357193707228, + "tokens_seen": 3146907648 + }, + { + "epoch": 0.91, + "learning_rate": 4.7272727272727275e-05, + "loss": 2.6227, + "theoretical_loss": 3.3086304729228866, + "tokens_seen": 3146973184 + }, + { + "epoch": 0.91, + "learning_rate": 4.725252525252525e-05, + "loss": 2.5344, + "theoretical_loss": 3.308625226614899, + "tokens_seen": 3147038720 + }, + { + "epoch": 0.91, + "learning_rate": 4.723232323232323e-05, + "loss": 2.3157, + "theoretical_loss": 3.3086199804467524, + "tokens_seen": 3147104256 + }, + { + "epoch": 0.91, + "learning_rate": 4.721212121212121e-05, + "loss": 2.5891, + "theoretical_loss": 3.308614734418441, + "tokens_seen": 3147169792 + }, + { + "epoch": 0.91, + "learning_rate": 4.71919191919192e-05, + "loss": 2.6042, + "theoretical_loss": 3.3086094885299575, + "tokens_seen": 3147235328 + }, + { + "epoch": 0.91, + "learning_rate": 4.7171717171717174e-05, + "loss": 2.4514, + "theoretical_loss": 3.3086042427812954, + "tokens_seen": 3147300864 + }, + { + "epoch": 0.91, + "learning_rate": 4.715151515151515e-05, + "loss": 2.3182, + "theoretical_loss": 3.3085989971724485, + "tokens_seen": 3147366400 + }, + { + "epoch": 0.91, + "learning_rate": 4.713131313131313e-05, + "loss": 2.2918, + "theoretical_loss": 3.3085937517034094, + "tokens_seen": 3147431936 + }, + { + "epoch": 0.91, + "learning_rate": 4.711111111111111e-05, + "loss": 2.5044, + "theoretical_loss": 3.3085885063741722, + "tokens_seen": 3147497472 + }, + { + "epoch": 0.91, + "learning_rate": 4.7090909090909097e-05, + "loss": 2.3367, + "theoretical_loss": 3.30858326118473, + "tokens_seen": 3147563008 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1724634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3333449363708496, + "objective/train/theoretical_loss": 3.3085780161350757, + "objective/train/tokens_used": 1506487776, + "theoretical_loss": 3.3085780161350757, + "tokens_seen": 3147628544 + }, + { + "epoch": 0.91, + "learning_rate": 4.707070707070707e-05, + "loss": 2.4599, + "theoretical_loss": 3.3085780161350757, + "tokens_seen": 3147628544 + }, + { + "epoch": 0.91, + "learning_rate": 4.705050505050505e-05, + "loss": 2.5799, + "theoretical_loss": 3.308572771225203, + "tokens_seen": 3147694080 + }, + { + "epoch": 0.91, + "learning_rate": 4.7030303030303035e-05, + "loss": 2.5505, + "theoretical_loss": 3.308567526455106, + "tokens_seen": 3147759616 + }, + { + "epoch": 0.91, + "learning_rate": 4.701010101010101e-05, + "loss": 2.5101, + "theoretical_loss": 3.308562281824777, + "tokens_seen": 3147825152 + }, + { + "epoch": 0.91, + "learning_rate": 4.698989898989899e-05, + "loss": 2.5005, + "theoretical_loss": 3.30855703733421, + "tokens_seen": 3147890688 + }, + { + "epoch": 0.91, + "learning_rate": 4.6969696969696966e-05, + "loss": 2.3201, + "theoretical_loss": 3.3085517929833976, + "tokens_seen": 3147956224 + }, + { + "epoch": 0.91, + "learning_rate": 4.694949494949495e-05, + "loss": 2.6669, + "theoretical_loss": 3.3085465487723345, + "tokens_seen": 3148021760 + }, + { + "epoch": 0.91, + "learning_rate": 4.6929292929292934e-05, + "loss": 2.4014, + "theoretical_loss": 3.308541304701013, + "tokens_seen": 3148087296 + }, + { + "epoch": 0.91, + "learning_rate": 4.690909090909091e-05, + "loss": 2.5685, + "theoretical_loss": 3.308536060769426, + "tokens_seen": 3148152832 + }, + { + "epoch": 0.91, + "learning_rate": 4.688888888888889e-05, + "loss": 2.7379, + "theoretical_loss": 3.3085308169775685, + "tokens_seen": 3148218368 + }, + { + "epoch": 0.91, + "learning_rate": 4.686868686868687e-05, + "loss": 2.42, + "theoretical_loss": 3.308525573325433, + "tokens_seen": 3148283904 + }, + { + "epoch": 0.91, + "learning_rate": 4.684848484848485e-05, + "loss": 2.435, + "theoretical_loss": 3.308520329813012, + "tokens_seen": 3148349440 + }, + { + "epoch": 0.91, + "learning_rate": 4.6828282828282834e-05, + "loss": 2.5593, + "theoretical_loss": 3.3085150864403, + "tokens_seen": 3148414976 + }, + { + "epoch": 0.91, + "learning_rate": 4.6808080808080804e-05, + "loss": 2.4342, + "theoretical_loss": 3.3085098432072906, + "tokens_seen": 3148480512 + }, + { + "epoch": 0.91, + "learning_rate": 4.678787878787879e-05, + "loss": 2.6365, + "theoretical_loss": 3.3085046001139764, + "tokens_seen": 3148546048 + }, + { + "epoch": 0.91, + "learning_rate": 4.676767676767677e-05, + "loss": 2.625, + "theoretical_loss": 3.308499357160351, + "tokens_seen": 3148611584 + }, + { + "epoch": 0.91, + "learning_rate": 4.674747474747475e-05, + "loss": 2.2848, + "theoretical_loss": 3.3084941143464075, + "tokens_seen": 3148677120 + }, + { + "epoch": 0.91, + "learning_rate": 4.6727272727272727e-05, + "loss": 2.6712, + "theoretical_loss": 3.3084888716721403, + "tokens_seen": 3148742656 + }, + { + "epoch": 0.91, + "learning_rate": 4.6707070707070704e-05, + "loss": 2.5308, + "theoretical_loss": 3.308483629137541, + "tokens_seen": 3148808192 + }, + { + "epoch": 0.91, + "learning_rate": 4.668686868686869e-05, + "loss": 2.5516, + "theoretical_loss": 3.308478386742605, + "tokens_seen": 3148873728 + }, + { + "epoch": 0.91, + "learning_rate": 4.666666666666667e-05, + "loss": 2.5738, + "theoretical_loss": 3.308473144487324, + "tokens_seen": 3148939264 + }, + { + "epoch": 0.91, + "learning_rate": 4.664646464646464e-05, + "loss": 2.1946, + "theoretical_loss": 3.3084679023716923, + "tokens_seen": 3149004800 + }, + { + "epoch": 0.91, + "learning_rate": 4.6626262626262626e-05, + "loss": 2.6653, + "theoretical_loss": 3.308462660395703, + "tokens_seen": 3149070336 + }, + { + "epoch": 0.91, + "learning_rate": 4.660606060606061e-05, + "loss": 2.5705, + "theoretical_loss": 3.3084574185593496, + "tokens_seen": 3149135872 + }, + { + "epoch": 0.91, + "learning_rate": 4.658585858585859e-05, + "loss": 2.5897, + "theoretical_loss": 3.3084521768626254, + "tokens_seen": 3149201408 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1725801, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.423551559448242, + "objective/train/theoretical_loss": 3.3084469353055237, + "objective/train/tokens_used": 1508126176, + "theoretical_loss": 3.3084469353055237, + "tokens_seen": 3149266944 + }, + { + "epoch": 0.91, + "learning_rate": 4.656565656565657e-05, + "loss": 2.5694, + "theoretical_loss": 3.3084469353055237, + "tokens_seen": 3149266944 + }, + { + "epoch": 0.91, + "learning_rate": 4.654545454545454e-05, + "loss": 2.4356, + "theoretical_loss": 3.308441693888038, + "tokens_seen": 3149332480 + }, + { + "epoch": 0.91, + "learning_rate": 4.6525252525252525e-05, + "loss": 2.5484, + "theoretical_loss": 3.3084364526101613, + "tokens_seen": 3149398016 + }, + { + "epoch": 0.91, + "learning_rate": 4.650505050505051e-05, + "loss": 2.5537, + "theoretical_loss": 3.3084312114718877, + "tokens_seen": 3149463552 + }, + { + "epoch": 0.91, + "learning_rate": 4.6484848484848487e-05, + "loss": 2.7322, + "theoretical_loss": 3.30842597047321, + "tokens_seen": 3149529088 + }, + { + "epoch": 0.91, + "learning_rate": 4.6464646464646464e-05, + "loss": 2.6085, + "theoretical_loss": 3.3084207296141215, + "tokens_seen": 3149594624 + }, + { + "epoch": 0.91, + "learning_rate": 4.644444444444444e-05, + "loss": 2.3938, + "theoretical_loss": 3.3084154888946156, + "tokens_seen": 3149660160 + }, + { + "epoch": 0.91, + "learning_rate": 4.6424242424242425e-05, + "loss": 2.4876, + "theoretical_loss": 3.3084102483146864, + "tokens_seen": 3149725696 + }, + { + "epoch": 0.91, + "learning_rate": 4.640404040404041e-05, + "loss": 2.4827, + "theoretical_loss": 3.3084050078743266, + "tokens_seen": 3149791232 + }, + { + "epoch": 0.91, + "learning_rate": 4.638383838383838e-05, + "loss": 2.6805, + "theoretical_loss": 3.3083997675735297, + "tokens_seen": 3149856768 + }, + { + "epoch": 0.91, + "learning_rate": 4.636363636363636e-05, + "loss": 2.7177, + "theoretical_loss": 3.3083945274122892, + "tokens_seen": 3149922304 + }, + { + "epoch": 0.91, + "learning_rate": 4.634343434343435e-05, + "loss": 2.4718, + "theoretical_loss": 3.3083892873905985, + "tokens_seen": 3149987840 + }, + { + "epoch": 0.91, + "learning_rate": 4.6323232323232324e-05, + "loss": 2.6717, + "theoretical_loss": 3.3083840475084507, + "tokens_seen": 3150053376 + }, + { + "epoch": 0.91, + "learning_rate": 4.630303030303031e-05, + "loss": 2.5897, + "theoretical_loss": 3.30837880776584, + "tokens_seen": 3150118912 + }, + { + "epoch": 0.91, + "learning_rate": 4.628282828282828e-05, + "loss": 2.4068, + "theoretical_loss": 3.308373568162758, + "tokens_seen": 3150184448 + }, + { + "epoch": 0.91, + "learning_rate": 4.626262626262626e-05, + "loss": 2.682, + "theoretical_loss": 3.3083683286992, + "tokens_seen": 3150249984 + }, + { + "epoch": 0.91, + "learning_rate": 4.624242424242425e-05, + "loss": 2.4561, + "theoretical_loss": 3.3083630893751588, + "tokens_seen": 3150315520 + }, + { + "epoch": 0.91, + "learning_rate": 4.6222222222222224e-05, + "loss": 2.4564, + "theoretical_loss": 3.3083578501906272, + "tokens_seen": 3150381056 + }, + { + "epoch": 0.91, + "learning_rate": 4.62020202020202e-05, + "loss": 2.5892, + "theoretical_loss": 3.3083526111455988, + "tokens_seen": 3150446592 + }, + { + "epoch": 0.91, + "learning_rate": 4.6181818181818185e-05, + "loss": 2.5721, + "theoretical_loss": 3.3083473722400676, + "tokens_seen": 3150512128 + }, + { + "epoch": 0.91, + "learning_rate": 4.616161616161616e-05, + "loss": 2.4268, + "theoretical_loss": 3.308342133474026, + "tokens_seen": 3150577664 + }, + { + "epoch": 0.91, + "learning_rate": 4.6141414141414146e-05, + "loss": 2.6015, + "theoretical_loss": 3.3083368948474683, + "tokens_seen": 3150643200 + }, + { + "epoch": 0.91, + "learning_rate": 4.6121212121212117e-05, + "loss": 2.4212, + "theoretical_loss": 3.3083316563603877, + "tokens_seen": 3150708736 + }, + { + "epoch": 0.91, + "learning_rate": 4.61010101010101e-05, + "loss": 2.5021, + "theoretical_loss": 3.308326418012777, + "tokens_seen": 3150774272 + }, + { + "epoch": 0.91, + "learning_rate": 4.6080808080808085e-05, + "loss": 2.5819, + "theoretical_loss": 3.30832117980463, + "tokens_seen": 3150839808 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1726538, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5869221687316895, + "objective/train/theoretical_loss": 3.3083159417359402, + "objective/train/tokens_used": 1509764576, + "theoretical_loss": 3.3083159417359402, + "tokens_seen": 3150905344 + }, + { + "epoch": 0.91, + "learning_rate": 4.606060606060606e-05, + "loss": 2.4537, + "theoretical_loss": 3.3083159417359402, + "tokens_seen": 3150905344 + }, + { + "epoch": 0.91, + "learning_rate": 4.604040404040404e-05, + "loss": 2.4596, + "theoretical_loss": 3.3083107038067014, + "tokens_seen": 3150970880 + }, + { + "epoch": 0.91, + "learning_rate": 4.6020202020202016e-05, + "loss": 2.5351, + "theoretical_loss": 3.308305466016906, + "tokens_seen": 3151036416 + }, + { + "epoch": 0.91, + "learning_rate": 4.6e-05, + "loss": 2.5452, + "theoretical_loss": 3.3083002283665475, + "tokens_seen": 3151101952 + }, + { + "epoch": 0.91, + "learning_rate": 4.5979797979797984e-05, + "loss": 2.6433, + "theoretical_loss": 3.30829499085562, + "tokens_seen": 3151167488 + }, + { + "epoch": 0.91, + "learning_rate": 4.595959595959596e-05, + "loss": 2.3762, + "theoretical_loss": 3.3082897534841162, + "tokens_seen": 3151233024 + }, + { + "epoch": 0.91, + "learning_rate": 4.593939393939394e-05, + "loss": 2.4158, + "theoretical_loss": 3.30828451625203, + "tokens_seen": 3151298560 + }, + { + "epoch": 0.91, + "learning_rate": 4.591919191919192e-05, + "loss": 2.4434, + "theoretical_loss": 3.308279279159355, + "tokens_seen": 3151364096 + }, + { + "epoch": 0.91, + "learning_rate": 4.58989898989899e-05, + "loss": 2.3821, + "theoretical_loss": 3.3082740422060835, + "tokens_seen": 3151429632 + }, + { + "epoch": 0.91, + "learning_rate": 4.5878787878787883e-05, + "loss": 2.5416, + "theoretical_loss": 3.30826880539221, + "tokens_seen": 3151495168 + }, + { + "epoch": 0.91, + "learning_rate": 4.5858585858585854e-05, + "loss": 2.4174, + "theoretical_loss": 3.3082635687177273, + "tokens_seen": 3151560704 + }, + { + "epoch": 0.91, + "learning_rate": 4.583838383838384e-05, + "loss": 2.2646, + "theoretical_loss": 3.308258332182629, + "tokens_seen": 3151626240 + }, + { + "epoch": 0.91, + "learning_rate": 4.581818181818182e-05, + "loss": 2.4322, + "theoretical_loss": 3.3082530957869083, + "tokens_seen": 3151691776 + }, + { + "epoch": 0.91, + "learning_rate": 4.57979797979798e-05, + "loss": 2.6364, + "theoretical_loss": 3.308247859530559, + "tokens_seen": 3151757312 + }, + { + "epoch": 0.91, + "learning_rate": 4.5777777777777776e-05, + "loss": 2.5817, + "theoretical_loss": 3.308242623413574, + "tokens_seen": 3151822848 + }, + { + "epoch": 0.91, + "learning_rate": 4.575757575757576e-05, + "loss": 2.5294, + "theoretical_loss": 3.308237387435947, + "tokens_seen": 3151888384 + }, + { + "epoch": 0.91, + "learning_rate": 4.573737373737374e-05, + "loss": 2.5568, + "theoretical_loss": 3.308232151597671, + "tokens_seen": 3151953920 + }, + { + "epoch": 0.91, + "learning_rate": 4.571717171717172e-05, + "loss": 2.5941, + "theoretical_loss": 3.3082269158987403, + "tokens_seen": 3152019456 + }, + { + "epoch": 0.91, + "learning_rate": 4.56969696969697e-05, + "loss": 2.6122, + "theoretical_loss": 3.3082216803391473, + "tokens_seen": 3152084992 + }, + { + "epoch": 0.91, + "learning_rate": 4.5676767676767676e-05, + "loss": 2.3577, + "theoretical_loss": 3.308216444918886, + "tokens_seen": 3152150528 + }, + { + "epoch": 0.91, + "learning_rate": 4.565656565656566e-05, + "loss": 2.4073, + "theoretical_loss": 3.3082112096379497, + "tokens_seen": 3152216064 + }, + { + "epoch": 0.91, + "learning_rate": 4.563636363636364e-05, + "loss": 2.3733, + "theoretical_loss": 3.3082059744963317, + "tokens_seen": 3152281600 + }, + { + "epoch": 0.91, + "learning_rate": 4.561616161616162e-05, + "loss": 2.4627, + "theoretical_loss": 3.308200739494025, + "tokens_seen": 3152347136 + }, + { + "epoch": 0.91, + "learning_rate": 4.559595959595959e-05, + "loss": 2.6435, + "theoretical_loss": 3.3081955046310236, + "tokens_seen": 3152412672 + }, + { + "epoch": 0.91, + "learning_rate": 4.5575757575757575e-05, + "loss": 2.5496, + "theoretical_loss": 3.3081902699073207, + "tokens_seen": 3152478208 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1727599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.214886426925659, + "objective/train/theoretical_loss": 3.3081850353229094, + "objective/train/tokens_used": 1511402976, + "theoretical_loss": 3.3081850353229094, + "tokens_seen": 3152543744 + }, + { + "epoch": 0.91, + "learning_rate": 4.555555555555556e-05, + "loss": 2.4493, + "theoretical_loss": 3.3081850353229094, + "tokens_seen": 3152543744 + }, + { + "epoch": 0.91, + "learning_rate": 4.5535353535353536e-05, + "loss": 2.728, + "theoretical_loss": 3.3081798008777836, + "tokens_seen": 3152609280 + }, + { + "epoch": 0.91, + "learning_rate": 4.5515151515151513e-05, + "loss": 2.4059, + "theoretical_loss": 3.3081745665719366, + "tokens_seen": 3152674816 + }, + { + "epoch": 0.91, + "learning_rate": 4.54949494949495e-05, + "loss": 2.6717, + "theoretical_loss": 3.3081693324053614, + "tokens_seen": 3152740352 + }, + { + "epoch": 0.91, + "learning_rate": 4.5474747474747475e-05, + "loss": 2.4909, + "theoretical_loss": 3.308164098378052, + "tokens_seen": 3152805888 + }, + { + "epoch": 0.91, + "learning_rate": 4.545454545454546e-05, + "loss": 2.6006, + "theoretical_loss": 3.308158864490001, + "tokens_seen": 3152871424 + }, + { + "epoch": 0.91, + "learning_rate": 4.543434343434343e-05, + "loss": 2.2881, + "theoretical_loss": 3.308153630741203, + "tokens_seen": 3152936960 + }, + { + "epoch": 0.91, + "learning_rate": 4.541414141414141e-05, + "loss": 2.5644, + "theoretical_loss": 3.3081483971316503, + "tokens_seen": 3153002496 + }, + { + "epoch": 0.91, + "learning_rate": 4.53939393939394e-05, + "loss": 2.4017, + "theoretical_loss": 3.3081431636613363, + "tokens_seen": 3153068032 + }, + { + "epoch": 0.91, + "learning_rate": 4.5373737373737374e-05, + "loss": 2.606, + "theoretical_loss": 3.3081379303302554, + "tokens_seen": 3153133568 + }, + { + "epoch": 0.91, + "learning_rate": 4.535353535353536e-05, + "loss": 2.4798, + "theoretical_loss": 3.3081326971384, + "tokens_seen": 3153199104 + }, + { + "epoch": 0.91, + "learning_rate": 4.5333333333333335e-05, + "loss": 2.3501, + "theoretical_loss": 3.3081274640857643, + "tokens_seen": 3153264640 + }, + { + "epoch": 0.91, + "learning_rate": 4.531313131313131e-05, + "loss": 2.7317, + "theoretical_loss": 3.308122231172341, + "tokens_seen": 3153330176 + }, + { + "epoch": 0.91, + "learning_rate": 4.5292929292929296e-05, + "loss": 2.5632, + "theoretical_loss": 3.3081169983981233, + "tokens_seen": 3153395712 + }, + { + "epoch": 0.91, + "learning_rate": 4.5272727272727274e-05, + "loss": 2.2928, + "theoretical_loss": 3.3081117657631056, + "tokens_seen": 3153461248 + }, + { + "epoch": 0.91, + "learning_rate": 4.525252525252525e-05, + "loss": 2.4417, + "theoretical_loss": 3.3081065332672805, + "tokens_seen": 3153526784 + }, + { + "epoch": 0.91, + "learning_rate": 4.5232323232323235e-05, + "loss": 2.6083, + "theoretical_loss": 3.3081013009106424, + "tokens_seen": 3153592320 + }, + { + "epoch": 0.91, + "learning_rate": 4.521212121212121e-05, + "loss": 2.4616, + "theoretical_loss": 3.3080960686931835, + "tokens_seen": 3153657856 + }, + { + "epoch": 0.91, + "learning_rate": 4.5191919191919196e-05, + "loss": 2.2754, + "theoretical_loss": 3.3080908366148973, + "tokens_seen": 3153723392 + }, + { + "epoch": 0.91, + "learning_rate": 4.5171717171717166e-05, + "loss": 2.5295, + "theoretical_loss": 3.3080856046757785, + "tokens_seen": 3153788928 + }, + { + "epoch": 0.91, + "learning_rate": 4.515151515151515e-05, + "loss": 2.4313, + "theoretical_loss": 3.308080372875819, + "tokens_seen": 3153854464 + }, + { + "epoch": 0.91, + "learning_rate": 4.5131313131313134e-05, + "loss": 2.5287, + "theoretical_loss": 3.308075141215013, + "tokens_seen": 3153920000 + }, + { + "epoch": 0.91, + "learning_rate": 4.511111111111111e-05, + "loss": 2.5478, + "theoretical_loss": 3.3080699096933537, + "tokens_seen": 3153985536 + }, + { + "epoch": 0.91, + "learning_rate": 4.5090909090909095e-05, + "loss": 2.7657, + "theoretical_loss": 3.3080646783108345, + "tokens_seen": 3154051072 + }, + { + "epoch": 0.91, + "learning_rate": 4.507070707070707e-05, + "loss": 2.3992, + "theoretical_loss": 3.308059447067449, + "tokens_seen": 3154116608 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1728703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3344175815582275, + "objective/train/theoretical_loss": 3.3080542159631907, + "objective/train/tokens_used": 1513041376, + "theoretical_loss": 3.3080542159631907, + "tokens_seen": 3154182144 + }, + { + "epoch": 0.91, + "learning_rate": 4.505050505050505e-05, + "loss": 2.3596, + "theoretical_loss": 3.3080542159631907, + "tokens_seen": 3154182144 + }, + { + "epoch": 0.91, + "learning_rate": 4.5030303030303034e-05, + "loss": 2.3545, + "theoretical_loss": 3.3080489849980523, + "tokens_seen": 3154247680 + }, + { + "epoch": 0.91, + "learning_rate": 4.501010101010101e-05, + "loss": 2.4463, + "theoretical_loss": 3.308043754172028, + "tokens_seen": 3154313216 + }, + { + "epoch": 0.91, + "learning_rate": 4.498989898989899e-05, + "loss": 2.463, + "theoretical_loss": 3.3080385234851106, + "tokens_seen": 3154378752 + }, + { + "epoch": 0.91, + "learning_rate": 4.496969696969697e-05, + "loss": 2.5546, + "theoretical_loss": 3.308033292937294, + "tokens_seen": 3154444288 + }, + { + "epoch": 0.91, + "learning_rate": 4.494949494949495e-05, + "loss": 2.6124, + "theoretical_loss": 3.3080280625285714, + "tokens_seen": 3154509824 + }, + { + "epoch": 0.91, + "learning_rate": 4.492929292929293e-05, + "loss": 2.3598, + "theoretical_loss": 3.3080228322589362, + "tokens_seen": 3154575360 + }, + { + "epoch": 0.91, + "learning_rate": 4.490909090909091e-05, + "loss": 2.2859, + "theoretical_loss": 3.308017602128382, + "tokens_seen": 3154640896 + }, + { + "epoch": 0.91, + "learning_rate": 4.488888888888889e-05, + "loss": 2.3961, + "theoretical_loss": 3.308012372136902, + "tokens_seen": 3154706432 + }, + { + "epoch": 0.91, + "learning_rate": 4.486868686868687e-05, + "loss": 2.6163, + "theoretical_loss": 3.308007142284489, + "tokens_seen": 3154771968 + }, + { + "epoch": 0.91, + "learning_rate": 4.484848484848485e-05, + "loss": 2.4108, + "theoretical_loss": 3.3080019125711377, + "tokens_seen": 3154837504 + }, + { + "epoch": 0.91, + "learning_rate": 4.4828282828282826e-05, + "loss": 2.3477, + "theoretical_loss": 3.307996682996841, + "tokens_seen": 3154903040 + }, + { + "epoch": 0.91, + "learning_rate": 4.480808080808081e-05, + "loss": 2.4156, + "theoretical_loss": 3.307991453561592, + "tokens_seen": 3154968576 + }, + { + "epoch": 0.91, + "learning_rate": 4.478787878787879e-05, + "loss": 2.4959, + "theoretical_loss": 3.3079862242653846, + "tokens_seen": 3155034112 + }, + { + "epoch": 0.91, + "learning_rate": 4.476767676767677e-05, + "loss": 2.44, + "theoretical_loss": 3.3079809951082115, + "tokens_seen": 3155099648 + }, + { + "epoch": 0.91, + "learning_rate": 4.474747474747475e-05, + "loss": 2.5769, + "theoretical_loss": 3.307975766090067, + "tokens_seen": 3155165184 + }, + { + "epoch": 0.91, + "learning_rate": 4.4727272727272725e-05, + "loss": 2.5634, + "theoretical_loss": 3.3079705372109434, + "tokens_seen": 3155230720 + }, + { + "epoch": 0.91, + "learning_rate": 4.470707070707071e-05, + "loss": 2.3751, + "theoretical_loss": 3.3079653084708354, + "tokens_seen": 3155296256 + }, + { + "epoch": 0.91, + "learning_rate": 4.4686868686868686e-05, + "loss": 2.5129, + "theoretical_loss": 3.3079600798697353, + "tokens_seen": 3155361792 + }, + { + "epoch": 0.91, + "learning_rate": 4.466666666666667e-05, + "loss": 2.5421, + "theoretical_loss": 3.3079548514076373, + "tokens_seen": 3155427328 + }, + { + "epoch": 0.91, + "learning_rate": 4.464646464646465e-05, + "loss": 2.5352, + "theoretical_loss": 3.307949623084535, + "tokens_seen": 3155492864 + }, + { + "epoch": 0.91, + "learning_rate": 4.4626262626262625e-05, + "loss": 2.5649, + "theoretical_loss": 3.307944394900421, + "tokens_seen": 3155558400 + }, + { + "epoch": 0.91, + "learning_rate": 4.460606060606061e-05, + "loss": 2.5088, + "theoretical_loss": 3.307939166855289, + "tokens_seen": 3155623936 + }, + { + "epoch": 0.91, + "learning_rate": 4.460606060606061e-05, + "loss": 2.2653, + "theoretical_loss": 3.307933938949132, + "tokens_seen": 3155689472 + }, + { + "epoch": 0.91, + "learning_rate": 4.4585858585858586e-05, + "loss": 2.6617, + "theoretical_loss": 3.3079287111819444, + "tokens_seen": 3155755008 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1729376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.470989942550659, + "objective/train/theoretical_loss": 3.307923483553719, + "objective/train/tokens_used": 1514679776, + "theoretical_loss": 3.307923483553719, + "tokens_seen": 3155820544 + }, + { + "epoch": 0.91, + "learning_rate": 4.456565656565656e-05, + "loss": 2.5243, + "theoretical_loss": 3.307923483553719, + "tokens_seen": 3155820544 + }, + { + "epoch": 0.91, + "learning_rate": 4.454545454545455e-05, + "loss": 2.3407, + "theoretical_loss": 3.3079182560644496, + "tokens_seen": 3155886080 + }, + { + "epoch": 0.91, + "learning_rate": 4.4525252525252524e-05, + "loss": 2.5822, + "theoretical_loss": 3.307913028714129, + "tokens_seen": 3155951616 + }, + { + "epoch": 0.91, + "learning_rate": 4.450505050505051e-05, + "loss": 2.3692, + "theoretical_loss": 3.3079078015027514, + "tokens_seen": 3156017152 + }, + { + "epoch": 0.91, + "learning_rate": 4.448484848484849e-05, + "loss": 2.5143, + "theoretical_loss": 3.3079025744303094, + "tokens_seen": 3156082688 + }, + { + "epoch": 0.91, + "learning_rate": 4.446464646464646e-05, + "loss": 2.7139, + "theoretical_loss": 3.307897347496797, + "tokens_seen": 3156148224 + }, + { + "epoch": 0.91, + "learning_rate": 4.4444444444444447e-05, + "loss": 2.3544, + "theoretical_loss": 3.3078921207022076, + "tokens_seen": 3156213760 + }, + { + "epoch": 0.91, + "learning_rate": 4.4424242424242424e-05, + "loss": 2.4888, + "theoretical_loss": 3.3078868940465345, + "tokens_seen": 3156279296 + }, + { + "epoch": 0.91, + "learning_rate": 4.440404040404041e-05, + "loss": 2.6135, + "theoretical_loss": 3.307881667529771, + "tokens_seen": 3156344832 + }, + { + "epoch": 0.91, + "learning_rate": 4.4383838383838385e-05, + "loss": 2.3677, + "theoretical_loss": 3.3078764411519104, + "tokens_seen": 3156410368 + }, + { + "epoch": 0.91, + "learning_rate": 4.436363636363636e-05, + "loss": 2.5025, + "theoretical_loss": 3.3078712149129466, + "tokens_seen": 3156475904 + }, + { + "epoch": 0.91, + "learning_rate": 4.4343434343434346e-05, + "loss": 2.3018, + "theoretical_loss": 3.307865988812873, + "tokens_seen": 3156541440 + }, + { + "epoch": 0.91, + "learning_rate": 4.432323232323232e-05, + "loss": 2.7169, + "theoretical_loss": 3.3078607628516825, + "tokens_seen": 3156606976 + }, + { + "epoch": 0.91, + "learning_rate": 4.43030303030303e-05, + "loss": 2.5412, + "theoretical_loss": 3.307855537029369, + "tokens_seen": 3156672512 + }, + { + "epoch": 0.91, + "learning_rate": 4.4282828282828284e-05, + "loss": 2.5646, + "theoretical_loss": 3.3078503113459257, + "tokens_seen": 3156738048 + }, + { + "epoch": 0.91, + "learning_rate": 4.426262626262626e-05, + "loss": 2.4471, + "theoretical_loss": 3.307845085801346, + "tokens_seen": 3156803584 + } + ], + "max_steps": 25000, + "num_train_epochs": 9223372036854775807, + "total_flos": 7.6305634689024e+17, + "trial_name": null, + "trial_params": null +}